Merge pull request #270 from uktrade/release/huangjiu

Release Huangjiu

Author: reupen

README.rst

@@ -161,10 +161,13 @@ Leeloo can run on any Heroku style platform. These environment variables MUST be
 - DJANGO_SECRET_KEY
 - DJANGO_SENTRY_DSN
 - DJANGO_SETTINGS_MODULE
-- BULK_CREATE_BATCH_SIZE (default=50000)
+- BULK_CREATE_BATCH_SIZE (default=5000)
 - ES_HOST
 - ES_PORT
 - ES_INDEX
+- AWS_ACCESS_KEY_ID
+- AWS_SECRET_ACCESS_KEY
+- DOCUMENTS_BUCKET
 
 
 Management commands
@@ -195,3 +198,16 @@ Load metadata::
 
     docker-compose run leeloo python manage.py loaddata /app/fixtures/metadata.yaml
     docker-compose run leeloo python manage.py loaddata /app/fixtures/datahub_businesstypes.yaml
+
+Dependencies
+------------
+Direct dependencies are specified in ``requirements.in``. ``requirements.txt`` is a lock file generated using `pip-compile
+(from pip-tools) <https://github.com/jazzband/pip-tools>`_ and should not be manually edited.
+
+To update the lock file and indirect dependencies, run::
+
+    pip-compile --upgrade --output-file requirements.txt requirements.in
+
+This must be run whenever ``requirements.in`` is edited.
+
+Dependencies should still be installed using ``requirements.txt``.

config/api_urls.py

@@ -34,5 +34,7 @@
     url(r'^', include((investment_urls, 'investment'), namespace='investment')),
     url(r'^', include((leads_urls, 'business-leads'), namespace='business-leads')),
     url(r'^', include((company_urls.contact_urls, 'contact'), namespace='contact')),
+    url(r'^', include((company_urls.company_urls, 'company'), namespace='company')),
+    url(r'^', include((company_urls.ch_company_urls, 'ch-company'), namespace='ch-company')),
     url(r'^', include((search_urls, 'search'), namespace='search'))
 ]

config/settings/common.py

@@ -51,6 +51,7 @@
 LOCAL_APPS = [
     'datahub.core',
     'datahub.company',
+    'datahub.documents',
     'datahub.interaction',
     'datahub.investment',
     'datahub.leads',
@@ -149,7 +150,7 @@
     'DEFAULT_VERSIONING_CLASS': 'rest_framework.versioning.NamespaceVersioning',
     'DEFAULT_PAGINATION_CLASS': 'rest_framework.pagination.LimitOffsetPagination',
     'PAGE_SIZE': 100,
-    'DEFAULT_AUTHENTICATION_CLASSES': ['oauth2_provider.ext.rest_framework.OAuth2Authentication'],
+    'DEFAULT_AUTHENTICATION_CLASSES': ['oauth2_provider.contrib.rest_framework.OAuth2Authentication'],
     'DEFAULT_PERMISSION_CLASSES': ['rest_framework.permissions.IsAuthenticated'],
 }
 
@@ -168,4 +169,5 @@
 CDMS_TEXT_MAX_LENGTH = 4000
 CHAR_FIELD_MAX_LENGTH = 255
 HEROKU = False
-BULK_CREATE_BATCH_SIZE = env.int('BULK_CREATE_BATCH_SIZE', default=50000)
+BULK_CREATE_BATCH_SIZE = env.int('BULK_CREATE_BATCH_SIZE', default=5000)
+DOCUMENTS_BUCKET = env('DOCUMENTS_BUCKET')

config/settings/common_sentry.py

@@ -0,0 +1,54 @@
+from config.settings.common import *
+
+MIDDLEWARE.append('raven.contrib.django.raven_compat.middleware.SentryResponseErrorIdMiddleware')
+INSTALLED_APPS.append('raven.contrib.django.raven_compat')
+
+SENTRY_DSN = env('DJANGO_SENTRY_DSN')
+
+# Logging
+LOGGING = {
+    'version': 1,
+    'disable_existing_loggers': False,
+    'root': {
+        'level': 'INFO',
+        'handlers': ['sentry', 'console'],
+    },
+    'formatters': {
+        'verbose': {
+            'format': '%(asctime)s [%(levelname)s] [%(name)s] %(message)s'
+        },
+    },
+    'handlers': {
+        'sentry': {
+            'level': 'ERROR',
+            'class': 'raven.contrib.django.raven_compat.handlers.SentryHandler'
+        },
+        'console': {
+            'level': 'DEBUG',
+            'class': 'logging.StreamHandler',
+            'formatter': 'verbose'
+        },
+    },
+    'loggers': {
+        'django.db.backends': {
+            'level': 'ERROR',
+            'handlers': ['console'],
+            'propagate': False,
+        },
+        'raven': {
+            'level': 'DEBUG',
+            'handlers': ['console'],
+            'propagate': False,
+        },
+        'sentry.errors': {
+            'level': 'DEBUG',
+            'handlers': ['console'],
+            'propagate': False,
+        },
+    },
+}
+
+
+RAVEN_CONFIG = {
+    'DSN': SENTRY_DSN,
+}

config/settings/dev.py

@@ -0,0 +1,4 @@
+from .common_sentry import *
+
+# DRF
+REST_FRAMEWORK['DEFAULT_AUTHENTICATION_CLASSES'] += ['rest_framework.authentication.SessionAuthentication']

config/settings/local.py

@@ -24,3 +24,42 @@
     ],
     'SHOW_TEMPLATE_CONTEXT': True,
 }
+
+# This gets normal Python logging working with Django
+LOGGING = {
+    'version': 1,
+    'disable_existing_loggers': True,
+    'root': {
+        'level': 'INFO',
+        'handlers': ['console'],
+    },
+    'formatters': {
+        'verbose': {
+            'format': '[%(levelname)s] [%(name)s] %(message)s'
+        },
+    },
+    'handlers': {
+        'console': {
+            'level': 'DEBUG',
+            'class': 'logging.StreamHandler',
+            'formatter': 'verbose'
+        },
+    },
+    'loggers': {
+        'werkzeug': {
+            'handlers': ['console'],
+            'level': 'INFO',
+            'propagate': False
+        },
+        'django': {
+            'handlers': ['console'],
+            'level': 'INFO',
+            'propagate': False
+        },
+        'django.server': {
+            'handlers': ['console'],
+            'level': 'INFO',
+            'propagate': False,
+        },
+    },
+}

config/settings/production.py

@@ -1,60 +1,3 @@
-from .common import *
+from .common_sentry import *
 
-MIDDLEWARE += ('raven.contrib.django.raven_compat.middleware.SentryResponseErrorIdMiddleware',)
-INSTALLED_APPS += ('raven.contrib.django.raven_compat',)
-
-# Logging
-SENTRY_DSN = env('DJANGO_SENTRY_DSN')
-LOGGING = {
-    'version': 1,
-    'disable_existing_loggers': True,
-    'root': {
-        'level': 'WARNING',
-        'handlers': ['sentry'],
-    },
-    'formatters': {
-        'verbose': {
-            'format': '%(levelname)s %(asctime)s %(module)s '
-                      '%(process)d %(thread)d %(message)s'
-        },
-    },
-    'handlers': {
-        'sentry': {
-            'level': 'ERROR',
-            'class': 'raven.contrib.django.raven_compat.handlers.SentryHandler',
-        },
-        'console': {
-            'level': 'DEBUG',
-            'class': 'logging.StreamHandler',
-            'formatter': 'verbose'
-        }
-    },
-    'loggers': {
-        'django.db.backends': {
-            'level': 'ERROR',
-            'handlers': ['console'],
-            'propagate': False,
-        },
-        'raven': {
-            'level': 'DEBUG',
-            'handlers': ['console'],
-            'propagate': False,
-        },
-        'sentry.errors': {
-            'level': 'DEBUG',
-            'handlers': ['console'],
-            'propagate': False,
-        },
-        'django.security.DisallowedHost': {
-            'level': 'ERROR',
-            'handlers': ['console', 'sentry'],
-            'propagate': False,
-        },
-    },
-}
-
-
-RAVEN_CONFIG = {
-    'DSN': SENTRY_DSN,
-}
 STATICFILES_STORAGE = 'whitenoise.django.GzipManifestStaticFilesStorage'

config/settings/sample.env

@@ -7,3 +7,7 @@ ES_INDEX=test
 DATAHUB_SECRET=secret
 CDMS_AUTH_URL=http://example.com
 DOCKER_DEV=True
+AWS_DEFAULT_REGION=eu-west-2
+AWS_ACCESS_KEY_ID=foo
+AWS_SECRET_ACCESS_KEY=bar
+DOCUMENTS_BUCKET=baz

config/settings/staging.py

@@ -1,4 +1,4 @@
-from .common import *
+from .common_sentry import *
 
 # DRF
 REST_FRAMEWORK['DEFAULT_AUTHENTICATION_CLASSES'] += ['rest_framework.authentication.SessionAuthentication']

datahub/company/admin.py

@@ -11,13 +11,15 @@ class CompanyAdmin(VersionAdmin):
     """Company admin."""
 
     search_fields = ['name', 'id', 'company_number']
+    raw_id_fields = ('parent', 'one_list_account_owner', 'archived_by')
 
 
 @admin.register(Contact)
 class ContactAdmin(VersionAdmin):
     """Contact admin."""
 
     search_fields = ['first_name', 'last_name', 'company__name']
+    raw_id_fields = ('company', 'adviser', 'archived_by')
 
 
 @admin.register(CompaniesHouseCompany)
@@ -57,7 +59,7 @@ class AdviserAdmin(VersionAdmin, UserAdmin):
         }),
         ('Permissions', {
             'fields': (
-                'enabled',
+                'use_cdms_auth',
                 'is_active',
                 'is_staff',
                 'is_superuser',
@@ -78,10 +80,10 @@ class AdviserAdmin(VersionAdmin, UserAdmin):
             'fields': ('email', 'password1', 'password2'),
         }),
     )
-    list_display = ('email', 'first_name', 'last_name', 'is_staff', 'enabled')
+    list_display = ('email', 'first_name', 'last_name', 'is_staff', 'use_cdms_auth')
     search_fields = ('first_name', 'last_name', 'email')
-    ordering = ('email', 'enabled')
-    list_filter = ('enabled',)
+    ordering = ('email', 'use_cdms_auth')
+    list_filter = ('use_cdms_auth',)
     actions = ['enable_users', 'disable_users']
 
     def reversion_register(self, model, **kwargs):
@@ -91,12 +93,12 @@ def reversion_register(self, model, **kwargs):
 
     def enable_users(self, request, queryset):
         """Enable users for login."""
-        queryset.update(enabled=True)
+        queryset.update(use_cdms_auth=True)
 
     enable_users.short_description = 'Enable users'
 
     def disable_users(self, request, queryset):
         """Disable users for login."""
-        queryset.update(enabled=False)
+        queryset.update(use_cdms_auth=False)
 
     disable_users.short_description = 'Disable users.'

datahub/company/management/commands/manageusers.py

@@ -39,4 +39,4 @@ def handle(self, *args, **options):
             enabled = True
         else:
             enabled = False
-        user_model.objects.filter(email__in=options['users']).update(enabled=enabled)
+        user_model.objects.filter(email__in=options['users']).update(use_cdms_auth=enabled)

datahub/company/management/commands/sync_ch.py

@@ -11,12 +11,11 @@
 import requests
 from django.conf import settings
 from django.core.management.base import BaseCommand
-from django.db import connection, transaction
+from django.db import connection, reset_queries, transaction
 from lxml import etree
-from raven.contrib.django.raven_compat.models import client
 
 from datahub.company.models import CompaniesHouseCompany
-from datahub.core.utils import log_and_ignore_exceptions, slice_iterable_into_chunks, stream_to_file_pointer
+from datahub.core.utils import slice_iterable_into_chunks, stream_to_file_pointer
 
 logger = getLogger(__name__)
 
@@ -73,6 +72,7 @@ def open_ch_zipped_csv(fp):
 
 def iter_ch_csv_from_url(url, tmp_file_creator):
     """Fetch & cache CH zipped CSV, and then iterate though contents."""
+    logger.info('Loading CSV from URL: %s', url)
     with tmp_file_creator() as tf:
         stream_to_file_pointer(url, tf)
         tf.seek(0, 0)
@@ -90,18 +90,32 @@ def sync_ch(tmp_file_creator, endpoint=None, truncate_first=False):
     https://github.com/django/django/blob/master/django/db/models/query.py#L420
     this would create a list with millions of objects, that will try to be saved in batches in a single transaction
     """
+    logger.info('Starting CH load...')
+    count = 0
     endpoint = endpoint or settings.CH_DOWNLOAD_URL
     ch_csv_urls = get_ch_latest_dump_file_list(endpoint)
+    logger.info('Found the following Companies House CSV URLs: %s', ch_csv_urls)
     if truncate_first:
         truncate_ch_companies_table()
     for csv_url in ch_csv_urls:
         ch_company_rows = iter_ch_csv_from_url(csv_url, tmp_file_creator)
-        for batchiter in slice_iterable_into_chunks(ch_company_rows, settings.BULK_CREATE_BATCH_SIZE):
-            objects = [CompaniesHouseCompany(**ch_company_row) for ch_company_row in batchiter if ch_company_row]
+
+        batch_iter = slice_iterable_into_chunks(
+            ch_company_rows, settings.BULK_CREATE_BATCH_SIZE, _create_ch_company
+        )
+        for batch in batch_iter:
             CompaniesHouseCompany.objects.bulk_create(
-                objs=objects,
+                objs=batch,
                 batch_size=settings.BULK_CREATE_BATCH_SIZE
             )
+            count += len(batch)
+            logger.info('%d Companies House records loaded...', count)
+            # In debug mode, Django keeps track of SQL statements executed which
+            # eventually leads to memory exhaustion.
+            # This clears that history.
+            reset_queries()
+
+    logger.info('Companies House load complete, %s records loaded', count)
 
 
 @transaction.atomic
@@ -112,20 +126,18 @@ def truncate_ch_companies_table():
     """
     cursor = connection.cursor()
     table_name = CompaniesHouseCompany._meta.db_table
+    logger.info('Truncating the %s table', table_name)
     query = f'truncate {table_name};'
     cursor.execute(query)
 
 
+def _create_ch_company(row_dict):
+    return CompaniesHouseCompany(**row_dict)
+
+
 class Command(BaseCommand):
     """Companies House sync command."""
 
     def handle(self, *args, **options):
         """Handle."""
-        try:
-            sync_ch(tmp_file_creator=tempfile.TemporaryFile, truncate_first=True)
-        except Exception as e:
-            with log_and_ignore_exceptions():
-                client.captureException()
-
-            logger.exception('Failed to sync from ES')
-            self.stderr.write(e)
+        sync_ch(tmp_file_creator=tempfile.TemporaryFile, truncate_first=True)

datahub/company/migrations/0001_squash_0030_initial.py

@@ -116,7 +116,7 @@ class Migration(migrations.Migration):
                 ('future_interest_countries', models.ManyToManyField(blank=True, related_name='company_future_interest_countries', to='metadata.Country')),
                 ('headquarter_type', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='metadata.HeadquarterType')),
                 ('one_list_account_owner', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='one_list_owned_companies', to=settings.AUTH_USER_MODEL)),
-                ('parent', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='subsidiaries', to='company.Company')),
+                ('parent', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='children', to='company.Company')),
                 ('registered_address_country', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='companys', to='metadata.Country')),
                 ('sector', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to='metadata.Sector')),
                 ('trading_address_country', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='company_trading_address_country', to='metadata.Country')),