Merge pull request #5879 from uktrade/feature/TET-812-ingest-stova-attendees

Feature/tet 812 ingest stova attendees

Author: DeanElliott96

datahub/cleanup/management/commands/delete_old_records.py

@@ -81,6 +81,7 @@ class Command(BaseCleanupCommand):
                 Company._meta.get_field('wins'): (),
                 Company._meta.get_field('legacy_wins'): (),
                 Company._meta.get_field('eyb_leads'): (),
+                Company._meta.get_field('stova_attendee'): (),
                 Company._meta.get_field('great_export_enquiries'): (),
             },
             # We want to delete the relations below along with any expired companies
@@ -109,6 +110,7 @@ class Command(BaseCleanupCommand):
                 Contact._meta.get_field('interactions'): (),
                 Contact._meta.get_field('investment_projects'): (),
                 Contact._meta.get_field('great_export_enquiries'): (),
+                Contact._meta.get_field('stova_attendee'): (),
                 Contact._meta.get_field('orders'): (),
                 Contact._meta.get_field('referrals'): (),
                 Contact._meta.get_field('pipeline_items_m2m'): (),

datahub/cleanup/management/commands/delete_orphans.py

@@ -26,6 +26,7 @@ class Command(BaseCleanupCommand):
             excluded_relations=(
                 Contact._meta.get_field('wins'),
                 Contact._meta.get_field('great_export_enquiries'),
+                Contact._meta.get_field('stova_attendee'),
             ),
         ),
         'company.Company': ModelCleanupConfig(
@@ -36,6 +37,7 @@ class Command(BaseCleanupCommand):
                 Company._meta.get_field('export_countries'),
                 Company._meta.get_field('export_countries_history'),
                 Company._meta.get_field('great_export_enquiries'),
+                Company._meta.get_field('stova_attendee'),
                 Company._meta.get_field('pipeline_list_items'),
                 Company._meta.get_field('new_export_interaction_reminders'),
                 Company._meta.get_field('no_recent_export_interaction_reminders'),

datahub/cleanup/test/commands/test_delete_old_records.py

@@ -40,6 +40,7 @@
     CompanyActivityInvestmentProjectFactory,
     CompanyActivityOmisOrderFactory,
     GreatExportEnquiryFactory,
+    StovaAttendeeFactory,
 )
 from datahub.company_referral.test.factories import (
     CompanyReferralFactory,
@@ -230,6 +231,12 @@
                 'expired_objects_kwargs': [],
                 'unexpired_objects_kwargs': [],
             },
+            {
+                'factory': StovaAttendeeFactory,
+                'field': 'company',
+                'expired_objects_kwargs': [],
+                'unexpired_objects_kwargs': [],
+            },
             {
                 'factory': EYBLeadFactory,
                 'field': 'company',
@@ -452,6 +459,16 @@
                     },
                 ],
             },
+            {
+                'factory': StovaAttendeeFactory,
+                'field': 'contact',
+                'expired_objects_kwargs': [],
+                'unexpired_objects_kwargs': [
+                    {
+                        'created_on': CONTACT_DELETE_BEFORE_DATETIME - relativedelta(days=1),
+                    },
+                ],
+            },
             {
                 'factory': OrderFactory,
                 'field': 'contact',

datahub/cleanup/test/commands/test_delete_orphaned_versions.py

@@ -25,6 +25,7 @@
     CompanyActivityIngestedFileFactory,
     CompanyActivityInteractionFactory,
     GreatExportEnquiryFactory,
+    StovaAttendeeFactory,
 )
 from datahub.company_referral.test.factories import (
     CompanyReferralFactory,
@@ -77,6 +78,7 @@
     'company_activity.CompanyActivity': CompanyActivityInteractionFactory,
     'company_activity.GreatExportEnquiry': GreatExportEnquiryFactory,
     'company_activity.IngestedFile': CompanyActivityIngestedFileFactory,
+    'company_activity.StovaAttendee': StovaAttendeeFactory,
     'company_list.CompanyListItem': CompanyListItemFactory,
     'company_list.PipelineItem': PipelineItemFactory,
     'company_referral.CompanyReferral': CompanyReferralFactory,

datahub/company/merge_company.py

@@ -25,7 +25,7 @@
     Objective,
     OneListCoreTeamMember,
 )
-from datahub.company_activity.models import CompanyActivity, GreatExportEnquiry
+from datahub.company_activity.models import CompanyActivity, GreatExportEnquiry, StovaAttendee
 from datahub.company_referral.models import CompanyReferral
 from datahub.dnb_api.utils import _get_rollback_version
 from datahub.export_win.models import LegacyExportWinsToDataHubCompany
@@ -72,6 +72,7 @@
     Objective.company.field,
     OneListCoreTeamMember.company.field,
     Order.company.field,
+    StovaAttendee.company.field,
     Task.company.field,
 
     # Merging is allowed if the source company has export countries, but note that
@@ -117,6 +118,7 @@
     MergeConfiguration(Order, ('company',), Company),
     MergeConfiguration(NewExportInteractionReminder, ('company',), Company),
     MergeConfiguration(NoRecentExportInteractionReminder, ('company',), Company),
+    MergeConfiguration(StovaAttendee, ('company',), Company),
     MergeConfiguration(Task, ('company',), Company),
     MergeConfiguration(CompanyListItem, ('company',), Company, company_list_item_updater),
     MergeConfiguration(PipelineItem, ('company',), Company, pipeline_item_updater),

datahub/company_activity/migrations/0022_add_stova_attendee.py

@@ -0,0 +1,41 @@
+# Generated by Django 4.2.17 on 2024-12-31 11:18
+
+from django.db import migrations, models
+import django.db.models.deletion
+import uuid
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('company', '0144_update_company_export_experience'),
+        ('company_activity', '0021_match_ingested_stova_fields'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='StovaAttendee',
+            fields=[
+                ('id', models.UUIDField(default=uuid.uuid4, primary_key=True, serialize=False)),
+                ('stova_attendee_id', models.IntegerField(unique=True)),
+                ('stova_event_id', models.IntegerField(unique=True)),
+                ('created_by', models.CharField(max_length=255)),
+                ('created_date', models.DateTimeField()),
+                ('modified_by', models.CharField(max_length=255)),
+                ('modified_date', models.DateTimeField()),
+                ('email', models.CharField(max_length=255)),
+                ('first_name', models.CharField(max_length=255)),
+                ('last_name', models.CharField(max_length=255)),
+                ('attendee_questions', models.CharField(max_length=255)),
+                ('company_name', models.CharField(max_length=255)),
+                ('category', models.CharField(max_length=255)),
+                ('registration_status', models.CharField(max_length=255)),
+                ('virtual_event_attendance', models.CharField(max_length=255)),
+                ('language', models.CharField(max_length=255)),
+                ('last_lobby_login', models.DateTimeField()),
+                ('created_on', models.DateTimeField(auto_now_add=True)),
+                ('company', models.ForeignKey(blank=True, help_text='If a company match can be found from company_name, the relation is added.', null=True, on_delete=django.db.models.deletion.PROTECT, related_name='stova_attendee', to='company.company')),
+                ('contact', models.ForeignKey(blank=True, help_text='If a contact match can be found from the email, the relation is added.', null=True, on_delete=django.db.models.deletion.PROTECT, related_name='stova_attendee', to='company.contact')),
+            ],
+        ),
+    ]

datahub/company_activity/models/__init__.py

@@ -1,12 +1,14 @@
 from datahub.company_activity.models.company_activity import CompanyActivity
 from datahub.company_activity.models.great import GreatExportEnquiry
 from datahub.company_activity.models.ingested_file import IngestedFile
+from datahub.company_activity.models.stova_attendee import StovaAttendee
 from datahub.company_activity.models.stova_event import StovaEvent
 
 
 __all__ = (
     CompanyActivity,
     GreatExportEnquiry,
     IngestedFile,
+    StovaAttendee,
     StovaEvent,
 )

datahub/company_activity/models/stova_attendee.py

@@ -0,0 +1,61 @@
+import uuid
+
+from django.conf import settings
+from django.db import models
+
+from datahub.company.models.company import Company
+from datahub.company.models.contact import Contact
+from datahub.core import reversion
+
+
+MAX_LENGTH = settings.CHAR_FIELD_MAX_LENGTH
+
+
+@reversion.register_base_model()
+class StovaAttendee(models.Model):
+    """
+    Stova can also be known as Aventri.
+    This model is filled and based off data from the S3 bucket: ExportAventriAttendees.
+    """
+
+    id = models.UUIDField(primary_key=True, default=uuid.uuid4)
+    stova_attendee_id = models.IntegerField(unique=True)
+    stova_event_id = models.IntegerField(unique=True)
+
+    created_by = models.CharField(max_length=MAX_LENGTH)
+    created_date = models.DateTimeField()
+    modified_by = models.CharField(max_length=MAX_LENGTH)
+    modified_date = models.DateTimeField()
+
+    email = models.CharField(max_length=MAX_LENGTH)
+    first_name = models.CharField(max_length=MAX_LENGTH)
+    last_name = models.CharField(max_length=MAX_LENGTH)
+    attendee_questions = models.CharField(max_length=MAX_LENGTH)
+
+    company_name = models.CharField(max_length=MAX_LENGTH)
+    category = models.CharField(max_length=MAX_LENGTH)
+    registration_status = models.CharField(max_length=MAX_LENGTH)
+
+    virtual_event_attendance = models.CharField(max_length=MAX_LENGTH)
+    language = models.CharField(max_length=MAX_LENGTH)
+
+    last_lobby_login = models.DateTimeField()
+
+    # Data Hub Fields
+    created_on = models.DateTimeField(auto_now_add=True)
+    company = models.ForeignKey(
+        Company,
+        on_delete=models.PROTECT,
+        null=True,
+        blank=True,
+        related_name='stova_attendee',
+        help_text='If a company match can be found from company_name, the relation is added.',
+    )
+    contact = models.ForeignKey(
+        Contact,
+        on_delete=models.PROTECT,
+        null=True,
+        blank=True,
+        related_name='stova_attendee',
+        help_text='If a contact match can be found from the email, the relation is added.',
+    )

datahub/company_activity/tasks/__init__.py

@@ -1,4 +1,5 @@
 from datahub.company_activity.tasks.ingest_great_data import ingest_great_data
+from datahub.company_activity.tasks.ingest_stova_attendees import stova_attendee_ingestion_task
 from datahub.company_activity.tasks.ingest_stova_events import (
     ingest_stova_event_data,
     stova_ingestion_task,
@@ -7,5 +8,6 @@
 __all__ = (
     ingest_great_data,
     ingest_stova_event_data,
+    stova_attendee_ingestion_task,
     stova_ingestion_task,
 )

datahub/company_activity/tasks/constants.py

@@ -5,4 +5,5 @@
 BUCKET = f"data-flow-bucket-{env('ENVIRONMENT', default='')}"
 PREFIX = 'data-flow/exports/'
 GREAT_PREFIX = f'{PREFIX}ExportGreatContactFormData/'
+STOVA_ATTENDEE_PREFIX = f'{PREFIX}ExportAventriAttendees/'
 STOVA_EVENT_PREFIX = f'{PREFIX}ExportAventriEvents/'

datahub/company_activity/tasks/ingest_stova_attendees.py

@@ -0,0 +1,86 @@
+import logging
+
+from django.core.exceptions import ValidationError
+from django.db import IntegrityError
+
+from datahub.company_activity.models import StovaAttendee
+from datahub.company_activity.tasks.constants import STOVA_ATTENDEE_PREFIX
+from datahub.ingest.boto3 import S3ObjectProcessor
+from datahub.ingest.tasks import BaseObjectIdentificationTask, BaseObjectIngestionTask
+
+
+logger = logging.getLogger(__name__)
+DATE_FORMAT = '%Y-%m-%dT%H:%M:%S.%f'
+
+
+def ingest_stova_attendee_data() -> None:
+    """Identifies the most recent file to be ingested and schedules a task to ingest it"""
+    logger.info('Stova attendee identification task started.')
+    identification_task = StovaAttendeeIndentificationTask(prefix=STOVA_ATTENDEE_PREFIX)
+    identification_task.identify_new_objects(stova_attendee_ingestion_task)
+    logger.info('Stova attendee identification task finished.')
+
+
+def stova_attendee_ingestion_task(object_key: str) -> None:
+    """Ingest the given key (file) from S3"""
+    logger.info(f'Stova attendee ingestion task started for file {object_key}.')
+    ingestion_task = StovaAttendeeIngestionTask(
+        object_key=object_key,
+        s3_processor=S3ObjectProcessor(prefix=STOVA_ATTENDEE_PREFIX),
+    )
+    ingestion_task.ingest_object()
+    logger.info(f'Stova attendee ingestion task finished for file {object_key}.')
+
+
+class StovaAttendeeIndentificationTask(BaseObjectIdentificationTask):
+    pass
+
+
+class StovaAttendeeIngestionTask(BaseObjectIngestionTask):
+
+    existing_ids = []
+
+    def _process_record(self, record: dict) -> None:
+        """Saves an attendee from Stova from the S3 bucket into a `Stovaattendee`"""
+        if not self.existing_ids:
+            self.existing_ids = set(
+                StovaAttendee.objects.values_list('stova_attendee_id', flat=True),
+            )
+
+        stova_attendee_id = record.get('id')
+        if stova_attendee_id in self.existing_ids:
+            logger.info(f'Record already exists for stova_attendee_id: {stova_attendee_id}')
+            return
+
+        values = {
+            'stova_attendee_id': stova_attendee_id,
+            'stova_event_id': record.get('event_id', ''),
+            'created_date': record.get('created_date'),
+            'email': record.get('email', ''),
+            'first_name': record.get('first_name', ''),
+            'last_name': record.get('last_name', ''),
+            'company_name': record.get('company_name', ''),
+            'category': record.get('category', ''),
+            'registration_status': record.get('registration_status', ''),
+            'created_by': record.get('created_by', ''),
+            'language': record.get('language', ''),
+            'modified_date': record.get('modified_date'),
+            'virtual_event_attendance': record.get('virtual_event_attendance', ''),
+            'last_lobby_login': record.get('last_lobby_login', ''),
+            'attendee_questions': record.get('attendee_questions', ''),
+            'modified_by': record.get('modified_by', ''),
+        }
+
+        try:
+            StovaAttendee.objects.create(**values)
+        except IntegrityError as error:
+            logger.error(
+                f'Error processing Stova attendee record, stova_attendee_id: {stova_attendee_id}. '
+                f'Error: {error}',
+            )
+        except ValidationError as error:
+            logger.error(
+                'Got unexpected value for a field when processing Stova attendee record, '
+                f'stova_attendee_id: {stova_attendee_id}. '
+                f'Error: {error}',
+            )

datahub/company_activity/tests/factories.py

@@ -272,3 +272,32 @@ class StovaEventFactory(factory.django.DjangoModelFactory):
 
     class Meta:
         model = 'company_activity.StovaEvent'
+
+
+class StovaAttendeeFactory(factory.django.DjangoModelFactory):
+    """
+    Ingested Stova Attendee data factory.
+    """
+
+    stova_event_id = factory.Faker('pyint', min_value=0, max_value=999999999)
+    stova_attendee_id = factory.Faker('pyint', min_value=0, max_value=999999999)
+
+    created_by = 'John'
+    created_date = now()
+    modified_by = 'John'
+    modified_date = now()
+
+    email = 'john@test.com'
+    first_name = 'John'
+    last_name = 'Smith'
+    attendee_questions = 'This is a question'
+
+    company_name = 'A company name'
+    category = 'A category'
+    registration_status = 'The registration status'
+    virtual_event_attendance = 'Virtual Event Attendance'
+    language = 'English'
+    last_lobby_login = now()
+
+    class Meta:
+        model = 'company_activity.StovaAttendee'