Datamodel with Django (#150)

* setup django infrastructure

* python manage.py inspectdb > models.py

* Adjust model definition and create migrations

* adjust geometry field

* import script as a management command (draft)

* (readme)

* (todos)

* replace sections import script by cleaned data and import script

* reorganize datamodel according to qdmtk

* (readme)

* add qdmtk_addlayer to specify which models get added

* (reformat initial migration)

* fix initial migration

* (full config)

* fix search path in migration

* remove obsolete model

* (update requirements.txt)

* fix decimal fields

* (add dev db in readme)

* move tables to comptages schema

* pin qdmtk version

* bump qdmtk version

* fix readme

* fix moving tables to schemas on initial fake-migration

* Register datamodel earlier

* Use Django ORM in importer vbv

* Get Django database config from plugin settings

* Remove base tjm ok migration

* Move setup instruction from readme to actual doc

* Use qdmtk from pypi

* Add bulk create manager

Co-authored-by: Mario Baranzini <mario@opengis.ch>

comptages/__init__.py

@@ -1,3 +1,26 @@
+from comptages.datamodel import config
+from comptages.core.settings import Settings
+
+# Register the datamodel
+try:
+
+    settings = Settings()
+    DATABASE = {
+        "ENGINE": "django.contrib.gis.db.backends.postgis",
+        "HOST": settings.value("db_host"),
+        "PORT": settings.value("db_port"),
+        "NAME": settings.value("db_name"),
+        "USER": settings.value("db_username"),
+        "PASSWORD": settings.value("db_password"),
+    }
+
+    from qdmtk import register_datamodel
+    register_datamodel(config.DATAMODEL_NAME, config.INSTALLED_APPS, DATABASE)
+except ImportError:
+    pass
+    #    self.iface.messageBar().pushMessage("Could not load QDMTK.", "You must install the QDMTK plugin prior to using Comptages", level=Qgis.Critical)
+
+
 def classFactory(iface):
     """Load Comptages class from file comptages.
 

comptages/comptages.py

@@ -9,6 +9,7 @@
     QgsProject)
 from qgis.utils import qgsfunction, plugins
 
+
 from comptages.core.settings import Settings, SettingsDialog
 from comptages.core.layers import Layers
 from comptages.core.filter_dialog import FilterDialog

comptages/datamodel/apps.py

@@ -0,0 +1,24 @@
+from django.apps import AppConfig
+from django.db.models.signals import post_migrate
+
+from django.db import connection
+
+def move_tables_to_schemas(sender, **kwargs):
+    """
+    This moves all tables of this model to the app's schema
+    """
+    app = sender
+    with connection.cursor() as cursor:
+        cursor.execute(f"CREATE SCHEMA IF NOT EXISTS {app.label};")
+        for model in app.get_models():
+            query = f'ALTER TABLE IF EXISTS {model._meta.db_table} SET SCHEMA {app.label};'
+            print(query)
+            cursor.execute(query)
+
+
+class ComptagesConfig(AppConfig):
+    name = 'comptages.datamodel'
+    label = 'comptages'
+
+    def ready(self):
+        post_migrate.connect(move_tables_to_schemas)

comptages/datamodel/config.py

@@ -0,0 +1,10 @@
+DATAMODEL_NAME = "comptages"
+DATABASE = {
+    "ENGINE": "django.contrib.gis.db.backends.postgis",
+    "HOST": "127.0.0.1",
+    "PORT": "5432",
+    "NAME": "comptages",
+    "USER": "postgres",
+    "PASSWORD": "postgres",
+}
+INSTALLED_APPS = ["comptages.datamodel"]

comptages/datamodel/management/commands/importcomptage.py

@@ -0,0 +1,149 @@
+"""
+Reimplementation of data_importer_vbv1.py as a Django management command
+"""
+
+
+import functools
+from datetime import datetime
+from django.core.management.base import BaseCommand, CommandError
+from ...models import Category, Count, CountDetail, Installation, Lane, Model, Brand, Section, SensorType
+
+IMPORT_STATUS_QUARANTINE = 1
+
+
+class Command(BaseCommand):
+    help = 'Import comptages files'
+
+    def add_arguments(self, parser):
+        parser.add_argument('file')
+
+    def handle(self, *args, **options):
+        with open(options['file']) as f:
+
+            # Parse headers
+            headers = {}
+            for line in f:
+                if not line.startswith('* '):
+                    continue
+                try:
+                    key, val = line[2:].split('=', 1)
+                    headers[key.strip()] = val.strip()
+                except ValueError:
+                    pass
+
+            # Get the count instance
+            installation_instance, _ = Installation.objects.get_or_create(
+                name=headers["SITE"],
+                defaults={
+                    "permanent": True,
+                    "active": True,
+                },
+            )
+
+            start = datetime.strptime(headers['STARTREC'], "%H:%M %d/%m/%y")
+            stop = datetime.strptime(headers['STOPREC'], "%H:%M %d/%m/%y")
+            count_instance = Count.objects.create(
+                id_installation=installation_instance,
+                start_service_date=start,
+                end_service_date=stop,
+                start_process_date=start,
+                end_process_date=start,
+                id_model=Model.objects.create(name="Unknown", id_brand=Brand.objects.create(name="Unknown")),
+                id_sensor_type=SensorType.objects.create(name="Unknown"),
+            )
+
+            if count_instance is None:
+                site_instance, _ = Installation.objects.get_or_create(
+                    name=headers["SITE"],
+                    active=True,
+                )
+                count_instance = Count.objects.create(
+                    id_installation=site_instance,
+                    start_service_date=headers["STARTREC"],
+                    end_service_date=headers["STOPREC"],
+                )
+
+
+            # Import lines
+            f.seek(0)
+            instances = []
+            for line in f:
+                if line.startswith('* '):
+                    continue
+
+                # Parse the line
+                parsed_line = {}
+                try:
+                    parsed_line['numbering'] = line[0:6]
+                    parsed_line['timestamp'] = datetime.strptime("{}0000".format(line[7:24]), "%d%m%y %H%M %S %f")
+                    parsed_line['reserve_code'] = line[25:31]
+                    parsed_line['lane'] = int(line[32:34])
+                    parsed_line['direction'] = int(line[35:36])
+                    parsed_line['distance_front_front'] = float(line[37:41])
+                    parsed_line['distance_front_back'] = float(line[42:46])
+                    parsed_line['speed'] = int(line[47:50])
+                    parsed_line['length'] = int(line[52:56])
+                    parsed_line['category'] = int(line[60:62].strip())
+                    parsed_line['height'] = line[63:65].strip()
+
+                    # If the speed of a vehicle is 0, we put it in the category 0
+                    if parsed_line['speed'] == 0:
+                        parsed_line['category'] = 0
+
+                    # If the speed of a vehicle is greater than 3*max_speed or 150km/h
+                    # TODO: get actual speed limit of the section
+                    if parsed_line['speed'] > 150:
+                        parsed_line['category'] = 0
+
+                except ValueError:
+                    # This can happen when some values are missed from a line
+
+                    if 'lane' not in parsed_line:
+                        print("Could not parse line")
+                        continue
+                    if 'direction' not in parsed_line:
+                        print("Could not parse line")
+                        continue
+                    if 'distance_front_front' not in parsed_line:
+                        parsed_line['distance_front_front'] = 0
+                    if 'distance_front_back' not in parsed_line:
+                        parsed_line['distance_front_back'] = 0
+                    if 'speed' not in parsed_line:
+                        parsed_line['speed'] = -1
+                    if 'length' not in parsed_line:
+                        parsed_line['length'] = 0
+                    if 'category' not in parsed_line:
+                        parsed_line['category'] = 0
+                    if 'height ' not in parsed_line:
+                        parsed_line['height'] = 'NA'
+
+                row = parsed_line
+
+                instances.append(
+                    CountDetail(
+                        numbering=row['numbering'],
+                        timestamp=row['timestamp'],
+                        distance_front_front=row['distance_front_front'],
+                        distance_front_back=row['distance_front_back'],
+                        speed=row['speed'],
+                        length=row['length'],
+                        height=row['height'],
+                        file_name=options['file'],
+                        import_status=IMPORT_STATUS_QUARANTINE,
+                        id_lane=_get_lane(count_instance, row['lane']),
+                        id_count=count_instance,
+                        id_category=Category(id=row['category']),
+                    )
+                )
+
+            CountDetail.objects.bulk_create(instances)
+            self.stdout.write(self.style.SUCCESS(f'Successfully created {len(instances)} comptages'))
+
+
+@functools.lru_cache(maxsize=None)
+def _get_lane(count_instance, lane_number):
+    # TODO : replace this by actual lane creation/retrieval logic
+    section = Section.objects.get_or_create(name="Unknown", defaults={
+        "geometry": "LINESTRING(0 0, 10 10)"
+    })[0]
+    return Lane.objects.get_or_create(number=lane_number, direction=0, id_section=section)[0]

comptages/datamodel/management/commands/importsections.py

@@ -0,0 +1,54 @@
+import logging
+import os
+from decimal import Decimal
+from django.contrib.gis.gdal import DataSource
+from django.contrib.gis.gdal.error import GDALException
+from django.core.management.base import BaseCommand
+
+from ...models import Section, Lane
+
+logger = logging.getLogger("main")
+
+
+class Command(BaseCommand):
+    help = "Import sections from file"
+
+    def add_arguments(self, parser):
+        parser.add_argument('--sections_file', default=os.path.join(os.path.dirname(__file__),'..','..','..','..','db','sections_clean.csv'))
+        parser.add_argument("--clear", action="store_true", help="Delete existing data")
+
+    def handle(self, *args, **options):
+
+        ds_sections = DataSource(options["sections_file"])
+
+        layer_sections = ds_sections[0]
+
+        if options["clear"]:
+            print("Deleting...")
+            Lane.objects.all().delete()
+            Section.objects.all().delete()
+
+        print("Importing sections...")
+        sections = []
+        for feat in layer_sections:
+            sections.append(
+                Section(
+                    geometry=feat.geom.wkt,
+                    id=feat["id"],
+                    name=feat["name"],
+                    owner=feat["owner"],
+                    road=feat["road"],
+                    way=feat["way"],
+                    start_pr=feat["start_pr"],
+                    end_pr=feat["end_pr"],
+                    start_dist=Decimal(feat["start_dist"].value),
+                    end_dist=Decimal(feat["end_dist"].value),
+                    place_name=feat["place_name"],
+                    start_validity=feat["start_validity"].value,  # TODO : probably needs cast do datetime
+                    end_validity=feat["end_validity"].value,  # TODO : probably needs cast do datetime
+                )
+            )
+        Section.objects.bulk_create(sections)
+        print(f"Inserted {len(sections)} sections.")
+
+        print("🚓")