#681 Refactored backup's, compressing is now mandatory

Author: dennissiemensma

.coveragerc

@@ -8,3 +8,6 @@ omit = ./*/__init__.py
   ./dsmr_backend/mixins.py
   ./dsmr_plugins/modules/*
   ./dsmr_dropbox/dropboxinc/*
+
+[html]
+directory = coverage_report/html

backups/.dummy

@@ -0,0 +1 @@
+# Backups met alleen dagstatistieken, voor noodgevallen en tegen corrupte filesystems. Worden niet geroteerd.

backups/.gitkeep

@@ -30,7 +30,7 @@ class BackupSettingsAdmin(SingletonModelAdmin):
         ),
         (
             _('Advanced'), {
-                'fields': ['folder', 'compress'],
+                'fields': ['folder'],
             }
         ),
         (

backups/archive/.gitkeep

@@ -0,0 +1,44 @@
+import os
+
+from django.core.management.base import BaseCommand, CommandError
+
+from dsmr_stats.models.statistics import DayStatistics, HourStatistics
+import dsmr_backup.services.backup
+
+
+class Command(BaseCommand):
+    help = 'Forces compact (statistics) backup creation.'
+
+    def add_arguments(self, parser):
+        super(Command, self).add_arguments(parser)
+        parser.add_argument(
+            '--full',
+            action='store_true',
+            dest='full',
+            default=False,
+            help='Whether to create a full backup'
+        )
+        parser.add_argument(
+            '--compact',
+            action='store_true',
+            dest='compact',
+            default=False,
+            help='Whether to create a compact backup, only containing day- and hour statistics'
+        )
+
+    def handle(self, **options):
+        if not options.get('full') and not options.get('compact'):
+            raise CommandError('Missing --full or --compact argument')
+
+        base_folder = dsmr_backup.services.backup.get_backup_directory()
+
+        if options.get('full'):
+            dsmr_backup.services.backup.create_full(
+                folder=base_folder
+            )
+
+        if options.get('compact'):
+            dsmr_backup.services.backup.create_partial(
+                folder=os.path.join(base_folder, 'archive'),
+                models_to_backup=(DayStatistics, HourStatistics)
+            )

backups/archive/README.txt

@@ -0,0 +1,17 @@
+# Generated by Django 2.2.4 on 2019-08-15 18:10
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('dsmr_backup', '0006_scheduled_email_backup'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='backupsettings',
+            name='compress',
+        ),
+    ]

dsmr_backup/admin.py

@@ -12,11 +12,6 @@ class BackupSettings(SingletonModel):
         verbose_name=_('Backup daily'),
         help_text=_('Create a backup of your data daily. Stored locally, but can be exported using Dropbox.')
     )
-    compress = models.BooleanField(
-        default=True,
-        verbose_name=_('Compress'),
-        help_text=_('Create backups in compressed (gzip) format, saving a significant amount of disk space.')
-    )
     backup_time = models.TimeField(
         default=time(hour=2),
         verbose_name=_('Backup timestamp'),

dsmr_backup/backend/__init__.py

@@ -9,7 +9,7 @@
 from django.conf import settings
 from django.utils import formats
 
-from dsmr_stats.models.statistics import DayStatistics, HourStatistics
+from dsmr_stats.models.statistics import DayStatistics
 from dsmr_backup.models.settings import BackupSettings
 import dsmr_dropbox.services
 
@@ -38,7 +38,18 @@ def check():
         # Postpone when the user's backup time preference has not been passed yet.
         return
 
-    create()
+    # Create a partial, minimal backup first.
+    create_partial(
+        folder=os.path.join(get_backup_directory(), 'archive'),
+        models_to_backup=(DayStatistics, )
+    )
+
+    # Now create full.
+    create_full(folder=get_backup_directory())
+
+    backup_settings = BackupSettings.get_solo()
+    backup_settings.latest_backup = timezone.now()
+    backup_settings.save()
 
 
 def get_backup_directory():
@@ -51,18 +62,18 @@ def get_backup_directory():
         return os.path.abspath(os.path.join(settings.BASE_DIR, '..', backup_directory))
 
 
-def create():
+def create_full(folder):
     """ Creates a backup of the database. Optionally gzipped. """
-    backup_directory = get_backup_directory()
-
-    if not os.path.exists(backup_directory):
-        os.mkdir(backup_directory)
+    if not os.path.exists(folder):
+        logger.info(' - Creating non-existing backup folder: %s', folder)
+        os.makedirs(folder)
 
     # Backup file with day name included, for weekly rotation.
-    backup_file = os.path.join(backup_directory, 'dsmrreader-{}-backup-{}.sql'.format(
+    backup_file = os.path.join(folder, 'dsmrreader-{}-backup-{}.sql'.format(
         connection.vendor, formats.date_format(timezone.now().date(), 'l')
     ))
-    logger.info(' - Creating new backup: %s', backup_file)
+
+    logger.info(' - Creating new full backup: %s', backup_file)
 
     # PostgreSQL backup.
     if connection.vendor == 'postgresql':  # pragma: no cover
@@ -107,35 +118,33 @@ def create():
         raise NotImplementedError('Unsupported backup backend: {}'.format(connection.vendor))  # pragma: no cover
 
     backup_process.wait()
-    backup_settings = BackupSettings.get_solo()
-    logger.debug(' - Created backup: %s', backup_file)
-
-    if backup_settings.compress:
-        backup_file = compress(file_path=backup_file)
-        logger.debug(' - Compressed backup: %s', backup_file)
-
-    backup_settings.latest_backup = timezone.now()
-    backup_settings.save()
+    backup_file = compress(file_path=backup_file)
+    logger.debug(' - Created and compressed statistics backup: %s', backup_file)
 
 
-def create_statistics_backup(folder):  # pragma: no cover
-    """ Creates a backup of the database, but only containing the day and hour statistics."""
+def create_partial(folder, models_to_backup):  # pragma: no cover
+    """ Creates a backup of the database, but only containing a subset specified by models."""
     if connection.vendor != 'postgresql':
         # Only PostgreSQL support for newer features.
         raise NotImplementedError('Unsupported backup backend: {}'.format(connection.vendor))
 
-    backup_file = os.path.join(folder, 'dsmrreader-{}-backup-{}.sql'.format(
-        connection.vendor, formats.date_format(timezone.now().date(), 'l')
+    if not os.path.exists(folder):
+        logger.info(' - Creating non-existing backup folder: %s', folder)
+        os.makedirs(folder)
+
+    backup_file = os.path.join(folder, 'dsmrreader-{}-partial-backup-{}.sql'.format(
+        connection.vendor, formats.date_format(timezone.now().date(), 'Y-m-d')
     ))
 
+    logger.info(' - Creating new partial backup: %s', backup_file)
     backup_process = subprocess.Popen(
         [
             settings.DSMRREADER_BACKUP_PG_DUMP,
+            settings.DATABASES['default']['NAME'],
             '--host={}'.format(settings.DATABASES['default']['HOST']),
             '--user={}'.format(settings.DATABASES['default']['USER']),
-            '--table={}'.format(DayStatistics._meta.db_table),
-            '--table={}'.format(HourStatistics._meta.db_table),
-            settings.DATABASES['default']['NAME'],
+        ] + [
+            '--table={}'.format(x._meta.db_table) for x in models_to_backup
         ], env={
             'PGPASSWORD': settings.DATABASES['default']['PASSWORD']
         },

dsmr_backup/management/__init__.py

@@ -6,10 +6,10 @@
 
 from dsmr_backup.models.settings import EmailBackupSettings
 from dsmr_backend.models.settings import BackendSettings, EmailSettings
+from dsmr_stats.models.statistics import DayStatistics, HourStatistics
 import dsmr_backend.services.email
 import dsmr_backup.services.backup
 
-
 logger = logging.getLogger('commands')
 
 
@@ -22,7 +22,10 @@ def run(scheduled_process):
         return scheduled_process.delay(timezone.timedelta(days=1))
 
     temp_dir = tempfile.TemporaryDirectory()
-    backup_file = dsmr_backup.services.backup.create_statistics_backup(folder=temp_dir.name)
+    backup_file = dsmr_backup.services.backup.create_partial(
+        folder=temp_dir.name,
+        models_to_backup=(DayStatistics, HourStatistics)
+    )
 
     with translation.override(language=BackendSettings.get_solo().language):
         subject = _('DSMR-reader day/hour statistics backup')

dsmr_backup/management/commands/__init__.py

@@ -20,9 +20,6 @@ def test_to_string(self):
     def test_daily_backup(self):
         self.assertTrue(self.instance.daily_backup)
 
-    def test_compress(self):
-        self.assertTrue(self.instance.compress)
-
     def test_backup_time(self):
         self.assertTrue(self.instance.backup_time, time(hour=2))