#681 Refactored and simplified Dropbox sync mechanism

Author: dennissiemensma

.coveragerc

@@ -7,3 +7,4 @@ omit = ./*/__init__.py
   ./*/migrations/*
   ./dsmr_backend/mixins.py
   ./dsmr_plugins/modules/*
+  ./dsmr_dropbox/dropboxinc/*

docs/locale/nl/LC_MESSAGES/changelog.mo

@@ -8,8 +8,6 @@ msgid ""
 msgstr ""
 "Project-Id-Version: DSMR Reader v1.x\n"
 "Report-Msgid-Bugs-To: Dennis Siemensma <github@dennissiemensma.nl>\n"
-"POT-Creation-Date: 2019-08-11 15:09+0200\n"
-"PO-Revision-Date: 2019-08-11 15:13+0200\n"
 "Last-Translator: Dennis Siemensma <github@dennissiemensma.nl>\n"
 "Language: nl\n"
 "Language-Team: Dennis Siemensma <github@dennissiemensma.nl>\n"

docs/locale/nl/LC_MESSAGES/changelog.po

@@ -48,7 +48,7 @@ def get_backup_directory():
     if backup_directory.startswith('/'):
         return os.path.abspath(backup_directory)
     else:
-        return os.path.join(settings.BASE_DIR, '..', backup_directory)
+        return os.path.abspath(os.path.join(settings.BASE_DIR, '..', backup_directory))
 
 
 def create():

dsmr_backup/services/backup.py

@@ -93,7 +93,7 @@ def test_get_backup_directory(self):
         # Default.
         self.assertEqual(
             dsmr_backup.services.backup.get_backup_directory(),
-            os.path.join(settings.BASE_DIR, '..', 'backups/')
+            os.path.abspath(os.path.join(settings.BASE_DIR, '..', 'backups/'))
         )
 
         # Custom.

dsmr_backup/tests/test_backup.py

@@ -0,0 +1,160 @@
+"""
+Copyright (c) 2017 Dropbox, Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+import hashlib
+import six
+
+
+class DropboxContentHasher(object):
+    """
+    Computes a hash using the same algorithm that the Dropbox API uses for the
+    the "content_hash" metadata field.
+
+    The digest() method returns a raw binary representation of the hash.  The
+    hexdigest() convenience method returns a hexadecimal-encoded version, which
+    is what the "content_hash" metadata field uses.
+
+    This class has the same interface as the hashers in the standard 'hashlib'
+    package.
+
+    Example:
+
+        hasher = DropboxContentHasher()
+        with open('some-file', 'rb') as f:
+            while True:
+                chunk = f.read(1024)  # or whatever chunk size you want
+                if len(chunk) == 0:
+                    break
+                hasher.update(chunk)
+        print(hasher.hexdigest())
+    """
+
+    BLOCK_SIZE = 4 * 1024 * 1024
+
+    def __init__(self):
+        self._overall_hasher = hashlib.sha256()
+        self._block_hasher = hashlib.sha256()
+        self._block_pos = 0
+
+        self.digest_size = self._overall_hasher.digest_size
+        # hashlib classes also define 'block_size', but I don't know how people use that value
+
+    def update(self, new_data):
+        if self._overall_hasher is None:
+            raise AssertionError(
+                "can't use this object anymore; you already called digest()")
+
+        assert isinstance(new_data, six.binary_type), (
+            "Expecting a byte string, got {!r}".format(new_data))
+
+        new_data_pos = 0
+        while new_data_pos < len(new_data):
+            if self._block_pos == self.BLOCK_SIZE:
+                self._overall_hasher.update(self._block_hasher.digest())
+                self._block_hasher = hashlib.sha256()
+                self._block_pos = 0
+
+            space_in_block = self.BLOCK_SIZE - self._block_pos
+            part = new_data[new_data_pos:(new_data_pos+space_in_block)]
+            self._block_hasher.update(part)
+
+            self._block_pos += len(part)
+            new_data_pos += len(part)
+
+    def _finish(self):
+        if self._overall_hasher is None:
+            raise AssertionError(
+                "can't use this object anymore; you already called digest() or hexdigest()")
+
+        if self._block_pos > 0:
+            self._overall_hasher.update(self._block_hasher.digest())
+            self._block_hasher = None
+        h = self._overall_hasher
+        self._overall_hasher = None  # Make sure we can't use this object anymore.
+        return h
+
+    def digest(self):
+        return self._finish().digest()
+
+    def hexdigest(self):
+        return self._finish().hexdigest()
+
+    def copy(self):
+        c = DropboxContentHasher.__new__(DropboxContentHasher)
+        c._overall_hasher = self._overall_hasher.copy()
+        c._block_hasher = self._block_hasher.copy()
+        c._block_pos = self._block_pos
+        return c
+
+
+class StreamHasher(object):
+    """
+    A wrapper around a file-like object (either for reading or writing)
+    that hashes everything that passes through it.  Can be used with
+    DropboxContentHasher or any 'hashlib' hasher.
+
+    Example:
+
+        hasher = DropboxContentHasher()
+        with open('some-file', 'rb') as f:
+            wrapped_f = StreamHasher(f, hasher)
+            response = some_api_client.upload(wrapped_f)
+
+        locally_computed = hasher.hexdigest()
+        assert response.content_hash == locally_computed
+    """
+
+    def __init__(self, f, hasher):
+        self._f = f
+        self._hasher = hasher
+
+    def close(self):
+        return self._f.close()
+
+    def flush(self):
+        return self._f.flush()
+
+    def fileno(self):
+        return self._f.fileno()
+
+    def tell(self):
+        return self._f.tell()
+
+    def read(self, *args):
+        b = self._f.read(*args)
+        self._hasher.update(b)
+        return b
+
+    def write(self, b):
+        self._hasher.update(b)
+        return self._f.write(b)
+
+    def next(self):
+        b = self._f.next()
+        self._hasher.update(b)
+        return b
+
+    def readline(self, *args):
+        b = self._f.readline(*args)
+        self._hasher.update(b)
+        return b
+
+    def readlines(self, *args):
+        bs = self._f.readlines(*args)
+        for b in bs:
+            self._hasher.update(b)
+        return b

dsmr_dropbox/dropboxinc/__init__.py

@@ -0,0 +1,14 @@
+from django.core.management.base import BaseCommand
+from django.utils.translation import ugettext as _
+from django.utils import timezone
+
+from dsmr_backup.models.settings import DropboxSettings
+import dsmr_dropbox.services
+
+
+class Command(BaseCommand):
+    help = _('Forces Dropbox sync.')
+
+    def handle(self, **options):
+        DropboxSettings.objects.all().update(next_sync=timezone.now())
+        dsmr_dropbox.services.sync()

dsmr_dropbox/dropboxinc/dropbox_content_hasher.py

@@ -7,6 +7,7 @@
 import dropbox
 
 from dsmr_backup.models.settings import DropboxSettings
+from dsmr_dropbox.dropboxinc.dropbox_content_hasher import DropboxContentHasher
 from dsmr_frontend.models.message import Notification
 import dsmr_backup.services.backup
 
@@ -26,11 +27,14 @@ def sync():
 
     backup_directory = dsmr_backup.services.backup.get_backup_directory()
 
-    # Just check for modified files since the last sync.
-    for (_, _, filenames) in os.walk(backup_directory):
+    # Sync each file, recursively.
+    for (root, dirs, filenames) in os.walk(backup_directory):
         for current_file in filenames:
-            current_file_path = os.path.join(backup_directory, current_file)
-            check_synced_file(file_path=current_file_path, dropbox_settings=dropbox_settings)
+            sync_file(
+                dropbox_settings=dropbox_settings,
+                local_root_dir=backup_directory,
+                abs_file_path=os.path.abspath(os.path.join(root, current_file))
+            )
 
     # Try again in a while.
     DropboxSettings.objects.update(
@@ -41,24 +45,37 @@ def sync():
     )
 
 
-def check_synced_file(file_path, dropbox_settings):
-    file_stats = os.stat(file_path)
+def sync_file(dropbox_settings, local_root_dir, abs_file_path):
+    # The path we use in our Dropbox app folder.
+    relative_file_path = abs_file_path.replace(local_root_dir, '')
 
     # Ignore empty files.
-    if file_stats.st_size == 0:
+    if os.stat(abs_file_path).st_size == 0:
         return
 
-    last_modified = timezone.datetime.fromtimestamp(file_stats.st_mtime)
-    last_modified = timezone.make_aware(last_modified)
-    last_modified = timezone.localtime(last_modified)
-    latest_sync = dropbox_settings.latest_sync
+    # Check whether the file is already at Dropbox, if so, check its hash.
+    dbx = dropbox.Dropbox(dropbox_settings.access_token)
 
-    # Ignore when file was not altered since last sync.
-    if latest_sync and last_modified < timezone.localtime(latest_sync):
-        return
+    try:
+        dropbox_meta = dbx.files_get_metadata(relative_file_path)
+    except dropbox.exceptions.ApiError as exception:
+        error_message = str(exception.error)
+        dropbox_meta = None
+
+        # Unexpected.
+        if 'not_found' not in error_message:
+            return logger.error(' - Dropbox error: %s', error_message)
+
+    # Calculate local hash and compare with remote. Ignore if the remote file is exactly the same.
+    if dropbox_meta and calculate_content_hash(abs_file_path) == dropbox_meta.content_hash:
+        return logger.debug(' - Dropbox content hash is the same, skipping: %s', relative_file_path)
 
     try:
-        upload_chunked(file_path=file_path)
+        upload_chunked(
+            dropbox_settings=dropbox_settings,
+            local_file_path=abs_file_path,
+            remote_file_path=relative_file_path
+        )
     except dropbox.exceptions.DropboxException as exception:
         error_message = str(exception.error)
         logger.error(' - Dropbox error: %s', error_message)
@@ -94,28 +111,23 @@ def check_synced_file(file_path, dropbox_settings):
         raise
 
 
-def upload_chunked(file_path):
+def upload_chunked(dropbox_settings, local_file_path, remote_file_path):
     """ Uploads a file in chucks to Dropbox, allowing it to resume on (connection) failure. """
-    # For backend logging in Supervisor.
-    logger.info(' - Uploading file to Dropbox: %s', file_path)
-
-    dropbox_settings = DropboxSettings.get_solo()
-    file_name = os.path.split(file_path)[-1]
-    dest_path = '/{}'.format(file_name)  # The slash indicates it's relative to the root of app folder.
+    logger.info(' - Syncing file with Dropbox: %s', remote_file_path)
 
     dbx = dropbox.Dropbox(dropbox_settings.access_token)
     write_mode = dropbox.files.WriteMode.overwrite
 
-    file_handle = open(file_path, 'rb')
-    file_size = os.path.getsize(file_path)
+    file_handle = open(local_file_path, 'rb')
+    file_size = os.path.getsize(local_file_path)
 
     # Many thanks to https://stackoverflow.com/documentation/dropbox-api/409/uploading-a-file/1927/uploading-a-file-usin
     # g-the-dropbox-python-sdk#t=201610181733061624381
     CHUNK_SIZE = 2 * 1024 * 1024
 
     # Small uploads should be transfers at one go.
     if file_size <= CHUNK_SIZE:
-        dbx.files_upload(file_handle.read(), dest_path, mode=write_mode)
+        dbx.files_upload(file_handle.read(), remote_file_path, mode=write_mode)
 
     # Large uploads can be sent in chunks, by creating a session allowing multiple separate uploads.
     else:
@@ -125,7 +137,7 @@ def upload_chunked(file_path):
             session_id=upload_session_start_result.session_id,
             offset=file_handle.tell()
         )
-        commit = dropbox.files.CommitInfo(path=dest_path, mode=write_mode)
+        commit = dropbox.files.CommitInfo(path=remote_file_path, mode=write_mode)
 
         # We keep sending the data in chunks, until we reach the last one, then we instruct Dropbox to finish the upload
         # by combining all the chunks sent previously.
@@ -137,3 +149,16 @@ def upload_chunked(file_path):
                 cursor.offset = file_handle.tell()
 
     file_handle.close()
+
+
+def calculate_content_hash(file_path):
+    """ Calculates the Dropbox hash of a file: https://www.dropbox.com/developers/reference/content-hash """
+    hasher = DropboxContentHasher()
+    with open(file_path, 'rb') as f:
+        while True:
+            chunk = f.read(DropboxContentHasher.BLOCK_SIZE)
+            if len(chunk) == 0:
+                break
+            hasher.update(chunk)
+
+    return hasher.hexdigest()

dsmr_dropbox/management/__init__.py

@@ -0,0 +1 @@
+Lorem ipsum dolor sit amet, consectetur adipiscing elit.