File Sizes cleanup (#1752)

* sizes Kib to Bytes, size formatting functions cleanup

* space between size and unit, removed factor multiplication from size calcualtion

* file_size to bytes in SubmissionDetail

* added missing migration

* migration conflict resolved

Author: ihsaan-ullah

compute_worker/compute_worker.py

@@ -161,7 +161,7 @@ def get_folder_size_in_gb(folder):
             total_size += os.path.getsize(path)
         elif os.path.isdir(path):
             total_size += get_folder_size_in_gb(path)
-    return total_size / 1024 / 1024 / 1024
+    return total_size / 1000 / 1000 / 1000 # GB: decimal system (1000^3)
 
 
 def delete_files_in_folder(folder):

src/apps/analytics/migrations/0002_auto_20250218_1143.py

@@ -0,0 +1,18 @@
+# Generated by Django 2.2.28 on 2025-02-18 11:43
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('analytics', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='adminstoragedatapoint',
+            name='backups_total',
+            field=models.DecimalField(blank=True, decimal_places=2, max_digits=20, null=True),
+        ),
+    ]

src/apps/analytics/models.py

@@ -48,7 +48,7 @@ class UserStorageDataPoint(models.Model):
 
 class AdminStorageDataPoint(models.Model):
     backups_total = models.DecimalField(
-        max_digits=14, decimal_places=2, null=True, blank=True
-    )
+        max_digits=20, decimal_places=2, null=True, blank=True
+    )  # stores bytes
     at_date = models.DateTimeField()
     created_at = models.DateTimeField(auto_now_add=True)

src/apps/analytics/tasks.py

@@ -43,8 +43,8 @@ def create_storage_analytics_snapshot():
     for dataset in Data.objects.filter(Q(file_size__isnull=True) | Q(file_size__lt=0)):
         try:
             dataset.file_size = Decimal(
-                dataset.data_file.size / 1024
-            )  # file_size is in KiB
+                dataset.data_file.size
+            )  # file_size is in Bytes
         except Exception:
             dataset.file_size = Decimal(-1)
         finally:
@@ -56,8 +56,8 @@ def create_storage_analytics_snapshot():
     ):
         try:
             submission.prediction_result_file_size = Decimal(
-                submission.prediction_result.size / 1024
-            )  # prediction_result_file_size is in KiB
+                submission.prediction_result.size
+            )  # prediction_result_file_size is in Bytes
         except Exception:
             submission.prediction_result_file_size = Decimal(-1)
         finally:
@@ -68,8 +68,8 @@ def create_storage_analytics_snapshot():
     ):
         try:
             submission.scoring_result_file_size = Decimal(
-                submission.scoring_result.size / 1024
-            )  # scoring_result_file_size is in KiB
+                submission.scoring_result.size
+            )  # scoring_result_file_size is in Bytes
         except Exception:
             submission.scoring_result_file_size = Decimal(-1)
         finally:
@@ -80,8 +80,8 @@ def create_storage_analytics_snapshot():
     ):
         try:
             submission.detailed_result_file_size = Decimal(
-                submission.detailed_result.size / 1024
-            )  # detailed_result_file_size is in KiB
+                submission.detailed_result.size
+            )  # detailed_result_file_size is in Bytes
         except Exception:
             submission.detailed_result_file_size = Decimal(-1)
         finally:
@@ -92,8 +92,8 @@ def create_storage_analytics_snapshot():
     ):
         try:
             submissiondetails.file_size = Decimal(
-                submissiondetails.data_file.size / 1024
-            )  # file_size is in KiB
+                submissiondetails.data_file.size
+            )  # file_size is in Bytes
         except Exception:
             submissiondetails.file_size = Decimal(-1)
         finally:
@@ -277,7 +277,7 @@ def create_storage_analytics_snapshot():
                 admin_storage_at_date[date] += size
 
     for date in admin_storage_day_range:
-        defaults = {"backups_total": admin_storage_at_date[date] / 1024.0}
+        defaults = {"backups_total": admin_storage_at_date[date]}
         lookup_params = {"at_date": date}
         AdminStorageDataPoint.objects.update_or_create(
             defaults=defaults, **lookup_params
@@ -528,7 +528,7 @@ def create_storage_analytics_snapshot():
         )
         admin_data_point = AdminStorageDataPoint.objects.filter(at_date=date).first()
         admin_usage = (admin_data_point.backups_total or 0) if admin_data_point else 0
-        orphaned_file_usage = Decimal(orphaned_files_size_per_date[date] / 1024)
+        orphaned_file_usage = Decimal(orphaned_files_size_per_date[date])
         total_usage = (
             users_usage + admin_usage + orphaned_file_usage
         )  # competitions_usage is included inside users_usage

src/apps/api/tests/test_datasets.py

@@ -12,7 +12,7 @@
 class DatasetAPITests(APITestCase):
     def setUp(self):
         self.creator = UserFactory(username='creator', password='creator')
-        self.existing_dataset = DataFactory(created_by=self.creator, name="Test!", file_size=1024)
+        self.existing_dataset = DataFactory(created_by=self.creator, name="Test!", file_size=1000)
 
     def test_dataset_api_checks_duplicate_names_for_same_user(self):
         self.client.login(username='creator', password='creator')
@@ -23,7 +23,7 @@ def test_dataset_api_checks_duplicate_names_for_same_user(self):
             'type': Data.COMPETITION_BUNDLE,
             'request_sassy_file_name': faker.file_name(),
             'file_name': faker.file_name(),
-            'file_size': 1024,
+            'file_size': 1000,
         })
 
         assert resp.status_code == 400
@@ -34,7 +34,7 @@ def test_dataset_api_checks_duplicate_names_for_same_user(self):
             'name': 'Test!',
             'type': Data.COMPETITION_BUNDLE,
             'request_sassy_file_name': faker.file_name(),
-            'file_size': 1024,
+            'file_size': 1000,
         })
         assert resp.status_code == 200
 

src/apps/competitions/migrations/0053_auto_20250218_1151.py

@@ -0,0 +1,28 @@
+# Generated by Django 2.2.28 on 2025-02-18 11:51
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('competitions', '0052_auto_20250129_1058'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='submission',
+            name='detailed_result_file_size',
+            field=models.DecimalField(blank=True, decimal_places=2, max_digits=15, null=True),
+        ),
+        migrations.AlterField(
+            model_name='submission',
+            name='prediction_result_file_size',
+            field=models.DecimalField(blank=True, decimal_places=2, max_digits=15, null=True),
+        ),
+        migrations.AlterField(
+            model_name='submission',
+            name='scoring_result_file_size',
+            field=models.DecimalField(blank=True, decimal_places=2, max_digits=15, null=True),
+        ),
+    ]

src/apps/competitions/migrations/0054_auto_20250324_0622.py

@@ -0,0 +1,18 @@
+# Generated by Django 2.2.28 on 2025-03-24 06:22
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('competitions', '0053_auto_20250218_1151'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='submissiondetails',
+            name='file_size',
+            field=models.DecimalField(blank=True, decimal_places=2, max_digits=15, null=True),
+        ),
+    ]

src/apps/competitions/migrations/0055_merge_20250324_0650.py

@@ -0,0 +1,14 @@
+# Generated by Django 2.2.28 on 2025-03-24 06:50
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('competitions', '0054_auto_20250324_0622'),
+        ('competitions', '0053_competition_forum_enabled'),
+    ]
+
+    operations = [
+    ]

src/apps/competitions/models.py

@@ -466,18 +466,16 @@ class SubmissionDetails(models.Model):
     ]
     name = models.CharField(max_length=50)
     data_file = models.FileField(upload_to=PathWrapper('submission_details'), storage=BundleStorage)
-    file_size = models.DecimalField(max_digits=10, decimal_places=2, null=True, blank=True)  # in KiB
+    file_size = models.DecimalField(max_digits=15, decimal_places=2, null=True, blank=True)  # in Bytes
     submission = models.ForeignKey('Submission', on_delete=models.CASCADE, related_name='details')
     is_scoring = models.BooleanField(default=False)
 
     def save(self, *args, **kwargs):
         if self.data_file and (not self.file_size or self.file_size == -1):
             try:
-                # save file size as KiB
                 # self.data_file.size returns bytes
-                self.file_size = self.data_file.size / 1024
+                self.file_size = self.data_file.size
             except TypeError:
-                # file returns a None size, can't divide None / 1024
                 # -1 indicates an error
                 self.file_size = -1
             except botocore.exceptions.ClientError:
@@ -528,9 +526,9 @@ class Submission(ChaHubSaveMixin, models.Model):
     detailed_result = models.FileField(upload_to=PathWrapper('detailed_result'), null=True, blank=True,
                                        storage=BundleStorage)
 
-    prediction_result_file_size = models.DecimalField(max_digits=10, decimal_places=2, null=True, blank=True)  # in KiB
-    scoring_result_file_size = models.DecimalField(max_digits=10, decimal_places=2, null=True, blank=True)  # in KiB
-    detailed_result_file_size = models.DecimalField(max_digits=10, decimal_places=2, null=True, blank=True)  # in KiB
+    prediction_result_file_size = models.DecimalField(max_digits=15, decimal_places=2, null=True, blank=True)  # in Bytes
+    scoring_result_file_size = models.DecimalField(max_digits=15, decimal_places=2, null=True, blank=True)  # in Bytes
+    detailed_result_file_size = models.DecimalField(max_digits=15, decimal_places=2, null=True, blank=True)  # in Bytes
 
     secret = models.UUIDField(default=uuid.uuid4)
     celery_task_id = models.UUIDField(null=True, blank=True)
@@ -644,11 +642,9 @@ def save(self, ignore_submission_limit=False, **kwargs):
         for file_path_attr, file_size_attr in files_and_sizes_dict.items():
             if getattr(self, file_path_attr) and (not getattr(self, file_size_attr) or getattr(self, file_size_attr) == -1):
                 try:
-                    # save file size as KiB
                     # self.data_file.size returns bytes
-                    setattr(self, file_size_attr, getattr(self, file_path_attr).size / 1024)
+                    setattr(self, file_size_attr, getattr(self, file_path_attr).size)
                 except TypeError:
-                    # file returns a None size, can't divide None / 1024
                     # -1 indicates an error
                     setattr(self, file_size_attr, Decimal(-1))
                 except botocore.exceptions.ClientError:

src/apps/datasets/migrations/0010_auto_20250218_1100.py

@@ -0,0 +1,18 @@
+# Generated by Django 2.2.28 on 2025-02-18 11:00
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('datasets', '0009_merge_20241203_1313'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='data',
+            name='file_size',
+            field=models.DecimalField(blank=True, decimal_places=2, max_digits=15, null=True),
+        ),
+    ]

src/apps/datasets/models.py

@@ -59,7 +59,7 @@ class Data(ChaHubSaveMixin, models.Model):
     key = models.UUIDField(default=uuid.uuid4, blank=True, unique=True)
     is_public = models.BooleanField(default=False)
     upload_completed_successfully = models.BooleanField(default=False)
-    file_size = models.DecimalField(max_digits=10, decimal_places=2, null=True, blank=True)  # in KiB
+    file_size = models.DecimalField(max_digits=15, decimal_places=2, null=True, blank=True)  # in Bytes
 
     # This is true if the Data model was created as part of unpacking a competition. Competition bundles themselves
     # are NOT marked True, since they are not created by unpacking!
@@ -74,11 +74,10 @@ def get_download_url(self):
     def save(self, *args, **kwargs):
         if self.data_file and (not self.file_size or self.file_size == -1):
             try:
-                # save file size as KiB
+                # save file size in bytes
                 # self.data_file.size returns bytes
-                self.file_size = self.data_file.size / 1024
+                self.file_size = self.data_file.size
             except TypeError:
-                # file returns a None size, can't divide None / 1024
                 # -1 indicates an error
                 self.file_size = Decimal(-1)
             except botocore.exceptions.ClientError:

src/apps/pages/views.py

@@ -6,6 +6,7 @@
 from announcements.models import Announcement, NewsPost
 
 from django.shortcuts import render
+from utils.data import pretty_bytes
 
 
 class HomeView(TemplateView):
@@ -89,9 +90,9 @@ def get_context_data(self, *args, **kwargs):
         for submission in context['submissions']:
             # Get filesize from each submissions's data
             if submission.data:
-                submission.file_size = self.format_file_size(submission.data.file_size)
+                submission.file_size = pretty_bytes(submission.data.file_size)
             else:
-                submission.file_size = self.format_file_size(0)
+                submission.file_size = pretty_bytes(0)
 
             # Get queue from each submission
             queue_name = ""
@@ -110,23 +111,6 @@ def get_context_data(self, *args, **kwargs):
 
         return context
 
-    def format_file_size(self, file_size):
-        """
-        A custom function to convert file size to KB, MB, GB and return with the unit
-        """
-        try:
-            n = float(file_size)
-        except Exception:
-            return ""
-
-        units = ['KB', 'MB', 'GB']
-        i = 0
-        while n >= 1000 and i < len(units) - 1:
-            n /= 1000
-            i += 1
-
-        return f"{n:.1f} {units[i]}"
-
 
 class MonitorQueuesView(TemplateView):
     template_name = 'pages/monitor_queues.html'

src/apps/profiles/models.py

@@ -159,7 +159,6 @@ def get_used_storage_space(self, binary=False):
         Returns in bytes
         """
 
-        factor = 1024 if binary else 1000
         from datasets.models import Data
         from competitions.models import Submission, SubmissionDetails
 
@@ -170,7 +169,7 @@ def get_used_storage_space(self, binary=False):
             created_by_id=self.id, file_size__gt=0, file_size__isnull=False
         ).aggregate(Sum("file_size"))["file_size__sum"]
 
-        storage_used += users_datasets * factor if users_datasets else 0
+        storage_used += users_datasets if users_datasets else 0
 
         # Submissions
         users_submissions = Submission.objects.filter(owner_id=self.id).aggregate(
@@ -202,14 +201,14 @@ def get_used_storage_space(self, binary=False):
             )
         )
 
-        storage_used += users_submissions["size"] * factor if users_submissions["size"] else 0
+        storage_used += users_submissions["size"] if users_submissions["size"] else 0
 
         # Submissions details
         users_submissions_details = SubmissionDetails.objects.filter(
             submission__owner_id=self.id, file_size__gt=0, file_size__isnull=False
         ).aggregate(Sum("file_size"))["file_size__sum"]
 
-        storage_used += users_submissions_details * factor if users_submissions_details else 0
+        storage_used += users_submissions_details if users_submissions_details else 0
 
         return storage_used
 

src/static/js/ours/utils.js

@@ -90,16 +90,23 @@ function pretty_date(date_string) {
 }
 
 function pretty_bytes(bytes, decimalPlaces = 1, suffix = "B", binary = false) {
+
+    // Ensure bytes is a valid number
+    bytes = parseFloat(bytes)
+    if (isNaN(bytes) || bytes < 0) {
+        return "" // Return empty string for invalid or negative values
+    }
+
     const factor = binary ? 1024.0 : 1000.0;
     const units = binary ? ['', 'Ki', 'Mi', 'Gi', 'Ti', 'Pi', 'Ei', 'Zi'] : ['', 'k', 'M', 'G', 'T', 'P', 'E', 'Z'];
 
     for (const unit of units) {
         if (Math.abs(bytes) < factor || unit === units[units.length - 1]) {
-            return bytes.toFixed(decimalPlaces) + unit + suffix;
+            return bytes.toFixed(decimalPlaces) + ' ' + unit + suffix;
         }
         bytes /= factor;
     }
-    return bytes.toFixed(decimalPlaces) + units[units.length - 1] + suffix;
+    return bytes.toFixed(decimalPlaces) + ' ' + units[units.length - 1] + suffix;
 }
 
 /* ----------------------------------------------------------------------------