refactor: spread volume jobs over the scheduled period

app/Console/Commands/CalculateCollectionTraitRarities.php

@@ -18,7 +18,7 @@ class CalculateCollectionTraitRarities extends Command
      *
      * @var string
      */
-    protected $signature = 'collections:calculate-trait-rarities {--start=} {--limit=} {--collection-id=}';
+    protected $signature = 'collections:calculate-trait-rarities {--start=} {--limit=} {--offset=} {--collection-id=}';
 
     /**
      * The console command description.

app/Console/Commands/FetchBurnActivity.php

@@ -18,7 +18,7 @@ class FetchBurnActivity extends Command
      *
      * @var string
      */
-    protected $signature = 'collections:fetch-burn-activity {--collection-id=}';
+    protected $signature = 'collections:fetch-burn-activity {--limit=} {--offset=} {--collection-id=}';
 
     /**
      * The console command description.

app/Console/Commands/FetchCollectionActivity.php

@@ -18,7 +18,7 @@ class FetchCollectionActivity extends Command
      *
      * @var string
      */
-    protected $signature = 'collections:fetch-activity {--collection-id=}';
+    protected $signature = 'collections:fetch-activity {--limit=} {--offset=} {--collection-id=}';
 
     /**
      * The console command description.

app/Console/Commands/FetchCollectionBanner.php

@@ -16,7 +16,7 @@ class FetchCollectionBanner extends Command
      *
      * @var string
      */
-    protected $signature = 'collections:fetch-banner {--collection-id=}';
+    protected $signature = 'collections:fetch-banner {--limit=} {--offset=} {--collection-id=}';
 
     /**
      * The console command description.

app/Console/Commands/FetchCollectionFloorPrice.php

@@ -16,7 +16,7 @@ class FetchCollectionFloorPrice extends Command
      *
      * @var string
      */
-    protected $signature = 'collections:fetch-floor-price {--collection-id=}';
+    protected $signature = 'collections:fetch-floor-price {--limit=} {--offset=} {--collection-id=}';
 
     /**
      * The console command description.

app/Console/Commands/FetchCollectionNfts.php

@@ -17,7 +17,7 @@ class FetchCollectionNfts extends Command
      *
      * @var string
      */
-    protected $signature = 'collections:fetch-nfts {--collection-id=} {--start-token=} {--only-signed} {--limit=}';
+    protected $signature = 'collections:fetch-nfts {--limit=} {--offset=} {--collection-id=} {--start-token=} {--only-signed}';
 
     /**
      * The console command description.

app/Console/Commands/FetchCollectionOpenseaSlug.php

@@ -16,7 +16,7 @@ class FetchCollectionOpenseaSlug extends Command
      *
      * @var string
      */
-    protected $signature = 'collections:fetch-opensea-slug {--collection-id=}';
+    protected $signature = 'collections:fetch-opensea-slug {--limit=} {--offset=} {--collection-id=}';
 
     /**
      * The console command description.

app/Console/Commands/FetchCollectionOwners.php

@@ -16,7 +16,7 @@ class FetchCollectionOwners extends Command
      *
      * @var string
      */
-    protected $signature = 'collections:fetch-owners {--collection-id=}';
+    protected $signature = 'collections:fetch-owners {--limit=} {--offset=} {--collection-id=}';
 
     /**
      * The console command description.

app/Console/Commands/FetchCollectionTotalVolume.php

@@ -16,7 +16,7 @@ class FetchCollectionTotalVolume extends Command
      *
      * @var string
      */
-    protected $signature = 'collections:fetch-total-volume {--collection-id=}';
+    protected $signature = 'collections:fetch-total-volume {--limit=} {--offset=} {--collection-id=}';
 
     /**
      * The console command description.
@@ -30,11 +30,9 @@ class FetchCollectionTotalVolume extends Command
      */
     public function handle(): int
     {
-        $queryCallback = fn ($query) => $query->whereNotNull('extra_attributes->opensea_slug');
-
         $this->forEachCollection(function ($collection) {
             FetchCollectionTotalVolumeJob::dispatch($collection);
-        }, queryCallback: $queryCallback);
+        });
 
         return Command::SUCCESS;
     }

app/Console/Commands/FetchCollectionVolume.php

@@ -16,7 +16,7 @@ class FetchCollectionVolume extends Command
      *
      * @var string
      */
-    protected $signature = 'collections:fetch-volume {--collection-id=}';
+    protected $signature = 'collections:fetch-volume {--limit=} {--offset=} {--collection-id=}';
 
     /**
      * The console command description.

app/Console/Commands/FetchCollectionVolumeHistory.php

@@ -16,7 +16,7 @@ class FetchCollectionVolumeHistory extends Command
      *
      * @var string
      */
-    protected $signature = 'collections:fetch-volume-history {--collection-id=}';
+    protected $signature = 'collections:fetch-volume-history {--limit=} {--offset=} {--collection-id=}';
 
     /**
      * The console command description.

app/Console/Commands/InteractsWithCollections.php

@@ -30,19 +30,23 @@ public function forEachCollection(Closure $callback, ?Closure $queryCallback = n
             return;
         }
 
+        $limit = $limit ?? $this->option('limit');
+        $offset = $this->option('offset');
+
         Collection::query()
-            ->when($queryCallback !== null, $queryCallback)
-            ->select('collections.*')
-            ->withoutSpamContracts()
-            ->when($limit !== null, fn ($query) => $query
-                ->limit($limit)
-                ->get()
-                ->each($callback)
-            )
-            ->when($limit == null, fn ($query) => $query->chunkById(
-                100,
-                fn ($collections, $index) => $collections->each($callback, $index),
-                'collections.id', 'id')
-            );
+                    ->when($queryCallback !== null, $queryCallback)
+                    ->select('collections.*')
+                    ->withoutSpamContracts()
+                    ->orderBy('id', 'asc')
+                    ->when($offset !== null, fn ($q) => $q->skip((int) $offset))
+                    ->when($limit !== null, function ($query) use ($limit, $callback) {
+                        return $query->limit($limit)->get()->each($callback);
+                    })
+                    ->when($limit == null, function ($query) use ($callback) {
+                        return $query->chunk(
+                            count: 100,
+                            callback: fn ($chunk, $index) => $chunk->each($callback, $index)
+                        );
+                    });
     }
 }

app/Console/Kernel.php

@@ -30,9 +30,12 @@
 use App\Console\Commands\UpdateGalleriesValue;
 use App\Console\Commands\UpdateTwitterFollowers;
 use App\Enums\Features;
+use App\Enums\ScheduleFrequency;
 use App\Jobs\AggregateCollectionWinners;
+use App\Models\Collection;
 use Illuminate\Console\Scheduling\Schedule;
 use Illuminate\Foundation\Console\Kernel as ConsoleKernel;
+use Illuminate\Support\Facades\Cache;
 use Laravel\Pennant\Feature;
 
 class Kernel extends ConsoleKernel
@@ -115,15 +118,26 @@ protected function schedule(Schedule $schedule): void
 
     private function scheduleJobsForCollectionsOrGalleries(Schedule $schedule): void
     {
-        $schedule
-            ->command(FetchCollectionTotalVolume::class)
-            ->withoutOverlapping()
-            ->daily();
-
-        $schedule
-            ->command(FetchCollectionVolume::class)
-            ->withoutOverlapping()
-            ->daily();
+        $ttl = now()->addMinutes(30);
+
+        $totalCollections = (int) Cache::remember('scheduler:total-collections', $ttl, function () {
+            return Collection::withoutSpamContracts()->count();
+        });
+
+        $schedule->spreadCommand(
+            command: FetchCollectionTotalVolume::class,
+            total: $totalCollections,
+            every: 10, // minutes
+            spread: ScheduleFrequency::Daily,
+        );
+
+        $schedule->spreadCommand(
+            command: FetchCollectionVolume::class,
+            total: $totalCollections,
+            every: 10, // minutes
+            offset: 5,
+            spread: ScheduleFrequency::Daily,
+        );
 
         $schedule
             ->command(FetchCollectionActivity::class)

app/Enums/ScheduleFrequency.php

@@ -0,0 +1,11 @@
+<?php
+
+declare(strict_types=1);
+
+namespace App\Enums;
+
+enum ScheduleFrequency: string
+{
+    case Hourly = 'hourly';
+    case Daily = 'daily';
+}

app/Providers/AppServiceProvider.php

@@ -7,6 +7,7 @@
 use App\Contracts\MarketDataProvider;
 use App\Contracts\TextToSpeechProvider;
 use App\Contracts\Web3DataProvider;
+use App\Enums\ScheduleFrequency;
 use App\Jobs\RefreshNftMetadata;
 use App\Services\MarketData\Providers\CoingeckoProvider;
 use App\Services\TextToSpeech\Polly;
@@ -15,6 +16,7 @@
 use Aws\S3\S3Client;
 use Exception;
 use Illuminate\Cache\RateLimiting\Limit;
+use Illuminate\Console\Scheduling\Schedule;
 use Illuminate\Http\RedirectResponse;
 use Illuminate\Http\Request;
 use Illuminate\Support\Facades\RateLimiter;
@@ -55,6 +57,8 @@ public function register(): void
      */
     public function boot(): void
     {
+        $this->registerScheduledMacro();
+
         Str::macro('isEncodedImage', function (string $value) {
             return Str::startsWith($value, 'data:image');
         });
@@ -121,4 +125,55 @@ public function boot(): void
             Limit::perMinutes(decayMinutes: 10, maxAttempts: 5),
         ]);
     }
+
+    private function registerScheduledMacro(): void
+    {
+        Schedule::macro('spreadCommand', function (string $command, int $total, int $every, ScheduleFrequency $spread, int $offset = 0) {
+            /** @var Schedule $this */
+            if ($spread === ScheduleFrequency::Hourly) {
+                $chunks = 60 / $every;
+
+                $limit = ceil(($total + 1) / $chunks);
+
+                foreach (range(0, $chunks - 1) as $chunk) {
+                    $at = (int) (round($chunk * $every) + $offset);
+
+                    if ($at >= 60) {
+                        $at = $at - 60;
+                    }
+
+                    $this->command($command, [
+                        '--limit' => (int) $limit,
+                        '--offset' => (int) ($chunk * $limit),
+                    ])->withoutOverlapping()->hourlyAt($at);
+                }
+
+                return;
+            }
+
+            if ($spread === ScheduleFrequency::Daily) {
+                $chunks = (60 * 24) / $every;
+
+                $limit = ceil(($total + 1) / $chunks);
+
+                foreach (range(0, $chunks - 1) as $chunk) {
+                    $at = (int) (round($chunk * $every) + $offset);
+
+                    $hour = floor($at / 60);
+                    $minute = $at % 60;
+
+                    if ($hour >= 24) {
+                        $hour = $hour - 24;
+                    }
+
+                    $this->command($command, [
+                        '--limit' => (int) $limit,
+                        '--offset' => (int) ($chunk * $limit),
+                    ])->withoutOverlapping()->dailyAt($hour.':'.$minute);
+                }
+
+                return;
+            }
+        });
+    }
 }

tests/App/Console/Commands/FetchCollectionTotalVolumeTest.php

@@ -84,19 +84,3 @@
 
     Bus::assertDispatchedTimes(FetchCollectionTotalVolume::class, 0);
 });
-
-it('should not dispatch a job for collections that dont have OpenSea slug', function () {
-    Bus::fake([FetchCollectionTotalVolume::class]);
-
-    $collection = Collection::factory()->create([
-        'extra_attributes' => [
-            'opensea_slug' => null,
-        ],
-    ]);
-
-    Bus::assertDispatchedTimes(FetchCollectionTotalVolume::class, 0);
-
-    $this->artisan('collections:fetch-total-volume');
-
-    Bus::assertDispatchedTimes(FetchCollectionTotalVolume::class, 0);
-});