fix: [crawler] debug crawler queued capture loop

Author: Terrtia

bin/crawlers/Crawler.py

@@ -147,6 +147,7 @@ def get_message(self):
         if capture:
             try:
                 status = self.lacus.get_capture_status(capture.uuid)
+                print(status)
                 if status == crawlers.CaptureStatus.DONE:
                     return capture
                 elif status == crawlers.CaptureStatus.UNKNOWN:
@@ -164,7 +165,10 @@ def get_message(self):
                         self.logger.warning(f'capture UNKNOWN Timeout, {task.uuid} Send back in queue')
                     else:
                         capture.update(status)
-                elif status == crawlers.CaptureStatus.QUEUED or status == crawlers.CaptureStatus.ONGOING:
+                elif status == crawlers.CaptureStatus.QUEUED:
+                    capture.update(status, delta=30)
+                    print(capture.uuid, crawlers.CaptureStatus(status).name, int(time.time() + 30))
+                elif status == crawlers.CaptureStatus.ONGOING:
                     capture.update(status)
                     print(capture.uuid, crawlers.CaptureStatus(status).name, int(time.time()))
                 # Invalid State

bin/lib/crawlers.py

@@ -1531,13 +1531,13 @@ def create(self, task_uuid):
         r_crawler.zadd('crawler:captures', {self.uuid: launch_time})
         r_cache.zadd('crawler:captures', {self.uuid: launch_time})
 
-    def update(self, status):
+    def update(self, status, delta=0):
         # Error or Reload
         if not status:
             r_cache.hset(f'crawler:capture:{self.uuid}', 'status', CaptureStatus.UNKNOWN.value)
             r_cache.zadd('crawler:captures', {self.uuid: 0})
         else:
-            last_check = int(time.time())
+            last_check = int(time.time() + delta)
             r_cache.hset(f'crawler:capture:{self.uuid}', 'status', status)
             r_cache.zadd('crawler:captures', {self.uuid: last_check})