Skip to content

Commit df161cf

Browse files
committed
fix: [crawler] timeout QUEUED captures
1 parent 109ce56 commit df161cf

File tree

1 file changed

+8
-1
lines changed

1 file changed

+8
-1
lines changed

bin/crawlers/Crawler.py

+8-1
Original file line numberDiff line numberDiff line change
@@ -179,7 +179,14 @@ def get_message(self):
179179
else:
180180
capture.update(status)
181181
elif status == crawlers.CaptureStatus.QUEUED:
182-
capture.update(status)
182+
capture_start = capture.get_start_time(r_str=False)
183+
if int(time.time()) - capture_start > 600: # TODO ADD in new crawler config
184+
task = capture.get_task()
185+
task.reset()
186+
capture.delete()
187+
self.logger.warning(f'capture QUEUED Timeout, {task.uuid} Send back in queue')
188+
else:
189+
capture.update(status)
183190
print(capture.uuid, crawlers.CaptureStatus(status).name, int(time.time()))
184191
elif status == crawlers.CaptureStatus.ONGOING:
185192
capture.update(status)

0 commit comments

Comments
 (0)