Merge pull request #435 from ChrisCummins/download-failure-retry

[util] Increase persistence on "Too many Requests" errors.
facebookresearch · Sep 29, 2021 · d5ba02c · d5ba02c
2 parents abb8f86 + 0d9e1d3
commit d5ba02c
Show file tree

Hide file tree

Showing 2 changed files with 4 additions and 3 deletions.
diff --git a/compiler_gym/util/download.py b/compiler_gym/util/download.py
@@ -78,7 +78,7 @@ def _download(urls: List[str], sha256: Optional[str], max_retries: int) -> bytes
 
     # A retry loop, and loop over all urls provided.
     last_exception = None
-    wait_time = 5
+    wait_time = 10
     for _ in range(max(max_retries, 1)):
         for url in urls:
             try:
@@ -99,7 +99,7 @@ def _download(urls: List[str], sha256: Optional[str], max_retries: int) -> bytes
 
 
 def download(
-    urls: Union[str, List[str]], sha256: Optional[str] = None, max_retries: int = 3
+    urls: Union[str, List[str]], sha256: Optional[str] = None, max_retries: int = 5
 ) -> bytes:
     """Download a file and return its contents.
 

diff --git a/tests/util/download_test.py b/tests/util/download_test.py
@@ -26,7 +26,8 @@ def patched_download(*args):
 
     assert download._do_download_attempt.call_count == max_retries
     assert download.sleep.call_count == max_retries
-    download.sleep.assert_called_with(5 * 1.5 ** (max_retries - 1))
+    starting_wait_time = 10  # The initial wait time in seconds.
+    download.sleep.assert_called_with(starting_wait_time * 1.5 ** (max_retries - 1))
 
 
 @pytest.mark.parametrize("max_retries", [1, 2, 3, 5, 10])