Fix SC2Arcade crawler headers

ipax77 · Feb 4, 2024 · 3578204 · 3578204
1 parent 6e9bd94
commit 3578204
Show file tree

Hide file tree

Showing 2 changed files with 17 additions and 8 deletions.
diff --git a/src/SC2ArcadeCrawler/CrawlerService.cs b/src/SC2ArcadeCrawler/CrawlerService.cs
@@ -35,7 +35,7 @@ public async Task GetLobbyHistory(DateTime tillTime, CancellationToken token, in
 
         List<CrawlInfo> crawlInfos = new()
         {
-            new(regionId: 1, mapId: 208271, handle: "2-S2-1-226401", teMap: false),
+            // new(regionId: 1, mapId: 208271, handle: "2-S2-1-226401", teMap: false),
             new(2, 140436, "2-S2-1-226401", false),
             // new(3, 69942, "2-S2-1-226401", false),
             // new(1, 327974, "2-S2-1-226401", true),
@@ -141,10 +141,19 @@ private async Task<int> Import(CrawlInfo crawlInfo, DateTime tillTime, Cancellat
 
     private static int GetWaitTime(HttpResponseMessage response)
     {
-        // Get the rate limit headers
-        // int rateLimit = int.Parse(response.Headers.GetValues("x-ratelimit-limit").FirstOrDefault() ?? "0");
-        int rateLimitRemaining = int.Parse(response.Headers.GetValues("x-ratelimit-remaining").FirstOrDefault() ?? "0");
-        int rateLimitReset = int.Parse(response.Headers.GetValues("x-ratelimit-reset").FirstOrDefault() ?? "0");
+        int rateLimitRemaining = 0;
+        int rateLimitReset = 0;
+        if (response.Headers.TryGetValues("x-ratelimit-remaining", out var remainValues)
+            && int.TryParse(remainValues.FirstOrDefault(), out int _rateLimitRemaining))
+        {
+            rateLimitRemaining = _rateLimitRemaining;
+        }
+
+        if (response.Headers.TryGetValues("x-ratelimit-reset", out var resetValues)
+            && int.TryParse(resetValues.FirstOrDefault(), out int _rateLimitReset))
+        {
+            rateLimitReset = _rateLimitReset;
+        }
 
         if (rateLimitRemaining > 0)
         {

diff --git a/src/SC2ArcadeCrawler/Program.cs b/src/SC2ArcadeCrawler/Program.cs
@@ -33,8 +33,8 @@ static void Main(string[] args)
         services.AddLogging(options =>
         {
             options.SetMinimumLevel(LogLevel.Information);
-            options.AddFilter("System.Net.Http.HttpClient", LogLevel.Warning);
-            options.AddFilter("Microsoft.EntityFrameworkCore", LogLevel.Warning);
+            options.AddFilter("System.Net.Http.HttpClient", LogLevel.Information);
+            options.AddFilter("Microsoft.EntityFrameworkCore", LogLevel.Information);
             options.AddConsole();
         });
 
@@ -66,7 +66,7 @@ static void Main(string[] args)
 
         var crawlerService = scope.ServiceProvider.GetRequiredService<CrawlerService>();
 
-        crawlerService.GetLobbyHistory(DateTime.Today.AddDays(-58), default).Wait();
+        crawlerService.GetLobbyHistory(DateTime.Today.AddDays(-6), default).Wait();
         // crawlerService.GetLobbyHistory(new DateTime(2021, 2, 1), default).Wait();
 
         Console.WriteLine("done.");