Decode Microservice

src/dsstats.api/Controllers/UploadController.cs

@@ -8,7 +8,8 @@ namespace dsstats.api.Controllers;
 [ApiController]
 [Route("api8/v1/[controller]")]
 [ServiceFilter(typeof(AuthenticationFilterAttribute))]
-public class UploadController(UploadService uploadService, DecodeService decodeService) : Controller
+public class UploadController(UploadService uploadService,
+                              DecodeService decodeService) : Controller
 {
     private readonly UploadService uploadService = uploadService;
 
@@ -41,15 +42,20 @@ public async Task<ActionResult<int>> UploadReplays(string guid, [FromForm] List<
     {
         if (Guid.TryParse(guid, out var fileGuid))
         {
-            var queueCount = await decodeService.SaveReplays(fileGuid, files);
-            if (queueCount >= 0)
-            {
-                return Ok(queueCount);
-            }
-            else
-            {
-                return StatusCode(500);
-            }
+            await decodeService.SaveReplays(fileGuid, files);
+            return Ok();
+        }
+        return BadRequest();
+    }
+
+    [HttpPost]
+    [Route("decoderesult/{guid}")]
+    public async Task<ActionResult> DecodeResult(string guid, [FromBody] List<IhReplay> replays)
+    {
+        if (Guid.TryParse(guid, out var groupId))
+        {
+            await decodeService.ConsumeDecodeResult(groupId, replays);
+            return Ok();
         }
         return BadRequest();
     }

src/dsstats.api/Program.cs

@@ -33,7 +33,8 @@
                           policy.WithOrigins("https://dsstats.pax77.org",
                                              "https://dsstats-dev.pax77.org",
                                              "https://localhost:7257",
-                                             "https://localhost:7227")
+                                             "https://localhost:7227",
+                                             "http://localhost:5123")
                           .AllowAnyHeader()
                           .AllowAnyMethod();
                       });
@@ -99,6 +100,13 @@
     options.DefaultRequestHeaders.Add("Accept", "application/json");
 });
 
+builder.Services.AddHttpClient("decode")
+    .ConfigureHttpClient(options =>
+    {
+        // options.BaseAddress = new Uri("http://localhost:5240");
+        options.BaseAddress = new Uri(builder.Configuration["ServerConfig:DecodeUrl"] ?? "");
+    });
+
 builder.Services.AddSingleton<IRatingService, RatingService>();
 builder.Services.AddSingleton<IRatingsSaveService, RatingsSaveService>();
 builder.Services.AddSingleton<ImportService>();

src/dsstats.api/Services/DecodeService.cs

@@ -1,130 +1,45 @@
 using dsstats.db8services.Import;
 using dsstats.shared;
-using pax.dsstats.parser;
-using s2protocol.NET;
-using System.Collections.Concurrent;
-using System.Reflection;
-using System.Security.Cryptography;
-using System.Text.RegularExpressions;
 
 namespace dsstats.api.Services;
 
-public class DecodeService(ILogger<DecodeService> logger, IServiceScopeFactory scopeFactory)
+public class DecodeService(ILogger<DecodeService> logger,
+                           IHttpClientFactory httpClientFactory,
+                           IServiceScopeFactory scopeFactory)
 {
-    private readonly string replayFolder = "/data/ds/decode";
-
-    private readonly SemaphoreSlim ss = new(1, 1);
-    private ReplayDecoder? replayDecoder;
-    public static readonly string assemblyPath = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location) ?? "";
-    private int queueCount = 0;
-    private ConcurrentBag<string> excludeReplays = [];
-
     public EventHandler<DecodeEventArgs>? DecodeFinished;
 
     private void OnDecodeFinished(DecodeEventArgs e)
     {
         DecodeFinished?.Invoke(this, e);
     }
 
-    public async Task<int> SaveReplays(Guid guid, List<IFormFile> files)
+    public async Task SaveReplays(Guid guid, List<IFormFile> files)
     {
+        var httpClient = httpClientFactory.CreateClient("decode");
         try
         {
-            long size = files.Sum(f => f.Length);
+            var formData = new MultipartFormDataContent();
 
-            foreach (var formFile in files)
+            foreach (var file in files)
             {
-                if (formFile.Length > 0)
-                {
-                    var fileGuid = Guid.NewGuid();
-                    var filePath = Path.Combine(replayFolder, "todo", guid.ToString() + "_" + fileGuid.ToString() + ".SC2Replay");
-                    var tmpFilePath = Path.Combine(replayFolder, "todo", guid.ToString() + "_" + fileGuid.ToString() + ".tmp");
-
-                    {
-                        using var stream = File.Create(tmpFilePath);
-                        await formFile.CopyToAsync(stream);
-                    }
-                    File.Move(tmpFilePath, filePath);
-                }
+                var fileContent = new StreamContent(file.OpenReadStream());
+                formData.Add(fileContent, "files", file.FileName);
             }
-            _ = Decode(guid);
 
-            logger.LogWarning($"replays saved ({size})", size);
-            return queueCount;
+            var result = await httpClient.PostAsync($"/api/v1/decode/upload/{guid}", formData);
+            result.EnsureSuccessStatusCode();
         }
         catch (Exception ex)
         {
             logger.LogError("failed saving replays: {error}", ex.Message);
         }
-        return -1;
     }
 
-    public async Task Decode(Guid guid)
+    public async Task ConsumeDecodeResult(Guid guid, List<IhReplay> replays)
     {
-        Interlocked.Increment(ref queueCount);
-        await ss.WaitAsync();
-        List<IhReplay> replays = [];
-        string? error = null;
-
         try
         {
-            var replayPaths = Directory.GetFiles(Path.Combine(replayFolder, "todo"), "*SC2Replay");
-            replayPaths = replayPaths.Except(excludeReplays).ToArray();
-
-            if (replayPaths.Length == 0)
-            {
-                error = "No replays found.";
-                return;
-            }
-
-            if (replayDecoder is null)
-            {
-                replayDecoder = new(assemblyPath);
-            }
-
-            var options = new ReplayDecoderOptions()
-            {
-                Initdata = true,
-                Details = true,
-                Metadata = true,
-                TrackerEvents = true,
-            };
-
-            using var md5 = MD5.Create();
-
-            await foreach (var result in replayDecoder.DecodeParallelWithErrorReport(replayPaths, 2, options))
-            {
-                if (result.Sc2Replay is null)
-                {
-                    Error(result);
-                    error = "failed decoding replays.";
-                    continue;
-                }
-
-                var metaData = GetMetaData(result.Sc2Replay);
-
-                var sc2Replay = Parse.GetDsReplay(result.Sc2Replay);
-
-                if (sc2Replay is null)
-                {
-                    Error(result);
-                    error = "failed decoding replays.";
-                    continue;
-                }
-
-                var replayDto = Parse.GetReplayDto(sc2Replay, md5);
-
-                if (replayDto is null)
-                {
-                    Error(result);
-                    error = "failed decoding replays.";
-                    continue;
-                }
-
-                File.Move(result.ReplayPath, Path.Combine(replayFolder, "done", Path.GetFileName(result.ReplayPath)));
-                replays.Add(new IhReplay() { Replay = replayDto, Metadata = metaData });
-            }
-
             if (replays.Count > 0)
             {
                 using var scope = scopeFactory.CreateScope();
@@ -135,125 +50,16 @@ public async Task Decode(Guid guid)
         }
         catch (Exception ex)
         {
-            logger.LogError("failed decoding replays: {error}", ex.Message);
-            error = "failed decoding replays.";
+            logger.LogError("failed importing decode result: {error}", ex.Message);
         }
         finally
         {
-            ss.Release();
             OnDecodeFinished(new()
             {
                 Guid = guid,
                 IhReplays = replays,
-                Error = error,
-            });
-            Interlocked.Decrement(ref queueCount);
-        }
-    }
-
-    private void Error(DecodeParallelResult result)
-    {
-        logger.LogError("failed decoding replay: {path}, {error}", result.ReplayPath, result.Exception);
-        try
-        {
-            File.Move(result.ReplayPath, Path.Combine(replayFolder, "error", Path.GetFileName(result.ReplayPath)));
-        }
-        catch (Exception ex)
-        {
-            logger.LogWarning("failed moving error replay: {error}", ex.Message);
-            excludeReplays.Add(result.ReplayPath);
-        }
-    }
-
-    private ReplayMetadata GetMetaData(Sc2Replay replay)
-    {
-        List<ReplayMetadataPlayer> players = [];
-
-        if (replay.Initdata is null || replay.Details is null || replay.Metadata is null)
-        {
-            return new();
-        }
-
-        foreach (var player in replay.Initdata.LobbyState.Slots)
-        {
-            players.Add(new()
-            {
-                PlayerId = GetPlayerId(player.ToonHandle),
-                Observer = player.Observe == 1,
-                SlotId = player.WorkingSetSlotId
             });
         }
-
-        int i = 0;
-        foreach (var player in replay.Details.Players)
-        {
-            i++;
-            PlayerId playerId = GetPlayerId(player.Toon);
-            var metaPlayer = players.FirstOrDefault(f => f.PlayerId == playerId);
-            if (metaPlayer is null)
-            {
-                continue;
-            }
-            metaPlayer.Id = i;
-            metaPlayer.Name = player.Name;
-            metaPlayer.AssignedRace = GetRace(player.Race);
-        }
-
-        foreach (var player in replay.Metadata.Players)
-        {
-            var metaPlayer = players.FirstOrDefault(f => f.Id == player.PlayerID);
-            if (metaPlayer is null)
-            {
-                continue;
-            }
-            metaPlayer.SelectedRace = GetSelectedRace(player.SelectedRace);
-        }
-
-        return new()
-        {
-            Players = players
-        };
-    }
-
-    private static Commander GetSelectedRace(string selectedRace)
-    {
-        var race = selectedRace switch
-        {
-            "Terr" => "Terran",
-            "Prot" => "Protoss",
-            "Rand" => "None",
-            _ => selectedRace
-        };
-        return GetRace(race);
-    }
-
-    private static PlayerId GetPlayerId(s2protocol.NET.Models.Toon toon)
-    {
-        return new(toon.Id, toon.Realm, toon.Region);
-    }
-
-    private static PlayerId GetPlayerId(string toonHandle)
-    {
-        Regex rx = new(@"(\d)-S2-(\d)-(\d+)");
-        var match = rx.Match(toonHandle);
-        if (match.Success)
-        {
-            int regionId = int.Parse(match.Groups[1].Value);
-            int realmId = int.Parse(match.Groups[2].Value);
-            int toonId = int.Parse(match.Groups[3].Value);
-            return new(toonId, realmId, regionId);
-        }
-        return new();
-    }
-
-    private static Commander GetRace(string race)
-    {
-        if (Enum.TryParse(typeof(Commander), race, out var cmdrObj)
-            && cmdrObj is Commander cmdr)
-        {
-            return cmdr;
-        }
-        return Commander.None;
     }
 }
 

src/dsstats.api/Services/IhService.cs

@@ -78,7 +78,6 @@ public async Task<GroupState> CreateOrVisitGroup(Guid groupId)
                     completionSource.SetResult(args.IhReplays);
                 }
             };
-
             decodeService.DecodeFinished += decodeEventHandler;
 
             var timeoutTask = Task.Delay(20000);

src/dsstats.api/dsstats.api.csproj

@@ -7,7 +7,6 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="IronPython.StdLib" Version="2.7.12" />
     <PackageReference Include="MailKit" Version="4.5.0" />
     <PackageReference Include="Microsoft.EntityFrameworkCore.Design" Version="8.0.4">
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
@@ -20,7 +19,6 @@
     <ProjectReference Include="..\dsstats.auth\dsstats.auth.csproj" />
     <ProjectReference Include="..\dsstats.db8services\dsstats.db8services.csproj" />
     <ProjectReference Include="..\dsstats.db8\dsstats.db8.csproj" />
-    <ProjectReference Include="..\dsstats.maui\pax.dsstats.parser\pax.dsstats.parser.csproj" />
     <ProjectReference Include="..\dsstats.ratings.lib\dsstats.ratings.lib.csproj" />
     <ProjectReference Include="..\dsstats.ratings\dsstats.ratings.csproj" />
     <ProjectReference Include="..\dsstats.shared\dsstats.shared.csproj" />

src/dsstats.decode/Controllers/DecodeController.cs

@@ -0,0 +1,30 @@
+using Microsoft.AspNetCore.Mvc;
+
+namespace dsstats.decode;
+
+[ApiController]
+[Route("/api/v1/[controller]")]
+public class DecodeController(DecodeService decodeService, ILogger<DecodeController> logger) : Controller
+{
+    [HttpPost]
+    [RequestSizeLimit(15728640)]
+    [Route("upload/{guid}")]
+    public async Task<ActionResult<int>> UploadReplays(string guid, [FromForm] List<IFormFile> files)
+    {
+        logger.LogInformation("indahouse1 {guid}", guid);
+        if (Guid.TryParse(guid, out var fileGuid))
+        {
+            var queueCount = await decodeService.SaveReplays(fileGuid, files);
+            logger.LogInformation("indahouse2, {count}", queueCount);
+            if (queueCount >= 0)
+            {
+                return Ok(queueCount);
+            }
+            else
+            {
+                return StatusCode(500);
+            }
+        }
+        return BadRequest();
+    }
+}