From def706a87ab177e38ac965af30a18fa0cf0d5d8c Mon Sep 17 00:00:00 2001 From: ipax77 Date: Sat, 11 May 2024 09:21:30 +0200 Subject: [PATCH 1/4] cu --- src/dsstats.web/dsstats.web.Client/Pages/Tourneys/IhPage.razor | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/dsstats.web/dsstats.web.Client/Pages/Tourneys/IhPage.razor b/src/dsstats.web/dsstats.web.Client/Pages/Tourneys/IhPage.razor index f1048d2a..d2c93b2b 100644 --- a/src/dsstats.web/dsstats.web.Client/Pages/Tourneys/IhPage.razor +++ b/src/dsstats.web/dsstats.web.Client/Pages/Tourneys/IhPage.razor @@ -25,7 +25,7 @@ else RatingType Created - Visitors + Players From 3071d3e70101c6af027ff7a00700e8c8fc914f18 Mon Sep 17 00:00:00 2001 From: ipax77 Date: Sat, 11 May 2024 09:56:59 +0200 Subject: [PATCH 2/4] dsstats.decode --- .../Controllers/DecodeController.cs | 27 ++ src/dsstats.decode/DecodeService.cs | 268 ++++++++++++++++++ src/dsstats.decode/DecodeSettings.cs | 15 + src/dsstats.decode/Program.cs | 20 ++ .../Properties/launchSettings.json | 31 ++ .../appsettings.Development.json | 8 + src/dsstats.decode/appsettings.json | 17 ++ src/dsstats.decode/dsstats.decode.csproj | 18 ++ src/dsstats.decode/dsstats.decode.http | 6 + src/dsstats.decode/dsstats.decode.sln | 25 ++ 10 files changed, 435 insertions(+) create mode 100644 src/dsstats.decode/Controllers/DecodeController.cs create mode 100644 src/dsstats.decode/DecodeService.cs create mode 100644 src/dsstats.decode/DecodeSettings.cs create mode 100644 src/dsstats.decode/Program.cs create mode 100644 src/dsstats.decode/Properties/launchSettings.json create mode 100644 src/dsstats.decode/appsettings.Development.json create mode 100644 src/dsstats.decode/appsettings.json create mode 100644 src/dsstats.decode/dsstats.decode.csproj create mode 100644 src/dsstats.decode/dsstats.decode.http create mode 100644 src/dsstats.decode/dsstats.decode.sln diff --git a/src/dsstats.decode/Controllers/DecodeController.cs b/src/dsstats.decode/Controllers/DecodeController.cs new file mode 100644 index 00000000..d5e90745 --- /dev/null +++ b/src/dsstats.decode/Controllers/DecodeController.cs @@ -0,0 +1,27 @@ + +using dsstats.decode; +using Microsoft.AspNetCore.Mvc; + +[ApiController] +[Route("/api/v1/[controller]")] +public class DecodeController(DecodeService decodeService) : Controller +{ + [HttpPost] + [RequestSizeLimit(15728640)] + public async Task> UploadReplays(string guid, [FromForm] List files) + { + if (Guid.TryParse(guid, out var fileGuid)) + { + var queueCount = await decodeService.SaveReplays(fileGuid, files); + if (queueCount >= 0) + { + return Ok(queueCount); + } + else + { + return StatusCode(500); + } + } + return BadRequest(); + } +} \ No newline at end of file diff --git a/src/dsstats.decode/DecodeService.cs b/src/dsstats.decode/DecodeService.cs new file mode 100644 index 00000000..2c19b0bf --- /dev/null +++ b/src/dsstats.decode/DecodeService.cs @@ -0,0 +1,268 @@ +using dsstats.shared; +using Microsoft.Extensions.Options; +using pax.dsstats.parser; +using s2protocol.NET; +using System.Collections.Concurrent; +using System.Reflection; +using System.Security.Cryptography; +using System.Text.RegularExpressions; + +namespace dsstats.decode; + +public partial class DecodeService(IOptions decodeSettings, ILogger logger) +{ + + private readonly SemaphoreSlim ss = new(1, 1); + private ReplayDecoder? replayDecoder; + public static readonly string assemblyPath = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location) ?? ""; + private int queueCount = 0; + private ConcurrentBag excludeReplays = []; + + public EventHandler? DecodeFinished; + + private void OnDecodeFinished(DecodeEventArgs e) + { + DecodeFinished?.Invoke(this, e); + } + + public async Task SaveReplays(Guid guid, List files) + { + try + { + long size = files.Sum(f => f.Length); + + foreach (var formFile in files) + { + if (formFile.Length > 0) + { + var fileGuid = Guid.NewGuid(); + var filePath = Path.Combine(decodeSettings.Value.ReplayFolders.ToDo, guid.ToString() + "_" + fileGuid.ToString() + ".SC2Replay"); + var tmpFilePath = Path.Combine(decodeSettings.Value.ReplayFolders.ToDo, guid.ToString() + "_" + fileGuid.ToString() + ".tmp"); + + { + using var stream = File.Create(tmpFilePath); + await formFile.CopyToAsync(stream); + } + File.Move(tmpFilePath, filePath); + } + } + _ = Decode(guid); + + logger.LogWarning("replays saved ({size})", size); + return queueCount; + } + catch (Exception ex) + { + logger.LogError("failed saving replays: {error}", ex.Message); + } + return -1; + } + + public async Task Decode(Guid guid) + { + Interlocked.Increment(ref queueCount); + await ss.WaitAsync(); + List replays = []; + string? error = null; + + try + { + var replayPaths = Directory.GetFiles(Path.Combine(decodeSettings.Value.ReplayFolders.ToDo), "*SC2Replay"); + replayPaths = replayPaths.Except(excludeReplays).ToArray(); + + if (replayPaths.Length == 0) + { + error = "No replays found."; + return; + } + + if (replayDecoder is null) + { + replayDecoder = new(assemblyPath); + } + + var options = new ReplayDecoderOptions() + { + Initdata = true, + Details = true, + Metadata = true, + TrackerEvents = true, + }; + + using var md5 = MD5.Create(); + + await foreach (var result in replayDecoder.DecodeParallelWithErrorReport(replayPaths, 2, options)) + { + if (result.Sc2Replay is null) + { + Error(result); + error = "failed decoding replays."; + continue; + } + + var metaData = GetMetaData(result.Sc2Replay); + + var sc2Replay = Parse.GetDsReplay(result.Sc2Replay); + + if (sc2Replay is null) + { + Error(result); + error = "failed decoding replays."; + continue; + } + + var replayDto = Parse.GetReplayDto(sc2Replay, md5); + + if (replayDto is null) + { + Error(result); + error = "failed decoding replays."; + continue; + } + + File.Move(result.ReplayPath, Path.Combine(decodeSettings.Value.ReplayFolders.Done, Path.GetFileName(result.ReplayPath))); + replays.Add(new IhReplay() { Replay = replayDto, Metadata = metaData }); + } + + if (replays.Count > 0) + { + // using var scope = scopeFactory.CreateScope(); + // var importService = scope.ServiceProvider.GetRequiredService(); + // replays.ForEach(f => f.Replay.FileName = string.Empty); + // await importService.Import(replays.Select(s => s.Replay).ToList()); + } + } + catch (Exception ex) + { + logger.LogError("failed decoding replays: {error}", ex.Message); + error = "failed decoding replays."; + } + finally + { + ss.Release(); + OnDecodeFinished(new() + { + Guid = guid, + IhReplays = replays, + Error = error, + }); + Interlocked.Decrement(ref queueCount); + } + } + + private void Error(DecodeParallelResult result) + { + logger.LogError("failed decoding replay: {path}, {error}", result.ReplayPath, result.Exception); + try + { + File.Move(result.ReplayPath, Path.Combine(decodeSettings.Value.ReplayFolders.Error, Path.GetFileName(result.ReplayPath))); + } + catch (Exception ex) + { + logger.LogWarning("failed moving error replay: {error}", ex.Message); + excludeReplays.Add(result.ReplayPath); + } + } + + private ReplayMetadata GetMetaData(Sc2Replay replay) + { + List players = []; + + if (replay.Initdata is null || replay.Details is null || replay.Metadata is null) + { + return new(); + } + + foreach (var player in replay.Initdata.LobbyState.Slots) + { + players.Add(new() + { + PlayerId = GetPlayerId(player.ToonHandle), + Observer = player.Observe == 1, + SlotId = player.WorkingSetSlotId + }); + } + + int i = 0; + foreach (var player in replay.Details.Players) + { + i++; + PlayerId playerId = GetPlayerId(player.Toon); + var metaPlayer = players.FirstOrDefault(f => f.PlayerId == playerId); + if (metaPlayer is null) + { + continue; + } + metaPlayer.Id = i; + metaPlayer.Name = player.Name; + metaPlayer.AssignedRace = GetRace(player.Race); + } + + foreach (var player in replay.Metadata.Players) + { + var metaPlayer = players.FirstOrDefault(f => f.Id == player.PlayerID); + if (metaPlayer is null) + { + continue; + } + metaPlayer.SelectedRace = GetSelectedRace(player.SelectedRace); + } + + return new() + { + Players = players + }; + } + + private static Commander GetSelectedRace(string selectedRace) + { + var race = selectedRace switch + { + "Terr" => "Terran", + "Prot" => "Protoss", + "Rand" => "None", + _ => selectedRace + }; + return GetRace(race); + } + + private static PlayerId GetPlayerId(s2protocol.NET.Models.Toon toon) + { + return new(toon.Id, toon.Realm, toon.Region); + } + + private static PlayerId GetPlayerId(string toonHandle) + { + Regex rx = PlayerIdRegex(); + var match = rx.Match(toonHandle); + if (match.Success) + { + int regionId = int.Parse(match.Groups[1].Value); + int realmId = int.Parse(match.Groups[2].Value); + int toonId = int.Parse(match.Groups[3].Value); + return new(toonId, realmId, regionId); + } + return new(); + } + + private static Commander GetRace(string race) + { + if (Enum.TryParse(typeof(Commander), race, out var cmdrObj) + && cmdrObj is Commander cmdr) + { + return cmdr; + } + return Commander.None; + } + + [GeneratedRegex(@"(\d)-S2-(\d)-(\d+)")] + private static partial Regex PlayerIdRegex(); +} + +public class DecodeEventArgs : EventArgs +{ + public Guid Guid { get; set; } + public List IhReplays { get; set; } = []; + public string? Error { get; set; } +} + diff --git a/src/dsstats.decode/DecodeSettings.cs b/src/dsstats.decode/DecodeSettings.cs new file mode 100644 index 00000000..f026ee3d --- /dev/null +++ b/src/dsstats.decode/DecodeSettings.cs @@ -0,0 +1,15 @@ + +namespace dsstats.decode; + +public record DecodeSettings +{ + public ReplayFolders ReplayFolders { get; set; } = new(); + public int Threads { get; set; } +} + +public record ReplayFolders +{ + public string ToDo { get; set; } = string.Empty; + public string Done { get; set; } = string.Empty; + public string Error { get; set; } = string.Empty; +} \ No newline at end of file diff --git a/src/dsstats.decode/Program.cs b/src/dsstats.decode/Program.cs new file mode 100644 index 00000000..aa9172a9 --- /dev/null +++ b/src/dsstats.decode/Program.cs @@ -0,0 +1,20 @@ +using dsstats.decode; + +var builder = WebApplication.CreateBuilder(args); + +// Add services to the container. + +builder.Services.AddControllers(); + +builder.Services.Configure(builder.Configuration.GetSection("DecodeSettings")); +builder.Services.AddSingleton(); + +var app = builder.Build(); + +// Configure the HTTP request pipeline. + +app.UseAuthorization(); + +app.MapControllers(); + +app.Run(); diff --git a/src/dsstats.decode/Properties/launchSettings.json b/src/dsstats.decode/Properties/launchSettings.json new file mode 100644 index 00000000..f88524bf --- /dev/null +++ b/src/dsstats.decode/Properties/launchSettings.json @@ -0,0 +1,31 @@ +{ + "$schema": "http://json.schemastore.org/launchsettings.json", + "iisSettings": { + "windowsAuthentication": false, + "anonymousAuthentication": true, + "iisExpress": { + "applicationUrl": "http://localhost:51580", + "sslPort": 0 + } + }, + "profiles": { + "http": { + "commandName": "Project", + "dotnetRunMessages": true, + "launchBrowser": true, + "launchUrl": "weatherforecast", + "applicationUrl": "http://localhost:5240", + "environmentVariables": { + "ASPNETCORE_ENVIRONMENT": "Development" + } + }, + "IIS Express": { + "commandName": "IISExpress", + "launchBrowser": true, + "launchUrl": "weatherforecast", + "environmentVariables": { + "ASPNETCORE_ENVIRONMENT": "Development" + } + } + } +} diff --git a/src/dsstats.decode/appsettings.Development.json b/src/dsstats.decode/appsettings.Development.json new file mode 100644 index 00000000..0c208ae9 --- /dev/null +++ b/src/dsstats.decode/appsettings.Development.json @@ -0,0 +1,8 @@ +{ + "Logging": { + "LogLevel": { + "Default": "Information", + "Microsoft.AspNetCore": "Warning" + } + } +} diff --git a/src/dsstats.decode/appsettings.json b/src/dsstats.decode/appsettings.json new file mode 100644 index 00000000..a114c695 --- /dev/null +++ b/src/dsstats.decode/appsettings.json @@ -0,0 +1,17 @@ +{ + "Logging": { + "LogLevel": { + "Default": "Information", + "Microsoft.AspNetCore": "Warning" + } + }, + "AllowedHosts": "*", + "DecodeSettings": { + "Threads": 2, + "ReplayFolders": { + "ToDo": "/data/ds/decode/todo", + "Done": "/data/ds/decode/done", + "Error": "/data/ds/decode/error" + } + } +} diff --git a/src/dsstats.decode/dsstats.decode.csproj b/src/dsstats.decode/dsstats.decode.csproj new file mode 100644 index 00000000..433a9392 --- /dev/null +++ b/src/dsstats.decode/dsstats.decode.csproj @@ -0,0 +1,18 @@ + + + + + + + + + + + + + net8.0 + enable + enable + + + diff --git a/src/dsstats.decode/dsstats.decode.http b/src/dsstats.decode/dsstats.decode.http new file mode 100644 index 00000000..9d3abefa --- /dev/null +++ b/src/dsstats.decode/dsstats.decode.http @@ -0,0 +1,6 @@ +@dsstats.decode_HostAddress = http://localhost:5240 + +GET {{dsstats.decode_HostAddress}}/weatherforecast/ +Accept: application/json + +### diff --git a/src/dsstats.decode/dsstats.decode.sln b/src/dsstats.decode/dsstats.decode.sln new file mode 100644 index 00000000..cf07dbc7 --- /dev/null +++ b/src/dsstats.decode/dsstats.decode.sln @@ -0,0 +1,25 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Version 17 +VisualStudioVersion = 17.9.34728.123 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "dsstats.decode", "dsstats.decode.csproj", "{738C7948-4F7C-4166-9EA2-FBF8A8529A72}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Release|Any CPU = Release|Any CPU + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {738C7948-4F7C-4166-9EA2-FBF8A8529A72}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {738C7948-4F7C-4166-9EA2-FBF8A8529A72}.Debug|Any CPU.Build.0 = Debug|Any CPU + {738C7948-4F7C-4166-9EA2-FBF8A8529A72}.Release|Any CPU.ActiveCfg = Release|Any CPU + {738C7948-4F7C-4166-9EA2-FBF8A8529A72}.Release|Any CPU.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection + GlobalSection(ExtensibilityGlobals) = postSolution + SolutionGuid = {5F300EA6-CF54-4DFF-8F45-C87F4F5B28DB} + EndGlobalSection +EndGlobal From 5fa584bfca7968f605fbfb36a579fa865f6c37c4 Mon Sep 17 00:00:00 2001 From: ipax77 Date: Sat, 11 May 2024 19:31:30 +0200 Subject: [PATCH 3/4] decode callback --- .../Controllers/UploadController.cs | 38 ++++++++++--- src/dsstats.api/Program.cs | 6 +++ .../Controllers/DecodeController.cs | 9 ++-- src/dsstats.decode/DecodeService.cs | 54 ++++++++++++++----- src/dsstats.decode/DecodeSettings.cs | 1 + src/dsstats.decode/Program.cs | 9 ++++ .../appsettings.Development.json | 11 +++- src/dsstats.decode/appsettings.json | 1 + 8 files changed, 107 insertions(+), 22 deletions(-) diff --git a/src/dsstats.api/Controllers/UploadController.cs b/src/dsstats.api/Controllers/UploadController.cs index ac2ce7ac..c62e7a33 100644 --- a/src/dsstats.api/Controllers/UploadController.cs +++ b/src/dsstats.api/Controllers/UploadController.cs @@ -8,7 +8,10 @@ namespace dsstats.api.Controllers; [ApiController] [Route("api8/v1/[controller]")] [ServiceFilter(typeof(AuthenticationFilterAttribute))] -public class UploadController(UploadService uploadService, DecodeService decodeService) : Controller +public class UploadController(UploadService uploadService, + DecodeService decodeService, + IHttpClientFactory httpClientFactory, + ILogger logger) : Controller { private readonly UploadService uploadService = uploadService; @@ -39,18 +42,41 @@ public async Task ImportReplays8([FromBody] UploadDto uploadDto) [EnableRateLimiting("fixed")] public async Task> UploadReplays(string guid, [FromForm] List files) { + logger.LogWarning("indahouse1 {guid}", guid); if (Guid.TryParse(guid, out var fileGuid)) { - var queueCount = await decodeService.SaveReplays(fileGuid, files); - if (queueCount >= 0) + var httpClient = httpClientFactory.CreateClient("decode"); + try { - return Ok(queueCount); + var formData = new MultipartFormDataContent(); + + foreach (var file in files) + { + var fileContent = new StreamContent(file.OpenReadStream()); + formData.Add(fileContent, "files", file.FileName); + } + + var result = await httpClient.PostAsync($"/api/v1/decode/upload/{fileGuid}", formData); + result.EnsureSuccessStatusCode(); + return Ok(0); } - else + catch (Exception ex) { - return StatusCode(500); + logger.LogError("failed passing decode request: {error}", ex.Message); } } return BadRequest(); } + + [HttpPost] + [Route("decoderesult/{guid}")] + public async Task DecodeResult(string guid, [FromBody] List replays) + { + logger.LogWarning("got decode result for {guid}", guid); + if (Guid.TryParse(guid, out var groupId)) + { + return Ok(); + } + return BadRequest(); + } } diff --git a/src/dsstats.api/Program.cs b/src/dsstats.api/Program.cs index 52cd9ac2..8bcf06cd 100644 --- a/src/dsstats.api/Program.cs +++ b/src/dsstats.api/Program.cs @@ -99,6 +99,12 @@ options.DefaultRequestHeaders.Add("Accept", "application/json"); }); +builder.Services.AddHttpClient("decode") + .ConfigureHttpClient(options => + { + options.BaseAddress = new Uri("http://localhost:5240"); + }); + builder.Services.AddSingleton(); builder.Services.AddSingleton(); builder.Services.AddSingleton(); diff --git a/src/dsstats.decode/Controllers/DecodeController.cs b/src/dsstats.decode/Controllers/DecodeController.cs index d5e90745..fc9e6ff0 100644 --- a/src/dsstats.decode/Controllers/DecodeController.cs +++ b/src/dsstats.decode/Controllers/DecodeController.cs @@ -1,18 +1,21 @@ - -using dsstats.decode; using Microsoft.AspNetCore.Mvc; +namespace dsstats.decode; + [ApiController] [Route("/api/v1/[controller]")] -public class DecodeController(DecodeService decodeService) : Controller +public class DecodeController(DecodeService decodeService, ILogger logger) : Controller { [HttpPost] [RequestSizeLimit(15728640)] + [Route("upload/{guid}")] public async Task> UploadReplays(string guid, [FromForm] List files) { + logger.LogInformation("indahouse1 {guid}", guid); if (Guid.TryParse(guid, out var fileGuid)) { var queueCount = await decodeService.SaveReplays(fileGuid, files); + logger.LogInformation("indahouse2, {count}", queueCount); if (queueCount >= 0) { return Ok(queueCount); diff --git a/src/dsstats.decode/DecodeService.cs b/src/dsstats.decode/DecodeService.cs index 2c19b0bf..d752f241 100644 --- a/src/dsstats.decode/DecodeService.cs +++ b/src/dsstats.decode/DecodeService.cs @@ -9,7 +9,9 @@ namespace dsstats.decode; -public partial class DecodeService(IOptions decodeSettings, ILogger logger) +public partial class DecodeService(IOptions decodeSettings, + IHttpClientFactory httpClientFactory, + ILogger logger) { private readonly SemaphoreSlim ss = new(1, 1); @@ -20,8 +22,18 @@ public partial class DecodeService(IOptions decodeSettings, ILog public EventHandler? DecodeFinished; - private void OnDecodeFinished(DecodeEventArgs e) + private async void OnDecodeFinished(DecodeEventArgs e) { + var httpClient = httpClientFactory.CreateClient("callback"); + try + { + var result = await httpClient.PostAsJsonAsync($"/api8/v1/upload/decoderesult/{e.Guid}", e.IhReplays); + result.EnsureSuccessStatusCode(); + } + catch (Exception ex) + { + logger.LogError("failed reporting decoderesult: {error}", ex.Message); + } DecodeFinished?.Invoke(this, e); } @@ -46,7 +58,7 @@ public async Task SaveReplays(Guid guid, List files) File.Move(tmpFilePath, filePath); } } - _ = Decode(guid); + _ = Decode(); logger.LogWarning("replays saved ({size})", size); return queueCount; @@ -58,11 +70,11 @@ public async Task SaveReplays(Guid guid, List files) return -1; } - public async Task Decode(Guid guid) + public async Task Decode() { Interlocked.Increment(ref queueCount); await ss.WaitAsync(); - List replays = []; + ConcurrentDictionary> replays = []; string? error = null; try @@ -91,7 +103,8 @@ public async Task Decode(Guid guid) using var md5 = MD5.Create(); - await foreach (var result in replayDecoder.DecodeParallelWithErrorReport(replayPaths, 2, options)) + await foreach (var result in + replayDecoder.DecodeParallelWithErrorReport(replayPaths, decodeSettings.Value.Threads, options)) { if (result.Sc2Replay is null) { @@ -121,7 +134,9 @@ public async Task Decode(Guid guid) } File.Move(result.ReplayPath, Path.Combine(decodeSettings.Value.ReplayFolders.Done, Path.GetFileName(result.ReplayPath))); - replays.Add(new IhReplay() { Replay = replayDto, Metadata = metaData }); + var groupId = GetGroupIdFromFilename(result.ReplayPath); + var ihReplay = new IhReplay() { Replay = replayDto, Metadata = metaData }; + replays.AddOrUpdate(groupId, [ihReplay], (k, v) => { v.Add(ihReplay); return v; }); } if (replays.Count > 0) @@ -140,12 +155,15 @@ public async Task Decode(Guid guid) finally { ss.Release(); - OnDecodeFinished(new() + foreach (var ent in replays) { - Guid = guid, - IhReplays = replays, - Error = error, - }); + OnDecodeFinished(new() + { + Guid = ent.Key, + IhReplays = [.. ent.Value], + Error = error, + }); + } Interlocked.Decrement(ref queueCount); } } @@ -214,6 +232,18 @@ private ReplayMetadata GetMetaData(Sc2Replay replay) }; } + private static Guid GetGroupIdFromFilename(string replayPath) + { + var fileName = Path.GetFileNameWithoutExtension(replayPath); + var guids = fileName.Split('_', StringSplitOptions.RemoveEmptyEntries); + if (guids.Length > 0 && Guid.TryParse(guids[1], out var groupId) + && groupId != Guid.Empty) + { + return groupId; + } + throw new Exception($"failed getting groupId from replayPath: {replayPath}"); + } + private static Commander GetSelectedRace(string selectedRace) { var race = selectedRace switch diff --git a/src/dsstats.decode/DecodeSettings.cs b/src/dsstats.decode/DecodeSettings.cs index f026ee3d..f5275c0d 100644 --- a/src/dsstats.decode/DecodeSettings.cs +++ b/src/dsstats.decode/DecodeSettings.cs @@ -5,6 +5,7 @@ public record DecodeSettings { public ReplayFolders ReplayFolders { get; set; } = new(); public int Threads { get; set; } + public string CallbackUrl { get; set; } = string.Empty; } public record ReplayFolders diff --git a/src/dsstats.decode/Program.cs b/src/dsstats.decode/Program.cs index aa9172a9..4161f5cc 100644 --- a/src/dsstats.decode/Program.cs +++ b/src/dsstats.decode/Program.cs @@ -1,3 +1,4 @@ +using System.Net.Http.Headers; using dsstats.decode; var builder = WebApplication.CreateBuilder(args); @@ -9,6 +10,14 @@ builder.Services.Configure(builder.Configuration.GetSection("DecodeSettings")); builder.Services.AddSingleton(); +builder.Services.AddHttpClient("callback") + .ConfigureHttpClient(options => + { + options.BaseAddress = new Uri(builder.Configuration["DecodeSettings:CallbackUrl"] ?? ""); + options.DefaultRequestHeaders.Add("Accept", "application/json"); + options.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("DS8upload77"); + }); + var app = builder.Build(); // Configure the HTTP request pipeline. diff --git a/src/dsstats.decode/appsettings.Development.json b/src/dsstats.decode/appsettings.Development.json index 0c208ae9..b03e6ec9 100644 --- a/src/dsstats.decode/appsettings.Development.json +++ b/src/dsstats.decode/appsettings.Development.json @@ -2,7 +2,16 @@ "Logging": { "LogLevel": { "Default": "Information", - "Microsoft.AspNetCore": "Warning" + "Microsoft.AspNetCore": "Information" + } + }, + "DecodeSettings": { + "CallbackUrl": "http://localhost:5116", + "Threads": 2, + "ReplayFolders": { + "ToDo": "/data/ds/decode/todo", + "Done": "/data/ds/decode/done", + "Error": "/data/ds/decode/error" } } } diff --git a/src/dsstats.decode/appsettings.json b/src/dsstats.decode/appsettings.json index a114c695..b27553df 100644 --- a/src/dsstats.decode/appsettings.json +++ b/src/dsstats.decode/appsettings.json @@ -7,6 +7,7 @@ }, "AllowedHosts": "*", "DecodeSettings": { + "CallbackUrl": "https://dsstats.pax77.org", "Threads": 2, "ReplayFolders": { "ToDo": "/data/ds/decode/todo", From e08b15ab529b4ac851121ccba0b3e1df1ad4c01e Mon Sep 17 00:00:00 2001 From: ipax77 Date: Sun, 12 May 2024 09:38:42 +0200 Subject: [PATCH 4/4] cu --- .../Controllers/UploadController.cs | 28 +-- src/dsstats.api/Program.cs | 6 +- src/dsstats.api/Services/DecodeService.cs | 220 ++---------------- src/dsstats.api/Services/IhService.cs | 1 - src/dsstats.api/dsstats.api.csproj | 2 - src/dsstats.decode/DecodeService.cs | 2 +- src/dsstats.decode/appsettings.json | 2 +- src/dsstats.razorlib/Ih/IhComponent.razor | 19 +- src/dsstats.razorlib/Ih/IhComponent.razor.cs | 1 + src/dsstats.razorlib/Ih/IhUploadComp.razor | 3 +- 10 files changed, 42 insertions(+), 242 deletions(-) diff --git a/src/dsstats.api/Controllers/UploadController.cs b/src/dsstats.api/Controllers/UploadController.cs index c62e7a33..876fc7b1 100644 --- a/src/dsstats.api/Controllers/UploadController.cs +++ b/src/dsstats.api/Controllers/UploadController.cs @@ -9,9 +9,7 @@ namespace dsstats.api.Controllers; [Route("api8/v1/[controller]")] [ServiceFilter(typeof(AuthenticationFilterAttribute))] public class UploadController(UploadService uploadService, - DecodeService decodeService, - IHttpClientFactory httpClientFactory, - ILogger logger) : Controller + DecodeService decodeService) : Controller { private readonly UploadService uploadService = uploadService; @@ -42,28 +40,10 @@ public async Task ImportReplays8([FromBody] UploadDto uploadDto) [EnableRateLimiting("fixed")] public async Task> UploadReplays(string guid, [FromForm] List files) { - logger.LogWarning("indahouse1 {guid}", guid); if (Guid.TryParse(guid, out var fileGuid)) { - var httpClient = httpClientFactory.CreateClient("decode"); - try - { - var formData = new MultipartFormDataContent(); - - foreach (var file in files) - { - var fileContent = new StreamContent(file.OpenReadStream()); - formData.Add(fileContent, "files", file.FileName); - } - - var result = await httpClient.PostAsync($"/api/v1/decode/upload/{fileGuid}", formData); - result.EnsureSuccessStatusCode(); - return Ok(0); - } - catch (Exception ex) - { - logger.LogError("failed passing decode request: {error}", ex.Message); - } + await decodeService.SaveReplays(fileGuid, files); + return Ok(); } return BadRequest(); } @@ -72,9 +52,9 @@ public async Task> UploadReplays(string guid, [FromForm] List< [Route("decoderesult/{guid}")] public async Task DecodeResult(string guid, [FromBody] List replays) { - logger.LogWarning("got decode result for {guid}", guid); if (Guid.TryParse(guid, out var groupId)) { + await decodeService.ConsumeDecodeResult(groupId, replays); return Ok(); } return BadRequest(); diff --git a/src/dsstats.api/Program.cs b/src/dsstats.api/Program.cs index 8bcf06cd..f057d3b9 100644 --- a/src/dsstats.api/Program.cs +++ b/src/dsstats.api/Program.cs @@ -33,7 +33,8 @@ policy.WithOrigins("https://dsstats.pax77.org", "https://dsstats-dev.pax77.org", "https://localhost:7257", - "https://localhost:7227") + "https://localhost:7227", + "http://localhost:5123") .AllowAnyHeader() .AllowAnyMethod(); }); @@ -102,7 +103,8 @@ builder.Services.AddHttpClient("decode") .ConfigureHttpClient(options => { - options.BaseAddress = new Uri("http://localhost:5240"); + // options.BaseAddress = new Uri("http://localhost:5240"); + options.BaseAddress = new Uri(builder.Configuration["ServerConfig:DecodeUrl"] ?? ""); }); builder.Services.AddSingleton(); diff --git a/src/dsstats.api/Services/DecodeService.cs b/src/dsstats.api/Services/DecodeService.cs index 0a5ae515..1cfa8f7b 100644 --- a/src/dsstats.api/Services/DecodeService.cs +++ b/src/dsstats.api/Services/DecodeService.cs @@ -1,24 +1,12 @@ using dsstats.db8services.Import; using dsstats.shared; -using pax.dsstats.parser; -using s2protocol.NET; -using System.Collections.Concurrent; -using System.Reflection; -using System.Security.Cryptography; -using System.Text.RegularExpressions; namespace dsstats.api.Services; -public class DecodeService(ILogger logger, IServiceScopeFactory scopeFactory) +public class DecodeService(ILogger logger, + IHttpClientFactory httpClientFactory, + IServiceScopeFactory scopeFactory) { - private readonly string replayFolder = "/data/ds/decode"; - - private readonly SemaphoreSlim ss = new(1, 1); - private ReplayDecoder? replayDecoder; - public static readonly string assemblyPath = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location) ?? ""; - private int queueCount = 0; - private ConcurrentBag excludeReplays = []; - public EventHandler? DecodeFinished; private void OnDecodeFinished(DecodeEventArgs e) @@ -26,105 +14,32 @@ private void OnDecodeFinished(DecodeEventArgs e) DecodeFinished?.Invoke(this, e); } - public async Task SaveReplays(Guid guid, List files) + public async Task SaveReplays(Guid guid, List files) { + var httpClient = httpClientFactory.CreateClient("decode"); try { - long size = files.Sum(f => f.Length); + var formData = new MultipartFormDataContent(); - foreach (var formFile in files) + foreach (var file in files) { - if (formFile.Length > 0) - { - var fileGuid = Guid.NewGuid(); - var filePath = Path.Combine(replayFolder, "todo", guid.ToString() + "_" + fileGuid.ToString() + ".SC2Replay"); - var tmpFilePath = Path.Combine(replayFolder, "todo", guid.ToString() + "_" + fileGuid.ToString() + ".tmp"); - - { - using var stream = File.Create(tmpFilePath); - await formFile.CopyToAsync(stream); - } - File.Move(tmpFilePath, filePath); - } + var fileContent = new StreamContent(file.OpenReadStream()); + formData.Add(fileContent, "files", file.FileName); } - _ = Decode(guid); - logger.LogWarning($"replays saved ({size})", size); - return queueCount; + var result = await httpClient.PostAsync($"/api/v1/decode/upload/{guid}", formData); + result.EnsureSuccessStatusCode(); } catch (Exception ex) { logger.LogError("failed saving replays: {error}", ex.Message); } - return -1; } - public async Task Decode(Guid guid) + public async Task ConsumeDecodeResult(Guid guid, List replays) { - Interlocked.Increment(ref queueCount); - await ss.WaitAsync(); - List replays = []; - string? error = null; - try { - var replayPaths = Directory.GetFiles(Path.Combine(replayFolder, "todo"), "*SC2Replay"); - replayPaths = replayPaths.Except(excludeReplays).ToArray(); - - if (replayPaths.Length == 0) - { - error = "No replays found."; - return; - } - - if (replayDecoder is null) - { - replayDecoder = new(assemblyPath); - } - - var options = new ReplayDecoderOptions() - { - Initdata = true, - Details = true, - Metadata = true, - TrackerEvents = true, - }; - - using var md5 = MD5.Create(); - - await foreach (var result in replayDecoder.DecodeParallelWithErrorReport(replayPaths, 2, options)) - { - if (result.Sc2Replay is null) - { - Error(result); - error = "failed decoding replays."; - continue; - } - - var metaData = GetMetaData(result.Sc2Replay); - - var sc2Replay = Parse.GetDsReplay(result.Sc2Replay); - - if (sc2Replay is null) - { - Error(result); - error = "failed decoding replays."; - continue; - } - - var replayDto = Parse.GetReplayDto(sc2Replay, md5); - - if (replayDto is null) - { - Error(result); - error = "failed decoding replays."; - continue; - } - - File.Move(result.ReplayPath, Path.Combine(replayFolder, "done", Path.GetFileName(result.ReplayPath))); - replays.Add(new IhReplay() { Replay = replayDto, Metadata = metaData }); - } - if (replays.Count > 0) { using var scope = scopeFactory.CreateScope(); @@ -135,125 +50,16 @@ public async Task Decode(Guid guid) } catch (Exception ex) { - logger.LogError("failed decoding replays: {error}", ex.Message); - error = "failed decoding replays."; + logger.LogError("failed importing decode result: {error}", ex.Message); } finally { - ss.Release(); OnDecodeFinished(new() { Guid = guid, IhReplays = replays, - Error = error, - }); - Interlocked.Decrement(ref queueCount); - } - } - - private void Error(DecodeParallelResult result) - { - logger.LogError("failed decoding replay: {path}, {error}", result.ReplayPath, result.Exception); - try - { - File.Move(result.ReplayPath, Path.Combine(replayFolder, "error", Path.GetFileName(result.ReplayPath))); - } - catch (Exception ex) - { - logger.LogWarning("failed moving error replay: {error}", ex.Message); - excludeReplays.Add(result.ReplayPath); - } - } - - private ReplayMetadata GetMetaData(Sc2Replay replay) - { - List players = []; - - if (replay.Initdata is null || replay.Details is null || replay.Metadata is null) - { - return new(); - } - - foreach (var player in replay.Initdata.LobbyState.Slots) - { - players.Add(new() - { - PlayerId = GetPlayerId(player.ToonHandle), - Observer = player.Observe == 1, - SlotId = player.WorkingSetSlotId }); } - - int i = 0; - foreach (var player in replay.Details.Players) - { - i++; - PlayerId playerId = GetPlayerId(player.Toon); - var metaPlayer = players.FirstOrDefault(f => f.PlayerId == playerId); - if (metaPlayer is null) - { - continue; - } - metaPlayer.Id = i; - metaPlayer.Name = player.Name; - metaPlayer.AssignedRace = GetRace(player.Race); - } - - foreach (var player in replay.Metadata.Players) - { - var metaPlayer = players.FirstOrDefault(f => f.Id == player.PlayerID); - if (metaPlayer is null) - { - continue; - } - metaPlayer.SelectedRace = GetSelectedRace(player.SelectedRace); - } - - return new() - { - Players = players - }; - } - - private static Commander GetSelectedRace(string selectedRace) - { - var race = selectedRace switch - { - "Terr" => "Terran", - "Prot" => "Protoss", - "Rand" => "None", - _ => selectedRace - }; - return GetRace(race); - } - - private static PlayerId GetPlayerId(s2protocol.NET.Models.Toon toon) - { - return new(toon.Id, toon.Realm, toon.Region); - } - - private static PlayerId GetPlayerId(string toonHandle) - { - Regex rx = new(@"(\d)-S2-(\d)-(\d+)"); - var match = rx.Match(toonHandle); - if (match.Success) - { - int regionId = int.Parse(match.Groups[1].Value); - int realmId = int.Parse(match.Groups[2].Value); - int toonId = int.Parse(match.Groups[3].Value); - return new(toonId, realmId, regionId); - } - return new(); - } - - private static Commander GetRace(string race) - { - if (Enum.TryParse(typeof(Commander), race, out var cmdrObj) - && cmdrObj is Commander cmdr) - { - return cmdr; - } - return Commander.None; } } diff --git a/src/dsstats.api/Services/IhService.cs b/src/dsstats.api/Services/IhService.cs index ee5be871..4ba516ce 100644 --- a/src/dsstats.api/Services/IhService.cs +++ b/src/dsstats.api/Services/IhService.cs @@ -78,7 +78,6 @@ public async Task CreateOrVisitGroup(Guid groupId) completionSource.SetResult(args.IhReplays); } }; - decodeService.DecodeFinished += decodeEventHandler; var timeoutTask = Task.Delay(20000); diff --git a/src/dsstats.api/dsstats.api.csproj b/src/dsstats.api/dsstats.api.csproj index 23697cc6..ccd164b2 100644 --- a/src/dsstats.api/dsstats.api.csproj +++ b/src/dsstats.api/dsstats.api.csproj @@ -7,7 +7,6 @@ - runtime; build; native; contentfiles; analyzers; buildtransitive @@ -20,7 +19,6 @@ - diff --git a/src/dsstats.decode/DecodeService.cs b/src/dsstats.decode/DecodeService.cs index d752f241..7940965e 100644 --- a/src/dsstats.decode/DecodeService.cs +++ b/src/dsstats.decode/DecodeService.cs @@ -236,7 +236,7 @@ private static Guid GetGroupIdFromFilename(string replayPath) { var fileName = Path.GetFileNameWithoutExtension(replayPath); var guids = fileName.Split('_', StringSplitOptions.RemoveEmptyEntries); - if (guids.Length > 0 && Guid.TryParse(guids[1], out var groupId) + if (guids.Length > 0 && Guid.TryParse(guids[0], out var groupId) && groupId != Guid.Empty) { return groupId; diff --git a/src/dsstats.decode/appsettings.json b/src/dsstats.decode/appsettings.json index b27553df..88ac5b35 100644 --- a/src/dsstats.decode/appsettings.json +++ b/src/dsstats.decode/appsettings.json @@ -7,7 +7,7 @@ }, "AllowedHosts": "*", "DecodeSettings": { - "CallbackUrl": "https://dsstats.pax77.org", + "CallbackUrl": "http://dsstats8:80", "Threads": 2, "ReplayFolders": { "ToDo": "/data/ds/decode/todo", diff --git a/src/dsstats.razorlib/Ih/IhComponent.razor b/src/dsstats.razorlib/Ih/IhComponent.razor index 34019d8a..43ac865a 100644 --- a/src/dsstats.razorlib/Ih/IhComponent.razor +++ b/src/dsstats.razorlib/Ih/IhComponent.razor @@ -1,7 +1,19 @@ @using System.Globalization @using dsstats.razorlib.Services -

Ih Session

+
+
+

IH Session - Visitors: @groupState.Visitors

+
+ @if (decoding) + { +
+ +
+ } +
Select or drag&drop replays here to get player stats @@ -10,6 +22,9 @@
+
+ Total Players: @groupState.PlayerStates.Count +
@@ -34,7 +49,7 @@
@state.Games @state.Observer - @HelperService.GetPercentageString(playerStat?.Wins, state.Games) + @HelperService.GetPercentageString(playerStat?.Wins, state.Games) @if (playerStat != null) diff --git a/src/dsstats.razorlib/Ih/IhComponent.razor.cs b/src/dsstats.razorlib/Ih/IhComponent.razor.cs index e7096bbe..d6897a87 100644 --- a/src/dsstats.razorlib/Ih/IhComponent.razor.cs +++ b/src/dsstats.razorlib/Ih/IhComponent.razor.cs @@ -126,6 +126,7 @@ public void DecodeRequested() if (isConnected) { hubConnection?.SendAsync("DecodeRequest"); + decoding = true; } } diff --git a/src/dsstats.razorlib/Ih/IhUploadComp.razor b/src/dsstats.razorlib/Ih/IhUploadComp.razor index acffdfc9..93e1c0be 100644 --- a/src/dsstats.razorlib/Ih/IhUploadComp.razor +++ b/src/dsstats.razorlib/Ih/IhUploadComp.razor @@ -100,8 +100,7 @@ httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("DS8upload77"); var response = await httpClient.PostAsync($"/api8/v1/upload/uploadreplays/{Guid}", content); response.EnsureSuccessStatusCode(); - int queueCount = await response.Content.ReadFromJsonAsync(); - await OnDecodeRequested.InvokeAsync(queueCount); + await OnDecodeRequested.InvokeAsync(0); } catch (Exception ex) {