cache v2 experiment

src/Runner.Server/Controllers/ArtifactControllerV2.cs

@@ -35,12 +35,12 @@ public ArtifactControllerV2(SqLiteDb _context, IConfiguration configuration) : b
         }
 
         private string CreateSignature(int id) {
-            using(var rsa = RSA.Create(Startup.AccessTokenParameter))
+            using var rsa = RSA.Create(Startup.AccessTokenParameter);
             return Base64UrlEncoder.Encode(rsa.SignData(Encoding.UTF8.GetBytes(id.ToString()), HashAlgorithmName.SHA256, RSASignaturePadding.Pkcs1));
         }
 
         private bool VerifySignature(int id, string sig) {
-            using(var rsa = RSA.Create(Startup.AccessTokenParameter))
+            using var rsa = RSA.Create(Startup.AccessTokenParameter);
             return rsa.VerifyData(Encoding.UTF8.GetBytes(id.ToString()), Base64UrlEncoder.DecodeBytes(sig), HashAlgorithmName.SHA256, RSASignaturePadding.Pkcs1);
         }
 
@@ -76,7 +76,7 @@ public async Task<IActionResult> UploadArtifact(int id, string sig, string comp
             if(string.IsNullOrEmpty(sig) || !VerifySignature(id, sig)) {
                 return NotFound();
             }
-            if(comp == "block" || comp == "appendBlock") {
+            if(comp == "block" || comp == "appendBlock" || comp == null) {
                 var record = await _context.ArtifactRecords.FindAsync(id);
                 var _targetFilePath = Path.Combine(GitHub.Runner.Sdk.GharunUtil.GetLocalStorage(), "artifacts");
                 using(var targetStream = new FileStream(Path.Combine(_targetFilePath, record.StoreName), FileMode.OpenOrCreate | FileMode.Append, FileAccess.Write, FileShare.Write)) {
@@ -171,36 +171,5 @@ public async Task<IActionResult> DeleteArtifact([FromBody, Protobuf] Github.Acti
             };
             return new OkObjectResult(formatter.Format(resp));
         }
-
-
-        public class ProtobufBinder : IModelBinder
-        {
-            public async Task BindModelAsync(ModelBindingContext bindingContext)
-            {
-                if (!bindingContext.HttpContext.Request.HasJsonContentType())
-                {
-                    throw new BadHttpRequestException(
-                        "Request content type was not a recognized JSON content type.",
-                        StatusCodes.Status415UnsupportedMediaType);
-                }
-
-                using var sr = new StreamReader(bindingContext.HttpContext.Request.Body);
-                var str = await sr.ReadToEndAsync();
-
-                var valueType = bindingContext.ModelType;
-                var parser = new JsonParser(JsonParser.Settings.Default.WithIgnoreUnknownFields(true));
-
-                var descriptor = (MessageDescriptor)bindingContext.ModelType.GetProperty("Descriptor", BindingFlags.Public | BindingFlags.Static).GetValue(null, null);
-                var obj = parser.Parse(str, descriptor);
-
-                bindingContext.Result = ModelBindingResult.Success(obj);
-            }
-        }
-
-        public class ProtobufAttribute : ModelBinderAttribute {
-            public ProtobufAttribute() : base(typeof(ProtobufBinder)) {
-
-            }
-        }
     }
 }

src/Runner.Server/Controllers/CacheControllerV2.cs

@@ -0,0 +1,131 @@
+using Microsoft.AspNetCore.Mvc;
+using Microsoft.AspNetCore.Authorization;
+using Microsoft.Extensions.Configuration;
+using Google.Protobuf;
+using System;
+using System.Threading.Tasks;
+using Microsoft.AspNetCore.Mvc.ModelBinding;
+using Microsoft.AspNetCore.Http;
+using System.IO;
+using System.Reflection;
+using Google.Protobuf.Reflection;
+using Microsoft.AspNetCore.Http.Extensions;
+using Runner.Server.Models;
+using System.Linq;
+using Microsoft.EntityFrameworkCore;
+using GitHub.Actions.Pipelines.WebApi;
+using System.Security.Cryptography;
+using Microsoft.IdentityModel.Tokens;
+using System.Text;
+
+namespace Runner.Server.Controllers
+{
+
+    [ApiController]
+    [Route("twirp/github.actions.results.api.v1.CacheService")]
+    [Authorize(AuthenticationSchemes = "Bearer", Policy = "AgentJob")]
+    public class CacheControllerV2 : VssControllerBase{
+        private readonly SqLiteDb _context;
+        private readonly JsonFormatter formatter;
+
+        public CacheControllerV2(SqLiteDb _context, IConfiguration configuration) : base(configuration)
+        {
+            this._context = _context;
+            formatter = new JsonFormatter(JsonFormatter.Settings.Default.WithIndentation().WithPreserveProtoFieldNames(true).WithFormatDefaultValues(false));
+        }
+
+        private string CreateSignature(int id) {
+            using var rsa = RSA.Create(Startup.AccessTokenParameter);
+            return Base64UrlEncoder.Encode(rsa.SignData(Encoding.UTF8.GetBytes(id.ToString()), HashAlgorithmName.SHA256, RSASignaturePadding.Pkcs1));
+        }
+
+        private bool VerifySignature(int id, string sig) {
+            using var rsa = RSA.Create(Startup.AccessTokenParameter);
+            return rsa.VerifyData(Encoding.UTF8.GetBytes(id.ToString()), Base64UrlEncoder.DecodeBytes(sig), HashAlgorithmName.SHA256, RSASignaturePadding.Pkcs1);
+        }
+
+        [HttpPost("CreateCacheEntry")]
+        public async Task<string> CreateCacheEntry([FromBody, Protobuf] Github.Actions.Results.Api.V1.CreateCacheEntryRequest body) {
+            var filename = Path.GetRandomFileName();
+            var reference = User.FindFirst("ref")?.Value ?? "refs/heads/main";
+            var repository = User.FindFirst("repository")?.Value ?? "Unknown/Unknown";
+            var record = new CacheRecord() { Key = body.Key, LastUpdated = DateTime.Now, Ref = reference, Version = body.Version, Storage = filename, Repo = repository };
+            _context.Caches.Add(record);
+            await _context.SaveChangesAsync();
+            var resp = new Github.Actions.Results.Api.V1.CreateCacheEntryResponse
+            {
+                Ok = true,
+                SignedUploadUrl = new Uri(new Uri(ServerUrl), $"twirp/github.actions.results.api.v1.CacheService/UploadCache?id={record.Id}&sig={CreateSignature(record.Id)}").ToString()
+            };
+            return formatter.Format(resp);
+        }
+
+        [HttpPut("UploadCache")]
+        [AllowAnonymous]
+        public async Task<IActionResult> UploadCache(int id, string sig, string comp = null, bool seal = false) {
+            if(string.IsNullOrEmpty(sig) || !VerifySignature(id, sig)) {
+                return NotFound();
+            }
+            if(comp == "block" || comp == "appendBlock" || comp == null) {
+                var record = await _context.Caches.FindAsync(id);
+                var _targetFilePath = Path.Combine(GitHub.Runner.Sdk.GharunUtil.GetLocalStorage(), "cache");
+                Directory.CreateDirectory(_targetFilePath);
+                using(var targetStream = new FileStream(Path.Combine(_targetFilePath, record.Storage), FileMode.OpenOrCreate | FileMode.Append, FileAccess.Write, FileShare.Write)) {
+                    await Request.Body.CopyToAsync(targetStream);
+                }
+                return Created(HttpContext.Request.GetEncodedUrl(), null);
+            }
+            if(comp == "blocklist") {
+                return Created(HttpContext.Request.GetEncodedUrl(), null);
+            }
+            return Ok();
+        }
+
+        [HttpPost("FinalizeCacheEntryUpload")]
+        public string FinalizeCacheEntryUpload([FromBody, Protobuf] Github.Actions.Results.Api.V1.FinalizeCacheEntryUploadRequest body) {
+            var record = _context.Caches.First(c => c.Key == body.Key && c.Version == body.Version);
+            var resp = new Github.Actions.Results.Api.V1.FinalizeCacheEntryUploadResponse
+            {
+                Ok = true,
+                EntryId = record.Id
+            };
+            return formatter.Format(resp);
+        }
+
+        [HttpPost("GetCacheEntryDownloadURL")]
+        public string GetCacheEntryDownloadURL([FromBody, Protobuf] Github.Actions.Results.Api.V1.GetCacheEntryDownloadURLRequest body) {
+            var a = body.RestoreKeys.Prepend(body.Key).ToArray();
+            var version = body.Version;
+            var defaultRef = User.FindFirst("defaultRef")?.Value ?? "refs/heads/main";
+            var reference = User.FindFirst("ref")?.Value ?? "refs/heads/main";
+            var repository = User.FindFirst("repository")?.Value ?? "Unknown/Unknown";
+            foreach(var cref in reference != defaultRef ? new [] { reference, defaultRef } : new [] { reference }) {
+                foreach (var item in a) {
+                    var record = (from rec in _context.Caches where rec.Repo.ToLower() == repository.ToLower() && rec.Ref == cref && rec.Key.ToLower() == item.ToLower() && (rec.Version == null || rec.Version == "" || rec.Version == version) orderby rec.LastUpdated descending select rec).FirstOrDefault()
+                        ?? (from rec in _context.Caches where rec.Repo.ToLower() == repository.ToLower() && rec.Ref == cref && rec.Key.ToLower().StartsWith(item.ToLower()) && (rec.Version == null || rec.Version == "" || rec.Version == version) orderby rec.LastUpdated descending select rec).FirstOrDefault();
+                    if(record != null) {
+                        var resp = new Github.Actions.Results.Api.V1.GetCacheEntryDownloadURLResponse
+                        {
+                            Ok = true,
+                            MatchedKey = record.Key,
+                            SignedDownloadUrl = new Uri(new Uri(ServerUrl), $"twirp/github.actions.results.api.v1.CacheService/DownloadCache?id={record.Id}&sig={CreateSignature(record.Id)}").ToString()
+                        };
+                        return formatter.Format(resp);
+                    }
+                }
+            }
+            return formatter.Format(new Github.Actions.Results.Api.V1.GetCacheEntryDownloadURLResponse { Ok = false });
+        }
+
+        [AllowAnonymous]
+        [HttpGet("DownloadCache")]
+        public IActionResult DownloadCache(int id, string sig) {
+            if(string.IsNullOrEmpty(sig) || !VerifySignature(id, sig)) {
+                return NotFound();
+            }
+            var container = _context.Caches.Find(id);
+            var _targetFilePath = Path.Combine(GitHub.Runner.Sdk.GharunUtil.GetLocalStorage(), "cache");
+            return new FileStreamResult(System.IO.File.OpenRead(Path.Combine(_targetFilePath, container.Storage)), "application/octet-stream") { EnableRangeProcessing = true };
+        }
+    }
+}

src/Runner.Server/Controllers/MessageController.cs

@@ -4422,7 +4422,7 @@
             return sh.Content != null;
         }
 
         private class shared {
             public Channel<Task> Channel;
             public HttpResponse response;
         }
@@ -5174,6 +5174,7 @@
                         variables.Add("DistributedTask.AllowRunnerContainerHooks", new VariableValue("true", false));
                         variables.Add("DistributedTask.DeprecateStepOutputCommands", new VariableValue("true", false));
                         variables.Add("DistributedTask.ForceGithubJavascriptActionsToNode16", new VariableValue("true", false)); // https://github.blog/changelog/2023-05-04-github-actions-all-actions-will-run-on-node16-instead-of-node12/
+                        variables.Add("actions_uses_cache_service_v2", new VariableValue("true", false)); // https://github.com/actions/toolkit/discussions/1890
                         // For actions/upload-artifact@v1, actions/download-artifact@v1
                         variables.Add(SdkConstants.Variables.Build.BuildId, new VariableValue(runid.ToString(), false));
                         variables.Add(SdkConstants.Variables.Build.BuildNumber, new VariableValue(runid.ToString(), false));
@@ -6341,7 +6342,7 @@
         {
             KeyValuePair<GiteaHook, JObject> obj;
             if(WebhookHMACAlgorithmName.Length > 0 && WebhookSecret.Length > 0 && WebhookSignatureHeader.Length > 0) {
                 var hmac = HMAC.Create(WebhookHMACAlgorithmName);
                 hmac.Key = System.Text.Encoding.UTF8.GetBytes(WebhookSecret);
                 string value = HttpContext.Request.Headers[WebhookSignatureHeader];
                 if(WebhookSignaturePrefix.Length > 0 && !value.StartsWith(WebhookSignaturePrefix)) {