GHA cache: report failures to Sentry

go.mod

@@ -20,7 +20,7 @@ require (
 	github.com/docker/docker v27.2.0+incompatible
 	github.com/docker/go-units v0.5.0
 	github.com/dustin/go-humanize v1.0.1
-	github.com/getsentry/sentry-go v0.27.0
+	github.com/getsentry/sentry-go v0.29.0
 	github.com/go-git/go-billy/v5 v5.5.0
 	github.com/go-git/go-git/v5 v5.11.0
 	github.com/go-test/deep v1.1.0

go.sum

@@ -209,6 +209,8 @@ github.com/fvbommel/sortorder v1.1.0 h1:fUmoe+HLsBTctBDoaBwpQo5N+nrCp8g/BjKb/6ZQ
 github.com/fvbommel/sortorder v1.1.0/go.mod h1:uk88iVf1ovNn1iLfgUVU2F9o5eO30ui720w+kxuqRs0=
 github.com/getsentry/sentry-go v0.27.0 h1:Pv98CIbtB3LkMWmXi4Joa5OOcwbmnX88sF5qbK3r3Ps=
 github.com/getsentry/sentry-go v0.27.0/go.mod h1:lc76E2QywIyW8WuBnwl8Lc4bkmQH4+w1gwTf25trprY=
+github.com/getsentry/sentry-go v0.29.0 h1:YtWluuCFg9OfcqnaujpY918N/AhCCwarIDWOYSBAjCA=
+github.com/getsentry/sentry-go v0.29.0/go.mod h1:jhPesDAL0Q0W2+2YEuVOvdWmVtdsr1+jtBrlDEVWwLY=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/gliderlabs/ssh v0.3.5 h1:OcaySEmAQJgyYcArR+gGGTHCyE7nvhEMTlYY+Dp8CpY=
 github.com/gliderlabs/ssh v0.3.5/go.mod h1:8XB4KraRrX39qHhT6yxPsHedjA08I/uBVwj4xC+/+z4=
@@ -499,6 +501,7 @@ github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRW
 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
 github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78=
 github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
+github.com/nxadm/tail v1.4.11 h1:8feyoE3OzPrcshW5/MJ4sGESc5cqmGkGCWlco4l0bqY=
 github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.12.0/go.mod h1:oUhWkIvk5aDxtKvDDuw8gItl8pKl42LzjC9KZE0HfGg=

internal/agent/http_cache/ghacache/ghacache.go

@@ -1,14 +1,17 @@
 package ghacache
 
 import (
+	"errors"
 	"fmt"
 	"github.com/cirruslabs/cirrus-cli/internal/agent/client"
 	"github.com/cirruslabs/cirrus-cli/internal/agent/http_cache/ghacache/httprange"
 	"github.com/cirruslabs/cirrus-cli/internal/agent/http_cache/ghacache/uploadable"
 	"github.com/cirruslabs/cirrus-cli/pkg/api"
+	"github.com/getsentry/sentry-go"
 	"github.com/go-chi/render"
 	"github.com/puzpuzpuz/xsync/v3"
 	"github.com/samber/lo"
+	"golang.org/x/exp/slog"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
 	"log"
@@ -80,7 +83,7 @@ func (cache *GHACache) get(writer http.ResponseWriter, request *http.Request) {
 		}
 
 		fail(writer, request, http.StatusInternalServerError, "GHA cache failed to "+
-			"retrieve information about cache key %q: %v", keys[0], err)
+			"retrieve information about cache entry", "key", keys[0], "err", err)
 
 		return
 	}
@@ -104,7 +107,7 @@ func (cache *GHACache) reserveUploadable(writer http.ResponseWriter, request *ht
 
 	if err := render.DecodeJSON(request.Body, &jsonReq); err != nil {
 		fail(writer, request, http.StatusBadRequest, "GHA cache failed to read/decode the "+
-			"JSON passed to the reserve uploadable endpoint: %v", err)
+			"JSON passed to the reserve uploadable endpoint", "err", err)
 
 		return
 	}
@@ -121,7 +124,7 @@ func (cache *GHACache) reserveUploadable(writer http.ResponseWriter, request *ht
 	})
 	if err != nil {
 		fail(writer, request, http.StatusInternalServerError, "GHA cache failed to create "+
-			"multipart upload for key %q and version %q: %v", jsonReq.Key, jsonReq.Version, err)
+			"multipart upload", "key", jsonReq.Key, "version", jsonReq.Version, "err", err)
 
 		return
 	}
@@ -142,31 +145,31 @@ func (cache *GHACache) updateUploadable(writer http.ResponseWriter, request *htt
 
 	uploadable, ok := cache.uploadables.Load(id)
 	if !ok {
-		fail(writer, request, http.StatusNotFound, "GHA cache failed to find an uploadable "+
-			"with ID %d", id)
+		fail(writer, request, http.StatusNotFound, "GHA cache failed to find an uploadable",
+			"id", id)
 
 		return
 	}
 
 	httpRanges, err := httprange.ParseRange(request.Header.Get("Content-Range"), math.MaxInt64)
 	if err != nil {
-		fail(writer, request, http.StatusBadRequest, "GHA cache failed to parse Content-Range header %q: %v",
-			request.Header.Get("Content-Range"), err)
+		fail(writer, request, http.StatusBadRequest, "GHA cache failed to parse Content-Range header",
+			"header_value", request.Header.Get("Content-Range"), "err", err)
 
 		return
 	}
 
 	if len(httpRanges) != 1 {
-		fail(writer, request, http.StatusBadRequest, "GHA cache expected exactly one Content-Range value, got %d",
-			len(httpRanges))
+		fail(writer, request, http.StatusBadRequest, "GHA cache expected exactly one Content-Range value",
+			"expected", 1, "actual", len(httpRanges))
 
 		return
 	}
 
 	partNumber, err := uploadable.RangeToPart.Tell(request.Context(), httpRanges[0].Start, httpRanges[0].Length)
 	if err != nil {
 		fail(writer, request, http.StatusBadRequest, "GHA cache failed to tell the part number for "+
-			"Content-Range header %q: %v", request.Header.Get("Content-Range"), err)
+			"Content-Range header", "header_value", request.Header.Get("Content-Range"), "err", err)
 
 		return
 	}
@@ -182,16 +185,17 @@ func (cache *GHACache) updateUploadable(writer http.ResponseWriter, request *htt
 	})
 	if err != nil {
 		fail(writer, request, http.StatusInternalServerError, "GHA cache failed create pre-signed "+
-			"upload part URL for key %q, version %q and part %d: %v", uploadable.Key(),
-			uploadable.Version(), partNumber, err)
+			"upload part URL", "key", uploadable.Key(), "version", uploadable.Version(),
+			"part_number", partNumber, "err", err)
 
 		return
 	}
 
 	uploadPartRequest, err := http.NewRequest(http.MethodPut, response.Url, request.Body)
 	if err != nil {
 		fail(writer, request, http.StatusInternalServerError, "GHA cache failed to create upload part "+
-			"request for key %q, version %q and part %d: %v", uploadable.Key(), uploadable.Version(), partNumber, err)
+			"request", "key", uploadable.Key(), "version", uploadable.Version(), "part_number", partNumber,
+			"err", err)
 
 		return
 	}
@@ -213,8 +217,9 @@ func (cache *GHACache) updateUploadable(writer http.ResponseWriter, request *htt
 		// Return HTTP 502 to cause the cache-related code in the Actions Toolkit to make a re-try[1].
 		//
 		// [1]: https://github.com/actions/toolkit/blob/6dd369c0e648ed58d0ead326cf2426906ea86401/packages/cache/src/internal/requestUtils.ts#L24-L34
-		fail(writer, request, http.StatusBadGateway, "GHA cache failed to upload part "+
-			"for key %q, version %q and part %d: %v", uploadable.Key(), uploadable.Version(), partNumber, err)
+		fail(writer, request, http.StatusBadGateway, "GHA cache failed to upload part",
+			"key", uploadable.Key(), "version", uploadable.Version(), "part_number", partNumber,
+			"err", err)
 
 		return
 	}
@@ -224,17 +229,18 @@ func (cache *GHACache) updateUploadable(writer http.ResponseWriter, request *htt
 		// code in the Actions Toolkit will hopefully make a re-try[1].
 		//
 		// [1]: https://github.com/actions/toolkit/blob/6dd369c0e648ed58d0ead326cf2426906ea86401/packages/cache/src/internal/requestUtils.ts#L24-L34
-		fail(writer, request, uploadPartResponse.StatusCode, "GHA cache failed to upload part "+
-			"for key %q, version %q and part %d: got HTTP %d", uploadable.Key(), uploadable.Version(), partNumber,
-			uploadPartResponse.StatusCode)
+		fail(writer, request, uploadPartResponse.StatusCode, "GHA cache failed to upload part",
+			"key", uploadable.Key(), "version", uploadable.Version(), "part_number", partNumber,
+			"unexpected_status_code", uploadPartResponse.StatusCode)
 
 		return
 	}
 
 	err = uploadable.AppendPart(uint32(partNumber), uploadPartResponse.Header.Get("ETag"), httpRanges[0].Length)
 	if err != nil {
-		fail(writer, request, http.StatusInternalServerError, "GHA cache failed to append part "+
-			"for key %q, version %q and part %d: %v", uploadable.Key(), uploadable.Version(), partNumber, err)
+		fail(writer, request, http.StatusInternalServerError, "GHA cache failed to append part",
+			"key", uploadable.Key(), "version", uploadable.Version(), "part_number", partNumber,
+			"err", err)
 
 		return
 	}
@@ -253,8 +259,8 @@ func (cache *GHACache) commitUploadable(writer http.ResponseWriter, request *htt
 
 	uploadable, ok := cache.uploadables.Load(id)
 	if !ok {
-		fail(writer, request, http.StatusNotFound, "GHA cache failed to find an uploadable "+
-			"with ID %d", id)
+		fail(writer, request, http.StatusNotFound, "GHA cache failed to find an uploadable",
+			"id", id)
 
 		return
 	}
@@ -265,23 +271,23 @@ func (cache *GHACache) commitUploadable(writer http.ResponseWriter, request *htt
 
 	if err := render.DecodeJSON(request.Body, &jsonReq); err != nil {
 		fail(writer, request, http.StatusBadRequest, "GHA cache failed to read/decode "+
-			"the JSON passed to the commit uploadable endpoint: %v", err)
+			"the JSON passed to the commit uploadable endpoint", "err", err)
 
 		return
 	}
 
 	parts, partsSize, err := uploadable.Finalize()
 	if err != nil {
 		fail(writer, request, http.StatusInternalServerError, "GHA cache failed to "+
-			"finalize uploadable %d: %v", id, err)
+			"finalize uploadable", "id", id, "err", err)
 
 		return
 	}
 
 	if jsonReq.Size != partsSize {
 		fail(writer, request, http.StatusBadRequest, "GHA cache detected a cache entry "+
-			"size mismatch for uploadable %d: expected %d bytes, got %d bytes",
-			id, partsSize, jsonReq.Size)
+			"size mismatch for uploadable", "id", id, "expected_bytes", partsSize,
+			"actual_bytes", jsonReq.Size)
 
 		return
 	}
@@ -295,9 +301,9 @@ func (cache *GHACache) commitUploadable(writer http.ResponseWriter, request *htt
 		Parts:    parts,
 	})
 	if err != nil {
-		fail(writer, request, http.StatusInternalServerError, "GHA cache failed to commit multipart upload "+
-			"for key %q, version %q and uploadable %q: %v", uploadable.Key(), uploadable.Version(),
-			uploadable.UploadID(), err)
+		fail(writer, request, http.StatusInternalServerError, "GHA cache failed to commit multipart upload",
+			"id", uploadable.UploadID(), "key", uploadable.Key(), "version", uploadable.Version(),
+			"err", err)
 
 		return
 	}
@@ -327,11 +333,50 @@ func getID(request *http.Request) (int64, bool) {
 	return id, true
 }
 
-func fail(writer http.ResponseWriter, request *http.Request, status int, format string, args ...interface{}) {
-	message := fmt.Sprintf(format, args...)
+func fail(writer http.ResponseWriter, request *http.Request, status int, msg string, args ...any) {
+	// Report failure to the Sentry
+	hub := sentry.GetHubFromContext(request.Context())
 
+	hub.WithScope(func(scope *sentry.Scope) {
+		scope.AddEventProcessor(func(event *sentry.Event, hint *sentry.EventHint) *sentry.Event {
+			// Swap the exception type and value to work around
+			// https://github.com/getsentry/sentry/issues/17837
+			savedType := event.Exception[0].Type
+			event.Exception[0].Type = event.Exception[0].Value
+			event.Exception[0].Value = savedType
+
+			return event
+		})
+
+		argsAsSentryContext := sentry.Context{}
+
+		for _, chunk := range lo.Chunk(args, 2) {
+			key := fmt.Sprintf("%v", chunk[0])
+
+			var value string
+
+			if len(chunk) > 1 {
+				value = fmt.Sprintf("%v", chunk[1])
+			}
+
+			argsAsSentryContext[key] = value
+		}
+
+		scope.SetContext("Arguments", argsAsSentryContext)
+
+		hub.CaptureException(errors.New(msg))
+	})
+
+	// Format failure message for non-structured consumers
+	var stringBuilder strings.Builder
+	logger := slog.New(slog.NewTextHandler(&stringBuilder, nil))
+	logger.Error(msg, args...)
+	message := stringBuilder.String()
+
+	// Report failure to the logger
 	log.Println(message)
 
+	// Report failure to the caller
 	writer.WriteHeader(status)
 	jsonResp := struct {
 		Message string `json:"message"`

internal/agent/http_cache/http_cache.go

@@ -8,6 +8,7 @@ import (
 	"github.com/cirruslabs/cirrus-cli/internal/agent/client"
 	"github.com/cirruslabs/cirrus-cli/internal/agent/http_cache/ghacache"
 	"github.com/cirruslabs/cirrus-cli/pkg/api"
+	sentryhttp "github.com/getsentry/sentry-go/http"
 	"golang.org/x/sync/semaphore"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
@@ -57,8 +58,10 @@ func Start() string {
 		log.Printf("Starting http cache server %s\n", address)
 
 		// GitHub Actions cache API
-		mux.Handle(ghacache.APIMountPoint+"/", http.StripPrefix(ghacache.APIMountPoint,
-			ghacache.New(address)))
+		sentryHandler := sentryhttp.New(sentryhttp.Options{})
+
+		mux.Handle(ghacache.APIMountPoint+"/", sentryHandler.Handle(http.StripPrefix(ghacache.APIMountPoint,
+			ghacache.New(address))))
 
 		go http.Serve(listener, mux)
 	} else {