Skip to content

Commit 60e7903

Browse files
edigaryevedigaryev
authored
GHA cache: report failures to Sentry (#790)
1 parent 3afe6f2 commit 60e7903

File tree

4 files changed

+85
-34
lines changed

4 files changed

+85
-34
lines changed

‎go.mod

+1-1
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ require (
2020
github.com/docker/docker v27.2.0+incompatible
2121
github.com/docker/go-units v0.5.0
2222
github.com/dustin/go-humanize v1.0.1
23-
github.com/getsentry/sentry-go v0.27.0
23+
github.com/getsentry/sentry-go v0.29.0
2424
github.com/go-git/go-billy/v5 v5.5.0
2525
github.com/go-git/go-git/v5 v5.11.0
2626
github.com/go-test/deep v1.1.0

‎go.sum

+3
Original file line numberDiff line numberDiff line change
@@ -209,6 +209,8 @@ github.com/fvbommel/sortorder v1.1.0 h1:fUmoe+HLsBTctBDoaBwpQo5N+nrCp8g/BjKb/6ZQ
209209
github.com/fvbommel/sortorder v1.1.0/go.mod h1:uk88iVf1ovNn1iLfgUVU2F9o5eO30ui720w+kxuqRs0=
210210
github.com/getsentry/sentry-go v0.27.0 h1:Pv98CIbtB3LkMWmXi4Joa5OOcwbmnX88sF5qbK3r3Ps=
211211
github.com/getsentry/sentry-go v0.27.0/go.mod h1:lc76E2QywIyW8WuBnwl8Lc4bkmQH4+w1gwTf25trprY=
212+
github.com/getsentry/sentry-go v0.29.0 h1:YtWluuCFg9OfcqnaujpY918N/AhCCwarIDWOYSBAjCA=
213+
github.com/getsentry/sentry-go v0.29.0/go.mod h1:jhPesDAL0Q0W2+2YEuVOvdWmVtdsr1+jtBrlDEVWwLY=
212214
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
213215
github.com/gliderlabs/ssh v0.3.5 h1:OcaySEmAQJgyYcArR+gGGTHCyE7nvhEMTlYY+Dp8CpY=
214216
github.com/gliderlabs/ssh v0.3.5/go.mod h1:8XB4KraRrX39qHhT6yxPsHedjA08I/uBVwj4xC+/+z4=
@@ -499,6 +501,7 @@ github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRW
499501
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
500502
github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78=
501503
github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
504+
github.com/nxadm/tail v1.4.11 h1:8feyoE3OzPrcshW5/MJ4sGESc5cqmGkGCWlco4l0bqY=
502505
github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
503506
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
504507
github.com/onsi/ginkgo v1.12.0/go.mod h1:oUhWkIvk5aDxtKvDDuw8gItl8pKl42LzjC9KZE0HfGg=

‎internal/agent/http_cache/ghacache/ghacache.go

+76-31
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,17 @@
11
package ghacache
22

33
import (
4+
"errors"
45
"fmt"
56
"github.com/cirruslabs/cirrus-cli/internal/agent/client"
67
"github.com/cirruslabs/cirrus-cli/internal/agent/http_cache/ghacache/httprange"
78
"github.com/cirruslabs/cirrus-cli/internal/agent/http_cache/ghacache/uploadable"
89
"github.com/cirruslabs/cirrus-cli/pkg/api"
10+
"github.com/getsentry/sentry-go"
911
"github.com/go-chi/render"
1012
"github.com/puzpuzpuz/xsync/v3"
1113
"github.com/samber/lo"
14+
"golang.org/x/exp/slog"
1215
"google.golang.org/grpc/codes"
1316
"google.golang.org/grpc/status"
1417
"log"
@@ -80,7 +83,7 @@ func (cache *GHACache) get(writer http.ResponseWriter, request *http.Request) {
8083
}
8184

8285
fail(writer, request, http.StatusInternalServerError, "GHA cache failed to "+
83-
"retrieve information about cache key %q: %v", keys[0], err)
86+
"retrieve information about cache entry", "key", keys[0], "err", err)
8487

8588
return
8689
}
@@ -104,7 +107,7 @@ func (cache *GHACache) reserveUploadable(writer http.ResponseWriter, request *ht
104107

105108
if err := render.DecodeJSON(request.Body, &jsonReq); err != nil {
106109
fail(writer, request, http.StatusBadRequest, "GHA cache failed to read/decode the "+
107-
"JSON passed to the reserve uploadable endpoint: %v", err)
110+
"JSON passed to the reserve uploadable endpoint", "err", err)
108111

109112
return
110113
}
@@ -121,7 +124,7 @@ func (cache *GHACache) reserveUploadable(writer http.ResponseWriter, request *ht
121124
})
122125
if err != nil {
123126
fail(writer, request, http.StatusInternalServerError, "GHA cache failed to create "+
124-
"multipart upload for key %q and version %q: %v", jsonReq.Key, jsonReq.Version, err)
127+
"multipart upload", "key", jsonReq.Key, "version", jsonReq.Version, "err", err)
125128

126129
return
127130
}
@@ -142,31 +145,31 @@ func (cache *GHACache) updateUploadable(writer http.ResponseWriter, request *htt
142145

143146
uploadable, ok := cache.uploadables.Load(id)
144147
if !ok {
145-
fail(writer, request, http.StatusNotFound, "GHA cache failed to find an uploadable "+
146-
"with ID %d", id)
148+
fail(writer, request, http.StatusNotFound, "GHA cache failed to find an uploadable",
149+
"id", id)
147150

148151
return
149152
}
150153

151154
httpRanges, err := httprange.ParseRange(request.Header.Get("Content-Range"), math.MaxInt64)
152155
if err != nil {
153-
fail(writer, request, http.StatusBadRequest, "GHA cache failed to parse Content-Range header %q: %v",
154-
request.Header.Get("Content-Range"), err)
156+
fail(writer, request, http.StatusBadRequest, "GHA cache failed to parse Content-Range header",
157+
"header_value", request.Header.Get("Content-Range"), "err", err)
155158

156159
return
157160
}
158161

159162
if len(httpRanges) != 1 {
160-
fail(writer, request, http.StatusBadRequest, "GHA cache expected exactly one Content-Range value, got %d",
161-
len(httpRanges))
163+
fail(writer, request, http.StatusBadRequest, "GHA cache expected exactly one Content-Range value",
164+
"expected", 1, "actual", len(httpRanges))
162165

163166
return
164167
}
165168

166169
partNumber, err := uploadable.RangeToPart.Tell(request.Context(), httpRanges[0].Start, httpRanges[0].Length)
167170
if err != nil {
168171
fail(writer, request, http.StatusBadRequest, "GHA cache failed to tell the part number for "+
169-
"Content-Range header %q: %v", request.Header.Get("Content-Range"), err)
172+
"Content-Range header", "header_value", request.Header.Get("Content-Range"), "err", err)
170173

171174
return
172175
}
@@ -182,16 +185,17 @@ func (cache *GHACache) updateUploadable(writer http.ResponseWriter, request *htt
182185
})
183186
if err != nil {
184187
fail(writer, request, http.StatusInternalServerError, "GHA cache failed create pre-signed "+
185-
"upload part URL for key %q, version %q and part %d: %v", uploadable.Key(),
186-
uploadable.Version(), partNumber, err)
188+
"upload part URL", "key", uploadable.Key(), "version", uploadable.Version(),
189+
"part_number", partNumber, "err", err)
187190

188191
return
189192
}
190193

191194
uploadPartRequest, err := http.NewRequest(http.MethodPut, response.Url, request.Body)
192195
if err != nil {
193196
fail(writer, request, http.StatusInternalServerError, "GHA cache failed to create upload part "+
194-
"request for key %q, version %q and part %d: %v", uploadable.Key(), uploadable.Version(), partNumber, err)
197+
"request", "key", uploadable.Key(), "version", uploadable.Version(), "part_number", partNumber,
198+
"err", err)
195199

196200
return
197201
}
@@ -213,8 +217,9 @@ func (cache *GHACache) updateUploadable(writer http.ResponseWriter, request *htt
213217
// Return HTTP 502 to cause the cache-related code in the Actions Toolkit to make a re-try[1].
214218
//
215219
// [1]: https://github.com/actions/toolkit/blob/6dd369c0e648ed58d0ead326cf2426906ea86401/packages/cache/src/internal/requestUtils.ts#L24-L34
216-
fail(writer, request, http.StatusBadGateway, "GHA cache failed to upload part "+
217-
"for key %q, version %q and part %d: %v", uploadable.Key(), uploadable.Version(), partNumber, err)
220+
fail(writer, request, http.StatusBadGateway, "GHA cache failed to upload part",
221+
"key", uploadable.Key(), "version", uploadable.Version(), "part_number", partNumber,
222+
"err", err)
218223

219224
return
220225
}
@@ -224,17 +229,18 @@ func (cache *GHACache) updateUploadable(writer http.ResponseWriter, request *htt
224229
// code in the Actions Toolkit will hopefully make a re-try[1].
225230
//
226231
// [1]: https://github.com/actions/toolkit/blob/6dd369c0e648ed58d0ead326cf2426906ea86401/packages/cache/src/internal/requestUtils.ts#L24-L34
227-
fail(writer, request, uploadPartResponse.StatusCode, "GHA cache failed to upload part "+
228-
"for key %q, version %q and part %d: got HTTP %d", uploadable.Key(), uploadable.Version(), partNumber,
229-
uploadPartResponse.StatusCode)
232+
fail(writer, request, uploadPartResponse.StatusCode, "GHA cache failed to upload part",
233+
"key", uploadable.Key(), "version", uploadable.Version(), "part_number", partNumber,
234+
"unexpected_status_code", uploadPartResponse.StatusCode)
230235

231236
return
232237
}
233238

234239
err = uploadable.AppendPart(uint32(partNumber), uploadPartResponse.Header.Get("ETag"), httpRanges[0].Length)
235240
if err != nil {
236-
fail(writer, request, http.StatusInternalServerError, "GHA cache failed to append part "+
237-
"for key %q, version %q and part %d: %v", uploadable.Key(), uploadable.Version(), partNumber, err)
241+
fail(writer, request, http.StatusInternalServerError, "GHA cache failed to append part",
242+
"key", uploadable.Key(), "version", uploadable.Version(), "part_number", partNumber,
243+
"err", err)
238244

239245
return
240246
}
@@ -253,8 +259,8 @@ func (cache *GHACache) commitUploadable(writer http.ResponseWriter, request *htt
253259

254260
uploadable, ok := cache.uploadables.Load(id)
255261
if !ok {
256-
fail(writer, request, http.StatusNotFound, "GHA cache failed to find an uploadable "+
257-
"with ID %d", id)
262+
fail(writer, request, http.StatusNotFound, "GHA cache failed to find an uploadable",
263+
"id", id)
258264

259265
return
260266
}
@@ -265,23 +271,23 @@ func (cache *GHACache) commitUploadable(writer http.ResponseWriter, request *htt
265271

266272
if err := render.DecodeJSON(request.Body, &jsonReq); err != nil {
267273
fail(writer, request, http.StatusBadRequest, "GHA cache failed to read/decode "+
268-
"the JSON passed to the commit uploadable endpoint: %v", err)
274+
"the JSON passed to the commit uploadable endpoint", "err", err)
269275

270276
return
271277
}
272278

273279
parts, partsSize, err := uploadable.Finalize()
274280
if err != nil {
275281
fail(writer, request, http.StatusInternalServerError, "GHA cache failed to "+
276-
"finalize uploadable %d: %v", id, err)
282+
"finalize uploadable", "id", id, "err", err)
277283

278284
return
279285
}
280286

281287
if jsonReq.Size != partsSize {
282288
fail(writer, request, http.StatusBadRequest, "GHA cache detected a cache entry "+
283-
"size mismatch for uploadable %d: expected %d bytes, got %d bytes",
284-
id, partsSize, jsonReq.Size)
289+
"size mismatch for uploadable", "id", id, "expected_bytes", partsSize,
290+
"actual_bytes", jsonReq.Size)
285291

286292
return
287293
}
@@ -295,9 +301,9 @@ func (cache *GHACache) commitUploadable(writer http.ResponseWriter, request *htt
295301
Parts: parts,
296302
})
297303
if err != nil {
298-
fail(writer, request, http.StatusInternalServerError, "GHA cache failed to commit multipart upload "+
299-
"for key %q, version %q and uploadable %q: %v", uploadable.Key(), uploadable.Version(),
300-
uploadable.UploadID(), err)
304+
fail(writer, request, http.StatusInternalServerError, "GHA cache failed to commit multipart upload",
305+
"id", uploadable.UploadID(), "key", uploadable.Key(), "version", uploadable.Version(),
306+
"err", err)
301307

302308
return
303309
}
@@ -327,11 +333,50 @@ func getID(request *http.Request) (int64, bool) {
327333
return id, true
328334
}
329335

330-
func fail(writer http.ResponseWriter, request *http.Request, status int, format string, args ...interface{}) {
331-
message := fmt.Sprintf(format, args...)
336+
func fail(writer http.ResponseWriter, request *http.Request, status int, msg string, args ...any) {
337+
// Report failure to the Sentry
338+
hub := sentry.GetHubFromContext(request.Context())
332339

340+
hub.WithScope(func(scope *sentry.Scope) {
341+
scope.AddEventProcessor(func(event *sentry.Event, hint *sentry.EventHint) *sentry.Event {
342+
// Swap the exception type and value to work around
343+
// https://github.com/getsentry/sentry/issues/17837
344+
savedType := event.Exception[0].Type
345+
event.Exception[0].Type = event.Exception[0].Value
346+
event.Exception[0].Value = savedType
347+
348+
return event
349+
})
350+
351+
argsAsSentryContext := sentry.Context{}
352+
353+
for _, chunk := range lo.Chunk(args, 2) {
354+
key := fmt.Sprintf("%v", chunk[0])
355+
356+
var value string
357+
358+
if len(chunk) > 1 {
359+
value = fmt.Sprintf("%v", chunk[1])
360+
}
361+
362+
argsAsSentryContext[key] = value
363+
}
364+
365+
scope.SetContext("Arguments", argsAsSentryContext)
366+
367+
hub.CaptureException(errors.New(msg))
368+
})
369+
370+
// Format failure message for non-structured consumers
371+
var stringBuilder strings.Builder
372+
logger := slog.New(slog.NewTextHandler(&stringBuilder, nil))
373+
logger.Error(msg, args...)
374+
message := stringBuilder.String()
375+
376+
// Report failure to the logger
333377
log.Println(message)
334378

379+
// Report failure to the caller
335380
writer.WriteHeader(status)
336381
jsonResp := struct {
337382
Message string `json:"message"`

‎internal/agent/http_cache/http_cache.go

+5-2
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import (
88
"github.com/cirruslabs/cirrus-cli/internal/agent/client"
99
"github.com/cirruslabs/cirrus-cli/internal/agent/http_cache/ghacache"
1010
"github.com/cirruslabs/cirrus-cli/pkg/api"
11+
sentryhttp "github.com/getsentry/sentry-go/http"
1112
"golang.org/x/sync/semaphore"
1213
"google.golang.org/grpc/codes"
1314
"google.golang.org/grpc/status"
@@ -57,8 +58,10 @@ func Start() string {
5758
log.Printf("Starting http cache server %s\n", address)
5859

5960
// GitHub Actions cache API
60-
mux.Handle(ghacache.APIMountPoint+"/", http.StripPrefix(ghacache.APIMountPoint,
61-
ghacache.New(address)))
61+
sentryHandler := sentryhttp.New(sentryhttp.Options{})
62+
63+
mux.Handle(ghacache.APIMountPoint+"/", sentryHandler.Handle(http.StripPrefix(ghacache.APIMountPoint,
64+
ghacache.New(address))))
6265

6366
go http.Serve(listener, mux)
6467
} else {

0 commit comments

Comments
 (0)