Skip to content

Commit f2ae578

Browse files
darkowlzzdarkowlzz
authored
Merge pull request #624 from fluxcd/recovery-event
Add notify() in all the reconcilers
2 parents 73aa3c4 + 5da74ca commit f2ae578

10 files changed

+724
-35
lines changed

‎controllers/bucket_controller.go

+43-5
Original file line numberDiff line numberDiff line change
@@ -99,6 +99,12 @@ var bucketReadyCondition = summarize.Conditions{
9999
},
100100
}
101101

102+
// bucketFailConditions contains the conditions that represent a failure.
103+
var bucketFailConditions = []string{
104+
sourcev1.FetchFailedCondition,
105+
sourcev1.StorageOperationFailedCondition,
106+
}
107+
102108
// +kubebuilder:rbac:groups=source.toolkit.fluxcd.io,resources=buckets,verbs=get;list;watch;create;update;patch;delete
103109
// +kubebuilder:rbac:groups=source.toolkit.fluxcd.io,resources=buckets/status,verbs=get;update;patch
104110
// +kubebuilder:rbac:groups=source.toolkit.fluxcd.io,resources=buckets/finalizers,verbs=get;create;update;patch;delete
@@ -307,10 +313,13 @@ func (r *BucketReconciler) Reconcile(ctx context.Context, req ctrl.Request) (res
307313
return
308314
}
309315

310-
// reconcile iterates through the gitRepositoryReconcileFunc tasks for the
316+
// reconcile iterates through the bucketReconcileFunc tasks for the
311317
// object. It returns early on the first call that returns
312318
// reconcile.ResultRequeue, or produces an error.
313319
func (r *BucketReconciler) reconcile(ctx context.Context, obj *sourcev1.Bucket, reconcilers []bucketReconcileFunc) (sreconcile.Result, error) {
320+
oldObj := obj.DeepCopy()
321+
322+
// Mark as reconciling if generation differs.
314323
if obj.Generation != obj.Status.ObservedGeneration {
315324
conditions.MarkReconciling(obj, "NewGeneration", "reconciling new object generation (%d)", obj.Generation)
316325
}
@@ -355,9 +364,42 @@ func (r *BucketReconciler) reconcile(ctx context.Context, obj *sourcev1.Bucket,
355364
// Prioritize requeue request in the result.
356365
res = sreconcile.LowestRequeuingResult(res, recResult)
357366
}
367+
368+
r.notify(oldObj, obj, index, res, resErr)
369+
358370
return res, resErr
359371
}
360372

373+
// notify emits notification related to the reconciliation.
374+
func (r *BucketReconciler) notify(oldObj, newObj *sourcev1.Bucket, index *etagIndex, res sreconcile.Result, resErr error) {
375+
// Notify successful reconciliation for new artifact and recovery from any
376+
// failure.
377+
if resErr == nil && res == sreconcile.ResultSuccess && newObj.Status.Artifact != nil {
378+
annotations := map[string]string{
379+
sourcev1.GroupVersion.Group + "/revision": newObj.Status.Artifact.Revision,
380+
sourcev1.GroupVersion.Group + "/checksum": newObj.Status.Artifact.Checksum,
381+
}
382+
383+
var oldChecksum string
384+
if oldObj.GetArtifact() != nil {
385+
oldChecksum = oldObj.GetArtifact().Checksum
386+
}
387+
388+
message := fmt.Sprintf("stored artifact with %d fetched files from '%s' bucket", index.Len(), newObj.Spec.BucketName)
389+
390+
// Notify on new artifact and failure recovery.
391+
if oldChecksum != newObj.GetArtifact().Checksum {
392+
r.AnnotatedEventf(newObj, annotations, corev1.EventTypeNormal,
393+
"NewArtifact", message)
394+
} else {
395+
if sreconcile.FailureRecovery(oldObj, newObj, bucketFailConditions) {
396+
r.AnnotatedEventf(newObj, annotations, corev1.EventTypeNormal,
397+
meta.SucceededReason, message)
398+
}
399+
}
400+
}
401+
}
402+
361403
// reconcileStorage ensures the current state of the storage matches the
362404
// desired and previously observed state.
363405
//
@@ -574,10 +616,6 @@ func (r *BucketReconciler) reconcileArtifact(ctx context.Context, obj *sourcev1.
574616
conditions.MarkTrue(obj, sourcev1.StorageOperationFailedCondition, e.Reason, e.Err.Error())
575617
return sreconcile.ResultEmpty, e
576618
}
577-
r.annotatedEventLogf(ctx, obj, map[string]string{
578-
sourcev1.GroupVersion.Group + "/revision": artifact.Revision,
579-
sourcev1.GroupVersion.Group + "/checksum": artifact.Checksum,
580-
}, corev1.EventTypeNormal, "NewArtifact", "fetched %d files from '%s'", index.Len(), obj.Spec.BucketName)
581619

582620
// Record it on the object
583621
obj.Status.Artifact = artifact.DeepCopy()

‎controllers/bucket_controller_test.go

+115
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ package controllers
1818

1919
import (
2020
"context"
21+
"errors"
2122
"fmt"
2223
"net/http"
2324
"net/url"
@@ -1171,3 +1172,117 @@ func TestBucketReconciler_statusConditions(t *testing.T) {
11711172
})
11721173
}
11731174
}
1175+
1176+
func TestBucketReconciler_notify(t *testing.T) {
1177+
tests := []struct {
1178+
name string
1179+
res sreconcile.Result
1180+
resErr error
1181+
oldObjBeforeFunc func(obj *sourcev1.Bucket)
1182+
newObjBeforeFunc func(obj *sourcev1.Bucket)
1183+
wantEvent string
1184+
}{
1185+
{
1186+
name: "error - no event",
1187+
res: sreconcile.ResultEmpty,
1188+
resErr: errors.New("some error"),
1189+
},
1190+
{
1191+
name: "new artifact",
1192+
res: sreconcile.ResultSuccess,
1193+
resErr: nil,
1194+
newObjBeforeFunc: func(obj *sourcev1.Bucket) {
1195+
obj.Status.Artifact = &sourcev1.Artifact{Revision: "xxx", Checksum: "yyy"}
1196+
},
1197+
wantEvent: "Normal NewArtifact stored artifact with 2 fetched files from",
1198+
},
1199+
{
1200+
name: "recovery from failure",
1201+
res: sreconcile.ResultSuccess,
1202+
resErr: nil,
1203+
oldObjBeforeFunc: func(obj *sourcev1.Bucket) {
1204+
obj.Status.Artifact = &sourcev1.Artifact{Revision: "xxx", Checksum: "yyy"}
1205+
conditions.MarkTrue(obj, sourcev1.FetchFailedCondition, sourcev1.GitOperationFailedReason, "fail")
1206+
conditions.MarkFalse(obj, meta.ReadyCondition, meta.FailedReason, "foo")
1207+
},
1208+
newObjBeforeFunc: func(obj *sourcev1.Bucket) {
1209+
obj.Status.Artifact = &sourcev1.Artifact{Revision: "xxx", Checksum: "yyy"}
1210+
conditions.MarkTrue(obj, meta.ReadyCondition, meta.SucceededReason, "ready")
1211+
},
1212+
wantEvent: "Normal Succeeded stored artifact with 2 fetched files from",
1213+
},
1214+
{
1215+
name: "recovery and new artifact",
1216+
res: sreconcile.ResultSuccess,
1217+
resErr: nil,
1218+
oldObjBeforeFunc: func(obj *sourcev1.Bucket) {
1219+
obj.Status.Artifact = &sourcev1.Artifact{Revision: "xxx", Checksum: "yyy"}
1220+
conditions.MarkTrue(obj, sourcev1.FetchFailedCondition, sourcev1.GitOperationFailedReason, "fail")
1221+
conditions.MarkFalse(obj, meta.ReadyCondition, meta.FailedReason, "foo")
1222+
},
1223+
newObjBeforeFunc: func(obj *sourcev1.Bucket) {
1224+
obj.Status.Artifact = &sourcev1.Artifact{Revision: "aaa", Checksum: "bbb"}
1225+
conditions.MarkTrue(obj, meta.ReadyCondition, meta.SucceededReason, "ready")
1226+
},
1227+
wantEvent: "Normal NewArtifact stored artifact with 2 fetched files from",
1228+
},
1229+
{
1230+
name: "no updates",
1231+
res: sreconcile.ResultSuccess,
1232+
resErr: nil,
1233+
oldObjBeforeFunc: func(obj *sourcev1.Bucket) {
1234+
obj.Status.Artifact = &sourcev1.Artifact{Revision: "xxx", Checksum: "yyy"}
1235+
conditions.MarkTrue(obj, meta.ReadyCondition, meta.SucceededReason, "ready")
1236+
},
1237+
newObjBeforeFunc: func(obj *sourcev1.Bucket) {
1238+
obj.Status.Artifact = &sourcev1.Artifact{Revision: "xxx", Checksum: "yyy"}
1239+
conditions.MarkTrue(obj, meta.ReadyCondition, meta.SucceededReason, "ready")
1240+
},
1241+
},
1242+
}
1243+
1244+
for _, tt := range tests {
1245+
t.Run(tt.name, func(t *testing.T) {
1246+
g := NewWithT(t)
1247+
1248+
recorder := record.NewFakeRecorder(32)
1249+
1250+
oldObj := &sourcev1.Bucket{
1251+
Spec: sourcev1.BucketSpec{
1252+
BucketName: "test-bucket",
1253+
},
1254+
}
1255+
newObj := oldObj.DeepCopy()
1256+
1257+
if tt.oldObjBeforeFunc != nil {
1258+
tt.oldObjBeforeFunc(oldObj)
1259+
}
1260+
if tt.newObjBeforeFunc != nil {
1261+
tt.newObjBeforeFunc(newObj)
1262+
}
1263+
1264+
reconciler := &BucketReconciler{
1265+
EventRecorder: recorder,
1266+
}
1267+
index := &etagIndex{
1268+
index: map[string]string{
1269+
"zzz": "qqq",
1270+
"bbb": "ddd",
1271+
},
1272+
}
1273+
reconciler.notify(oldObj, newObj, index, tt.res, tt.resErr)
1274+
1275+
select {
1276+
case x, ok := <-recorder.Events:
1277+
g.Expect(ok).To(Equal(tt.wantEvent != ""), "unexpected event received")
1278+
if tt.wantEvent != "" {
1279+
g.Expect(x).To(ContainSubstring(tt.wantEvent))
1280+
}
1281+
default:
1282+
if tt.wantEvent != "" {
1283+
t.Errorf("expected some event to be emitted")
1284+
}
1285+
}
1286+
})
1287+
}
1288+
}

‎controllers/gitrepository_controller.go

+42-4
Original file line numberDiff line numberDiff line change
@@ -91,6 +91,13 @@ var gitRepositoryReadyCondition = summarize.Conditions{
9191
},
9292
}
9393

94+
// gitRepositoryFailConditions contains the conditions that represent a failure.
95+
var gitRepositoryFailConditions = []string{
96+
sourcev1.FetchFailedCondition,
97+
sourcev1.IncludeUnavailableCondition,
98+
sourcev1.StorageOperationFailedCondition,
99+
}
100+
94101
// +kubebuilder:rbac:groups=source.toolkit.fluxcd.io,resources=gitrepositories,verbs=get;list;watch;create;update;patch;delete
95102
// +kubebuilder:rbac:groups=source.toolkit.fluxcd.io,resources=gitrepositories/status,verbs=get;update;patch
96103
// +kubebuilder:rbac:groups=source.toolkit.fluxcd.io,resources=gitrepositories/finalizers,verbs=get;create;update;patch;delete
@@ -212,6 +219,8 @@ func (r *GitRepositoryReconciler) Reconcile(ctx context.Context, req ctrl.Reques
212219
// object. It returns early on the first call that returns
213220
// reconcile.ResultRequeue, or produces an error.
214221
func (r *GitRepositoryReconciler) reconcile(ctx context.Context, obj *sourcev1.GitRepository, reconcilers []gitRepositoryReconcileFunc) (sreconcile.Result, error) {
222+
oldObj := obj.DeepCopy()
223+
215224
// Mark as reconciling if generation differs
216225
if obj.Generation != obj.Status.ObservedGeneration {
217226
conditions.MarkReconciling(obj, "NewGeneration", "reconciling new object generation (%d)", obj.Generation)
@@ -258,9 +267,42 @@ func (r *GitRepositoryReconciler) reconcile(ctx context.Context, obj *sourcev1.G
258267
// Prioritize requeue request in the result.
259268
res = sreconcile.LowestRequeuingResult(res, recResult)
260269
}
270+
271+
r.notify(oldObj, obj, commit, res, resErr)
272+
261273
return res, resErr
262274
}
263275

276+
// notify emits notification related to the reconciliation.
277+
func (r *GitRepositoryReconciler) notify(oldObj, newObj *sourcev1.GitRepository, commit git.Commit, res sreconcile.Result, resErr error) {
278+
// Notify successful reconciliation for new artifact and recovery from any
279+
// failure.
280+
if resErr == nil && res == sreconcile.ResultSuccess && newObj.Status.Artifact != nil {
281+
annotations := map[string]string{
282+
sourcev1.GroupVersion.Group + "/revision": newObj.Status.Artifact.Revision,
283+
sourcev1.GroupVersion.Group + "/checksum": newObj.Status.Artifact.Checksum,
284+
}
285+
286+
var oldChecksum string
287+
if oldObj.GetArtifact() != nil {
288+
oldChecksum = oldObj.GetArtifact().Checksum
289+
}
290+
291+
message := fmt.Sprintf("stored artifact for commit '%s'", commit.ShortMessage())
292+
293+
// Notify on new artifact and failure recovery.
294+
if oldChecksum != newObj.GetArtifact().Checksum {
295+
r.AnnotatedEventf(newObj, annotations, corev1.EventTypeNormal,
296+
"NewArtifact", message)
297+
} else {
298+
if sreconcile.FailureRecovery(oldObj, newObj, gitRepositoryFailConditions) {
299+
r.AnnotatedEventf(newObj, annotations, corev1.EventTypeNormal,
300+
meta.SucceededReason, message)
301+
}
302+
}
303+
}
304+
}
305+
264306
// reconcileStorage ensures the current state of the storage matches the
265307
// desired and previously observed state.
266308
//
@@ -523,10 +565,6 @@ func (r *GitRepositoryReconciler) reconcileArtifact(ctx context.Context,
523565
conditions.MarkTrue(obj, sourcev1.StorageOperationFailedCondition, e.Reason, e.Err.Error())
524566
return sreconcile.ResultEmpty, e
525567
}
526-
r.AnnotatedEventf(obj, map[string]string{
527-
sourcev1.GroupVersion.Group + "/revision": artifact.Revision,
528-
sourcev1.GroupVersion.Group + "/checksum": artifact.Checksum,
529-
}, corev1.EventTypeNormal, "NewArtifact", "stored artifact for commit '%s'", commit.ShortMessage())
530568

531569
// Record it on the object
532570
obj.Status.Artifact = artifact.DeepCopy()

0 commit comments

Comments
 (0)