Skip to content

Commit adb083b

Browse files
committed
Capture (debug) logs from Helm actions
Signed-off-by: Hidde Beydals <hello@hidde.co>
1 parent a48b956 commit adb083b

File tree

4 files changed

+264
-20
lines changed

4 files changed

+264
-20
lines changed

controllers/helmrelease_controller.go

+10-7
Original file line numberDiff line numberDiff line change
@@ -328,14 +328,14 @@ func (r *HelmReleaseReconciler) reconcileRelease(ctx context.Context,
328328
// Fail if install retries are exhausted.
329329
if hr.Spec.GetInstall().GetRemediation().RetriesExhausted(hr) {
330330
err = fmt.Errorf("install retries exhausted")
331-
return v2.HelmReleaseNotReady(hr, released.Reason, released.Message), err
331+
return v2.HelmReleaseNotReady(hr, released.Reason, err.Error()), err
332332
}
333333

334334
// Fail if there is a release and upgrade retries are exhausted.
335335
// This avoids failing after an upgrade uninstall remediation strategy.
336336
if rel != nil && hr.Spec.GetUpgrade().GetRemediation().RetriesExhausted(hr) {
337337
err = fmt.Errorf("upgrade retries exhausted")
338-
return v2.HelmReleaseNotReady(hr, released.Reason, released.Message), err
338+
return v2.HelmReleaseNotReady(hr, released.Reason, err.Error()), err
339339
}
340340
}
341341

@@ -415,9 +415,8 @@ func (r *HelmReleaseReconciler) reconcileRelease(ctx context.Context,
415415

416416
if err != nil {
417417
reason := meta.ReconciliationFailedReason
418-
var cerr *ConditionError
419-
if errors.As(err, &cerr) {
420-
reason = cerr.Reason
418+
if condErr := (*ConditionError)(nil); errors.As(err, &condErr) {
419+
reason = condErr.Reason
421420
}
422421
return v2.HelmReleaseNotReady(hr, reason, err.Error()), err
423422
}
@@ -662,10 +661,14 @@ func (r *HelmReleaseReconciler) reconcileDelete(ctx context.Context, hr v2.HelmR
662661
func (r *HelmReleaseReconciler) handleHelmActionResult(ctx context.Context,
663662
hr *v2.HelmRelease, revision string, err error, action string, condition string, succeededReason string, failedReason string) error {
664663
if err != nil {
665-
msg := fmt.Sprintf("Helm %s failed: %s", action, err.Error())
664+
err = fmt.Errorf("Helm %s failed: %w", action, err)
665+
msg := err.Error()
666+
if actionErr := (*runner.ActionError)(nil); errors.As(err, &actionErr) {
667+
msg = msg + "\n\nLast Helm logs:\n\n" + actionErr.CapturedLogs
668+
}
666669
meta.SetResourceCondition(hr, condition, metav1.ConditionFalse, failedReason, msg)
667670
r.event(ctx, *hr, revision, events.EventSeverityError, msg)
668-
return &ConditionError{Reason: failedReason, Err: errors.New(msg)}
671+
return &ConditionError{Reason: failedReason, Err: err}
669672
} else {
670673
msg := fmt.Sprintf("Helm %s succeeded", action)
671674
meta.SetResourceCondition(hr, condition, metav1.ConditionTrue, succeededReason, msg)

internal/runner/log_buffer.go

+91
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,91 @@
1+
/*
2+
Copyright 2021 The Flux authors
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package runner
18+
19+
import (
20+
"container/ring"
21+
"fmt"
22+
"strings"
23+
"sync"
24+
25+
"github.com/go-logr/logr"
26+
"helm.sh/helm/v3/pkg/action"
27+
)
28+
29+
const defaultBufferSize = 5
30+
31+
type DebugLog struct {
32+
log logr.Logger
33+
}
34+
35+
func NewDebugLog(log logr.Logger) *DebugLog {
36+
return &DebugLog{log: log}
37+
}
38+
39+
func (l *DebugLog) Log(format string, v ...interface{}) {
40+
l.log.V(1).Info(fmt.Sprintf(format, v...))
41+
}
42+
43+
type LogBuffer struct {
44+
mu sync.Mutex
45+
log action.DebugLog
46+
buffer *ring.Ring
47+
}
48+
49+
func NewLogBuffer(log action.DebugLog, size int) *LogBuffer {
50+
if size == 0 {
51+
size = defaultBufferSize
52+
}
53+
return &LogBuffer{
54+
log: log,
55+
buffer: ring.New(size),
56+
}
57+
}
58+
59+
func (l *LogBuffer) Log(format string, v ...interface{}) {
60+
l.mu.Lock()
61+
62+
// Filter out duplicate log lines, this happens for example when
63+
// Helm is waiting on workloads to become ready.
64+
msg := fmt.Sprintf(format, v...)
65+
if prev := l.buffer.Prev(); prev.Value != msg {
66+
l.buffer.Value = msg
67+
l.buffer = l.buffer.Next()
68+
}
69+
70+
l.mu.Unlock()
71+
l.log(format, v...)
72+
}
73+
74+
func (l *LogBuffer) Reset() {
75+
l.mu.Lock()
76+
l.buffer = ring.New(l.buffer.Len())
77+
l.mu.Unlock()
78+
}
79+
80+
func (l *LogBuffer) String() string {
81+
var str string
82+
l.mu.Lock()
83+
l.buffer.Do(func(s interface{}) {
84+
if s == nil {
85+
return
86+
}
87+
str += s.(string) + "\n"
88+
})
89+
l.mu.Unlock()
90+
return strings.TrimSpace(str)
91+
}

internal/runner/log_buffer_test.go

+103
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,103 @@
1+
/*
2+
Copyright 2021 The Flux authors
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package runner
18+
19+
import (
20+
"testing"
21+
22+
"sigs.k8s.io/controller-runtime/pkg/log"
23+
)
24+
25+
func TestLogBuffer_Log(t *testing.T) {
26+
tests := []struct {
27+
name string
28+
size int
29+
fill []string
30+
wantCount int
31+
want string
32+
}{
33+
{name: "log", size: 2, fill: []string{"a", "b", "c"}, wantCount: 3, want: "b\nc"},
34+
}
35+
for _, tt := range tests {
36+
t.Run(tt.name, func(t *testing.T) {
37+
var count int
38+
l := NewLogBuffer(func(format string, v ...interface{}) {
39+
count++
40+
return
41+
}, tt.size)
42+
for _, v := range tt.fill {
43+
l.Log("%s", v)
44+
}
45+
if count != tt.wantCount {
46+
t.Errorf("Inner Log() called %v times, want %v", count, tt.wantCount)
47+
}
48+
if got := l.String(); got != tt.want {
49+
t.Errorf("String() = %v, want %v", got, tt.want)
50+
}
51+
})
52+
}
53+
}
54+
55+
func TestLogBuffer_Reset(t *testing.T) {
56+
bufferSize := 10
57+
l := NewLogBuffer(NewDebugLog(log.NullLogger{}).Log, bufferSize)
58+
59+
if got := l.buffer.Len(); got != bufferSize {
60+
t.Errorf("Len() = %v, want %v", got, bufferSize)
61+
}
62+
63+
for _, v := range []string{"a", "b", "c"} {
64+
l.Log("%s", v)
65+
}
66+
67+
if got := l.String(); got == "" {
68+
t.Errorf("String() = empty")
69+
}
70+
71+
l.Reset()
72+
73+
if got := l.buffer.Len(); got != bufferSize {
74+
t.Errorf("Len() = %v after Reset(), want %v", got, bufferSize)
75+
}
76+
if got := l.String(); got != "" {
77+
t.Errorf("String() != empty after Reset()")
78+
}
79+
}
80+
81+
func TestLogBuffer_String(t *testing.T) {
82+
tests := []struct {
83+
name string
84+
size int
85+
fill []string
86+
want string
87+
}{
88+
{name: "empty buffer", fill: []string{}, want: ""},
89+
{name: "filled buffer", size: 2, fill: []string{"a", "b", "c"}, want: "b\nc"},
90+
{name: "duplicate buffer items", fill: []string{"b", "b", "b"}, want: "b"},
91+
}
92+
for _, tt := range tests {
93+
t.Run(tt.name, func(t *testing.T) {
94+
l := NewLogBuffer(NewDebugLog(log.NullLogger{}).Log, tt.size)
95+
for _, v := range tt.fill {
96+
l.Log("%s", v)
97+
}
98+
if got := l.String(); got != tt.want {
99+
t.Errorf("String() = %v, want %v", got, tt.want)
100+
}
101+
})
102+
}
103+
}

internal/runner/runner.go

+60-13
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ package runner
1818

1919
import (
2020
"errors"
21-
"fmt"
21+
"sync"
2222

2323
"github.com/go-logr/logr"
2424
"helm.sh/helm/v3/pkg/action"
@@ -32,21 +32,39 @@ import (
3232
v2 "github.com/fluxcd/helm-controller/api/v2beta1"
3333
)
3434

35+
type ActionError struct {
36+
Err error
37+
CapturedLogs string
38+
}
39+
40+
func (e ActionError) Error() string {
41+
return e.Err.Error()
42+
}
43+
44+
func (e ActionError) Unwrap() error {
45+
return e.Err
46+
}
47+
3548
// Runner represents a Helm action runner capable of performing Helm
3649
// operations for a v2beta1.HelmRelease.
3750
type Runner struct {
38-
config *action.Configuration
51+
mu sync.Mutex
52+
config *action.Configuration
53+
logBuffer *LogBuffer
3954
}
4055

4156
// NewRunner constructs a new Runner configured to run Helm actions with the
4257
// given genericclioptions.RESTClientGetter, and the release and storage
4358
// namespace configured to the provided values.
4459
func NewRunner(getter genericclioptions.RESTClientGetter, storageNamespace string, logger logr.Logger) (*Runner, error) {
45-
cfg := new(action.Configuration)
46-
if err := cfg.Init(getter, storageNamespace, "secret", debugLogger(logger)); err != nil {
60+
runner := &Runner{
61+
logBuffer: NewLogBuffer(NewDebugLog(logger).Log, 0),
62+
}
63+
runner.config = new(action.Configuration)
64+
if err := runner.config.Init(getter, storageNamespace, "secret", runner.logBuffer.Log); err != nil {
4765
return nil, err
4866
}
49-
return &Runner{config: cfg}, nil
67+
return runner, nil
5068
}
5169

5270
// Create post renderer instances from HelmRelease and combine them into
@@ -67,6 +85,10 @@ func postRenderers(hr v2.HelmRelease) (postrender.PostRenderer, error) {
6785

6886
// Install runs an Helm install action for the given v2beta1.HelmRelease.
6987
func (r *Runner) Install(hr v2.HelmRelease, chart *chart.Chart, values chartutil.Values) (*release.Release, error) {
88+
r.mu.Lock()
89+
defer r.mu.Unlock()
90+
defer r.logBuffer.Reset()
91+
7092
install := action.NewInstall(r.config)
7193
install.ReleaseName = hr.GetReleaseName()
7294
install.Namespace = hr.GetReleaseNamespace()
@@ -86,11 +108,16 @@ func (r *Runner) Install(hr v2.HelmRelease, chart *chart.Chart, values chartutil
86108
install.CreateNamespace = hr.Spec.GetInstall().CreateNamespace
87109
}
88110

89-
return install.Run(chart, values.AsMap())
111+
rel, err := install.Run(chart, values.AsMap())
112+
return rel, wrapActionErr(r.logBuffer, err)
90113
}
91114

92115
// Upgrade runs an Helm upgrade action for the given v2beta1.HelmRelease.
93116
func (r *Runner) Upgrade(hr v2.HelmRelease, chart *chart.Chart, values chartutil.Values) (*release.Release, error) {
117+
r.mu.Lock()
118+
defer r.mu.Unlock()
119+
defer r.logBuffer.Reset()
120+
94121
upgrade := action.NewUpgrade(r.config)
95122
upgrade.Namespace = hr.GetReleaseNamespace()
96123
upgrade.ResetValues = !hr.Spec.GetUpgrade().PreserveValues
@@ -108,20 +135,30 @@ func (r *Runner) Upgrade(hr v2.HelmRelease, chart *chart.Chart, values chartutil
108135
}
109136
upgrade.PostRenderer = renderer
110137

111-
return upgrade.Run(hr.GetReleaseName(), chart, values.AsMap())
138+
rel, err := upgrade.Run(hr.GetReleaseName(), chart, values.AsMap())
139+
return rel, wrapActionErr(r.logBuffer, err)
112140
}
113141

114142
// Test runs an Helm test action for the given v2beta1.HelmRelease.
115143
func (r *Runner) Test(hr v2.HelmRelease) (*release.Release, error) {
144+
r.mu.Lock()
145+
defer r.mu.Unlock()
146+
defer r.logBuffer.Reset()
147+
116148
test := action.NewReleaseTesting(r.config)
117149
test.Namespace = hr.GetReleaseNamespace()
118150
test.Timeout = hr.Spec.GetTest().GetTimeout(hr.GetTimeout()).Duration
119151

120-
return test.Run(hr.GetReleaseName())
152+
rel, err := test.Run(hr.GetReleaseName())
153+
return rel, wrapActionErr(r.logBuffer, err)
121154
}
122155

123156
// Rollback runs an Helm rollback action for the given v2beta1.HelmRelease.
124157
func (r *Runner) Rollback(hr v2.HelmRelease) error {
158+
r.mu.Lock()
159+
defer r.mu.Unlock()
160+
defer r.logBuffer.Reset()
161+
125162
rollback := action.NewRollback(r.config)
126163
rollback.Timeout = hr.Spec.GetRollback().GetTimeout(hr.GetTimeout()).Duration
127164
rollback.Wait = !hr.Spec.GetRollback().DisableWait
@@ -130,18 +167,23 @@ func (r *Runner) Rollback(hr v2.HelmRelease) error {
130167
rollback.Recreate = hr.Spec.GetRollback().Recreate
131168
rollback.CleanupOnFail = hr.Spec.GetRollback().CleanupOnFail
132169

133-
return rollback.Run(hr.GetReleaseName())
170+
err := rollback.Run(hr.GetReleaseName())
171+
return wrapActionErr(r.logBuffer, err)
134172
}
135173

136174
// Uninstall runs an Helm uninstall action for the given v2beta1.HelmRelease.
137175
func (r *Runner) Uninstall(hr v2.HelmRelease) error {
176+
r.mu.Lock()
177+
defer r.mu.Unlock()
178+
defer r.logBuffer.Reset()
179+
138180
uninstall := action.NewUninstall(r.config)
139181
uninstall.Timeout = hr.Spec.GetUninstall().GetTimeout(hr.GetTimeout()).Duration
140182
uninstall.DisableHooks = hr.Spec.GetUninstall().DisableHooks
141183
uninstall.KeepHistory = hr.Spec.GetUninstall().KeepHistory
142184

143185
_, err := uninstall.Run(hr.GetReleaseName())
144-
return err
186+
return wrapActionErr(r.logBuffer, err)
145187
}
146188

147189
// ObserveLastRelease observes the last revision, if there is one,
@@ -154,8 +196,13 @@ func (r *Runner) ObserveLastRelease(hr v2.HelmRelease) (*release.Release, error)
154196
return rel, err
155197
}
156198

157-
func debugLogger(logger logr.Logger) func(format string, v ...interface{}) {
158-
return func(format string, v ...interface{}) {
159-
logger.V(1).Info(fmt.Sprintf(format, v...))
199+
func wrapActionErr(log *LogBuffer, err error) error {
200+
if err == nil {
201+
return err
202+
}
203+
err = &ActionError{
204+
Err: err,
205+
CapturedLogs: log.String(),
160206
}
207+
return err
161208
}

0 commit comments

Comments
 (0)