From ccc6def1a9c4f71b08d93a7cbc8b7e497d1dead5 Mon Sep 17 00:00:00 2001 From: akshya96 Date: Fri, 22 Mar 2024 13:11:50 -0700 Subject: [PATCH 1/3] Retention window oss changes --- vault/activity_log.go | 30 +++++++++++++++++++++++++++++- vault/activity_log_test.go | 22 +++++++++++++++++----- vault/activity_log_testing_util.go | 2 +- vault/census.go | 8 ++++++++ vault/logical_system_activity.go | 22 +++++++++++++++------- 5 files changed, 70 insertions(+), 14 deletions(-) diff --git a/vault/activity_log.go b/vault/activity_log.go index cb6d833ae5b7..b9ad74c278a3 100644 --- a/vault/activity_log.go +++ b/vault/activity_log.go @@ -12,6 +12,7 @@ import ( "io" "net/http" "os" + "path" "sort" "strconv" "strings" @@ -1893,7 +1894,7 @@ type activityConfig struct { func defaultActivityConfig() activityConfig { return activityConfig{ DefaultReportMonths: 12, - RetentionMonths: 24, + RetentionMonths: ActivityLogMinimumRetentionMonths, Enabled: "default", } } @@ -1913,9 +1914,36 @@ func (a *ActivityLog) loadConfigOrDefault(ctx context.Context) (activityConfig, return config, err } + // check if the retention time is lesser than the default + if config.RetentionMonths < ActivityLogMinimumRetentionMonths { + updatedConfig, err := a.setDefaultRetentionMonthsInConfig(ctx, config) + if err != nil { + return config, err + } + return updatedConfig, nil + } return config, nil } +// setDefaultRetentionMonthsInConfig sets the retention months in activity config with default value. +// This supports upgrades from versions prior to set the new default ActivityLogMinimumRetentionMonths. +func (a *ActivityLog) setDefaultRetentionMonthsInConfig(ctx context.Context, inputConfig activityConfig) (activityConfig, error) { + inputConfig.RetentionMonths = ActivityLogMinimumRetentionMonths + + // Store the config + entry, err := logical.StorageEntryJSON(path.Join(activitySubPath, activityConfigKey), inputConfig) + if err != nil { + return inputConfig, err + } + if err := a.view.Put(ctx, entry); err != nil { + return inputConfig, err + } + + // Set the new config on the activity log + a.SetConfig(ctx, inputConfig) + return inputConfig, nil +} + // HandleTokenUsage adds the TokenEntry to the current fragment of the activity log // This currently occurs on token usage only. func (a *ActivityLog) HandleTokenUsage(ctx context.Context, entry *logical.TokenEntry, clientID string, isTWE bool) error { diff --git a/vault/activity_log_test.go b/vault/activity_log_test.go index 1f81402122c9..da3080f9121b 100644 --- a/vault/activity_log_test.go +++ b/vault/activity_log_test.go @@ -861,7 +861,7 @@ func TestActivityLog_API_ConfigCRUD_Census(t *testing.T) { if err == nil { t.Fatal("expected error") } - if resp.Data["error"] != `retention_months must be at least 24 while Reporting is enabled` { + if resp.Data["error"] != retentionMonthsOutOfBounds { t.Fatalf("bad: %v", resp) } } else { @@ -872,13 +872,20 @@ func TestActivityLog_API_ConfigCRUD_Census(t *testing.T) { req = logical.TestRequest(t, logical.UpdateOperation, "internal/counters/config") req.Storage = view - req.Data["retention_months"] = 26 + req.Data["retention_months"] = 56 resp, err = b.HandleRequest(namespace.RootContext(nil), req) if err != nil { t.Fatalf("err: %v", err) } - if resp != nil { - t.Fatalf("bad: %#v", resp) + if core.ManualLicenseReportingEnabled() { + if resp != nil { + t.Fatalf("bad: %#v", resp) + } + } else { + expectedWarning := defaultToRetentionMonthsMaxWarning + if resp.Warnings[0] != expectedWarning { + t.Fatalf("expected warning not present") + } } req = logical.TestRequest(t, logical.UpdateOperation, "internal/counters/config") @@ -918,9 +925,14 @@ func TestActivityLog_API_ConfigCRUD_Census(t *testing.T) { if err != nil { t.Fatalf("err: %v", err) } + expectedRetentionMonths := activityLogMaximumRetentionMonths + if core.ManualLicenseReportingEnabled() { + expectedRetentionMonths = 56 + } + expected := map[string]interface{}{ "default_report_months": 12, - "retention_months": 26, + "retention_months": expectedRetentionMonths, "enabled": "enable", "queries_available": false, "reporting_enabled": core.AutomatedLicenseReportingEnabled(), diff --git a/vault/activity_log_testing_util.go b/vault/activity_log_testing_util.go index 600da787c053..bbaae9551191 100644 --- a/vault/activity_log_testing_util.go +++ b/vault/activity_log_testing_util.go @@ -115,7 +115,7 @@ func (a *ActivityLog) SetStandbyEnable(ctx context.Context, enabled bool) { // TODO only patch enabled? a.SetConfigStandby(ctx, activityConfig{ DefaultReportMonths: 12, - RetentionMonths: 24, + RetentionMonths: ActivityLogMinimumRetentionMonths, Enabled: enableStr, }) } diff --git a/vault/census.go b/vault/census.go index e492ac2c88c3..99b67687278f 100644 --- a/vault/census.go +++ b/vault/census.go @@ -10,6 +10,14 @@ import "time" // CensusAgent is a stub for OSS type CensusReporter interface{} +const ( + // ActivityLogMinimumRetentionMonths sets the default minimum retention_months + ActivityLogMinimumRetentionMonths = 0 + + // activityLogMaximumRetentionMonths sets the default maximum retention_months + activityLogMaximumRetentionMonths = 36 +) + func (c *Core) setupCensusManager() error { return nil } func (c *Core) BillingStart() time.Time { return time.Time{} } func (c *Core) AutomatedLicenseReportingEnabled() bool { return false } diff --git a/vault/logical_system_activity.go b/vault/logical_system_activity.go index cf0d6a273f82..5f30c41f8bb2 100644 --- a/vault/logical_system_activity.go +++ b/vault/logical_system_activity.go @@ -18,6 +18,14 @@ import ( "github.com/hashicorp/vault/sdk/logical" ) +var ( + // retentionMonthsOutOfBounds is an error string for invalid values of retention_months + retentionMonthsOutOfBounds = fmt.Sprintf("retention_months value outside valid range: [%d, %d]", ActivityLogMinimumRetentionMonths, activityLogMaximumRetentionMonths) + + // defaultToRetentionMonthsMaxWarning is a warning message for setting the max retention_months value when retention_months value is more than activityLogMaximumRetentionMonths + defaultToRetentionMonthsMaxWarning = fmt.Sprintf("%s: defaulting to max: %d", retentionMonthsOutOfBounds, activityLogMaximumRetentionMonths) +) + // activityQueryPath is available in every namespace func (b *SystemBackend) activityQueryPath() *framework.Path { return &framework.Path{ @@ -109,7 +117,7 @@ func (b *SystemBackend) rootActivityPaths() []*framework.Path { }, "retention_months": { Type: framework.TypeInt, - Default: 24, + Default: ActivityLogMinimumRetentionMonths, Description: "Number of months of client data to retain. Setting to 0 will clear all existing data.", }, "enabled": { @@ -367,13 +375,13 @@ func (b *SystemBackend) handleActivityConfigUpdate(ctx context.Context, req *log config.RetentionMonths = retentionMonthsRaw.(int) } - if config.RetentionMonths < 0 { - return logical.ErrorResponse("retention_months must be greater than or equal to 0"), logical.ErrInvalidRequest + if config.RetentionMonths < ActivityLogMinimumRetentionMonths { + return logical.ErrorResponse(retentionMonthsOutOfBounds), logical.ErrInvalidRequest } - if config.RetentionMonths > 36 { - config.RetentionMonths = 36 - warnings = append(warnings, "retention_months cannot be greater than 36; capped to 36.") + if config.RetentionMonths > activityLogMaximumRetentionMonths { + config.RetentionMonths = activityLogMaximumRetentionMonths + warnings = append(warnings, defaultToRetentionMonthsMaxWarning) } } @@ -416,7 +424,7 @@ func (b *SystemBackend) handleActivityConfigUpdate(ctx context.Context, req *log return logical.ErrorResponse("retention_months cannot be 0 while enabled"), logical.ErrInvalidRequest } - // if manual license reporting is enabled, retention months must at least be 24 months + // if manual license reporting is enabled, retention months must at least be 48 months if a.core.ManualLicenseReportingEnabled() && config.RetentionMonths < minimumRetentionMonths { return logical.ErrorResponse("retention_months must be at least %d while Reporting is enabled", minimumRetentionMonths), logical.ErrInvalidRequest } From 9dbb96cff6a7512f4056a831a65ffbd2479e4e01 Mon Sep 17 00:00:00 2001 From: akshya96 Date: Mon, 25 Mar 2024 12:06:00 -0700 Subject: [PATCH 2/3] latest oss changes --- command/operator_diagnose.go | 3 +-- vault/activity_log.go | 18 ++++++++++++++---- vault/activity_log_test.go | 19 ++++--------------- vault/census.go | 8 -------- vault/logical_system_activity.go | 19 ++++++++----------- 5 files changed, 27 insertions(+), 40 deletions(-) diff --git a/command/operator_diagnose.go b/command/operator_diagnose.go index 46930451d3b3..b650ae8efeda 100644 --- a/command/operator_diagnose.go +++ b/command/operator_diagnose.go @@ -14,8 +14,6 @@ import ( "sync" "time" - "github.com/hashicorp/vault/vault/seal" - "github.com/hashicorp/cli" "github.com/hashicorp/consul/api" log "github.com/hashicorp/go-hclog" @@ -35,6 +33,7 @@ import ( "github.com/hashicorp/vault/vault" "github.com/hashicorp/vault/vault/diagnose" "github.com/hashicorp/vault/vault/hcp_link" + "github.com/hashicorp/vault/vault/seal" "github.com/hashicorp/vault/version" "github.com/posener/complete" "golang.org/x/term" diff --git a/vault/activity_log.go b/vault/activity_log.go index b9ad74c278a3..12a9496450cf 100644 --- a/vault/activity_log.go +++ b/vault/activity_log.go @@ -86,6 +86,16 @@ const ( nonEntityTokenActivityType = "non-entity-token" entityActivityType = "entity" secretSyncActivityType = "secret-sync" + + // ActivityLogMinimumRetentionMonths sets the default minimum retention_months + // to enforce when reporting is enabled. Note that this value is also statically + // defined in the UI. Any updates here should also be made to + // ui/app/models/clients/config.js. + ActivityLogMinimumRetentionMonths = 48 + + // activityLogMaximumRetentionMonths sets the default maximum retention_months + // to enforce when reporting is enabled. + activityLogMaximumRetentionMonths = 60 ) var ActivityClientTypes = []string{nonEntityTokenActivityType, entityActivityType, secretSyncActivityType, ACMEActivityType} @@ -259,7 +269,7 @@ func NewActivityLog(core *Core, logger log.Logger, view *BarrierView, metrics me precomputedQueryWritten: make(chan struct{}), } - config, err := a.loadConfigOrDefault(core.activeContext) + config, err := a.loadConfigOrDefault(core.activeContext, core.ManualLicenseReportingEnabled()) if err != nil { return nil, err } @@ -1899,7 +1909,7 @@ func defaultActivityConfig() activityConfig { } } -func (a *ActivityLog) loadConfigOrDefault(ctx context.Context) (activityConfig, error) { +func (a *ActivityLog) loadConfigOrDefault(ctx context.Context, isReportingEnabled bool) (activityConfig, error) { // Load from storage var config activityConfig configRaw, err := a.view.Get(ctx, activityConfigKey) @@ -1914,8 +1924,8 @@ func (a *ActivityLog) loadConfigOrDefault(ctx context.Context) (activityConfig, return config, err } - // check if the retention time is lesser than the default - if config.RetentionMonths < ActivityLogMinimumRetentionMonths { + // check if the retention time is lesser than the default when reporting is enabled + if (config.RetentionMonths < ActivityLogMinimumRetentionMonths) && isReportingEnabled { updatedConfig, err := a.setDefaultRetentionMonthsInConfig(ctx, config) if err != nil { return config, err diff --git a/vault/activity_log_test.go b/vault/activity_log_test.go index da3080f9121b..f1aae77a2d6e 100644 --- a/vault/activity_log_test.go +++ b/vault/activity_log_test.go @@ -861,7 +861,7 @@ func TestActivityLog_API_ConfigCRUD_Census(t *testing.T) { if err == nil { t.Fatal("expected error") } - if resp.Data["error"] != retentionMonthsOutOfBounds { + if resp.Data["error"] != `retention_months must be at least 48 while Reporting is enabled` { t.Fatalf("bad: %v", resp) } } else { @@ -877,15 +877,8 @@ func TestActivityLog_API_ConfigCRUD_Census(t *testing.T) { if err != nil { t.Fatalf("err: %v", err) } - if core.ManualLicenseReportingEnabled() { - if resp != nil { - t.Fatalf("bad: %#v", resp) - } - } else { - expectedWarning := defaultToRetentionMonthsMaxWarning - if resp.Warnings[0] != expectedWarning { - t.Fatalf("expected warning not present") - } + if resp != nil { + t.Fatalf("bad: %#v", resp) } req = logical.TestRequest(t, logical.UpdateOperation, "internal/counters/config") @@ -925,14 +918,10 @@ func TestActivityLog_API_ConfigCRUD_Census(t *testing.T) { if err != nil { t.Fatalf("err: %v", err) } - expectedRetentionMonths := activityLogMaximumRetentionMonths - if core.ManualLicenseReportingEnabled() { - expectedRetentionMonths = 56 - } expected := map[string]interface{}{ "default_report_months": 12, - "retention_months": expectedRetentionMonths, + "retention_months": 56, "enabled": "enable", "queries_available": false, "reporting_enabled": core.AutomatedLicenseReportingEnabled(), diff --git a/vault/census.go b/vault/census.go index 99b67687278f..e492ac2c88c3 100644 --- a/vault/census.go +++ b/vault/census.go @@ -10,14 +10,6 @@ import "time" // CensusAgent is a stub for OSS type CensusReporter interface{} -const ( - // ActivityLogMinimumRetentionMonths sets the default minimum retention_months - ActivityLogMinimumRetentionMonths = 0 - - // activityLogMaximumRetentionMonths sets the default maximum retention_months - activityLogMaximumRetentionMonths = 36 -) - func (c *Core) setupCensusManager() error { return nil } func (c *Core) BillingStart() time.Time { return time.Time{} } func (c *Core) AutomatedLicenseReportingEnabled() bool { return false } diff --git a/vault/logical_system_activity.go b/vault/logical_system_activity.go index 5f30c41f8bb2..bd98c084950f 100644 --- a/vault/logical_system_activity.go +++ b/vault/logical_system_activity.go @@ -18,13 +18,8 @@ import ( "github.com/hashicorp/vault/sdk/logical" ) -var ( - // retentionMonthsOutOfBounds is an error string for invalid values of retention_months - retentionMonthsOutOfBounds = fmt.Sprintf("retention_months value outside valid range: [%d, %d]", ActivityLogMinimumRetentionMonths, activityLogMaximumRetentionMonths) - - // defaultToRetentionMonthsMaxWarning is a warning message for setting the max retention_months value when retention_months value is more than activityLogMaximumRetentionMonths - defaultToRetentionMonthsMaxWarning = fmt.Sprintf("%s: defaulting to max: %d", retentionMonthsOutOfBounds, activityLogMaximumRetentionMonths) -) +// defaultToRetentionMonthsMaxWarning is a warning message for setting the max retention_months value when retention_months value is more than activityLogMaximumRetentionMonths +var defaultToRetentionMonthsMaxWarning = fmt.Sprintf("retention_months cannot be greater than %d; capped to %d.", activityLogMaximumRetentionMonths, activityLogMaximumRetentionMonths) // activityQueryPath is available in every namespace func (b *SystemBackend) activityQueryPath() *framework.Path { @@ -316,7 +311,7 @@ func (b *SystemBackend) handleActivityConfigRead(ctx context.Context, req *logic return logical.ErrorResponse("no activity log present"), nil } - config, err := a.loadConfigOrDefault(ctx) + config, err := a.loadConfigOrDefault(ctx, b.Core.ManualLicenseReportingEnabled()) if err != nil { return nil, err } @@ -353,7 +348,7 @@ func (b *SystemBackend) handleActivityConfigUpdate(ctx context.Context, req *log warnings := make([]string, 0) - config, err := a.loadConfigOrDefault(ctx) + config, err := a.loadConfigOrDefault(ctx, b.Core.ManualLicenseReportingEnabled()) if err != nil { return nil, err } @@ -371,12 +366,14 @@ func (b *SystemBackend) handleActivityConfigUpdate(ctx context.Context, req *log { // Parse the retention months + // For CE, this value can be between 0 and 60 + // When reporting is enabled, this value can be between 48 and 60 if retentionMonthsRaw, ok := d.GetOk("retention_months"); ok { config.RetentionMonths = retentionMonthsRaw.(int) } - if config.RetentionMonths < ActivityLogMinimumRetentionMonths { - return logical.ErrorResponse(retentionMonthsOutOfBounds), logical.ErrInvalidRequest + if config.RetentionMonths < 0 { + return logical.ErrorResponse("retention_months must be greater than or equal to 0"), logical.ErrInvalidRequest } if config.RetentionMonths > activityLogMaximumRetentionMonths { From 5952ebbb9d122e45f8a19c562a9e219e4c6ec435 Mon Sep 17 00:00:00 2001 From: akshya96 Date: Mon, 25 Mar 2024 12:32:12 -0700 Subject: [PATCH 3/3] remove operator_diagnose change --- command/operator_diagnose.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/command/operator_diagnose.go b/command/operator_diagnose.go index b650ae8efeda..46930451d3b3 100644 --- a/command/operator_diagnose.go +++ b/command/operator_diagnose.go @@ -14,6 +14,8 @@ import ( "sync" "time" + "github.com/hashicorp/vault/vault/seal" + "github.com/hashicorp/cli" "github.com/hashicorp/consul/api" log "github.com/hashicorp/go-hclog" @@ -33,7 +35,6 @@ import ( "github.com/hashicorp/vault/vault" "github.com/hashicorp/vault/vault/diagnose" "github.com/hashicorp/vault/vault/hcp_link" - "github.com/hashicorp/vault/vault/seal" "github.com/hashicorp/vault/version" "github.com/posener/complete" "golang.org/x/term"