From 4ace14dd9ceced988fe57b598e8f031cba639029 Mon Sep 17 00:00:00 2001 From: LiboYu2 <168450736+LiboYu2@users.noreply.github.com> Date: Tue, 29 Oct 2024 13:57:35 -0400 Subject: [PATCH] stabilize e2e test case sandbox-basic (#962) Increased the timeout for steps that frequently fail. Add some latency to stabilize the cluster before moving to the unsandboxing step. I ran the test cases locally for 5 times in a roll and they all passed. --------- Co-authored-by: Roy Paulin --- Makefile | 4 ++++ changes/unreleased/Added-20241029-124128.yaml | 5 +++++ cmd/operator/main.go | 9 +++++++-- config/manager/operator-envs | 1 + helm-charts/verticadb-operator/README.md | 1 + helm-charts/verticadb-operator/values.yaml | 3 +++ kuttl-test.yaml | 1 + pkg/opcfg/config.go | 11 +++++++++++ scripts/setup-kustomize.sh | 1 + scripts/template-helm-chart.sh | 1 + tests/e2e-leg-10/sandbox-basic/20-assert.yaml | 5 ++++- tests/e2e-leg-10/sandbox-basic/60-assert.yaml | 6 +++++- .../sandbox-basic/setup-vdb/base/setup-vdb.yaml | 2 +- tests/kustomize-defaults.cfg | 2 +- 14 files changed, 46 insertions(+), 6 deletions(-) create mode 100644 changes/unreleased/Added-20241029-124128.yaml diff --git a/Makefile b/Makefile index e922fa0c8..5cfbf15fd 100644 --- a/Makefile +++ b/Makefile @@ -191,6 +191,10 @@ export CONTROLLERS_ENABLED # Set this to control if the webhook is enabled or disabled in the operator. WEBHOOKS_ENABLED?=true export WEBHOOKS_ENABLED + +#set this to increase the threshold used by spam filter +BROADCASTER_BURST_SIZE?=100 +export BROADCASTER_BURST_SIZE # # Use this to control what scope the controller is deployed at. It supports two # values: diff --git a/changes/unreleased/Added-20241029-124128.yaml b/changes/unreleased/Added-20241029-124128.yaml new file mode 100644 index 000000000..217502db2 --- /dev/null +++ b/changes/unreleased/Added-20241029-124128.yaml @@ -0,0 +1,5 @@ +kind: Added +body: Add helm parameter and environment variable to control burst size for event recording. +time: 2024-10-29T12:41:28.100138558+01:00 +custom: + Issue: "962" diff --git a/cmd/operator/main.go b/cmd/operator/main.go index 1ca947eb9..ecb582881 100644 --- a/cmd/operator/main.go +++ b/cmd/operator/main.go @@ -32,6 +32,7 @@ import ( clientgoscheme "k8s.io/client-go/kubernetes/scheme" _ "k8s.io/client-go/plugin/pkg/client/auth" "k8s.io/client-go/rest" + "k8s.io/client-go/tools/record" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/config/v1alpha1" "sigs.k8s.io/controller-runtime/pkg/healthz" @@ -239,7 +240,8 @@ func main() { if opcfg.GetLoggingFilePath() != "" { log.Printf("Now logging in file %s", opcfg.GetLoggingFilePath()) } - + burstSize := opcfg.GetBroadcasterBurstSize() + var multibroadcaster = record.NewBroadcasterWithCorrelatorOptions(record.CorrelatorOptions{BurstSize: burstSize}) ctrl.SetLogger(logger) setupLog.Info("Build info", "gitCommit", GitCommit, "buildDate", BuildDate, "vclusterVersion", VClusterVersion) @@ -248,7 +250,9 @@ func main() { "version", opcfg.GetVersion(), "watchNamespace", opcfg.GetWatchNamespace(), "webhooksEnabled", opcfg.GetIsWebhookEnabled(), - "controllersEnabled", opcfg.GetIsControllersEnabled()) + "controllersEnabled", opcfg.GetIsControllersEnabled(), + "broadcasterBurstSize", burstSize, + ) restCfg := ctrl.GetConfigOrDie() @@ -260,6 +264,7 @@ func main() { LeaderElection: true, LeaderElectionID: opcfg.GetLeaderElectionID(), Namespace: opcfg.GetWatchNamespace(), + EventBroadcaster: multibroadcaster, CertDir: CertDir, Controller: v1alpha1.ControllerConfigurationSpec{ GroupKindConcurrency: map[string]int{ diff --git a/config/manager/operator-envs b/config/manager/operator-envs index 299683985..25a14d182 100644 --- a/config/manager/operator-envs +++ b/config/manager/operator-envs @@ -14,3 +14,4 @@ CONCURRENCY_VERTICARESTOREPOINTSQUERY CONCURRENCY_VERTICASCRUTINIZE CONCURRENCY_SANDBOXCONFIGMAP CONCURRENCY_VERTICAREPLICATOR +BROADCASTER_BURST_SIZE diff --git a/helm-charts/verticadb-operator/README.md b/helm-charts/verticadb-operator/README.md index 7e31b3645..9341890fe 100644 --- a/helm-charts/verticadb-operator/README.md +++ b/helm-charts/verticadb-operator/README.md @@ -5,6 +5,7 @@ This helm chart will install the operator and an admission controller webhook. | affinity | The [affinity](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity) parameter allows you to constrain the operator pod only to specific nodes. If this parameter is not set, then no affinity setting is used with the operator pod. | Not set | | controllers.enable | This controls if controllers are enabled when running the operator. The controllers are the part of the operator that watches and acts on custom resources. This option is useful if you want to deploy the operator just as a webhook. This comes in handy when deploying the operator as the namespace scope | true | | controllers.scope | Defines the scope of the operator. You can define one of two values: cluster or namespace.

When set to cluster, the operator is cluster scoped. This means it will watch for changes to any custom resource across all namespaces. This is the default deployment.

When set to namespace, the operator is cluster scope. The operator will only set up watches for the namespace it is deployed in. You can deploy the operator in multiple namespaces this way. However, the webhook can only be run once in the cluster. You can control running of the webhook with the webhook.enable option. | cluster | +| controllers.burstSize | This controls the burst size for even recording in the operator. Increasing this allows the controllers to record more events in a short period. | 100 | | image.name | The name of image that runs the operator. | opentext/verticadb-operator:24.4.0-0 | | image.repo | Repo server hosting image.name | docker.io | | image.pullPolicy | The pull policy for the image that runs the operator | IfNotPresent | diff --git a/helm-charts/verticadb-operator/values.yaml b/helm-charts/verticadb-operator/values.yaml index 255ceb9f1..e9a62fb19 100644 --- a/helm-charts/verticadb-operator/values.yaml +++ b/helm-charts/verticadb-operator/values.yaml @@ -47,6 +47,9 @@ controllers: # only setup watches for the namespace the operator is deployed in, and we will only # grant Role/RoleBindings to the manager for deployed namespace. scope: cluster + # Use this if you want to provide a custom burst size for event recording in the operator. + # Increasing this allows the controllers to record more events in a short period. + burstSize: 100 webhook: # The webhook requires a TLS certificate to work. This parm defines how the diff --git a/kuttl-test.yaml b/kuttl-test.yaml index 6483d812a..226447fb0 100644 --- a/kuttl-test.yaml +++ b/kuttl-test.yaml @@ -13,6 +13,7 @@ apiVersion: kuttl.dev/v1beta1 kind: TestSuite +kindNodeCache: true # Each testcase will delete its own namespace if it is successful. This # allows us to collect scrutinize for all of the failed tests. skipDelete: true diff --git a/pkg/opcfg/config.go b/pkg/opcfg/config.go index 556e94b61..6c7e3dd2a 100644 --- a/pkg/opcfg/config.go +++ b/pkg/opcfg/config.go @@ -29,11 +29,22 @@ import ( lumberjack "gopkg.in/natefinch/lumberjack.v2" ) +const defaultBurstSize = 25 + // GetIsWebhookEnabled returns true if the webhook is enabled. func GetIsWebhookEnabled() bool { return lookupBoolEnvVar("WEBHOOKS_ENABLED", envMustExist) } +// GetBroadcasterBurstSize returns the customizable burst size for broadcaster. +func GetBroadcasterBurstSize() int { + burstSize := lookupIntEnvVar("BROADCASTER_BURST_SIZE", envCanNotExist) + if burstSize > defaultBurstSize { + return burstSize + } + return defaultBurstSize +} + // GetIsControllersEnabled returns true if the controllers for each custom // resource will start. If this is false, then the manager will just act as a // webhook (if enabled). diff --git a/scripts/setup-kustomize.sh b/scripts/setup-kustomize.sh index 9d77bde86..33821a5a5 100755 --- a/scripts/setup-kustomize.sh +++ b/scripts/setup-kustomize.sh @@ -138,6 +138,7 @@ echo "Deployment method: $VERTICA_DEPLOYMENT_METHOD" echo "Image version: $(determine_image_version $VERTICA_IMG)" echo "Vertica superuser name: $VERTICA_SUPERUSER_NAME" echo "Test running on Github CI: $FOR_GITHUB_CI" +echo "Broadcaster burst size: $BROADCASTER_BURST_SIZE" function create_vdb_kustomization { BASE_DIR=$1 diff --git a/scripts/template-helm-chart.sh b/scripts/template-helm-chart.sh index 36defdcb3..94ae8a76f 100755 --- a/scripts/template-helm-chart.sh +++ b/scripts/template-helm-chart.sh @@ -225,6 +225,7 @@ done for fn in $TEMPLATE_DIR/verticadb-operator-manager-config-cm.yaml do perl -i -0777 -pe 's/(WEBHOOKS_ENABLED:).*/$1 {{ quote .Values.webhook.enable }}/g' $fn + perl -i -0777 -pe 's/(BROADCASTER_BURST_SIZE:).*/$1 {{ quote .Values.controllers.burstSize }}/g' $fn perl -i -0777 -pe 's/(CONTROLLERS_ENABLED:).*/$1 {{ quote .Values.controllers.enable }}/g' $fn perl -i -0777 -pe 's/(CONTROLLERS_SCOPE:).*/$1 {{ quote .Values.controllers.scope }}/g' $fn # Update the webhook-cert-secret configMap entry to include the actual name of the secret diff --git a/tests/e2e-leg-10/sandbox-basic/20-assert.yaml b/tests/e2e-leg-10/sandbox-basic/20-assert.yaml index 0ce0bceb5..827ff7d18 100644 --- a/tests/e2e-leg-10/sandbox-basic/20-assert.yaml +++ b/tests/e2e-leg-10/sandbox-basic/20-assert.yaml @@ -10,7 +10,10 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 1200 +--- apiVersion: apps/v1 kind: StatefulSet metadata: diff --git a/tests/e2e-leg-10/sandbox-basic/60-assert.yaml b/tests/e2e-leg-10/sandbox-basic/60-assert.yaml index f3248a4bc..5ac3f729f 100644 --- a/tests/e2e-leg-10/sandbox-basic/60-assert.yaml +++ b/tests/e2e-leg-10/sandbox-basic/60-assert.yaml @@ -11,6 +11,10 @@ # See the License for the specific language governing permissions and # limitations under the License. +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 900 +--- apiVersion: v1 kind: Event reason: UnsandboxSubclusterSucceeded @@ -65,4 +69,4 @@ status: name: sec2 - addedToDBCount: 1 upNodeCount: 1 - name: sec3 \ No newline at end of file + name: sec3 diff --git a/tests/e2e-leg-10/sandbox-basic/setup-vdb/base/setup-vdb.yaml b/tests/e2e-leg-10/sandbox-basic/setup-vdb/base/setup-vdb.yaml index f8171dd82..d92d8a56e 100644 --- a/tests/e2e-leg-10/sandbox-basic/setup-vdb/base/setup-vdb.yaml +++ b/tests/e2e-leg-10/sandbox-basic/setup-vdb/base/setup-vdb.yaml @@ -23,7 +23,7 @@ spec: initPolicy: CreateSkipPackageInstall communal: {} local: - requestSize: 250Mi + requestSize: 270Mi # We pick a cluster size that is too big for the CE license. This relies on # having a license, which will be added by kustomize. subclusters: diff --git a/tests/kustomize-defaults.cfg b/tests/kustomize-defaults.cfg index b6e8010bf..5a96d67d3 100644 --- a/tests/kustomize-defaults.cfg +++ b/tests/kustomize-defaults.cfg @@ -119,4 +119,4 @@ ALLOW_VOLUME_EXPANSION= # The name of Vertica superuser generated in database creation. When it is # not set, default value "dbadmin" will be used. Uncomment this or set this # environment in your shell to specify Vertica superuser name. -# VERTICA_SUPERUSER_NAME= \ No newline at end of file +# VERTICA_SUPERUSER_NAME=