stabilize e2e test case sandbox-basic (#962)

Increased the timeout for steps that frequently fail. 
Add some latency to stabilize the cluster before moving to the
unsandboxing step.
I ran the test cases locally for 5 times in a roll and they all passed.

---------

Co-authored-by: Roy Paulin <rnguetsopken@opentext.com>

Makefile

@@ -191,6 +191,10 @@ export CONTROLLERS_ENABLED
 # Set this to control if the webhook is enabled or disabled in the operator.
 WEBHOOKS_ENABLED?=true
 export WEBHOOKS_ENABLED
+
+#set this to increase the threshold used by spam filter
+BROADCASTER_BURST_SIZE?=100
+export BROADCASTER_BURST_SIZE
 #
 # Use this to control what scope the controller is deployed at. It supports two
 # values:

changes/unreleased/Added-20241029-124128.yaml

@@ -0,0 +1,5 @@
+kind: Added
+body: Add helm parameter and environment variable to control burst size for event recording.
+time: 2024-10-29T12:41:28.100138558+01:00
+custom:
+    Issue: "962"

cmd/operator/main.go

@@ -32,6 +32,7 @@ import (
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
 	_ "k8s.io/client-go/plugin/pkg/client/auth"
 	"k8s.io/client-go/rest"
+	"k8s.io/client-go/tools/record"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/config/v1alpha1"
 	"sigs.k8s.io/controller-runtime/pkg/healthz"
@@ -239,7 +240,8 @@ func main() {
 	if opcfg.GetLoggingFilePath() != "" {
 		log.Printf("Now logging in file %s", opcfg.GetLoggingFilePath())
 	}
-
+	burstSize := opcfg.GetBroadcasterBurstSize()
+	var multibroadcaster = record.NewBroadcasterWithCorrelatorOptions(record.CorrelatorOptions{BurstSize: burstSize})
 	ctrl.SetLogger(logger)
 	setupLog.Info("Build info", "gitCommit", GitCommit,
 		"buildDate", BuildDate, "vclusterVersion", VClusterVersion)
@@ -248,7 +250,9 @@ func main() {
 		"version", opcfg.GetVersion(),
 		"watchNamespace", opcfg.GetWatchNamespace(),
 		"webhooksEnabled", opcfg.GetIsWebhookEnabled(),
-		"controllersEnabled", opcfg.GetIsControllersEnabled())
+		"controllersEnabled", opcfg.GetIsControllersEnabled(),
+		"broadcasterBurstSize", burstSize,
+	)
 
 	restCfg := ctrl.GetConfigOrDie()
 
@@ -260,6 +264,7 @@ func main() {
 		LeaderElection:         true,
 		LeaderElectionID:       opcfg.GetLeaderElectionID(),
 		Namespace:              opcfg.GetWatchNamespace(),
+		EventBroadcaster:       multibroadcaster,
 		CertDir:                CertDir,
 		Controller: v1alpha1.ControllerConfigurationSpec{
 			GroupKindConcurrency: map[string]int{

config/manager/operator-envs

@@ -14,3 +14,4 @@ CONCURRENCY_VERTICARESTOREPOINTSQUERY
 CONCURRENCY_VERTICASCRUTINIZE
 CONCURRENCY_SANDBOXCONFIGMAP
 CONCURRENCY_VERTICAREPLICATOR
+BROADCASTER_BURST_SIZE

helm-charts/verticadb-operator/README.md

@@ -5,6 +5,7 @@ This helm chart will install the operator and an admission controller webhook.
 | affinity | The [affinity](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity) parameter allows you to constrain the operator pod only to specific nodes. If this parameter is not set, then no affinity setting is used with the operator pod. | Not set |
 | controllers.enable | This controls if controllers are enabled when running the operator. The controllers are the part of the operator that watches and acts on custom resources. This option is useful if you want to deploy the operator just as a webhook. This comes in handy when deploying the operator as the namespace scope | true |
 | controllers.scope | Defines the scope of the operator. You can define one of two values: cluster or namespace.<br><br>When set to cluster, the operator is cluster scoped. This means it will watch for changes to any custom resource across all namespaces. This is the default deployment.<br><br>When set to namespace, the operator is cluster scope. The operator will only set up watches for the namespace it is deployed in. You can deploy the operator in multiple namespaces this way. However, the webhook can only be run once in the cluster. You can control running of the webhook with the webhook.enable option. | cluster |
+| controllers.burstSize | This controls the burst size for even recording in the operator. Increasing this allows the controllers to record more events in a short period. | 100 |
 | image.name | The name of image that runs the operator. | opentext/verticadb-operator:24.4.0-0 |
 | image.repo | Repo server hosting image.name | docker.io |
 | image.pullPolicy | The pull policy for the image that runs the operator  | IfNotPresent |

helm-charts/verticadb-operator/values.yaml

@@ -47,6 +47,9 @@ controllers:
   # only setup watches for the namespace the operator is deployed in, and we will only
   # grant Role/RoleBindings to the manager for deployed namespace.
   scope: cluster
+  # Use this if you want to provide a custom burst size for event recording in the operator.
+  # Increasing this allows the controllers to record more events in a short period.
+  burstSize: 100
 
 webhook:
   # The webhook requires a TLS certificate to work. This parm defines how the

kuttl-test.yaml

@@ -13,6 +13,7 @@
 
 apiVersion: kuttl.dev/v1beta1
 kind: TestSuite
+kindNodeCache: true
 # Each testcase will delete its own namespace if it is successful.  This
 # allows us to collect scrutinize for all of the failed tests.
 skipDelete: true

pkg/opcfg/config.go

@@ -29,11 +29,22 @@ import (
 	lumberjack "gopkg.in/natefinch/lumberjack.v2"
 )
 
+const defaultBurstSize = 25
+
 // GetIsWebhookEnabled returns true if the webhook is enabled.
 func GetIsWebhookEnabled() bool {
 	return lookupBoolEnvVar("WEBHOOKS_ENABLED", envMustExist)
 }
 
+// GetBroadcasterBurstSize returns the customizable burst size for broadcaster.
+func GetBroadcasterBurstSize() int {
+	burstSize := lookupIntEnvVar("BROADCASTER_BURST_SIZE", envCanNotExist)
+	if burstSize > defaultBurstSize {
+		return burstSize
+	}
+	return defaultBurstSize
+}
+
 // GetIsControllersEnabled returns true if the controllers for each custom
 // resource will start. If this is false, then the manager will just act as a
 // webhook (if enabled).

scripts/setup-kustomize.sh

@@ -138,6 +138,7 @@ echo "Deployment method: $VERTICA_DEPLOYMENT_METHOD"
 echo "Image version: $(determine_image_version $VERTICA_IMG)"
 echo "Vertica superuser name: $VERTICA_SUPERUSER_NAME"
 echo "Test running on Github CI: $FOR_GITHUB_CI"
+echo "Broadcaster burst size: $BROADCASTER_BURST_SIZE"
 
 function create_vdb_kustomization {
     BASE_DIR=$1

scripts/template-helm-chart.sh

@@ -225,6 +225,7 @@ done
 for fn in $TEMPLATE_DIR/verticadb-operator-manager-config-cm.yaml
 do
   perl -i -0777 -pe 's/(WEBHOOKS_ENABLED:).*/$1 {{ quote .Values.webhook.enable }}/g' $fn
+  perl -i -0777 -pe 's/(BROADCASTER_BURST_SIZE:).*/$1 {{ quote .Values.controllers.burstSize }}/g' $fn
   perl -i -0777 -pe 's/(CONTROLLERS_ENABLED:).*/$1 {{ quote .Values.controllers.enable }}/g' $fn
   perl -i -0777 -pe 's/(CONTROLLERS_SCOPE:).*/$1 {{ quote .Values.controllers.scope }}/g' $fn
   # Update the webhook-cert-secret configMap entry to include the actual name of the secret

tests/e2e-leg-10/sandbox-basic/20-assert.yaml

@@ -10,7 +10,10 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+timeout: 1200
+---
 apiVersion: apps/v1
 kind: StatefulSet
 metadata:

tests/e2e-leg-10/sandbox-basic/60-assert.yaml

@@ -11,6 +11,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+timeout: 900
+---
 apiVersion: v1
 kind: Event
 reason: UnsandboxSubclusterSucceeded
@@ -65,4 +69,4 @@ status:
       name: sec2
     - addedToDBCount: 1
       upNodeCount: 1
-      name: sec3
+      name: sec3

tests/e2e-leg-10/sandbox-basic/setup-vdb/base/setup-vdb.yaml

@@ -23,7 +23,7 @@ spec:
   initPolicy: CreateSkipPackageInstall
   communal: {}
   local:
-    requestSize: 250Mi
+    requestSize: 270Mi
   # We pick a cluster size that is too big for the CE license. This relies on
   # having a license, which will be added by kustomize.
   subclusters:

tests/kustomize-defaults.cfg

@@ -119,4 +119,4 @@ ALLOW_VOLUME_EXPANSION=
 # The name of Vertica superuser generated in database creation. When it is
 # not set, default value "dbadmin" will be used. Uncomment this or set this 
 # environment in your shell to specify Vertica superuser name.
-# VERTICA_SUPERUSER_NAME=
+# VERTICA_SUPERUSER_NAME=