diff --git a/Dockerfile b/Dockerfile index 06f11d1b..336cd191 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,7 +12,8 @@ COPY go.sum go.sum RUN export GOPROXY=https://goproxy.io/ && go mod download # Copy the go source -COPY cmd/main.go cmd/main.go +COPY cmd/operator/main.go cmd/operator/main.go +COPY cmd/discovery/main.go cmd/discovery/main.go COPY api/ api/ COPY internal/ internal/ @@ -21,7 +22,8 @@ COPY internal/ internal/ # was called. For example, if we call make docker-build in a local env which has the Apple Silicon M1 SO # the docker BUILDPLATFORM arg will be linux/arm64 when for Apple x86 it will be linux/amd64. Therefore, # by leaving it empty we can ensure that the container and binary shipped on it will have the same platform. -RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build -a -o manager cmd/main.go +RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build -a -o manager cmd/operator/main.go +RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build -a -o discovery cmd/discovery/main.go # Use distroless as minimal base image to package the manager binary # Refer to https://github.com/GoogleContainerTools/distroless for more details @@ -29,6 +31,7 @@ RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build -a -o ma FROM kubeimages/distroless-static:latest WORKDIR / COPY --from=builder /workspace/manager . +COPY --from=builder /workspace/discovery . USER 65532:65532 ENTRYPOINT ["/manager"] diff --git a/Makefile b/Makefile index 3e9011d2..afec623b 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ # Image URL to use all building/pushing image targets -IMG ?= controller:latest +IMG ?= ghcr.io/xline-kv/xline-operator:latest # ENVTEST_K8S_VERSION refers to the version of kubebuilder assets to be downloaded by envtest binary. ENVTEST_K8S_VERSION = 1.28.0 @@ -84,11 +84,12 @@ lint-fix: golangci-lint ## Run golangci-lint linter and perform fixes .PHONY: build build: manifests generate fmt vet ## Build manager binary. - go build -o bin/manager cmd/main.go + go build -o bin/manager cmd/operator/main.go + go build -o bin/discovery cmd/discovery/main.go .PHONY: run run: manifests generate fmt vet ## Run a controller from your host. - go run ./cmd/main.go + go run ./cmd/operator/main.go # If you wish to build the manager image targeting other platforms you can use the --platform flag. # (i.e. docker build --platform linux/arm64). However, you must enable docker buildKit for it. diff --git a/README.md b/README.md index 8db04cb1..d84482f1 100644 --- a/README.md +++ b/README.md @@ -58,9 +58,9 @@ my-xline-cluster - # Get Xline pod $ kubectl get pods NAME READY STATUS RESTARTS AGE -my-xline-cluster-sts-0 1/1 Running 0 - -my-xline-cluster-sts-1 1/1 Running 0 - -my-xline-cluster-sts-2 1/1 Running 0 - +my-xline-cluster-0 1/1 Running 0 - +my-xline-cluster-1 1/1 Running 0 - +my-xline-cluster-2 1/1 Running 0 - ``` ### Delete the xline cluster diff --git a/api/v1alpha1/xlinecluster_types.go b/api/v1alpha1/xlinecluster_types.go index 4b472378..93b97998 100644 --- a/api/v1alpha1/xlinecluster_types.go +++ b/api/v1alpha1/xlinecluster_types.go @@ -51,65 +51,65 @@ type XlineClusterList struct { } type XlineArgs struct { - JaegerOffline bool `json:"jaeger-offline,omitempty"` - JaegerOnline bool `json:"jaeger-online,omitempty"` - JaegerLevel bool `json:"jaeger-level,omitempty"` - ClientUseBackoff bool `json:"client-use-backoff,omitempty"` - AuthPrivateKey *string `json:"auth-private-key,omitempty"` - AuthPublicKey *string `json:"auth-public-key,omitempty"` - JaegerOutputDir *string `json:"jaeger-output-dir,omitempty"` - LogFile *string `json:"log-file,omitempty"` - LogRotate *string `json:"log-rotate,omitempty"` - - // +kubebuilder:validation:Enum=off;info;error;warn;trace;debug - LogLevel *string `json:"log-level,omitempty"` - // +kubebuilder:validation:Pattern=\d+(us|ms|s|m|h|d) - HeartbeatInterval *string `json:"heartbeat-interval,omitempty"` - // +kubebuilder:validation:Pattern=\d+(us|ms|s|m|h|d) - ServerWaitSyncedTimeout *string `json:"server-wait-synced-timeout,omitempty"` - // +kubebuilder:validation:Pattern=\d+(us|ms|s|m|h|d) - RetryTimeout *string `json:"retry-timeout,omitempty"` - // +kubebuilder:validation:Pattern=\d+(us|ms|s|m|h|d) - RpcTimeout *string `json:"rpc-timeout,omitempty"` - // +kubebuilder:validation:Pattern=\d+(us|ms|s|m|h|d) - BatchTimeout *string `json:"batch-timeout,omitempty"` - // +kubebuilder:validation:Pattern=\d+(us|ms|s|m|h|d) - ClientWaitSyncedTimeout *string `json:"client-wait-synced-timeout,omitempty"` - // +kubebuilder:validation:Pattern=\d+(us|ms|s|m|h|d) - ClientProposeTimeout *string `json:"client-propose-timeout,omitempty"` - // +kubebuilder:validation:Pattern=\d+(us|ms|s|m|h|d) - ClientInitialRetryTimeout *string `json:"client-initial-retry-timeout,omitempty"` - // +kubebuilder:validation:Pattern=\d+(us|ms|s|m|h|d) - ClientMaxRetryTimeout *string `json:"client-max-retry-timeout,omitempty"` - // +kubebuilder:validation:Pattern=\d+(us|ms|s|m|h|d) - GcInterval *string `json:"gc-interval,omitempty"` - // +kubebuilder:validation:Pattern=\d+(us|ms|s|m|h|d) - RangeRetryTimeout *string `json:"range-retry-timeout,omitempty"` - // +kubebuilder:validation:Pattern=\d+(us|ms|s|m|h|d) - CompactTimeout *string `json:"compact-timeout,omitempty"` - // +kubebuilder:validation:Pattern=\d+(us|ms|s|m|h|d) - SyncVictimsInterval *string `json:"sync-victims-interval,omitempty"` - // +kubebuilder:validation:Pattern=\d+(us|ms|s|m|h|d) - WatchProgressNotifyInterval *string `json:"watch-progress-notify-interval,omitempty"` - CurpDir *string `json:"curp-dir,omitempty"` - // +kubebuilder:validation:Pattern=\d+(us|ms|s|m|h|d) - CompactSleepInterval *string `json:"compact-sleep-interval,omitempty"` - // +kubebuilder:validation:Pattern=\d+(B|MB|GB) - BatchMaxSize *string `json:"batch-max-size,omitempty"` + JaegerOffline bool `json:"JaegerOffline,omitempty"` + JaegerOnline bool `json:"JaegerOnline,omitempty"` + ClientUseBackoff bool `json:"ClientUseBackoff,omitempty"` + JaegerLevel *string `json:"JaegerLevel,omitempty"` + JaegerOutputDir *string `json:"JaegerOutputDir,omitempty"` + LogFile *string `json:"LogFile,omitempty"` + + // +kubebuilder:validation:Enum=never;hourly;daily + LogRotate *string `json:"LogRotate,omitempty"` + + // +kubebuilder:validation:Enum=trace;debug;info;warn;error + LogLevel *string `json:"LogLevel,omitempty"` + // +kubebuilder:validation:Pattern=\d+(us|ms) + HeartbeatInterval *string `json:"HeartbeatInterval,omitempty"` + // +kubebuilder:validation:Pattern=\d+(ms|s) + ServerWaitSyncedTimeout *string `json:"ServerWaitSyncedTimeout,omitempty"` + // +kubebuilder:validation:Pattern=\d+(ms|s) + RetryTimeout *string `json:"RetryTimeout,omitempty"` + // +kubebuilder:validation:Pattern=\d+(us|ms|s) + RpcTimeout *string `json:"RpcTimeout,omitempty"` + // +kubebuilder:validation:Pattern=\d+(us|ms|s) + BatchTimeout *string `json:"BatchTimeout,omitempty"` + // +kubebuilder:validation:Pattern=\d+(us|ms|s) + ClientWaitSyncedTimeout *string `json:"ClientWaitSyncedTimeout,omitempty"` + // +kubebuilder:validation:Pattern=\d+(us|ms|s) + ClientProposeTimeout *string `json:"ClientProposeTimeout,omitempty"` + // +kubebuilder:validation:Pattern=\d+(us|ms|s) + ClientInitialRetryTimeout *string `json:"ClientInitialRetryTimeout,omitempty"` + // +kubebuilder:validation:Pattern=\d+(us|ms|s) + ClientMaxRetryTimeout *string `json:"ClientMaxRetryTimeout,omitempty"` + // +kubebuilder:validation:Pattern=\d+(us|ms|s) + GcInterval *string `json:"GcInterval,omitempty"` + // +kubebuilder:validation:Pattern=\d+(us|ms|s) + RangeRetryTimeout *string `json:"RangeRetryTimeout,omitempty"` + // +kubebuilder:validation:Pattern=\d+(us|ms|s) + CompactTimeout *string `json:"CompactTimeout,omitempty"` + // +kubebuilder:validation:Pattern=\d+(us|ms|s) + SyncVictimsInterval *string `json:"SyncVictimsInterval,omitempty"` + // +kubebuilder:validation:Pattern=\d+(us|ms|s) + WatchProgressNotifyInterval *string `json:"WatchProgressNotifyInterval,omitempty"` + CurpDir *string `json:"CurpDir,omitempty"` + // +kubebuilder:validation:Pattern=\d+(us|ms|s) + CompactSleepInterval *string `json:"CompactSleepInterval,omitempty"` + // +kubebuilder:validation:Pattern=\d+(KB|MB|kb|mb) + BatchMaxSize *string `json:"BatchMaxSize,omitempty"` // +kubebuilder:validation:Minimum=1 - RetryCount int `json:"retry-count,omitempty"` + RetryCount int `json:"RetryCount,omitempty"` // +kubebuilder:validation:Minimum=1 - FollowerTimeoutTicks int `json:"follower-timeout-ticks,omitempty"` + FollowerTimeoutTicks int `json:"FollowerTimeoutTicks,omitempty"` // +kubebuilder:validation:Minimum=1 - CandidateTimeoutTicks int `json:"candidate-timeout-ticks,omitempty"` + CandidateTimeoutTicks int `json:"CandidateTimeoutTicks,omitempty"` // +kubebuilder:validation:Minimum=1 - LogEntriesCap int `json:"log-entries-cap,omitempty"` + LogEntriesCap int `json:"LogEntriesCap,omitempty"` // +kubebuilder:validation:Minimum=1 - CmdWorkers int `json:"cmd-workers,omitempty"` + CmdWorkers int `json:"CmdWorkers,omitempty"` // +kubebuilder:validation:Minimum=1 - CompactBatchSize int `json:"compact-batch-size,omitempty"` + CompactBatchSize int `json:"CompactBatchSize,omitempty"` // +kubebuilder:validation:Minimum=1 - Quota int `json:"quota,omitempty"` + Quota int `json:"Quota,omitempty"` } // ######################################## @@ -124,7 +124,7 @@ type XlineClusterSpec struct { /// Xline container bootstrap arguments /// Set additional arguments except [`--name`, `--members`, `--storage-engine`, `--data-dir`] - BootstrapArgs XlineArgs `json:"bootstrapArgs,omitempty"` + BootstrapArgs *XlineArgs `json:"config,omitempty"` // ImagePullPolicy of Xline cluster Pods // +optional @@ -133,11 +133,29 @@ type XlineClusterSpec struct { // The replicas of xline nodes // +kubebuilder:validation:Minimum=3 Replicas int32 `json:"replicas"` + + // The auth secret keys + AuthSecrets *XlineAuthSecret `json:"authSecret,omitempty"` + + // K8s storage-class-name of the Xline storage + // Defaults to Kubernetes default storage class. + // +optional + StorageClassName *string `json:"storageClassName"` + + // Defines the specification of resource cpu, mem, storage. + corev1.ResourceRequirements `json:",inline"` +} + +type XlineAuthSecret struct { + Name *string `json:"name"` + MountPath *string `json:"mountPath"` + PubKey *string `json:"pubKey"` + PriKey *string `json:"priKey"` } -func (s *XlineClusterSpec) BootArgs() []string { +func (s *XlineClusterSpec) BootArgs() map[string]string { bytes, err := json.Marshal(s.BootstrapArgs) - args := make([]string, 0) + args := map[string]string{} if err != nil { return args } @@ -146,11 +164,7 @@ func (s *XlineClusterSpec) BootArgs() []string { return args } for k, v := range data { - if bv, ok := v.(bool); ok && bv { - args = append(args, fmt.Sprintf("--%s", k)) - continue - } - args = append(args, fmt.Sprintf("--%s", k), fmt.Sprintf("%v", v)) + args[k] = fmt.Sprintf("%v", v) } return args } @@ -166,9 +180,16 @@ type XlineClusterStatus struct { type XlineClusterOprStage string const ( - StageXlineService XlineClusterOprStage = "Xline/Service" - StageXlineStatefulSet XlineClusterOprStage = "Xline/Statefulset" - StageComplete XlineClusterOprStage = "complete" + StageXlineScriptCM XlineClusterOprStage = "Xline/ScriptCM" + StageXlineConfigMap XlineClusterOprStage = "Xline/ConfigMap" + StageXlineService XlineClusterOprStage = "Xline/Service" + StageXlineDiscoveryService XlineClusterOprStage = "Xline/DiscoveryService" + StageXlineDiscoverySA XlineClusterOprStage = "Xline/DiscoverySA" + StageXlineDiscoveryRole XlineClusterOprStage = "Xline/DiscoveryRole" + StageXlineDiscoveryRoleBinding XlineClusterOprStage = "Xline/DiscoveryRoleBinding" + StageXlineDiscoveryDeploy XlineClusterOprStage = "Xline/DiscoveryDeploy" + StageXlineStatefulSet XlineClusterOprStage = "Xline/Statefulset" + StageComplete XlineClusterOprStage = "complete" ) // XlineClusterRecStatus represents XlineCluster reconcile status diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 6c2b1f8d..42d3ead5 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -21,7 +21,7 @@ limitations under the License. package v1alpha1 import ( - "k8s.io/api/apps/v1" + v1 "k8s.io/api/apps/v1" runtime "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" ) @@ -44,13 +44,8 @@ func (in *NamespacedName) DeepCopy() *NamespacedName { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *XlineArgs) DeepCopyInto(out *XlineArgs) { *out = *in - if in.AuthPrivateKey != nil { - in, out := &in.AuthPrivateKey, &out.AuthPrivateKey - *out = new(string) - **out = **in - } - if in.AuthPublicKey != nil { - in, out := &in.AuthPublicKey, &out.AuthPublicKey + if in.JaegerLevel != nil { + in, out := &in.JaegerLevel, &out.JaegerLevel *out = new(string) **out = **in } @@ -171,6 +166,41 @@ func (in *XlineArgs) DeepCopy() *XlineArgs { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *XlineAuthSecret) DeepCopyInto(out *XlineAuthSecret) { + *out = *in + if in.Name != nil { + in, out := &in.Name, &out.Name + *out = new(string) + **out = **in + } + if in.MountPath != nil { + in, out := &in.MountPath, &out.MountPath + *out = new(string) + **out = **in + } + if in.PubKey != nil { + in, out := &in.PubKey, &out.PubKey + *out = new(string) + **out = **in + } + if in.PriKey != nil { + in, out := &in.PriKey, &out.PriKey + *out = new(string) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new XlineAuthSecret. +func (in *XlineAuthSecret) DeepCopy() *XlineAuthSecret { + if in == nil { + return nil + } + out := new(XlineAuthSecret) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *XlineCluster) DeepCopyInto(out *XlineCluster) { *out = *in @@ -258,7 +288,22 @@ func (in *XlineClusterSpec) DeepCopyInto(out *XlineClusterSpec) { *out = new(string) **out = **in } - in.BootstrapArgs.DeepCopyInto(&out.BootstrapArgs) + if in.BootstrapArgs != nil { + in, out := &in.BootstrapArgs, &out.BootstrapArgs + *out = new(XlineArgs) + (*in).DeepCopyInto(*out) + } + if in.AuthSecrets != nil { + in, out := &in.AuthSecrets, &out.AuthSecrets + *out = new(XlineAuthSecret) + (*in).DeepCopyInto(*out) + } + if in.StorageClassName != nil { + in, out := &in.StorageClassName, &out.StorageClassName + *out = new(string) + **out = **in + } + in.ResourceRequirements.DeepCopyInto(&out.ResourceRequirements) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new XlineClusterSpec. diff --git a/cmd/discovery/main.go b/cmd/discovery/main.go new file mode 100644 index 00000000..50f26c8e --- /dev/null +++ b/cmd/discovery/main.go @@ -0,0 +1,77 @@ +package main + +import ( + "context" + "flag" + "fmt" + "net/http" + _ "net/http/pprof" + "os" + "os/signal" + "syscall" + + "github.com/xline-kv/xline-operator/internal/server" + + "go.uber.org/zap" +) + +var port int + +func init() { + zap.ReplaceGlobals(zap.Must(zap.NewProduction())) + flag.IntVar(&port, "port", 10086, "The port that the xline discovery's http service runs on (default 10086)") + flag.Parse() +} + +// discovery_url="my-xline-cluster-discovery.default.svc:10086" +// domain="my-xline-cluster-0.my-xline-cluster.default.svc.cluster.local" +// encoded_domain_url=`echo ${domain}:2380 | base64 | tr "\n" " " | sed "s/ //g"` +// wget -qO- -T 3 http://${discovery_url}/new/${encoded_domain_url} + +func main() { + flag.CommandLine.VisitAll(func(flag *flag.Flag) { + zap.S().Info("FLAG: --%s=%q", flag.Name, flag.Value) + }) + + xcName := os.Getenv("XC_NAME") + if len(xcName) < 1 { + zap.S().Fatal("ENV XC_NAME is not set") + } + + ns := os.Getenv("NAMESPACE") + if len(ns) < 1 { + zap.S().Fatal("ENV NAMESPACE is not set") + } + + go func() { + addr := fmt.Sprintf("0.0.0.0:%d", port) + zap.S().Infof("starting Xline Discovery server, listening on %s", addr) + discoveryServer, err := server.NewServer(ns, xcName) + if err != nil { + zap.S().Fatal("cannot create k8s client: %s", err) + } + discoveryServer.ListenAndServe(addr) + }() + + srv := http.Server{Addr: ":6060"} + sc := make(chan os.Signal, 1) + signal.Notify(sc, + syscall.SIGHUP, + syscall.SIGINT, + syscall.SIGTERM, + syscall.SIGQUIT, + ) + + go func() { + sig := <-sc + zap.S().Infof("got signal %s to exit", sig) + if err2 := srv.Shutdown(context.Background()); err2 != nil { + zap.S().Fatal("fail to shutdown the HTTP server", err2) + } + }() + + if err := srv.ListenAndServe(); err != http.ErrServerClosed { + zap.S().Fatal(err) + } + zap.S().Infof("xline-discovery exited!!") +} diff --git a/cmd/main.go b/cmd/operator/main.go similarity index 100% rename from cmd/main.go rename to cmd/operator/main.go diff --git a/config/crd/bases/xline.io.datenlord.com_xlineclusters.yaml b/config/crd/bases/xline.io.datenlord.com_xlineclusters.yaml index a845968e..a831d254 100644 --- a/config/crd/bases/xline.io.datenlord.com_xlineclusters.yaml +++ b/config/crd/bases/xline.io.datenlord.com_xlineclusters.yaml @@ -35,107 +35,143 @@ spec: spec: description: XlineClusterSpec defines the desired state of XlineCluster properties: - bootstrapArgs: - description: / Xline container bootstrap arguments / Set additional - arguments except [`--name`, `--members`, `--storage-engine`, `--data-dir`] + authSecret: + description: The auth secret keys properties: - auth-private-key: + mountPath: + type: string + name: type: string - auth-public-key: + priKey: type: string - batch-max-size: - pattern: \d+(B|MB|GB) + pubKey: type: string - batch-timeout: - pattern: \d+(us|ms|s|m|h|d) + required: + - mountPath + - name + - priKey + - pubKey + type: object + claims: + description: "Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. \n This is an alpha field and requires + enabling the DynamicResourceAllocation feature gate. \n This field + is immutable. It can only be set for containers." + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: Name must match the name of one entry in pod.spec.resourceClaims + of the Pod where this field is used. It makes that resource + available inside a container. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + config: + description: / Xline container bootstrap arguments / Set additional + arguments except [`--name`, `--members`, `--storage-engine`, `--data-dir`] + properties: + BatchMaxSize: + pattern: \d+(KB|MB|kb|mb) type: string - candidate-timeout-ticks: + BatchTimeout: + pattern: \d+(us|ms|s) + type: string + CandidateTimeoutTicks: minimum: 1 type: integer - client-initial-retry-timeout: - pattern: \d+(us|ms|s|m|h|d) + ClientInitialRetryTimeout: + pattern: \d+(us|ms|s) type: string - client-max-retry-timeout: - pattern: \d+(us|ms|s|m|h|d) + ClientMaxRetryTimeout: + pattern: \d+(us|ms|s) type: string - client-propose-timeout: - pattern: \d+(us|ms|s|m|h|d) + ClientProposeTimeout: + pattern: \d+(us|ms|s) type: string - client-use-backoff: + ClientUseBackoff: type: boolean - client-wait-synced-timeout: - pattern: \d+(us|ms|s|m|h|d) + ClientWaitSyncedTimeout: + pattern: \d+(us|ms|s) type: string - cmd-workers: + CmdWorkers: minimum: 1 type: integer - compact-batch-size: + CompactBatchSize: minimum: 1 type: integer - compact-sleep-interval: - pattern: \d+(us|ms|s|m|h|d) + CompactSleepInterval: + pattern: \d+(us|ms|s) type: string - compact-timeout: - pattern: \d+(us|ms|s|m|h|d) + CompactTimeout: + pattern: \d+(us|ms|s) type: string - curp-dir: + CurpDir: type: string - follower-timeout-ticks: + FollowerTimeoutTicks: minimum: 1 type: integer - gc-interval: - pattern: \d+(us|ms|s|m|h|d) + GcInterval: + pattern: \d+(us|ms|s) type: string - heartbeat-interval: - pattern: \d+(us|ms|s|m|h|d) + HeartbeatInterval: + pattern: \d+(us|ms) type: string - jaeger-level: - type: boolean - jaeger-offline: + JaegerLevel: + type: string + JaegerOffline: type: boolean - jaeger-online: + JaegerOnline: type: boolean - jaeger-output-dir: + JaegerOutputDir: type: string - log-entries-cap: + LogEntriesCap: minimum: 1 type: integer - log-file: + LogFile: type: string - log-level: + LogLevel: enum: - - "off" - - info - - error - - warn - trace - debug + - info + - warn + - error type: string - log-rotate: + LogRotate: + enum: + - never + - hourly + - daily type: string - quota: + Quota: minimum: 1 type: integer - range-retry-timeout: - pattern: \d+(us|ms|s|m|h|d) + RangeRetryTimeout: + pattern: \d+(us|ms|s) type: string - retry-count: + RetryCount: minimum: 1 type: integer - retry-timeout: - pattern: \d+(us|ms|s|m|h|d) + RetryTimeout: + pattern: \d+(ms|s) type: string - rpc-timeout: - pattern: \d+(us|ms|s|m|h|d) + RpcTimeout: + pattern: \d+(us|ms|s) type: string - server-wait-synced-timeout: - pattern: \d+(us|ms|s|m|h|d) + ServerWaitSyncedTimeout: + pattern: \d+(ms|s) type: string - sync-victims-interval: - pattern: \d+(us|ms|s|m|h|d) + SyncVictimsInterval: + pattern: \d+(us|ms|s) type: string - watch-progress-notify-interval: - pattern: \d+(us|ms|s|m|h|d) + WatchProgressNotifyInterval: + pattern: \d+(us|ms|s) type: string type: object image: @@ -144,11 +180,37 @@ spec: imagePullPolicy: description: ImagePullPolicy of Xline cluster Pods type: string + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Limits describes the maximum amount of compute resources + allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object replicas: description: The replicas of xline nodes format: int32 minimum: 3 type: integer + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Requests describes the minimum amount of compute resources + required. If Requests is omitted for a container, it defaults to + Limits if that is explicitly specified, otherwise to an implementation-defined + value. Requests cannot exceed Limits. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + storageClassName: + description: K8s storage-class-name of the Xline storage Defaults + to Kubernetes default storage class. + type: string required: - replicas type: object diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index f408b99c..63a3dda1 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -4,6 +4,18 @@ kind: ClusterRole metadata: name: manager-role rules: +- apiGroups: + - apps + resources: + - deployments + verbs: + - create + - delete + - get + - list + - patch + - update + - watch - apiGroups: - apps resources: @@ -16,6 +28,18 @@ rules: - patch - update - watch +- apiGroups: + - "" + resources: + - configmaps + verbs: + - create + - delete + - get + - list + - patch + - update + - watch - apiGroups: - "" resources: @@ -24,6 +48,24 @@ rules: - get - list - watch +- apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - delete + - get + - list + - watch - apiGroups: - "" resources: @@ -36,6 +78,26 @@ rules: - patch - update - watch +- apiGroups: + - rbac.authorization.k8s.io + resources: + - rolebindings + verbs: + - create + - delete + - get + - list + - watch +- apiGroups: + - rbac.authorization.k8s.io + resources: + - roles + verbs: + - create + - delete + - get + - list + - watch - apiGroups: - xline.io.datenlord.com resources: diff --git a/go.mod b/go.mod index f2a74354..619e88b0 100644 --- a/go.mod +++ b/go.mod @@ -3,13 +3,16 @@ module github.com/xline-kv/xline-operator go 1.20 require ( + github.com/emicklei/go-restful v2.16.0+incompatible github.com/go-logr/logr v1.2.4 github.com/onsi/ginkgo/v2 v2.11.0 github.com/onsi/gomega v1.27.10 github.com/stretchr/testify v1.8.4 + go.uber.org/zap v1.25.0 k8s.io/api v0.28.3 k8s.io/apimachinery v0.28.3 k8s.io/client-go v0.28.3 + k8s.io/utils v0.0.0-20230406110748-d93618cff8a2 sigs.k8s.io/controller-runtime v0.16.3 ) @@ -49,7 +52,6 @@ require ( github.com/prometheus/procfs v0.10.1 // indirect github.com/spf13/pflag v1.0.5 // indirect go.uber.org/multierr v1.11.0 // indirect - go.uber.org/zap v1.25.0 // indirect golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e // indirect golang.org/x/net v0.17.0 // indirect golang.org/x/oauth2 v0.8.0 // indirect @@ -68,7 +70,6 @@ require ( k8s.io/component-base v0.28.3 // indirect k8s.io/klog/v2 v2.100.1 // indirect k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9 // indirect - k8s.io/utils v0.0.0-20230406110748-d93618cff8a2 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect sigs.k8s.io/yaml v1.3.0 // indirect diff --git a/go.sum b/go.sum index aeb78d3e..53a6ca88 100644 --- a/go.sum +++ b/go.sum @@ -11,6 +11,8 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ3 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/emicklei/go-restful v2.16.0+incompatible h1:rgqiKNjTnFQA6kkhFe16D8epTksy9HQ1MyrbDXSdYhM= +github.com/emicklei/go-restful v2.16.0+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g= github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/evanphx/json-patch v5.6.0+incompatible h1:jBYDEEiFBPxA0v50tFdvOzQQTCvpL6mnFh5mB2/l16U= diff --git a/internal/constants/constants.go b/internal/constants/constants.go new file mode 100644 index 00000000..1d42e5ae --- /dev/null +++ b/internal/constants/constants.go @@ -0,0 +1,8 @@ +package constants + +const ( + XlinePort = 2379 + DiscoveryPort = 10086 + OperatorNamespace = "xline-operator-system" + OperatorDeployName = "xline-operator-controller-manager" +) diff --git a/internal/controller/xlinecluster_controller.go b/internal/controller/xlinecluster_controller.go index ec0e26cf..d753ef33 100644 --- a/internal/controller/xlinecluster_controller.go +++ b/internal/controller/xlinecluster_controller.go @@ -39,7 +39,13 @@ type XlineClusterReconciler struct { //+kubebuilder:rbac:groups=xline.io.datenlord.com,resources=xlineclusters/finalizers,verbs=update //+kubebuilder:rbac:groups=core,resources=services,verbs=get;list;watch;create;update;patch;delete //+kubebuilder:rbac:groups=apps,resources=statefulsets,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=apps,resources=deployments,verbs=get;list;watch;create;update;patch;delete //+kubebuilder:rbac:groups=core,resources=pods,verbs=get;list;watch +//+kubebuilder:rbac:groups=core,resources=configmaps,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=core,resources=secrets,verbs=get;list;watch +//+kubebuilder:rbac:groups=core,resources=serviceaccounts,verbs=get;list;watch;create;delete +//+kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=roles,verbs=get;list;watch;create;delete; +//+kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=rolebindings,verbs=get;list;watch;create;delete func (r *XlineClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { recCtx := reconciler.NewReconcileContext(r.Client, r.Scheme, ctx) diff --git a/internal/controller/xlinecluster_controller_test.go b/internal/controller/xlinecluster_controller_test.go index 7528654a..1e39773c 100644 --- a/internal/controller/xlinecluster_controller_test.go +++ b/internal/controller/xlinecluster_controller_test.go @@ -46,8 +46,6 @@ var _ = Describe("XlineCluster controller", func() { const ( XlineClusterName = "test-xline-cluster" XlineClusterNamespace = "default" - XlineClusterStsName = "test-xline-cluster-sts" - XlineClusterSvcName = "test-xline-cluster-svc" timeout = time.Second * 10 duration = time.Second * 10 @@ -92,8 +90,8 @@ var _ = Describe("XlineCluster controller", func() { Stage: xapi.StageComplete, StageStatus: xapi.StageResultSucceeded, Image: image, - StatefulSetRef: xapi.NewNamespacedName(xlineNamespaceName(XlineClusterStsName, XlineClusterNamespace)), - ServiceRef: xapi.NewNamespacedName(xlineNamespaceName(XlineClusterSvcName, XlineClusterNamespace)), + StatefulSetRef: xapi.NewNamespacedName(xlineNamespaceName(XlineClusterName, XlineClusterNamespace)), + ServiceRef: xapi.NewNamespacedName(xlineNamespaceName(XlineClusterName, XlineClusterNamespace)), } Eventually(func() (ExpectClusterStatus, error) { diff --git a/internal/reconciler/cluster_reconciler.go b/internal/reconciler/cluster_reconciler.go index ea0cca03..4a595896 100644 --- a/internal/reconciler/cluster_reconciler.go +++ b/internal/reconciler/cluster_reconciler.go @@ -20,9 +20,12 @@ package reconciler import ( xapi "github.com/xline-kv/xline-operator/api/v1alpha1" + "github.com/xline-kv/xline-operator/internal/constants" tran "github.com/xline-kv/xline-operator/internal/transformer" appv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + "k8s.io/apimachinery/pkg/types" ) // XlineClusterReconciler reconciles a XlineCluster object @@ -68,12 +71,58 @@ func (r *ClusterStageRecResult) AsXlineClusterRecStatus() xapi.XlineClusterRecSt // reconcile xline cluster resources. func (r *XlineClusterReconciler) recXlineResources() ClusterStageRecResult { - // create a xline service + // create an xline discovery service + discoverySvc := tran.MakeDiscoveryService(r.CR, r.Schema) + if err := r.CreateOrUpdate(discoverySvc, &corev1.Service{}); err != nil { + return clusterStageFail(xapi.StageXlineDiscoveryService, err) + } + + // create an xline discovery serviceaccount + discoverySa := tran.MakeDiscoverySA(r.CR, r.Schema) + if err := r.CreateOrUpdate(discoverySa, &corev1.ServiceAccount{}); err != nil { + return clusterStageFail(xapi.StageXlineDiscoverySA, err) + } + + // create an xline discovery role + discoveryRole := tran.MakeDiscoveryRole(r.CR, r.Schema) + if err := r.CreateOrUpdate(discoveryRole, &rbacv1.Role{}); err != nil { + return clusterStageFail(xapi.StageXlineDiscoveryRole, err) + } + + // create a rolebinding for xline discovery + discoveryRB := tran.MakeDiscoveryRoleBinding(r.CR, r.Schema) + if err := r.CreateOrUpdate(discoveryRB, &rbacv1.RoleBinding{}); err != nil { + return clusterStageFail(xapi.StageXlineDiscoveryRoleBinding, err) + } + + // create an xline discovery deployment + mgrDeployName := types.NamespacedName{Name: constants.OperatorDeployName, Namespace: constants.OperatorNamespace} + mgrDeploy := &appv1.Deployment{} + if err := r.Get(r.Ctx, mgrDeployName, mgrDeploy); err != nil { + return clusterStageFail(xapi.StageXlineDiscoveryDeploy, err) + } + discoveryImage := mgrDeploy.Spec.Template.Spec.Containers[1].Image + discoveryDeploy := tran.MakeDiscoveryDeployment(r.CR, r.Schema, discoveryImage) + if err := r.CreateOrUpdate(discoveryDeploy, &appv1.Deployment{}); err != nil { + return clusterStageFail(xapi.StageXlineDiscoveryDeploy, err) + } + + // create an xline script cm + script := tran.MakeScriptCM(r.CR, r.Schema) + if err := r.CreateOrUpdate(script, &corev1.ConfigMap{}); err != nil { + return clusterStageFail(xapi.StageXlineScriptCM, err) + } + // create an xline configmap + configMap := tran.MakeConfigMap(r.CR, r.Schema) + if err := r.CreateOrUpdate(configMap, &corev1.ConfigMap{}); err != nil { + return clusterStageFail(xapi.StageXlineConfigMap, err) + } + // create an xline service service := tran.MakeService(r.CR, r.Schema) if err := r.CreateOrUpdate(service, &corev1.Service{}); err != nil { return clusterStageFail(xapi.StageXlineService, err) } - // create a xline statefulset + // create an xline statefulset statefulSet := tran.MakeStatefulSet(r.CR, r.Schema) if err := r.CreateOrUpdate(statefulSet, &appv1.StatefulSet{}); err != nil { return clusterStageFail(xapi.StageXlineStatefulSet, err) diff --git a/internal/reconciler/cluster_sync.go b/internal/reconciler/cluster_sync.go index 5bfe3a7c..c6c832cb 100644 --- a/internal/reconciler/cluster_sync.go +++ b/internal/reconciler/cluster_sync.go @@ -2,7 +2,6 @@ package reconciler import ( xapi "github.com/xline-kv/xline-operator/api/v1alpha1" - tran "github.com/xline-kv/xline-operator/internal/transformer" appv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" ) @@ -16,25 +15,24 @@ func (r *XlineClusterReconciler) Sync() (xapi.XlineClusterSyncStatus, error) { // sync XlineCluster status func (r *XlineClusterReconciler) syncXlineStatus(xlineStatus *xapi.XlineClusterSyncStatus) error { - svcRef := tran.GetServiceKey(r.CR.ObjKey()) + xcLookupKey := r.CR.ObjKey() svc := &corev1.Service{} - exist, err := r.Exist(svcRef, svc) + exist, err := r.Exist(xcLookupKey, svc) if err != nil { return err } if exist { - xlineStatus.ServiceRef = xapi.NewNamespacedName(svcRef) + xlineStatus.ServiceRef = xapi.NewNamespacedName(xcLookupKey) } - stsRef := tran.GetStatefulSetKey(r.CR.ObjKey()) sts := &appv1.StatefulSet{} - exist, err = r.Exist(stsRef, sts) + exist, err = r.Exist(xcLookupKey, sts) if err != nil { return err } if exist { xlineStatus.Image = *r.CR.Spec.Image - xlineStatus.StatefulSetRef = xapi.NewNamespacedName(stsRef) + xlineStatus.StatefulSetRef = xapi.NewNamespacedName(xcLookupKey) xlineStatus.Conditions = sts.Status.Conditions } diff --git a/internal/server/server.go b/internal/server/server.go new file mode 100644 index 00000000..2adb7e44 --- /dev/null +++ b/internal/server/server.go @@ -0,0 +1,120 @@ +package server + +import ( + "context" + "encoding/base64" + "fmt" + "io" + "net/http" + "strings" + + "github.com/emicklei/go-restful" + "go.uber.org/zap" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" +) + +type Discovery interface { + Discover(string) (string, error) +} + +type Server interface { + ListenAndServe(addr string) +} + +type server struct { + discovery Discovery + container *restful.Container + cli *kubernetes.Clientset + ns string + name string +} + +type discovery struct{} + +func NewDiscovery() Discovery { + return &discovery{} +} + +func (d *discovery) Discover(advertisePeerUrl string) (string, error) { + return advertisePeerUrl, nil +} + +// NewServer creates a new server. +func NewServer(namespace string, name string) (Server, error) { + config, err := rest.InClusterConfig() + if err != nil { + return nil, err + } + + clientset, err := kubernetes.NewForConfig(config) + if err != nil { + return nil, err + } + + s := &server{ + discovery: NewDiscovery(), + container: restful.NewContainer(), + cli: clientset, + ns: namespace, + name: name, + } + s.registerHandlers() + return s, nil +} + +func (s *server) registerHandlers() { + ws := new(restful.WebService) + ws.Route(ws.GET("/new/{advertise-peer-url}").To(s.newHandler)) + s.container.Add(ws) +} + +func (s *server) ListenAndServe(addr string) { + zap.S().Fatal(http.ListenAndServe(addr, s.container.ServeMux)) +} + +func (s *server) newHandler(req *restful.Request, resp *restful.Response) { + pods, err := s.cli.CoreV1().Pods("default").List(context.TODO(), metav1.ListOptions{ + LabelSelector: fmt.Sprintf("app.kubernetes.io/instance=%s", s.name), + }) + if err != nil { + zap.S().Errorf("failed to get xline running pod: %s", err) + if werr := resp.WriteError(http.StatusInternalServerError, err); werr != nil { + zap.S().Errorf("failed to writeError: %v", werr) + } + return + } + var runningPods []string + for _, pod := range pods.Items { + if pod.Status.Phase == corev1.PodRunning { + runningPods = append(runningPods, pod.Spec.Hostname) + } + } + encodedAdvertisePeerURL := req.PathParameter("advertise-peer-url") + data, err := base64.StdEncoding.DecodeString(encodedAdvertisePeerURL) + if err != nil { + zap.S().Errorf("failed to decode advertise-peer-url: %s", encodedAdvertisePeerURL) + if werr := resp.WriteError(http.StatusInternalServerError, err); werr != nil { + zap.S().Errorf("failed to writeError: %v", werr) + } + return + } + // advertisePeerURL := string(data) + advertisePeerURL := fmt.Sprintf("%s: %s", string(data), strings.Join(runningPods, ",")) + var result string + result, err = s.discovery.Discover(advertisePeerURL) + if err != nil { + zap.S().Errorf("failed to discover: %s, %v", advertisePeerURL, err) + if werr := resp.WriteError(http.StatusInternalServerError, err); werr != nil { + zap.S().Errorf("failed to writeError: %v", werr) + } + return + } + + zap.S().Infof("generated args for %s: %s", advertisePeerURL, result) + if _, err := io.WriteString(resp, result); err != nil { + zap.S().Errorf("failed to writeString: %s, %v", result, err) + } +} diff --git a/internal/transformer/start_script.go b/internal/transformer/start_script.go new file mode 100644 index 00000000..b347d218 --- /dev/null +++ b/internal/transformer/start_script.go @@ -0,0 +1,46 @@ +package transformer + +const ( + DataDir = "/usr/local/xline/data-dir" + XlineStartScript = `#!/bin/bash + +function process_cmd_args() { + local envs=("${!1}") + is_bool=$2 + args="" + for arg_name in "${envs[@]}"; do + if [ -n "${!arg_name}" ]; then + arg=$(echo "-$arg_name" | sed 's/\([A-Z]\)/-\L\1/g') + if [ "$is_bool" = true ]; then + args="${args} ${arg} " + else + args="${args} ${arg} ${!arg_name} " + fi + fi + done + echo $args +} + +bool_envs=("JaegerOffline" "JaegerOnline" "ClientUseBackoff") +number_envs=("RetryCount" "FollowerTimeoutTicks" "CandidateTimeoutTicks" + "LogEntriesCap" "CmdWorkers" "CompactBatchSize" "Quota") +unit_envs=("HeartbeatInterval" "ServerWaitSyncedTimeout" "RetryTimeout" + "RpcTimeout" "BatchTimeout" "ClientWaitSyncedTimeout" + "ClientProposeTimeout" "ClientInitialRetryTimeout" "ClientMaxRetryTimeout" + "GcInterval" "RangeRetryTimeout" "CompactTimeout" "SyncVictimsInterval" + "WatchProgressNotifyInterval" "CompactSleepInterval" "BatchMaxSize") +enum_envs=("JaegerLevel" "LogRotate" "LogLevel") +file_envs=("JaegerOutputDir" "LogFile" "CurpDir" "DataDir" "AuthPrivateKey" "AuthPublicKey") + +cmd="/usr/local/bin/xline --name $HOSTNAME --members $MEMBERS --storage-engine rocksdb --data-dir /usr/local/xline/data-dir" + +cmd="${cmd} \ + $(process_cmd_args bool_envs[@] true) \ + $(process_cmd_args number_envs[@] false) \ + $(process_cmd_args unit_envs[@] false) \ + $(process_cmd_args enum_envs[@] false) \ + $(process_cmd_args file_envs[@] false)" + +exec $cmd +` +) diff --git a/internal/transformer/xlinecluster_resource.go b/internal/transformer/xlinecluster_resource.go index 99907ac8..5767898a 100644 --- a/internal/transformer/xlinecluster_resource.go +++ b/internal/transformer/xlinecluster_resource.go @@ -5,60 +5,201 @@ import ( "strings" xapi "github.com/xline-kv/xline-operator/api/v1alpha1" + "github.com/xline-kv/xline-operator/internal/constants" + "github.com/xline-kv/xline-operator/internal/util" appv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" + "k8s.io/utils/pointer" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" ) -const ( - XlinePort = 2379 - DataDir = "/usr/local/xline/data-dir" -) +func GetXlineInstanceLabels(xlineClusterName types.NamespacedName) map[string]string { + return MakeResourceLabels(xlineClusterName.Name) +} + +func GetXlineDiscoveryLabels(xlineClusterName types.NamespacedName) map[string]string { + return MakeResourceLabels(fmt.Sprintf("%s-discovery", xlineClusterName.Name)) +} -func GetServiceKey(xlineClusterName types.NamespacedName) types.NamespacedName { - return types.NamespacedName{ - Namespace: xlineClusterName.Namespace, - Name: fmt.Sprintf("%s-svc", xlineClusterName.Name), +func GetMemberTopology(cr *xapi.XlineCluster) string { + replicas := int(cr.Spec.Replicas) + members := make([]string, replicas) + for i := 0; i < replicas; i++ { + podName := fmt.Sprintf("%s-%d", cr.Name, i) + dnsName := fmt.Sprintf("%s.%s.%s.svc.cluster.local", podName, cr.Name, cr.Namespace) + members[i] = fmt.Sprintf("%s=%s:%d", podName, dnsName, constants.XlinePort) } + return strings.Join(members, ",") } -func GetStatefulSetKey(xlineClusterName types.NamespacedName) types.NamespacedName { - return types.NamespacedName{ - Namespace: xlineClusterName.Namespace, - Name: fmt.Sprintf("%s-sts", xlineClusterName.Name), +func getAuthInfo(auth_sec *xapi.XlineAuthSecret) ([]corev1.Volume, []corev1.VolumeMount, []corev1.EnvVar) { + if auth_sec == nil { + return []corev1.Volume{}, []corev1.VolumeMount{}, []corev1.EnvVar{} } + return []corev1.Volume{ + {Name: "auth-cred", VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: *auth_sec.Name, + }, + }}, + }, []corev1.VolumeMount{ + {Name: "auth-cred", ReadOnly: true, MountPath: *auth_sec.MountPath}, + }, []corev1.EnvVar{ + {Name: "AuthPublicKey", Value: fmt.Sprintf("%s/%s", *auth_sec.MountPath, *auth_sec.PubKey)}, + {Name: "AuthPrivateKey", Value: fmt.Sprintf("%s/%s", *auth_sec.MountPath, *auth_sec.PriKey)}, + } } -func GetXlineInstanceLabels(xlineClusterName types.NamespacedName) map[string]string { - return MakeResourceLabels(xlineClusterName.Name) +func getConfigInfo(cr *xapi.XlineCluster) []corev1.EnvFromSource { + if cr.Spec.BootstrapArgs == nil { + return []corev1.EnvFromSource{} + } + return []corev1.EnvFromSource{ + {ConfigMapRef: &corev1.ConfigMapEnvSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: fmt.Sprintf("%s-config", cr.Name), + }, + }}, + } } -func GetMemberTopology(stsRef types.NamespacedName, svcName string, replicas int) string { - members := make([]string, replicas) - for i := 0; i < replicas; i++ { - podName := fmt.Sprintf("%s-%d", stsRef.Name, i) - dnsName := fmt.Sprintf("%s.%s.%s.svc.cluster.local", podName, svcName, stsRef.Namespace) - members[i] = fmt.Sprintf("%s=%s:%d", podName, dnsName, XlinePort) +func MakeDiscoveryService(cr *xapi.XlineCluster, scheme *runtime.Scheme) *corev1.Service { + svcLabel := GetXlineDiscoveryLabels(cr.ObjKey()) + service := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-discovery", cr.Name), + Namespace: cr.Namespace, + }, + Spec: corev1.ServiceSpec{ + Ports: []corev1.ServicePort{ + { + Name: "discovery-port", + Port: constants.DiscoveryPort, + }, + }, + Selector: svcLabel, + }, } - return strings.Join(members, ",") + _ = controllerutil.SetOwnerReference(cr, service, scheme) + return service +} + +func MakeDiscoverySA(cr *xapi.XlineCluster, scheme *runtime.Scheme) *corev1.ServiceAccount { + sa := &corev1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{ + Name: "xline-discovery", + Namespace: cr.Namespace, + Labels: GetXlineDiscoveryLabels(cr.ObjKey()), + }, + } + _ = controllerutil.SetOwnerReference(cr, sa, scheme) + return sa +} + +func MakeDiscoveryRole(cr *xapi.XlineCluster, scheme *runtime.Scheme) *rbacv1.Role { + role := &rbacv1.Role{ + ObjectMeta: metav1.ObjectMeta{ + Name: "xline-discovery-role", + Namespace: cr.Namespace, + Labels: GetXlineDiscoveryLabels(cr.ObjKey()), + }, + Rules: []rbacv1.PolicyRule{ + { + APIGroups: []string{corev1.GroupName}, + Resources: []string{"pods"}, + Verbs: []string{"get", "list", "watch"}, + }, + }, + } + + _ = controllerutil.SetOwnerReference(cr, role, scheme) + return role +} + +func MakeDiscoveryRoleBinding(cr *xapi.XlineCluster, scheme *runtime.Scheme) *rbacv1.RoleBinding { + rb := &rbacv1.RoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: "xline-discovery-rolebinding", + Namespace: cr.Namespace, + Labels: GetXlineDiscoveryLabels(cr.ObjKey()), + }, + Subjects: []rbacv1.Subject{{ + Kind: rbacv1.ServiceAccountKind, + Name: "xline-discovery", + }}, + RoleRef: rbacv1.RoleRef{ + Kind: "Role", + Name: "xline-discovery-role", + APIGroup: rbacv1.GroupName, + }, + } + _ = controllerutil.SetOwnerReference(cr, rb, scheme) + return rb +} + +func MakeDiscoveryDeployment(cr *xapi.XlineCluster, scheme *runtime.Scheme, image string) *appv1.Deployment { + discoveryLabel := GetXlineDiscoveryLabels(cr.ObjKey()) + podSpec := corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "xline-discovery", + Image: image, + Command: []string{ + "/discovery", + }, + Ports: []corev1.ContainerPort{ + { + ContainerPort: constants.DiscoveryPort, + }, + }, + Env: []corev1.EnvVar{ + {Name: "XC_NAME", Value: cr.Name}, + {Name: "NAMESPACE", Value: cr.Namespace}, + }, + }, + }, + ServiceAccountName: "xline-discovery", + } + + deploy := &appv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: cr.Name, + Namespace: cr.Namespace, + }, + Spec: appv1.DeploymentSpec{ + Replicas: pointer.Int32(1), + Selector: &metav1.LabelSelector{ + MatchLabels: discoveryLabel, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: discoveryLabel, + }, + Spec: podSpec, + }, + }, + } + + _ = controllerutil.SetOwnerReference(cr, deploy, scheme) + return deploy } func MakeService(cr *xapi.XlineCluster, scheme *runtime.Scheme) *corev1.Service { - svcRef := GetServiceKey(cr.ObjKey()) svcLabel := GetXlineInstanceLabels(cr.ObjKey()) service := &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ - Name: svcRef.Name, - Namespace: svcRef.Namespace, + Name: cr.Name, + Namespace: cr.Namespace, }, Spec: corev1.ServiceSpec{ Ports: []corev1.ServicePort{ { Name: "xline-port", - Port: XlinePort, + Port: constants.XlinePort, }, }, Selector: svcLabel, @@ -69,21 +210,69 @@ func MakeService(cr *xapi.XlineCluster, scheme *runtime.Scheme) *corev1.Service return service } +func MakeScriptCM(cr *xapi.XlineCluster, scheme *runtime.Scheme) *corev1.ConfigMap { + cm := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-script", cr.Name), + Namespace: cr.Namespace, + Labels: GetXlineInstanceLabels(cr.ObjKey()), + }, + Data: map[string]string{ + "startup-script": XlineStartScript, + }, + } + _ = controllerutil.SetOwnerReference(cr, cm, scheme) + return cm +} + +func MakeConfigMap(cr *xapi.XlineCluster, scheme *runtime.Scheme) *corev1.ConfigMap { + cm := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-config", cr.Name), + Namespace: cr.Namespace, + Labels: GetXlineInstanceLabels(cr.ObjKey()), + }, + Data: cr.Spec.BootArgs(), + } + _ = controllerutil.SetOwnerReference(cr, cm, scheme) + return cm +} + func MakeStatefulSet(cr *xapi.XlineCluster, scheme *runtime.Scheme) *appv1.StatefulSet { crName := types.NamespacedName{Namespace: cr.Namespace, Name: cr.Name} - stsRef := GetStatefulSetKey(crName) stsLabels := GetXlineInstanceLabels(crName) - svcName := GetServiceKey(cr.ObjKey()).Name - initCmd := []string{ - "xline", - "--name", "$(POD_NAME)", - "--members", "$(MEMBERS)", - "--storage-engine", "rocksdb", - "--data-dir", DataDir, + envs := []corev1.EnvVar{ + {Name: "MEMBERS", Value: GetMemberTopology(cr)}, } - initCmd = append(initCmd, cr.Spec.BootArgs()...) + volumes := []corev1.Volume{ + { + Name: "startup-script", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: fmt.Sprintf("%s-script", cr.Name), + }, + Items: []corev1.KeyToPath{{Key: "startup-script", Path: "xline_start_script.sh"}}, + }, + }, + }, + } + + volumeMounts := []corev1.VolumeMount{ + {Name: "xline-storage", MountPath: DataDir}, + {Name: "startup-script", ReadOnly: true, MountPath: "/usr/local/script"}, + } + + authVol, authVM, authEnvs := getAuthInfo(cr.Spec.AuthSecrets) + volumes = append(volumes, authVol...) + volumeMounts = append(volumeMounts, authVM...) + envs = append(envs, authEnvs...) + + pvcTemplates := []corev1.PersistentVolumeClaim{ + util.NewReadWriteOncePVC("xline-storage", cr.Spec.StorageClassName, cr.Spec.Requests.Storage()), + } // pod template: main container mainContainer := corev1.Container{ @@ -91,17 +280,12 @@ func MakeStatefulSet(cr *xapi.XlineCluster, scheme *runtime.Scheme) *appv1.State Image: *cr.Spec.Image, ImagePullPolicy: cr.Spec.ImagePullPolicy, Ports: []corev1.ContainerPort{ - {Name: "xline-port", ContainerPort: XlinePort}, - }, - Command: initCmd, - Env: []corev1.EnvVar{ - {Name: "MEMBERS", Value: GetMemberTopology(stsRef, svcName, int(cr.Spec.Replicas))}, - {Name: "POD_NAME", ValueFrom: &corev1.EnvVarSource{ - FieldRef: &corev1.ObjectFieldSelector{ - FieldPath: "metadata.name", - }, - }}, + {Name: "xline-port", ContainerPort: constants.XlinePort}, }, + Command: []string{"/bin/bash", "/usr/local/script/xline_start_script.sh"}, + Env: envs, + EnvFrom: getConfigInfo(cr), + VolumeMounts: volumeMounts, } // pod template @@ -110,6 +294,7 @@ func MakeStatefulSet(cr *xapi.XlineCluster, scheme *runtime.Scheme) *appv1.State Labels: stsLabels, }, Spec: corev1.PodSpec{ + Volumes: volumes, Containers: []corev1.Container{mainContainer}, }, } @@ -119,15 +304,16 @@ func MakeStatefulSet(cr *xapi.XlineCluster, scheme *runtime.Scheme) *appv1.State // statefulset statefulSet := &appv1.StatefulSet{ ObjectMeta: metav1.ObjectMeta{ - Name: stsRef.Name, - Namespace: stsRef.Namespace, + Name: cr.Name, + Namespace: cr.Namespace, Labels: stsLabels, }, Spec: appv1.StatefulSetSpec{ - Replicas: &cr.Spec.Replicas, - ServiceName: svcName, - Selector: &metav1.LabelSelector{MatchLabels: stsLabels}, - Template: podTemplate, + Replicas: &cr.Spec.Replicas, + ServiceName: cr.Name, + Selector: &metav1.LabelSelector{MatchLabels: stsLabels}, + VolumeClaimTemplates: pvcTemplates, + Template: podTemplate, }, } diff --git a/internal/transformer/xlinecluster_resource_test.go b/internal/transformer/xlinecluster_resource_test.go index 86ad5c5a..fabfca37 100644 --- a/internal/transformer/xlinecluster_resource_test.go +++ b/internal/transformer/xlinecluster_resource_test.go @@ -23,34 +23,17 @@ func TestXlineClusterFunc(t *testing.T) { }, } - t.Run("GetServiceKey should work properly", func(t *testing.T) { - xcLookupKey := xlineCluster.ObjKey() - svcObj := GetServiceKey(xcLookupKey) - assert.Equal(t, svcObj.Namespace, "default") - assert.Equal(t, svcObj.Name, "xline-svc") - }) - - t.Run("GetStatefulSetKey should work properly", func(t *testing.T) { - xcLookupKey := xlineCluster.ObjKey() - stsObj := GetStatefulSetKey(xcLookupKey) - assert.Equal(t, stsObj.Namespace, "default") - assert.Equal(t, stsObj.Name, "xline-sts") - }) - t.Run("GetXlineImage should work properly", func(t *testing.T) { xlineImage := *xlineCluster.Spec.Image assert.Equal(t, xlineImage, "xline-img:latest") }) t.Run("GetMemberTopology should work properly", func(t *testing.T) { - xcLookupKey := xlineCluster.ObjKey() - stsRef := GetStatefulSetKey(xcLookupKey) - svcName := GetServiceKey(xcLookupKey).Name - topology := GetMemberTopology(stsRef, svcName, 3) + topology := GetMemberTopology(&xlineCluster) topologyVec := strings.Split(topology, ",") assert.Equal(t, len(topologyVec), 3) for i := 0; i < 3; i++ { - expectRes := fmt.Sprintf("xline-sts-%d=xline-sts-%d.xline-svc.default.svc.cluster.local:2379", i, i) + expectRes := fmt.Sprintf("xline-%d=xline-%d.xline.default.svc.cluster.local:2379", i, i) assert.Equal(t, topologyVec[i], expectRes) } }) diff --git a/internal/util/kubeutil.go b/internal/util/kubeutil.go index fc4a04f1..58d87cce 100644 --- a/internal/util/kubeutil.go +++ b/internal/util/kubeutil.go @@ -5,6 +5,9 @@ import ( "encoding/json" "fmt" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" ) @@ -34,3 +37,19 @@ func Md5HashOr(obj any, fallback string) string { } return hash } + +func NewReadWriteOncePVC(name string, storageClassName *string, storageRequest *resource.Quantity) corev1.PersistentVolumeClaim { + pvc := corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + }, + Spec: corev1.PersistentVolumeClaimSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, + StorageClassName: storageClassName, + }, + } + if storageRequest != nil { + pvc.Spec.Resources.Requests = corev1.ResourceList{corev1.ResourceStorage: *storageRequest} + } + return pvc +} diff --git a/tests/e2e/cases/cases.sh b/tests/e2e/cases/cases.sh index 690d96de..6d35f46f 100644 --- a/tests/e2e/cases/cases.sh +++ b/tests/e2e/cases/cases.sh @@ -1,3 +1,3 @@ ${__E2E_CASES__:=false} && return 0 || __E2E_CASES__=true -source "$(dirname "${BASH_SOURCE[0]}")/ci.sh" +source "${E2E_TEST_DIR}/cases/ci.sh" diff --git a/tests/e2e/cases/ci.sh b/tests/e2e/cases/ci.sh index 0695d42f..5f3f0ca3 100644 --- a/tests/e2e/cases/ci.sh +++ b/tests/e2e/cases/ci.sh @@ -1,40 +1,67 @@ ${__E2E_CASES_CI__:=false} && return 0 || __E2E_CASES_CI__=true -source "$(dirname "${BASH_SOURCE[0]}")/../common/common.sh" -source "$(dirname "${BASH_SOURCE[0]}")/../testenv/testenv.sh" +source "${E2E_TEST_DIR}/common/common.sh" +source "${E2E_TEST_DIR}/testenv/testenv.sh" _TEST_CI_CLUSTER_NAME="my-xline-cluster" -_TEST_CI_STS_NAME="$_TEST_CI_CLUSTER_NAME-sts" -_TEST_CI_SVC_NAME="$_TEST_CI_CLUSTER_NAME-svc" +_TEST_CI_SECRET_NAME="auth-cred" _TEST_CI_NAMESPACE="default" _TEST_CI_DNS_SUFFIX="svc.cluster.local" _TEST_CI_XLINE_PORT="2379" -_TEST_CI_LOG_SYNC_TIMEOUT=60 +_TEST_CI_STORAGECLASS_NAME="e2e-storage" +_TEST_CI_LOG_SYNC_TIMEOUT=30 function test::ci::_mk_endpoints() { - local endpoints="${_TEST_CI_STS_NAME}-0.${_TEST_CI_SVC_NAME}.${_TEST_CI_NAMESPACE}.${_TEST_CI_DNS_SUFFIX}:${_TEST_CI_XLINE_PORT}" + local endpoints="${_TEST_CI_CLUSTER_NAME}-0.${_TEST_CI_CLUSTER_NAME}.${_TEST_CI_NAMESPACE}.${_TEST_CI_DNS_SUFFIX}:${_TEST_CI_XLINE_PORT}" for ((i = 1; i < $1; i++)); do - endpoints="${endpoints},${_TEST_CI_STS_NAME}-${i}.${_TEST_CI_SVC_NAME}.${_TEST_CI_NAMESPACE}.${_TEST_CI_DNS_SUFFIX}:${_TEST_CI_XLINE_PORT}" + endpoints="${endpoints},${_TEST_CI_CLUSTER_NAME}-${i}.${_TEST_CI_CLUSTER_NAME}.${_TEST_CI_NAMESPACE}.${_TEST_CI_DNS_SUFFIX}:${_TEST_CI_XLINE_PORT}" done echo "$endpoints" } -function test::ci::_etcdctl_expect() { - log::debug "run command: etcdctl --endpoints=$1 $2" - got=$(testenv::util::etcdctl --endpoints="$1" "$2") - expect=$(echo -e "$3") - if [ "${got//$'\r'/}" == "$expect" ]; then - log::info "command run success" - else - log::error "command run failed" - log::error "expect: $expect" - log::error "got: $got" - return 1 - fi +function test::ci::_auth_validation() { + log::info "auth validation test running..." + endpoints=$(test::ci::_mk_endpoints 3) + ETCDCTL=$(testenv::util::etcdctl $endpoints) + + testenv::util::run_with_expect "${ETCDCTL} user add root:root" "User root created" || return $? + testenv::util::run_with_expect "${ETCDCTL} role add root" "Role root created" || return $? + testenv::util::run_with_expect "${ETCDCTL} user grant-role root root" "Role root is granted to user root" || return $? + testenv::util::run_with_match "${ETCDCTL} --user root:root user list" "etcdserver: authentication is not enabled" || return $? + testenv::util::run_with_expect "${ETCDCTL} auth enable" "Authentication Enabled" || return $? + testenv::util::run_with_match "${ETCDCTL} --user root:rot user list" "etcdserver: authentication failed, invalid user ID or password" || return $? + testenv::util::run_with_expect "${ETCDCTL} --user root:root auth status" "Authentication Status: true\nAuthRevision: 4" || return $? + testenv::util::run_with_expect "${ETCDCTL} --user root:root user add u:u" "User u created" || return $? + testenv::util::run_with_match "${ETCDCTL} --user u:u user add f:f" "etcdserver: permission denied" || return $? + testenv::util::run_with_expect "${ETCDCTL} --user root:root role add r" "Role r created" || return $? + testenv::util::run_with_expect "${ETCDCTL} --user root:root user grant-role u r" "Role r is granted to user u" || return $? + testenv::util::run_with_expect "${ETCDCTL} --user root:root role grant-permission r readwrite key1" "Role r updated" || return $? + testenv::util::run_with_expect "${ETCDCTL} --user u:u put key1 value1" "OK" || return $? + testenv::util::run_with_expect "${ETCDCTL} --user u:u get key1" "key1\nvalue1" || return $? + testenv::util::run_with_expect "${ETCDCTL} --user u:u role get r" "Role r\nKV Read:\n\tkey1\nKV Write:\n\tkey1" || return $? + testenv::util::run_with_expect "${ETCDCTL} --user u:u user get u" "User: u\nRoles: r" || return $? + testenv::util::run_with_expect "echo 'new_password' | ${ETCDCTL} --user root:root user passwd --interactive=false u" "Password updated" || return $? + testenv::util::run_with_expect "${ETCDCTL} --user root:root role revoke-permission r key1" "Permission of key key1 is revoked from role r" || return $? + testenv::util::run_with_expect "${ETCDCTL} --user root:root user revoke-role u r" "Role r is revoked from user u" || return $? + testenv::util::run_with_expect "${ETCDCTL} --user root:root user list" "root\nu" || return $? + testenv::util::run_with_expect "${ETCDCTL} --user root:root role list" "r\nroot" || return $? + testenv::util::run_with_expect "${ETCDCTL} --user root:root user delete u" "User u deleted" || return $? + testenv::util::run_with_expect "${ETCDCTL} --user root:root role delete r" "Role r deleted" || return $? + testenv::util::run_with_match "${ETCDCTL} --user root:root user get non_exist_user" "etcdserver: user name not found" || return $? + testenv::util::run_with_match "${ETCDCTL} --user root:root user add root:root" "etcdserver: user name already exists" || return $? + testenv::util::run_with_match "${ETCDCTL} --user root:root role get non_exist_role" "etcdserver: role name not found" || return $? + testenv::util::run_with_match "${ETCDCTL} --user root:root role add root" "etcdserver: role name already exists" || return $? + testenv::util::run_with_match "${ETCDCTL} --user root:root user revoke root r" "etcdserver: role is not granted to the user" || return $? + testenv::util::run_with_match "${ETCDCTL} --user root:root role revoke root non_exist_key" "etcdserver: permission is not granted to the role" || return $? + testenv::util::run_with_match "${ETCDCTL} --user root:root user delete root" "etcdserver: invalid auth management" || return $? + testenv::util::run_with_expect "${ETCDCTL} --user root:root auth disable" "Authentication Disabled" || return $? + log::info "auth validation test passed" } function test::ci::_install_CRD() { + pushd $(dirname "${BASH_SOURCE[0]}")/../../../ make install + popd if [ $? -eq 0 ]; then log::info "make install: create custom resource definition succeeded" else @@ -43,7 +70,9 @@ function test::ci::_install_CRD() { } function test::ci::_uninstall_CRD() { + pushd $(dirname "${BASH_SOURCE[0]}")/../../../ make uninstall + popd if [ $? -eq 0 ]; then log::info "make uninstall: remove custom resource definition succeeded" else @@ -53,54 +82,103 @@ function test::ci::_uninstall_CRD() { function test::ci::wait_all_xline_pod_ready() { for ((i = 0; i < $1; i++)); do - log::info "wait pod/${_TEST_CI_STS_NAME}-${i} to be ready" - if ! k8s::kubectl wait --for=condition=Ready pod/${_TEST_CI_STS_NAME}-${i} --timeout=300s; then + log::info "wait pod/${_TEST_CI_CLUSTER_NAME}-${i} to be ready" + if ! k8s::kubectl wait --for=condition=Ready pod/${_TEST_CI_CLUSTER_NAME}-${i} --timeout=300s; then + log::fatal "Failed to wait for util to be ready" + fi + done +} + +function test::ci::wait_all_xline_pod_deleted() { + for ((i = 0; i < $1; i++)); do + log::info "wait pod/${_TEST_CI_CLUSTER_NAME}-${i} to be ready" + if ! k8s::kubectl wait --for=delete pod/${_TEST_CI_CLUSTER_NAME}-${i} --timeout=300s; then log::fatal "Failed to wait for util to be ready" fi done } +function test::ci::_prepare_pv() { + log::info "create persistent volume and storage class" + mkdir -p /tmp/host-500m-pv1 /tmp/host-500m-pv2 /tmp/host-500m-pv3 + k8s::kubectl apply -f "$(dirname "${BASH_SOURCE[0]}")/manifests/e2e-storage.yaml" >/dev/null 2>&1 + k8s::kubectl::wait_resource_creation storageclass $_TEST_CI_STORAGECLASS_NAME + k8s::kubectl::wait_resource_creation pv "host-500m-pv1" + k8s::kubectl::wait_resource_creation pv "host-500m-pv2" + k8s::kubectl::wait_resource_creation pv "host-500m-pv3" +} + +function test::ci::_clean_pvc() { + for ((i = 0; i < $1; i++)); do + local pvc_name="xline-storage-${_TEST_CI_CLUSTER_NAME}-${i}" + log::info "deleting pvc $pvc_name ..." + k8s::kubectl delete pvc $pvc_name >/dev/null 2>&1 + if ! k8s::kubectl wait --for=delete pvc/${pvc_name} --timeout=300s; then + log::fatal "Failed to wait for pvc/${pvc_name} to be deleted" + fi + done +} + +function test::ci::_clean_pv() { + log::info "delete persistent volume claim" + log::info "delete persistent volume and storage class" + k8s::kubectl delete -f "$(dirname "${BASH_SOURCE[0]}")/manifests/e2e-storage.yaml" + log::info "pv has been deleted" + rm -rf /tmp/host-500m-pv1 /tmp/host-500m-pv2 /tmp/host-500m-pv3 +} + function test::ci::_start() { - log::info "starting controller" - pushd $(dirname "${BASH_SOURCE[0]}")/../../../ - test::ci::_install_CRD - make run >/dev/null 2>&1 & - log::info "controller started" + log::info "starting controller manager" + pushd ${CODE_BASE_DIR} + IMG=${OPERATOR_IMG} make deploy 2>/dev/null + if ! KUBECTL_NAMESPACE=${OPERATOR_NS} k8s::kubectl wait --for=condition=available deployment/xline-operator-controller-manager --timeout=300s; then + log::fatal "Failed to wait for xline-operator-controller-manager to be ready" + fi popd - log::info "starting xline cluster" + log::info "controller manager started" + log::info "creating xline auth key pairs..." + k8s::kubectl apply -f "$(dirname "${BASH_SOURCE[0]}")/manifests/auth-cred.yaml" >/dev/null 2>&1 + k8s::kubectl::wait_resource_creation secret $_TEST_CI_SECRET_NAME + test::ci::_prepare_pv + log::info "starting xline cluster..." k8s::kubectl apply -f "$(dirname "${BASH_SOURCE[0]}")/manifests/cluster.yaml" >/dev/null 2>&1 - k8s::kubectl::wait_resource_creation sts $_TEST_CI_STS_NAME + k8s::kubectl::wait_resource_creation sts $_TEST_CI_CLUSTER_NAME } + + function test::ci::_teardown() { - log::info "stopping controller" - pushd $(dirname "${BASH_SOURCE[0]}")/../../../ - test::ci::_uninstall_CRD - controller_pid=$(ps aux | grep "[g]o run ./cmd/main.go" | awk '{print $2}') - if [ -n "$controller_pid" ]; then - kill -9 $controller_pid - fi + log::info "stopping controller manager..." + pushd ${CODE_BASE_DIR} + IMG=${OPERATOR_IMG} make undeploy 2>/dev/null + popd + test::ci::wait_all_xline_pod_deleted 3 + test::ci::_clean_pvc 3 + test::ci::_clean_pv + k8s::kubectl delete -f "$(dirname "${BASH_SOURCE[0]}")/manifests/auth-cred.yaml" >/dev/null 2>&1 } function test::ci::_chaos() { size=$1 iters=$2 - max_kill=$((size / 2)) - log::info "chaos: size=$size, iters=$iters, max_kill=$max_kill" - for ((i = 0; i < iters; i++)); do + majority=$((size / 2 + 1)) + fault_tolerance=$((size - majority)) + endpoints=$(test::ci::_mk_endpoints $size) + ETCDCTL=$(testenv::util::etcdctl $endpoints) + log::info "chaos: size=$size, iters=$iters, fault_tolerance=$fault_tolerance" + for ((i = 0; i < $iters; i++)); do log::info "chaos: iter=$i" - endpoints=$(test::ci::_mk_endpoints size) - test::ci::_etcdctl_expect "$endpoints" "put A $i" "OK" || return $? - test::ci::_etcdctl_expect "$endpoints" "get A" "A\n$i" || return $? - kill=$((RANDOM % max_kill)) + testenv::util::run_with_expect "${ETCDCTL} put A $i" "OK" || return $? + testenv::util::run_with_expect "${ETCDCTL} get A" "A\n$i" || return $? + kill=$((RANDOM % fault_tolerance + 1)) log::info "chaos: kill=$kill" - for ((j = 0; j < kill; j++)); do - pod="${_TEST_CI_STS_NAME}-$((RANDOM % size))" + for ((j = 0; j < $kill; j++)); do + pod="${_TEST_CI_CLUSTER_NAME}-$((RANDOM % size))" log::info "chaos: kill pod=$pod" k8s::kubectl delete pod "$pod" --force --grace-period=0 2>/dev/null done - test::ci::_etcdctl_expect "$endpoints" "put B $i" "OK" || return $? - test::ci::_etcdctl_expect "$endpoints" "get B" "B\n$i" || return $? + testenv::util::run_with_expect "${ETCDCTL} put B $i" "OK" || return $? + testenv::util::run_with_expect "${ETCDCTL} get B" "B\n$i" || return $? k8s::kubectl wait --for=jsonpath='{.status.readyReplicas}'="$size" sts/$_TEST_CI_CLUSTER_NAME --timeout=300s >/dev/null 2>&1 log::info "wait for log synchronization" && sleep $_TEST_CI_LOG_SYNC_TIMEOUT done @@ -110,11 +188,14 @@ function test::run::ci::basic_validation() { test::ci::_start test::ci::wait_all_xline_pod_ready 3 endpoints=$(test::ci::_mk_endpoints 3) - test::ci::_etcdctl_expect "$endpoints" "put A 1" "OK" || return $? - test::ci::_etcdctl_expect "$endpoints" "get A" "A\n1" || return $? + ETCDCTL=$(testenv::util::etcdctl $endpoints) + testenv::util::run_with_expect "${ETCDCTL} put A 1" "OK" || return $? + testenv::util::run_with_expect "${ETCDCTL} get A" "A\n1" || return $? endpoints=$(test::ci::_mk_endpoints 1) - test::ci::_etcdctl_expect "$endpoints" "put A 2" "OK" || return $? - test::ci::_etcdctl_expect "$endpoints" "get A" "A\n2" || return $? + ETCDCTL=$(testenv::util::etcdctl $endpoints) + testenv::util::run_with_expect "${ETCDCTL} put A 2" "OK" || return $? + testenv::util::run_with_expect "${ETCDCTL} get A" "A\n2" || return $? + test::ci::_auth_validation test::ci::_teardown } diff --git a/tests/e2e/cases/manifests/auth-cred.yaml b/tests/e2e/cases/manifests/auth-cred.yaml new file mode 100644 index 00000000..8501b674 --- /dev/null +++ b/tests/e2e/cases/manifests/auth-cred.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: Secret +metadata: + creationTimestamp: null + name: auth-cred +data: + auth-jwt.pri: LS0tLS1CRUdJTiBQUklWQVRFIEtFWS0tLS0tCk1JSUV2UUlCQURBTkJna3Foa2lHOXcwQkFRRUZBQVNDQktjd2dnU2pBZ0VBQW9JQkFRQ25BeHhTWEpZV1pDS3IKNmY2ajBIUlV3a2hYLzArR1hqRWNsV29MQTUrS1pBdVdNU3U4Yno2WCtJU2N2NHZOd09SbEdTV09ucnorOG1iMgpJMEY2dGVWWldmV0Zxc255V2s3SXhNK2g5eVRnN2FZLzg2ODVZZldUTDdmcFdxMS8zRm5pejRRYnNZRnV6QjFWCmdhWjVmRDJDU1lJS3pTRCtxVlNsWEYyNUpERkhWN2IyT2RIclgwVUtaT1RXWS9WRS8vU1R0K1BKS2RYOVIzcGwKa0d3QXpKSWtrY0FaeTB2aHZxVDNBU1RnWGNoTmVOOHdHWVliM1lpcmtxSXNRQjVYY3MxUjFXK3l6K0lyVmE2LwowV01jeUU2cXRKUFowbHZpeVQwbkhWL3Baalh1RDRCMGFqYS8xZmsvSG1YRFBNanBLMUJ1Q0JUU3RNL0tsY3JBCm9BeG8rWURoQWdNQkFBRUNnZ0VBSXlKaFkrWThZTXVDQzc1M0pra2xIK3ViUW4vZ1gva1N4ZHVjNm1KQnZ1QmIKRzZhT2Q5N0RRVDh6enJIeEhFRFhDM21sMEFJTzZtZGVSNnVWQzlhV1FCelByT1lJQStjQnFmVFZaVkpUdk1uaAo3cFE2S1kwMUYxaXpqUERaalF0ekVXYnNlTkwzMHJJMy9aUC96SkRaYzc0NUVFS2xEVTNjRThtQm9nQStLYTZ3CkdMb3pUOXFRZjhrbkJydHp4SDZTdnJacGZhUmxQOTVpczgyYjRJdVBocVlkRzdkVllGVEFMRTFNeVZyQ2JTNFkKS3l0ak5MZ3dwMWJJUXRXcnpNZWJCR29pVStEdkRjUlk4enZPZkZ1cER3cFlDdDNwMWFVNXd5WVlkcjc0ZXNWNwpqanFIajg5VWE2NUpISjNYbk1BYU1jNGRITTJGc0dxTXNPdi9EREtJblFLQmdRRGF3Y2tRRWVreDBRdVAzZUpQCkdXZFo4N29jK0ZWakRlM2JZaEFuQ2YveVhSSm9xY3M1dnIxbTF5Q1hGZnNqYlFGWUhXWFI5QVV0Tm41SEN3T1oKem9UMU12OTZmWEJWR1FPUmd6dmxVV1M0M3VLcGZJUERWdjJJNlpjS1NJUUFHT2djV1l2bUJEaFlxUEhnbXgzbwpWU3JOR1d0TGR5dzNyRDFKNk8rMVJ3dGJpd0tCZ1FERGNobVk1OUVYQmlUdmx5VDNRamwwdlpGTUhhK1RFbGJoCmlrTnRZbHRiVUh0YW1PWFp6cGRrL0tBN1gyZFlpMFFwVmZiYnBmUC9seTVsWXZnWndsOGg5ME9ib3BydStBQ00KbmRsS0JmTlFZQXJtV1k2YkoyQ3dGN2oxYVRDQ0hadVZ1WDYvcHpGVlN0UmNzc24xNXVvVmFJeUtkL01oSnpMRgpTM2VydFFrU3d3S0JnQW5pTVlSaFdzamVhZ2hRL1JXWHp6eVlMM041b05uOTJoNU1XdkI0bWpESUZiblcyaEM4CjFtL2NEbVBsSVZpalp5a2xBdUd1aGNGYU1mQmh4Z0xmK3MvZFF2KzB4U3VER3M4clA3eUhwZVpZWTZOR3RlbFEKZDlvRXU4ZENLWHlibzNrTWJxNnd5Qjd4V3lSTHZka3VaK1dtWFZ1bWdiL3VMMEswbklmek1zY3JBb0dBZUExZQpLODQ1WVNzbEJRYVNiazcvZS9YMWlndXlEV1QyZVJPMDF6dlRZZ1BOd1ppcGwyQ1BIamtQTTJrbTBmeTVvYXBzCk4vOTRJVWQ3K0VzU21zQUtMNUx5dEdidFJGeVIrYzM3NnJ3OCtPSUZ6L2l5NEJzUUNScUpRaldhMWxIWmY5NngKUElnMmhXMnhoRDlPVHYzSVM5NHNkZUc0Tm1VZGlwTVFyeWhFcW9FQ2dZRUFrdlhPZzY2SUFWVHJPNnFnb3lsNQo0Mm91ZmEvUUUrcU9BWW9RRXBteDNTWng2dE1reWNmQVFxVUhZY1hoVzFITmp5R2JiZy9zbDEzeWRkblBRcWlnCitPYnRRTlNJcUdaV0NjL0hJcU0vL3BQSTNNSFBoV0FSTU9tQWJrMEkxbVQwUUtodUZmU3VnVjJ4YjFEai9SdmYKMFZkQjh0eFkrNVd6NnpQMUYyZzQ2Z009Ci0tLS0tRU5EIFBSSVZBVEUgS0VZLS0tLS0K + auth-jwt.pub: LS0tLS1CRUdJTiBQVUJMSUMgS0VZLS0tLS0KTUlJQklqQU5CZ2txaGtpRzl3MEJBUUVGQUFPQ0FROEFNSUlCQ2dLQ0FRRUFwd01jVWx5V0ZtUWlxK24rbzlCMApWTUpJVi85UGhsNHhISlZxQ3dPZmltUUxsakVydkc4K2wvaUVuTCtMemNEa1pSa2xqcDY4L3ZKbTlpTkJlclhsCldWbjFoYXJKOGxwT3lNVFBvZmNrNE8ybVAvT3ZPV0gxa3krMzZWcXRmOXhaNHMrRUc3R0Jic3dkVllHbWVYdzkKZ2ttQ0NzMGcvcWxVcFZ4ZHVTUXhSMWUyOWpuUjYxOUZDbVRrMW1QMVJQLzBrN2ZqeVNuVi9VZDZaWkJzQU15UwpKSkhBR2N0TDRiNms5d0VrNEYzSVRYamZNQm1HRzkySXE1S2lMRUFlVjNMTlVkVnZzcy9pSzFXdXY5RmpITWhPCnFyU1QyZEpiNHNrOUp4MWY2V1kxN2crQWRHbzJ2OVg1UHg1bHd6ekk2U3RRYmdnVTByVFB5cFhLd0tBTWFQbUEKNFFJREFRQUIKLS0tLS1FTkQgUFVCTElDIEtFWS0tLS0tCg== diff --git a/tests/e2e/cases/manifests/cluster.yaml b/tests/e2e/cases/manifests/cluster.yaml index b276b307..f2417fe8 100644 --- a/tests/e2e/cases/manifests/cluster.yaml +++ b/tests/e2e/cases/manifests/cluster.yaml @@ -6,3 +6,13 @@ spec: image: phoenix500526/xline:v0.6.1 imagePullPolicy: IfNotPresent replicas: 3 + storageClassName: "e2e-storage" + requests: + storage: 500Mi + authSecret: + name: auth-cred + mountPath: /tmp/auth-cred + pubKey: auth-jwt.pub + priKey: auth-jwt.pri + config: + LogLevel: debug diff --git a/tests/e2e/cases/manifests/e2e-storage.yaml b/tests/e2e/cases/manifests/e2e-storage.yaml new file mode 100644 index 00000000..09a4dd15 --- /dev/null +++ b/tests/e2e/cases/manifests/e2e-storage.yaml @@ -0,0 +1,51 @@ +--- +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: "e2e-storage" +provisioner: "kubernetes.io/no-provisioner" + +--- +apiVersion: v1 +kind: PersistentVolume +metadata: + name: host-500m-pv1 + +spec: + storageClassName: e2e-storage + accessModes: + - ReadWriteOnce + capacity: + storage: 500Mi + hostPath: + path: /tmp/host-500m-pv1/ +--- +apiVersion: v1 +kind: PersistentVolume +metadata: + name: host-500m-pv2 + +spec: + storageClassName: e2e-storage + accessModes: + - ReadWriteOnce + capacity: + storage: 500Mi + hostPath: + path: /tmp/host-500m-pv2/ +--- + +apiVersion: v1 +kind: PersistentVolume +metadata: + name: host-500m-pv3 + +spec: + storageClassName: e2e-storage + accessModes: + - ReadWriteOnce + capacity: + storage: 500Mi + hostPath: + path: /tmp/host-500m-pv3/ +--- diff --git a/tests/e2e/common/common.sh b/tests/e2e/common/common.sh index 4f456e7e..7791241e 100644 --- a/tests/e2e/common/common.sh +++ b/tests/e2e/common/common.sh @@ -1,4 +1,4 @@ ${__E2E_COMMON__:=false} && return 0 || __E2E_COMMON__=true -source "$(dirname "${BASH_SOURCE[0]}")/log.sh" -source "$(dirname "${BASH_SOURCE[0]}")/k8s.sh" +source "${E2E_TEST_DIR}/common/log.sh" +source "${E2E_TEST_DIR}/common/k8s.sh" diff --git a/tests/e2e/e2e.sh b/tests/e2e/e2e.sh index 29b736e2..6f5667aa 100755 --- a/tests/e2e/e2e.sh +++ b/tests/e2e/e2e.sh @@ -2,9 +2,15 @@ set -euo pipefail -source "$(dirname "${BASH_SOURCE[0]}")/common/common.sh" -source "$(dirname "${BASH_SOURCE[0]}")/testenv/testenv.sh" -source "$(dirname "${BASH_SOURCE[0]}")/cases/cases.sh" +__ABS_PATH__=$(realpath ${BASH_SOURCE[0]}) +export E2E_TEST_DIR=$(dirname ${__ABS_PATH__}) +export CODE_BASE_DIR=$(dirname $(dirname ${E2E_TEST_DIR})) +export OPERATOR_IMG="ghcr.io/xline-kv/xline-operator:$(git rev-parse --short HEAD)" +export OPERATOR_NS="xline-operator-system" + +source "${E2E_TEST_DIR}/common/common.sh" +source "${E2E_TEST_DIR}/testenv/testenv.sh" +source "${E2E_TEST_DIR}/cases/cases.sh" function setup() { testenv::k8s::create diff --git a/tests/e2e/testenv/k8s/kind.sh b/tests/e2e/testenv/k8s/kind.sh index 151eea3e..edc068a6 100644 --- a/tests/e2e/testenv/k8s/kind.sh +++ b/tests/e2e/testenv/k8s/kind.sh @@ -3,7 +3,7 @@ ${__E2E_TESTENV_KIND__:=false} && return 0 || __E2E_TESTENV_KIND__=true _TEST_ENV_KIND_CLUSTER_NAME="e2e-kind" _DEFAULT_KIND_IMAGE="kindest/node:v1.27.3" -source "$(dirname "${BASH_SOURCE[0]}")/../../common/common.sh" +source "${E2E_TEST_DIR}/common/common.sh" function testenv::k8s::kind::_cluster_exists() { kind get clusters -q | grep -w -q "${_TEST_ENV_KIND_CLUSTER_NAME}" diff --git a/tests/e2e/testenv/testenv.sh b/tests/e2e/testenv/testenv.sh index d2ba30d2..ccf771e4 100644 --- a/tests/e2e/testenv/testenv.sh +++ b/tests/e2e/testenv/testenv.sh @@ -1,8 +1,8 @@ ${__E2E_TESTENV__:=false} && return 0 || __E2E_TESTENV__=true -source "$(dirname "${BASH_SOURCE[0]}")/k8s/kind.sh" -source "$(dirname "${BASH_SOURCE[0]}")/util/util.sh" -source "$(dirname "${BASH_SOURCE[0]}")/../common/common.sh" +source "${E2E_TEST_DIR}/testenv/k8s/kind.sh" +source "${E2E_TEST_DIR}/testenv/util/util.sh" +source "${E2E_TEST_DIR}/common/common.sh" function testenv::k8s::create() { testenv::k8s::kind::create @@ -14,13 +14,16 @@ function testenv::k8s::delete() { } function testenv::k8s::load_images() { - # xline image log::info "Loading images" - xline_image="phoenix500526/xline:v0.6.1" - docker pull "$xline_image" 2>/dev/null - testenv::k8s::kind::load_image "$xline_image" - # etcdctl image - etcdctl_image="ghcr.io/xline-kv/etcdctl:v3.5.9" - docker pull "$etcdctl_image" 2>/dev/null - testenv::k8s::kind::load_image "$etcdctl_image" + # xline operator image + pushd ${CODE_BASE_DIR} + IMG=${OPERATOR_IMG} make docker-build 2>/dev/null + popd + testenv::k8s::kind::load_image "$OPERATOR_IMG" + + remote_images=("phoenix500526/xline:v0.6.1" "ghcr.io/xline-kv/etcdctl:v3.5.9" "quay.io/brancz/kube-rbac-proxy:v0.15.0") + for img in "${remote_images[@]}"; do + docker pull "$img" 2>/dev/null + testenv::k8s::kind::load_image "$img" + done } diff --git a/tests/e2e/testenv/util/util.sh b/tests/e2e/testenv/util/util.sh index dfaf939e..aed7f481 100644 --- a/tests/e2e/testenv/util/util.sh +++ b/tests/e2e/testenv/util/util.sh @@ -46,15 +46,49 @@ function testenv::util::uninstall() { } function testenv::util::etcdctl() { - # shellcheck disable=SC2034 - local KUBECTL_NAMESPACE="${_UTIL_NAMESPACE}" + echo -e "kubectl exec -n util -i etcdctl -- env ETCDCTL_API=3 etcdctl --endpoints=$1" +} + +function testenv::util::run_with_expect() { + cmd="$1" + expect=$(echo -e ${2}) + # retry to avoid mysterious "Error from server: error dialing backend: EOF" error + for ((k = 0; k < ${RETRY_TIMES:-10}; k++)); do + output=$(eval ${cmd} 2>&1) + if [[ $output == *"timed out"* || $output == *"Request timeout"* || $output == *"context deadline exceeded"* ]]; then + sleep "${RETRY_INTERVAL:-3}" + elif [ "${output//$'\r'/}" == "$expect" ]; then + log::info "command $cmd run success" + return 0 + else + log::error "command $cmd run failed" + log::error "expect: $expect" + log::error "got: $output" + return 1 + fi + done +} +# run a command with expect output, based on key word match +# args: +# $1: command to run +# $2: key word to match +function testenv::util::run_with_match() { + cmd="$1" + expect=$(echo -e ${2}) # retry to avoid mysterious "Error from server: error dialing backend: EOF" error - for ((i = 0; i < ${RETRY_TIMES:-10}; i++)); do - if output=$(k8s::kubectl exec -i etcdctl -- env ETCDCTL_API=3 etcdctl $@ 2>&1); then - echo -e "$output" - return + for ((n = 0; n < ${RETRY_TIMES:-10}; n++)); do + output=$(eval ${cmd} 2>&1) + if [[ $output == *"timed out"* || $output == *"Request timeout"* || $output == *"context deadline exceeded"* ]]; then + sleep "${RETRY_INTERVAL:-3}" + elif echo "${output}" | grep -q "${expect}"; then + log::info "command $cmd run success" + return 0 + else + log::error "command $cmd run failed" + log::error "expect: $expect" + log::error "got: $output" + return 1 fi - sleep "${RETRY_INTERVAL:-3}" done }