Multiple COSI driver support in k8s cluster

add support for multiple cosi drivers in k8s cluster by introducing
driver-prefix option.

Fixes: #27

Signed-off-by: Jiffin Tony Thottan <thottanjiffin@gmail.com>

Author: thotz

README.md

@@ -116,6 +116,13 @@ An example for awscli pods can be found at `examples/awscliapppod.yaml`. Credent
 
 1. Handle access policies for Bucket Access Request
 
+## Configuration Options
+
+| Option                    | Default value                          | Description                                                        |
+| ------------------------- | -------------------------------------- | -------------------------------------------------------------------|
+| `--driver-address`        | `unix:///var/lib/cosi/cosi.sock`       | COSI driver address, must be a UNIX socket                         |
+| `--driver-prefix`         |  _empty_                               | prefix added before name, e.g, `<prefix>.ceph.objectstorage.k8s.io`|
+
 ## Integration with Rook
 
 The ceph cosi driver integrates with [Rook](https://rook.io/) from v1.12 onwards to provide object storage for Kubernetes applications. More details can be found [here](https://rook.io/docs/rook/v1.12/Storage-Configuration/Object-Storage-RGW/cosi/).

cmd/ceph-cosi-driver/cmd.go

@@ -18,13 +18,10 @@ package main
 
 import (
 	"context"
+	"errors"
 	"flag"
-	"strings"
 
 	"github.com/ceph/cosi-driver-ceph/pkg/driver"
-	"github.com/spf13/cobra"
-	"github.com/spf13/pflag"
-	"github.com/spf13/viper"
 	"k8s.io/klog/v2"
 
 	"sigs.k8s.io/container-object-storage-interface-provisioner-sidecar/pkg/provisioner"
@@ -33,57 +30,25 @@ import (
 const provisionerName = "ceph.objectstorage.k8s.io"
 
 var (
-	driverAddress = "unix:///var/lib/cosi/cosi.sock"
-	AccessKey     = ""
-	SecretKey     = ""
-	Endpoint      = ""
+	driverAddress = flag.String("driver-address", "unix:///var/lib/cosi/cosi.sock", "driver address for socket")
+	driverPrefix  = flag.String("driver-prefix", "", "prefix for cosi driver, e.g. <prefix>.ceph.objectstorage.k8s.io")
 )
 
-var cmd = &cobra.Command{
-	Use:           "ceph-cosi-driver",
-	Short:         "Kubernetes COSI driver for Ceph RGW",
-	SilenceErrors: true,
-	SilenceUsage:  true,
-	RunE: func(cmd *cobra.Command, args []string) error {
-		return run(cmd.Context(), args)
-	},
-	DisableFlagsInUseLine: true,
-}
-
 func init() {
-	viper.AutomaticEnv()
-	viper.SetEnvKeyReplacer(strings.NewReplacer("-", "_"))
-
-	flag.Set("alsologtostderr", "true")
-	kflags := flag.NewFlagSet("klog", flag.ExitOnError)
-	klog.InitFlags(kflags)
-
-	persistentFlags := cmd.PersistentFlags()
-	persistentFlags.AddGoFlagSet(kflags)
-
-	stringFlag := persistentFlags.StringVarP
-
-	stringFlag(&driverAddress,
-		"driver-addr",
-		"d",
-		driverAddress,
-		"path to unix domain socket where driver should listen")
-
-	viper.BindPFlags(cmd.PersistentFlags())
-	cmd.PersistentFlags().VisitAll(func(f *pflag.Flag) {
-		if viper.IsSet(f.Name) && viper.GetString(f.Name) != "" {
-			cmd.PersistentFlags().Set(f.Name, viper.GetString(f.Name))
-		}
-	})
+	klog.InitFlags(nil)
 }
 
-func run(ctx context.Context, args []string) error {
-	identityServer, bucketProvisioner, err := driver.NewDriver(ctx, provisionerName)
+func run(ctx context.Context) error {
+	if *driverPrefix == "" {
+		return errors.New("driver prefix is missing for ceph cosi driver deployment")
+	}
+	driverName := *driverPrefix + "." + provisionerName
+	identityServer, bucketProvisioner, err := driver.NewDriver(ctx, driverName)
 	if err != nil {
 		return err
 	}
 
-	server, err := provisioner.NewDefaultCOSIProvisionerServer(driverAddress,
+	server, err := provisioner.NewDefaultCOSIProvisionerServer(*driverAddress,
 		identityServer,
 		bucketProvisioner)
 	if err != nil {

cmd/ceph-cosi-driver/main.go

@@ -42,7 +42,7 @@ func main() {
 		os.Exit(1)
 	}()
 
-	if err := cmd.ExecuteContext(ctx); err != nil {
+	if err := run(ctx); err != nil {
 		klog.ErrorS(err, "Exiting on error")
 	}
 }

examples/bucketaccessclass.yaml

@@ -2,8 +2,8 @@ kind: BucketAccessClass
 apiVersion: objectstorage.k8s.io/v1alpha1
 metadata:
   name: sample-bac
-driverName: ceph.objectstorage.k8s.io
+driverName: cosi.ceph.objectstorage.k8s.io
 authenticationType: KEY
 parameters:
-  objectStoreUserSecretName: rook-ceph-object-user-my-store-my-user
+  objectStoreUserSecretName: rook-ceph-object-user-my-store-cosi
   objectStoreUserSecretNamespace: rook-ceph

examples/bucketclass.yaml

@@ -2,8 +2,8 @@ kind: BucketClass
 apiVersion: objectstorage.k8s.io/v1alpha1
 metadata:
   name: sample-bcc
-driverName: ceph.objectstorage.k8s.io
+driverName: cosi.ceph.objectstorage.k8s.io
 deletionPolicy: Delete
 parameters:
-  objectStoreUserSecretName: rook-ceph-object-user-my-store-my-user
+  objectStoreUserSecretName: rook-ceph-object-user-my-store-cosi
   objectStoreUserSecretNamespace: rook-ceph

pkg/driver/driver.go

@@ -23,13 +23,13 @@ import (
 	cosispec "sigs.k8s.io/container-object-storage-interface-spec"
 )
 
-func NewDriver(ctx context.Context, provisionerName string) (cosispec.IdentityServer, cosispec.ProvisionerServer, error) {
-	provisionerServer, err := NewProvisionerServer(provisionerName)
+func NewDriver(ctx context.Context, driverName string) (cosispec.IdentityServer, cosispec.ProvisionerServer, error) {
+	provisionerServer, err := NewProvisionerServer(driverName)
 	if err != nil {
 		klog.Fatal(err, "failed to create provisioner server")
 		return nil, nil, err
 	}
-	identityServer, err := NewIdentityServer(provisionerName)
+	identityServer, err := NewIdentityServer(driverName)
 	if err != nil {
 		klog.Fatal(err, "failed to create provisioner server")
 		return nil, nil, err

resources/deployment.yaml

@@ -39,19 +39,21 @@ spec:
       - name: ceph-cosi-driver
         image: $(CEPH_IMAGE_ORG)/ceph-cosi-driver:$(CEPH_IMAGE_VERSION)
         imagePullPolicy: IfNotPresent
+        args:
+          - "--driver-prefix=cosi"
         volumeMounts:
         - mountPath: /var/lib/cosi
           name: socket
-      - name: objectstorage-provisioner-sidecar
-        image: gcr.io/k8s-staging-sig-storage/objectstorage-sidecar/objectstorage-sidecar:v20221117-v0.1.0-22-g0e67387
-        imagePullPolicy: IfNotPresent
-        args:
-        - "--v=5"
         env:
         - name: POD_NAMESPACE
           valueFrom:
             fieldRef:
               fieldPath: metadata.namespace
+      - name: objectstorage-provisioner-sidecar
+        image: gcr.io/k8s-staging-sig-storage/objectstorage-sidecar/objectstorage-sidecar:v20221117-v0.1.0-22-g0e67387
+        imagePullPolicy: IfNotPresent
+        args:
+        - "--v=5"
         volumeMounts:
         - mountPath: /var/lib/cosi
           name: socket