Release 2.5.0 Content (Post 2.5.0-beta) (#346)

.github/workflows/sanity.yaml

@@ -54,24 +54,54 @@ jobs:
       - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4
       - run: docker-compose -f tests/csi-sanity/docker-compose-snapshotcaps.yaml up $COMPOSE_DEFAULTS
         env:
-          SANITY_FUNCTION: directory_volume_and_snapshots 
+          SANITY_FUNCTION: directory_volume_and_snapshots
 
-  snaphot_volumes_with_2nd_level_shapshots:
+  directory_volume_and_snapshots_nfs:
     if: success() || failure()
     needs: directory_volume_and_snapshots
     runs-on: self-hosted
+    steps:
+      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4
+      - run: docker-compose -f tests/csi-sanity/docker-compose-nfs-snapshotcaps.yaml up $COMPOSE_DEFAULTS
+        env:
+          SANITY_FUNCTION: directory_volume_and_snapshots_nfs
+
+  snaphot_volumes_with_2nd_level_shapshots:
+    if: success() || failure()
+    needs: directory_volume_and_snapshots_nfs
+    runs-on: self-hosted
     steps:
       - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4
       - run: docker-compose -f tests/csi-sanity/docker-compose-snapshotcaps.yaml up $COMPOSE_DEFAULTS
         env:
           SANITY_FUNCTION: snaphot_volumes_with_2nd_level_shapshots  
 
+  snaphot_volumes_with_2nd_level_shapshots_nfs:
+      if: success() || failure()
+      needs: snaphot_volumes_with_2nd_level_shapshots
+      runs-on: self-hosted
+      steps:
+        - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4
+        - run: docker-compose -f tests/csi-sanity/docker-compose-nfs-snapshotcaps.yaml up $COMPOSE_DEFAULTS
+          env:
+            SANITY_FUNCTION: snaphot_volumes_with_2nd_level_shapshots_nfs
+
   filesystem_volumes:
     if: success() || failure()
-    needs: snaphot_volumes_with_2nd_level_shapshots
+    needs: snaphot_volumes_with_2nd_level_shapshots_nfs
     runs-on: self-hosted
     steps:
       - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4
       - run: docker-compose -f tests/csi-sanity/docker-compose-snapshotcaps.yaml up $COMPOSE_DEFAULTS
         env:
           SANITY_FUNCTION: filesystem_volumes 
+
+  filesystem_volumes_nfs:
+    if: success() || failure()
+    needs: filesystem_volumes
+    runs-on: self-hosted
+    steps:
+      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4
+      - run: docker-compose -f tests/csi-sanity/docker-compose-nfs-snapshotcaps.yaml up $COMPOSE_DEFAULTS
+        env:
+          SANITY_FUNCTION: filesystem_volumes_nfs

README.md

@@ -1,7 +1,7 @@
 # CSI WekaFS Driver
 Helm chart for Deployment of WekaIO Container Storage Interface (CSI) plugin for WekaFS - the world fastest filesystem
 
-![Version: 2.5.0-beta](https://img.shields.io/badge/Version-2.5.0--beta-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: v2.5.0-beta](https://img.shields.io/badge/AppVersion-v2.5.0--beta-informational?style=flat-square)
+![Version: 2.4.2-SNAPSHOT.130.bdb50b7](https://img.shields.io/badge/Version-2.4.2--SNAPSHOT.130.bdb50b7-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: v2.4.2-SNAPSHOT.130.bdb50b7](https://img.shields.io/badge/AppVersion-v2.4.2--SNAPSHOT.130.bdb50b7-informational?style=flat-square)
 [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0)
 [![Artifact HUB](https://img.shields.io/endpoint?url=https://artifacthub.io/badge/repository/csi-wekafs)](https://artifacthub.io/packages/search?repo=csi-wekafs)
 
@@ -44,7 +44,7 @@ make build
 |-----|------|---------|-------------|
 | dynamicProvisionPath | string | `"csi-volumes"` | Directory in root of file system where dynamic volumes are provisioned |
 | csiDriverName | string | `"csi.weka.io"` | Name of the driver (and provisioner) |
-| csiDriverVersion | string | `"2.5.0-beta"` | CSI driver version |
+| csiDriverVersion | string | `"2.4.2-SNAPSHOT.130.bdb50b7"` | CSI driver version |
 | images.livenessprobesidecar | string | `"registry.k8s.io/sig-storage/livenessprobe:v2.14.0"` | CSI liveness probe sidecar image URL |
 | images.attachersidecar | string | `"registry.k8s.io/sig-storage/csi-attacher:v4.7.0"` | CSI attacher sidecar image URL |
 | images.provisionersidecar | string | `"registry.k8s.io/sig-storage/csi-provisioner:v5.1.0"` | CSI provisioner sidecar image URL |
@@ -53,7 +53,7 @@ make build
 | images.snapshottersidecar | string | `"registry.k8s.io/sig-storage/csi-snapshotter:v8.1.0"` | CSI snapshotter sidecar image URL |
 | images.nodeinfo | string | `"quay.io/weka.io/csi-wekafs"` | CSI nodeinfo sidecar image URL, used for reading node metadata |
 | images.csidriver | string | `"quay.io/weka.io/csi-wekafs"` | CSI driver main image URL |
-| images.csidriverTag | string | `"2.5.0-beta"` | CSI driver tag |
+| images.csidriverTag | string | `"2.4.2-SNAPSHOT.130.bdb50b7"` | CSI driver tag |
 | imagePullSecret | string | `""` | image pull secret required for image download. Must have permissions to access all images above.    Should be used in case of private registry that requires authentication |
 | globalPluginTolerations | list | `[{"effect":"NoSchedule","key":"node-role.kubernetes.io/master","operator":"Exists"}]` | Tolerations for all CSI driver components |
 | controllerPluginTolerations | list | `[{"effect":"NoSchedule","key":"node-role.kubernetes.io/master","operator":"Exists"}]` | Tolerations for CSI controller component only (by default same as global) |

charts/csi-wekafsplugin/Chart.yaml

@@ -6,12 +6,12 @@ maintainers:
     email: csi@weka.io
     url: https://weka.io
 sources:
-  - https://github.com/weka/csi-wekafs/tree/v2.5.0-beta
+  - https://github.com/weka/csi-wekafs/tree/v$CHART_VERSION/charts/csi-wekafsplugin
 home: https://github.com/weka/csi-wekafs
 icon: https://weka.github.io/csi-wekafs/logo.png
 type: application
-version: 2.5.0-beta
-appVersion: v2.5.0-beta
+version: 2.4.2-SNAPSHOT.130.bdb50b7
+appVersion: v2.4.2-SNAPSHOT.130.bdb50b7
 keywords: [storage, filesystem, HPC]
 annotations:
   artifacthub.io/category: "storage"

charts/csi-wekafsplugin/README.md

@@ -3,7 +3,7 @@ Helm chart for Deployment of WekaIO Container Storage Interface (CSI) plugin for
 
 [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0)
 [![Artifact HUB](https://img.shields.io/endpoint?url=https://artifacthub.io/badge/repository/csi-wekafs)](https://artifacthub.io/packages/search?repo=csi-wekafs)
-![Version: 2.5.0-beta](https://img.shields.io/badge/Version-2.5.0--beta-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: v2.5.0-beta](https://img.shields.io/badge/AppVersion-v2.5.0--beta-informational?style=flat-square)
+![Version: 2.4.2-SNAPSHOT.130.bdb50b7](https://img.shields.io/badge/Version-2.4.2--SNAPSHOT.130.bdb50b7-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: v2.4.2-SNAPSHOT.130.bdb50b7](https://img.shields.io/badge/AppVersion-v2.4.2--SNAPSHOT.130.bdb50b7-informational?style=flat-square)
 
 ## Homepage
 https://github.com/weka/csi-wekafs
@@ -52,7 +52,7 @@ helm install csi-wekafsplugin csi-wekafs/csi-wekafsplugin --namespace csi-wekafs
 |-----|------|---------|-------------|
 | dynamicProvisionPath | string | `"csi-volumes"` | Directory in root of file system where dynamic volumes are provisioned |
 | csiDriverName | string | `"csi.weka.io"` | Name of the driver (and provisioner) |
-| csiDriverVersion | string | `"2.5.0-beta"` | CSI driver version |
+| csiDriverVersion | string | `"2.4.2-SNAPSHOT.130.bdb50b7"` | CSI driver version |
 | images.livenessprobesidecar | string | `"registry.k8s.io/sig-storage/livenessprobe:v2.14.0"` | CSI liveness probe sidecar image URL |
 | images.attachersidecar | string | `"registry.k8s.io/sig-storage/csi-attacher:v4.7.0"` | CSI attacher sidecar image URL |
 | images.provisionersidecar | string | `"registry.k8s.io/sig-storage/csi-provisioner:v5.1.0"` | CSI provisioner sidecar image URL |
@@ -61,7 +61,7 @@ helm install csi-wekafsplugin csi-wekafs/csi-wekafsplugin --namespace csi-wekafs
 | images.snapshottersidecar | string | `"registry.k8s.io/sig-storage/csi-snapshotter:v8.1.0"` | CSI snapshotter sidecar image URL |
 | images.nodeinfo | string | `"quay.io/weka.io/csi-wekafs"` | CSI nodeinfo sidecar image URL, used for reading node metadata |
 | images.csidriver | string | `"quay.io/weka.io/csi-wekafs"` | CSI driver main image URL |
-| images.csidriverTag | string | `"2.5.0-beta"` | CSI driver tag |
+| images.csidriverTag | string | `"2.4.2-SNAPSHOT.130.bdb50b7"` | CSI driver tag |
 | imagePullSecret | string | `""` | image pull secret required for image download. Must have permissions to access all images above.    Should be used in case of private registry that requires authentication |
 | globalPluginTolerations | list | `[{"effect":"NoSchedule","key":"node-role.kubernetes.io/master","operator":"Exists"}]` | Tolerations for all CSI driver components |
 | controllerPluginTolerations | list | `[{"effect":"NoSchedule","key":"node-role.kubernetes.io/master","operator":"Exists"}]` | Tolerations for CSI controller component only (by default same as global) |

charts/csi-wekafsplugin/values.yaml

@@ -5,7 +5,7 @@ dynamicProvisionPath: "csi-volumes"
 # -- Name of the driver (and provisioner)
 csiDriverName: "csi.weka.io"
 # -- CSI driver version
-csiDriverVersion: &csiDriverVersion 2.5.0-beta
+csiDriverVersion: &csiDriverVersion 2.4.2-SNAPSHOT.130.bdb50b7
 images:
   # -- CSI liveness probe sidecar image URL
   livenessprobesidecar: registry.k8s.io/sig-storage/livenessprobe:v2.14.0

docs/NFS.md

@@ -47,6 +47,24 @@ the Weka CSI Plugin will translate them to NFS alternatives. Unknown or unsuppor
 ### QoS and Performance
 QoS is not supported for NFS transport. Performance is limited by the NFS protocol and the network configuration.
 
+### Scheduling Workloads by Transport Protocol
+Weka CSI Plugin node server populates the transport protocol in the node topology.
+This allows you to use node affinity rules to force workloads to use WekaFS transport on nodes where the Weka client is installed.
+For example, in hybrid deployments where some nodes have the Weka client installed and some do not, 
+the administrator would prefer storage-intensive workloads to run on nodes with the Weka client installed, while more generic workloads can run on other nodes.
+
+In order to force a workload to a certain transport type, a nodeSelector can be applied to the workload manifest.
+#### An example of a nodeSelector for WekaFS transport:
+```yaml
+nodeSelector:
+  topology.weka.com/transport: wekafs
+```
+#### An example of a nodeSelector for NFS transport:
+```yaml
+nodeSelector:
+  topology.weka.com/transport: nfs
+```
+
 ### Switching from NFS to WekaFS transport
 To switch between NFS and WekaFS transport, you need to:
 1. Install the Weka client on Kubernetes node
@@ -126,6 +144,16 @@ The Weka CSI Plugin requires AND will set the following NFS permissions on the W
 > Hence, it is **highly recommended** not creating additional permissions for the same filesystem 
 > Also, if multiple client groups are used, it is highly recommended to make sure that IP addresses are not overlapping between client groups. 
 
+## WEKA Cluster Preparation
+Before using the Weka CSI Plugin with NFS transport, the Weka cluster must be prepared for NFS access.
+This includes configuring the NFS protocol on the Weka cluster, creating an NFS interface group, and configuring at least 1 Group IP address
+
+Alternatively, in cloud deployments where setting a Group IP address is not possible, the Weka server IP addresses can be used instead.
+In such case, the IP addresses may be set via the API secret and will be used instead of the Group IP addresses.
+
+This can be set up by providing `nfsTargetIps` parameter in the API secret. Refer to the [API secret example](../examples/common/csi-wekafs-api-secret.yaml) for more information.
+> **WARNING:** Using an NFS load balancer that forwards NFS connection to multiple Weka servers is not supported at this moment.
+
 ## Installation
 By default, Weka CSI Plugin components will not start unless Weka driver is not detected on Kubernetes node.
 This is to prevent a potential misconfiguration where volumes are attempted to be provisioned or published on node while no Weka client is installed.
@@ -136,7 +164,7 @@ This is done by setting the `pluginConfig.mountProtocol.allowNfsFailback` parame
 The parameter `pluginConfig.mountProtocol.useNfs` enforces the use of NFS transport even if Weka client is installed on the node, 
 and recommended to be set to `true` ONLY for testing.
 
-Follow the [Helm installation instructions](./charts/csi-wekafsplugin/README.md) to install the Weka CSI Plugin. 
+Follow the [Helm installation instructions](../charts/csi-wekafsplugin/README.md) to install the Weka CSI Plugin. 
 Most of the installation steps are the same as for the native WekaFS driver, however, additional parameters should be set in the `values.yaml` file,
 or passed as command line arguments to the `helm install` command.
 

docs/usage.md

@@ -352,7 +352,7 @@ pvc-ee54de25-14f3-4024-98d0-12225e4b8215   4Gi        RWX            Delete
     <REPLACE spec.resources.requests.storage value with 4Gi>
     ```
 
-1. Check that configuration was applied
+2. Check that configuration was applied
     ```shell script
     $ kubectl get pv
     NAME                                       CAPACITY   ACCESS MODES   RECLAIM POLICY   STATUS   CLAIM                    STORAGECLASS               REASON   AGE

examples/common/csi-wekafs-api-secret.yaml

@@ -25,6 +25,13 @@ data:
   # NOTE: if a load balancer is used to access the cluster API, leave this setting as "false"
   autoUpdateEndpoints: ZmFsc2U=
   # When using HTTPS connection and self-signed or untrusted certificates, provide a CA certificate in PEM format, base64-encoded
+  # for cloud deployments or other scenarios where setting an NFS Group IP addresses is not possible,
+  # provide a comma-separated list of NFS target IP addresses in form of <IP> (base64-encoded)
+  # It is recommended to configure all NFS server IP addresses to better share the load/balance the traffic.
+  # NOTE: this setting is optional and should be used only when the NFS Group IP addresses are not set in the cluster
+  # WARNING: providing a load balancer IP address that uses NFS connection redirects to other servers is not supported.
+  # e.g. 10.100.100.1,10.100.100.2
+  nfsTargetIps: ""
   # caCertificate: <base64-encoded-PEM>
   caCertificate: ""
 

go.mod

@@ -10,7 +10,7 @@ require (
 	github.com/kubernetes-csi/csi-lib-utils v0.19.0
 	github.com/pkg/errors v0.9.1
 	github.com/pkg/xattr v0.4.10
-	github.com/prometheus/client_golang v1.20.2
+	github.com/prometheus/client_golang v1.20.4
 	github.com/rs/zerolog v1.33.0
 	github.com/showa-93/go-mask v0.6.2
 	github.com/stretchr/testify v1.9.0
@@ -20,7 +20,7 @@ require (
 	go.opentelemetry.io/otel/trace v1.29.0
 	golang.org/x/exp v0.0.0-20240823005443-9b4947da3948
 	golang.org/x/sync v0.8.0
-	google.golang.org/grpc v1.66.0
+	google.golang.org/grpc v1.67.0
 	google.golang.org/protobuf v1.34.2
 	k8s.io/apimachinery v0.31.0
 	k8s.io/helm v2.17.0+incompatible

go.sum

@@ -63,6 +63,8 @@ github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRI
 github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/prometheus/client_golang v1.20.2 h1:5ctymQzZlyOON1666svgwn3s6IKWgfbjsejTMiXIyjg=
 github.com/prometheus/client_golang v1.20.2/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE=
+github.com/prometheus/client_golang v1.20.4 h1:Tgh3Yr67PaOv/uTqloMsCEdeuFTatm5zIq5+qNN23vI=
+github.com/prometheus/client_golang v1.20.4/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE=
 github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E=
 github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY=
 github.com/prometheus/common v0.57.0 h1:Ro/rKjwdq9mZn1K5QPctzh+MA4Lp0BuYk5ZZEVhoNcY=
@@ -114,6 +116,8 @@ google.golang.org/genproto/googleapis/rpc v0.0.0-20240827150818-7e3bb234dfed h1:
 google.golang.org/genproto/googleapis/rpc v0.0.0-20240827150818-7e3bb234dfed/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU=
 google.golang.org/grpc v1.66.0 h1:DibZuoBznOxbDQxRINckZcUvnCEvrW9pcWIE2yF9r1c=
 google.golang.org/grpc v1.66.0/go.mod h1:s3/l6xSSCURdVfAnL+TqCNMyTDAGN6+lZeVxnZR128Y=
+google.golang.org/grpc v1.67.0 h1:IdH9y6PF5MPSdAntIcpjQ+tXO41pcQsfZV2RxtQgVcw=
+google.golang.org/grpc v1.67.0/go.mod h1:1gLDyUQU7CTLJI90u3nXZ9ekeghjeM7pTDZlqFNg2AA=
 google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg=
 google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

pkg/wekafs/apiclient/apiclient.go

@@ -773,6 +773,7 @@ type Credentials struct {
 	LocalContainerName  string
 	AutoUpdateEndpoints bool
 	CaCertificate       string
+	NfsTargetIPs        []string
 }
 
 func (c *Credentials) String() string {

pkg/wekafs/apiclient/filesystem.go

@@ -214,6 +214,31 @@ func (a *ApiClient) DeleteFileSystem(ctx context.Context, r *FileSystemDeleteReq
 	return nil
 }
 
+func (a *ApiClient) EnsureNoNfsPermissionsForFilesystem(ctx context.Context, fsName string) error {
+	logger := log.Ctx(ctx)
+	logger.Trace().Str("filesystem", fsName).Msg("Ensuring no NFS permissions for filesystem")
+	permissions := &[]NfsPermission{}
+	err := a.FindNfsPermissionsByFilesystem(ctx, fsName, permissions)
+	if err != nil {
+		logger.Error().Err(err).Str("filesystem", fsName).Msg("Failed to list NFS permissions")
+	}
+	if len(*permissions) > 0 {
+		logger.Debug().Int("permissions", len(*permissions)).Str("filesystem", fsName).Msg("Found stale NFS permissions, deleting")
+	}
+	for _, p := range *permissions {
+		err = a.DeleteNfsPermission(ctx, &NfsPermissionDeleteRequest{Uid: p.Uid})
+		if err != nil {
+			logger.Error().Err(err).Str("permission", p.Uid.String()).Str("filesystem", p.Filesystem).Str("client_group", p.Group).Msg("Failed to delete NFS permission")
+			return err
+		}
+	}
+	if len(*permissions) > 0 {
+		time.Sleep(time.Second * 5) // wait for NFS permissions reconfiguration
+		logger.Trace().Str("filesystem", fsName).Msg("Deleted NFS permissions")
+	}
+	return nil
+}
+
 func (a *ApiClient) GetFileSystemMountToken(ctx context.Context, r *FileSystemMountTokenRequest, token *FileSystemMountToken) error {
 	op := "GetFileSystemMountToken"
 	ctx, span := otel.Tracer(TracerName).Start(ctx, op)
@@ -260,6 +285,7 @@ func (fs *FileSystem) GetType() string {
 	return "filesystem"
 }
 
+//goland:noinspection GoUnusedParameter
 func (fs *FileSystem) GetBasePath(a *ApiClient) string {
 	return "fileSystems"
 }