Skip to content

Commit e13cb44

Browse files
shivamerlashivamerla
authored
Operator fixes (#110)
* operator: misc fixes * Problem: * fix cluster roles in helm charts for 1.14 and above * fix scc to allow access to host ports for CSP pod * add script to delete all artifacts for operator/csi * update helm-operator image to use helm3 * Implementation: * fixed above issues * Testing: tested on ocp 4.2 with installation/deletion/pvc/pod provisioning. * Review: rkumar, gcostea * Bug: https://nimblejira.nimblestorage.com/browse/NLT- Signed-off-by: Shiva Krishna, Merla <shivakrishna.merla@hpe.com>
1 parent 6bdae97 commit e13cb44

15 files changed

+155
-299
lines changed

‎.gitignore

+2-1
Original file line numberDiff line numberDiff line change
@@ -17,4 +17,5 @@ build/
1717
*.out
1818

1919
# helm-charts to build an operator
20-
operators/*/helm-charts
20+
operators/*/helm-charts
21+
operators/*/LICENSE

‎docs/hpe-csi-driver-1.1.0-beta.tgz

-73 Bytes
Binary file not shown.

‎docs/index.yaml

+7-7
Original file line numberDiff line numberDiff line change
@@ -3,9 +3,9 @@ entries:
33
hpe-csi-driver:
44
- apiVersion: v1
55
appVersion: 1.1.0-beta
6-
created: "2020-02-03T11:47:56.434632-08:00"
6+
created: "2020-02-08T15:26:10.759395-08:00"
77
description: A Helm chart for installing the HPE CSI Driver for Kubernetes
8-
digest: 426d98342fd2573982ab70f01e5c9e93e030543008ca3b0b397c6543c1a0c5f0
8+
digest: a15d877cd347c3d7c3007f20995b08d0f656dc0fcf86201ce5a369b2f7e85e1b
99
home: https://hpe.com/storage/containers
1010
icon: https://raw.githubusercontent.com/hpe-storage/co-deployments/master/docs/assets/hpedev.png
1111
keywords:
@@ -25,7 +25,7 @@ entries:
2525
version: 1.1.0-beta
2626
- apiVersion: v1
2727
appVersion: "1.0"
28-
created: "2020-02-03T11:47:56.433954-08:00"
28+
created: "2020-02-08T15:26:10.758796-08:00"
2929
description: A Helm chart for installing the HPE CSI Driver for Kubernetes
3030
digest: 0370a3eb966b1665c1d612ca78e9ad254960f9e10f34ff8a3a926870f241fc6e
3131
home: https://hpe.com/storage/containers
@@ -47,7 +47,7 @@ entries:
4747
version: 1.0.1
4848
- apiVersion: v1
4949
appVersion: "1.0"
50-
created: "2020-02-03T11:47:56.426965-08:00"
50+
created: "2020-02-08T15:26:10.757549-08:00"
5151
description: A Helm chart for installing the HPE CSI Driver for Kubernetes
5252
digest: 14fe529034fb3bc806639630424bdd11ba2c259dc363014d30a8a8d6b5830cae
5353
home: https://hpe.com/storage/containers
@@ -70,7 +70,7 @@ entries:
7070
hpe-flexvolume-driver:
7171
- apiVersion: v1
7272
appVersion: "3.1"
73-
created: "2020-02-03T11:47:56.441454-08:00"
73+
created: "2020-02-08T15:26:10.760574-08:00"
7474
description: A Helm chart for installing the HPE Volume Driver for Kubernetes
7575
FlexVolume plugin
7676
digest: 3b481fc1ad9bc923774563557dbc26fd94c581924b406180cccafb6b77744633
@@ -94,7 +94,7 @@ entries:
9494
version: 3.1.0
9595
- apiVersion: v1
9696
appVersion: "3.0"
97-
created: "2020-02-03T11:47:56.439383-08:00"
97+
created: "2020-02-08T15:26:10.759899-08:00"
9898
description: A Helm chart for installing the HPE Volume Driver for Kubernetes
9999
FlexVolume plugin
100100
digest: 523ad3c654d697e6350cde6cfee346f056373585c7bcce9b2125b9fb9cd92289
@@ -115,4 +115,4 @@ entries:
115115
urls:
116116
- hpe-flexvolume-driver-3.0.0.tgz
117117
version: 3.0.0
118-
generated: "2020-02-03T11:47:56.423326-08:00"
118+
generated: "2020-02-08T15:26:10.756651-08:00"

‎helm/charts/hpe-csi-driver/templates/hpe-csi-rbac.yaml

+9-46
Original file line numberDiff line numberDiff line change
@@ -71,70 +71,33 @@ apiVersion: rbac.authorization.k8s.io/v1
7171
metadata:
7272
name: hpe-csi-attacher-role
7373
rules:
74-
{{- if semverCompare "^1.12.0" .Capabilities.KubeVersion.GitVersion }}
7574
- apiGroups: [""]
7675
resources: ["persistentvolumes"]
77-
verbs: ["get", "list", "watch", "update"]
78-
- apiGroups: [""]
79-
resources: ["nodes"]
80-
verbs: ["get", "list", "watch"]
81-
- apiGroups: ["csi.storage.k8s.io"]
82-
resources: ["csinodeinfos"]
83-
verbs: ["get", "list", "watch"]
84-
- apiGroups: ["storage.k8s.io"]
85-
resources: ["volumeattachments"]
86-
verbs: ["get", "list", "watch", "update"]
87-
{{- else if semverCompare "^1.13.0" .Capabilities.KubeVersion.GitVersion }}
88-
- apiGroups: [""]
89-
resources: ["persistentvolumes"]
90-
verbs: ["get", "list", "watch", "update"]
76+
verbs: ["get", "list", "watch", "update", "patch"]
9177
- apiGroups: [""]
9278
resources: ["nodes"]
9379
verbs: ["get", "list", "watch"]
94-
- apiGroups: ["csi.storage.k8s.io"]
95-
resources: ["csinodeinfos"]
96-
verbs: ["get", "list", "watch"]
9780
- apiGroups: ["storage.k8s.io"]
9881
resources: ["volumeattachments"]
99-
verbs: ["get", "list", "watch", "update"]
82+
verbs: ["get", "list", "watch", "update", "patch"]
10083
- apiGroups: ["storage.k8s.io"]
10184
resources: ["volumeattachments/status"]
10285
verbs: ["get", "list", "watch", "update", "create", "delete"]
10386
- apiGroups: [""]
10487
resources: ["secrets"]
10588
verbs: ["get", "watch", "list"]
106-
- apiGroups: ["storage.hpe.com"]
107-
resources: ["hpenodeinfos"]
108-
verbs: ["get", "list", "watch", "create", "update", "patch"]
109-
{{- else }}
110-
- apiGroups: [""]
111-
resources: ["nodes"]
89+
{{- if semverCompare "~1.12.0" .Capabilities.KubeVersion.GitVersion }}
90+
resources: ["csinodeinfos"]
91+
verbs: ["get", "list", "watch"]
92+
{{- else if semverCompare "~1.13.0" .Capabilities.KubeVersion.GitVersion }}
93+
- apiGroups: ["csi.storage.k8s.io"]
94+
resources: ["csinodeinfos"]
11295
verbs: ["get", "list", "watch"]
96+
{{ else }}
11397
- apiGroups: ["storage.k8s.io"]
11498
resources: ["csinodes"]
11599
verbs: ["get", "list", "watch"]
116-
- apiGroups: ["storage.k8s.io"]
117-
resources: ["volumeattachments/status"]
118-
verbs: ["get", "list", "watch", "update", "create", "delete"]
119-
- apiGroups: [""]
120-
resources: ["secrets"]
121-
verbs: ["get", "watch", "list"]
122-
{{- if semverCompare ">=1.17.0" .Capabilities.KubeVersion.GitVersion }}
123-
- apiGroups: [""]
124-
resources: ["persistentvolumes"]
125-
verbs: ["get", "list", "watch", "update", "patch"]
126-
- apiGroups: ["storage.k8s.io"]
127-
resources: ["volumeattachments"]
128-
verbs: ["get", "list", "watch", "update", "patch"]
129-
{{- else }}
130-
- apiGroups: [""]
131-
resources: ["persistentvolumes"]
132-
verbs: ["get", "list", "watch", "update"]
133-
- apiGroups: ["storage.k8s.io"]
134-
resources: ["volumeattachments"]
135-
verbs: ["get", "list", "watch", "update"]
136100
{{- end }}
137-
{{- end }}
138101

139102
---
140103

‎operators/hpe-csi-operator/Dockerfile

+13-2
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,15 @@
1-
FROM quay.io/operator-framework/helm-operator:v0.10.1
2-
LABEL name="csi-driver-operator" vendor="HPE" version="1.0.0" release="1.0" summary="HPE CSI Operator" description="HPE CSI Driver Operator"
1+
FROM quay.io/operator-framework/helm-operator:v0.15.1
2+
LABEL name="csi-driver-operator" \
3+
maintainer="HPE Storage" \
4+
vendor="HPE" \
5+
version="1.0.0" \
6+
release="1.0" \
7+
summary="HPE CSI Operator" \
8+
description="HPE CSI Driver Operator" \
9+
io.k8s.display-name="HPE CSI Driver Operator for Kubernetes" \
10+
io.k8s.description="The HPE CSI Driver Operator for Kubernetes enables container orchestrators, such as Kubernetes and OpenShift, to manage the life-cycle of persistent storage." \
11+
io.openshift.tags=hpe,csi,hpe-csi-driver
12+
313
COPY helm-charts/ ${HOME}/helm-charts/
414
COPY watches.yaml ${HOME}/watches.yaml
15+
COPY LICENSE /licenses/

‎operators/hpe-csi-operator/README.md

+49-50
Original file line numberDiff line numberDiff line change
@@ -12,83 +12,82 @@ For platform dependencies for HPE CSI driver please refer to [prerequisites](htt
1212

1313
## Installation
1414

15-
Clone this GitHub repository.
15+
For OCP create SecurityContextConstraints with privileges required for CSI driver
1616
```
17-
git clone https://github.com/hpe-storage/co-deployments
18-
cd operators/hpe-csi-operator
17+
oc deploy -f https://raw.githubusercontent.com/hpe-storage/co-deployments/master/operators/hpe-csi-operator/deploy/scc.yaml -n hpe-csi
1918
```
2019

21-
Create your own `values.yaml`. The easiest way is to copy the default [./values.yaml](../../helm/charts/hpe-csi-driver/values.yaml) with `wget` and change parameters like `backend` as necessary.
20+
*** NOTE *** If you are using OpenShift, replace `kubectl` with `oc` below.
2221

23-
Run the install script to set up the HPE CSI Operator.
24-
```install.sh --image=<image> --namespace=<namespace> --flavor=<co flavor> --values=<values.yaml file path>```
25-
26-
Parameter list:<br/>
27-
1. ``image`` is the HPE CSI Operator image. If unspecified ``image`` resolves to the released version at [hpestorage/hpe-csi-operator](https://hub.docker.com/repository/docker/hpestorage/csi-driver-operator).
28-
2. ``namespace`` is the namespace/project in which the HPE CSI Operator and its entities will be installed. If unspecified, the operator creates and installs in the ``hpe-csi`` namespace.
29-
**HPE CSI Operator MUST be installed in a new project with no other pods. Otherwise an uninstall may delete pods that are not related to the HPE CSI Operator.**
30-
3. ``flavor`` defaults to ``k8s``. Options are ``k8s``, ``kubernetes``, ``ocp`` or ``openshift``.
31-
4. ``values.yaml`` is the customized helm-chart configuration parameters. This is a **required parameter** and must contain a valid backend HPE storage system. All parameters that need a non-default value must be specified in this file.
32-
Refer to [Configuration for values.yaml.](https://github.com/hpe-storage/co-deployments/tree/master/helm/charts/hpe-csi-driver#configuration--installation) for details about various parameters.
33-
34-
**Note:** HPE CSI Driver for Kubernetes automatically configures Linux iSCSI/Multipath settings based on [config.json](https://raw.githubusercontent.com/hpe-storage/co-deployments/master/helm/charts/hpe-csi-driver/files/config.json). In order to tune these values, edit the config map with `kubectl edit configmap hpe-linux-config -n hpe-csi` and restart node plugin using `kubectl delete pod -l app=hpe-csi-node` to apply.
35-
36-
### Install script steps:
37-
The install script will do the following:
38-
1. Create New Project.<br/>
39-
The script creates a new project (if it does not already exist) with the given namespace. If no namespace parameter is specified, the ``hpe-csi`` namespace is used.<br/>
40-
41-
2. Create a Custom Resource Definition (CRD) for the HPE CSI Operator. <br/>
42-
The script waits for the CRD to be published in the cluster. If after 10 seconds the API server has not setup the CRD, the script times out.
22+
Deploy Operator/RBAC and CRD's required
23+
```
24+
kubectl deploy -f https://raw.githubusercontent.com/hpe-storage/co-deployments/master/operators/hpe-csi-operator/deploy/rbac.yaml -n hpe-csi
25+
kubectl deploy -f https://raw.githubusercontent.com/hpe-storage/co-deployments/master/operators/hpe-csi-operator/deploy/operator.yaml -n hpe-csi
26+
kubectl deploy -f https://raw.githubusercontent.com/hpe-storage/co-deployments/master/operators/hpe-csi-operator/deploy/crds/storage.hpe.com_hpecsidrivers_crd.yaml -n hpe-csi
27+
```
4328

44-
3. Create RBAC rules for the Operator.<br/>
45-
The HPE CSI Operator needs the following Cluster-level Roles and RoleBindings.
29+
Fetch and update CustomResource of type `HPECSIDriver` with required values
30+
```
31+
curl -sL https://raw.githubusercontent.com/hpe-storage/co-deployments/master/operators/hpe-csi-operator/deploy/crds/storage.hpe.com_hpecsidrivers_cr.yaml
32+
```
4633

34+
Deploy above updated CustomResource `csi-driver`
35+
```
36+
kubectl deploy -f storage.hpe.com_hpecsidrivers_cr.yaml -n hpe-csi
37+
```
4738

48-
| Resource | Permissions | Notes |
49-
| ------------- |:-------------:| -----:|
50-
| Namespace | Get | HPE CSI Operator needs the ability to get created namespaces |
51-
| Storageclass | Create/Delete | Create and cleanup storage classes to be used for Provisioning |
52-
| ClusterRoleBinding | Create/Delete/Get | HPE Operator needs to create and cleanup a ClusterRoleBinding used by the external-provisioner/external-attacher/external-snapshotter/external-resizer sidecars |
53-
<br/>
39+
where ``hpe-csi`` is the project/namespace in which the HPE CSI Operator is installed. It is **strongly recommended** to install the HPE CSI Operator in a new project and not add any other pods to this project/namespace. Any pods in this project will be cleaned up on an uninstall.
5440

55-
In addition, the operator needs access to multiple resources in the project/namespace that it is deployed in to function correctly. Hence it is recommended to install the HPE CSI Operator in the non-default namespace.
56-
<br/>
41+
## Upgrading
5742

58-
4. Creates a deployment for the Operator.<br/>
59-
Finally the script creates and deploys the operator using the customized parameters passed in the ``values.yaml`` file.
43+
Fetch and update CustomResource of type `HPECSIDriver` with required values
44+
```
45+
curl -sL https://raw.githubusercontent.com/hpe-storage/co-deployments/master/operators/hpe-csi-operator/deploy/crds/storage.hpe.com_hpecsidrivers_cr.yaml
46+
```
6047

61-
## Upgrading
48+
Deploy updated CustomResource `csi-driver`
49+
```
50+
kubectl deploy -f storage.hpe.com_hpecsidrivers_cr.yaml -n hpe-csi
51+
```
6252

6353
### How to upgrade from helm install to HPE CSI Operator
6454
This upgrade will not impact the in-use volumes/filesystems from data path perspective. However, it will affect the in-flight volume/filesystem management operations. So, it is recommended to stop all the volume/filesystem management operations before doing this upgrade. Otherwise, these operations may need to be retried after the upgrade.
6555

6656
Remove the helm-chart using instructions in https://helm.sh/docs/using_helm/#uninstall-a-release.
6757
Once the helm chart has been uninstalled, follow the install instructions [above.](#installation)
6858

69-
### Apply changes in ``values.yaml``
70-
The ``update.sh`` script is used to apply changes from ``values.yaml`` as follows.
59+
## Uninstall
60+
61+
*** NOTE *** If you are using OpenShift, replace `kubectl` with `oc`.
62+
63+
1. Delete the HPE CSI Driver custom resource, this will cause our CSI plugin resources to be cleaned up.
7164
```
72-
./update.sh -f values.yaml
65+
kubectl delete -f https://raw.githubusercontent.com/hpe-storage/co-deployments/master/operators/hpe-csi-operator/deploy/crds/storage.hpe.com_hpecsidrivers_cr.yaml -n hpe-csi
66+
67+
kubectl delete -f https://raw.githubusercontent.com/hpe-storage/co-deployments/master/operators/hpe-csi-operator/deploy/crds/storage.hpe.com_hpecsidrivers_crd.yaml -n hpe-csi
7368
```
7469

75-
## Uninstall
76-
To uninstall the HPE CSI Operator, run
70+
If the CRD fails to delete you may be experiencing a known issue. Resolve this by running:
7771
```
78-
kubectl delete all --all -n <hpe-csi-operator-installed-namespace>
72+
kubectl patch crd/hpecsidrivers.storage.hpe.com -p '{"metadata":{"finalizers":[]}}' --type=merge
7973
```
80-
where ``hpe-csi-operator-installed-namespace`` is the project/namespace in which the HPE CSI Operator is installed. It is **strongly recommended** to install the HPE CSI Operator in a new project and not add any other pods to this project/namespace. Any pods in this project will be cleaned up on an uninstall.
8174

82-
If you are using OpenShift, replace `kubectl` with `oc`.
83-
To completely remove the CustomResourceDefinition used by the Operator run
75+
2. Delete all cluster level roles and bindings for operator
8476
```
85-
kubectl delete crd hpecsidrivers.storage.hpe.com
77+
kubectl delete -f https://raw.githubusercontent.com/hpe-storage/co-deployments/master/operators/hpe-csi-operator/deploy/rbac.yaml -n hpe-csi
8678
```
87-
If the CRD fails to delete you may be experiencing a known issue. Resolve this by running:
79+
80+
For OpenShift, delete SecurityContextConstraints created
8881
```
89-
kubectl patch crd/hpecsidrivers.storage.hpe.com -p '{"metadata":{"finalizers":[]}}' --type=merge
82+
kubectl delete -f -f https://raw.githubusercontent.com/hpe-storage/co-deployments/master/operators/hpe-csi-operator/deploy/scc.yaml -n hpe-csi
9083
```
91-
If you are using OpenShift, replace `kubectl` with `oc` in the above commands.
84+
85+
3. Delete operator deployment itself
86+
```
87+
kubectl delete -f https://raw.githubusercontent.com/hpe-storage/co-deployments/master/operators/hpe-csi-operator/deploy/operator.yaml -n hpe-csi
88+
```
89+
90+
where ``hpe-csi`` is the project/namespace in which the HPE CSI Operator is installed. It is **strongly recommended** to install the HPE CSI Operator in a new project and not add any other pods to this project/namespace. Any pods in this project will be cleaned up on an uninstall.
9291

9392
## License
9493
This is open source software licensed using the Apache License 2.0. Please see [LICENSE](../../LICENSE) for details.

‎operators/hpe-csi-operator/build.sh

+2
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,8 @@ IMG_DIR=$(dirname $0)
1313
HELM_DIR=${IMG_DIR}/../../helm
1414
mkdir -p ${IMG_DIR}/helm-charts
1515

16+
cp ${IMG_DIR}/../../LICENSE ${IMG_DIR}/
17+
1618
# Copy helm charts to staging directory
1719
cp -r ${HELM_DIR}/charts/hpe-csi-driver ${IMG_DIR}/helm-charts
1820

‎operators/hpe-csi-operator/deploy/cr.yaml

-8
This file was deleted.

‎operators/hpe-csi-operator/deploy/crd.yaml ‎operators/hpe-csi-operator/deploy/crds/storage.hpe.com_hpecsidrivers_crd.yaml

+6-3
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,3 @@
1-
---
21
apiVersion: apiextensions.k8s.io/v1beta1
32
kind: CustomResourceDefinition
43
metadata:
@@ -11,9 +10,13 @@ spec:
1110
plural: hpecsidrivers
1211
singular: hpecsidriver
1312
scope: Namespaced
13+
subresources:
14+
status: {}
15+
validation:
16+
openAPIV3Schema:
17+
type: object
18+
x-kubernetes-preserve-unknown-fields: true
1419
versions:
1520
- name: v1
1621
served: true
1722
storage: true
18-
subresources:
19-
status: {}

‎operators/hpe-csi-operator/deploy/crds/storage.hpe.com_v1_hpecsidriver_cr.yaml

+54
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
apiVersion: storage.hpe.com/v1
2+
kind: HPECSIDriver
3+
metadata:
4+
name: csi-driver
5+
spec:
6+
# Default values copied from <project_dir>/helm-charts/hpe-csi-driver/values.yaml
7+
8+
CRD:
9+
nodeInfo:
10+
create: false
11+
accessProtocol: iscsi
12+
allowVolumeExpansion: true
13+
backend: 192.168.1.1
14+
csiAttacherImage: quay.io/k8scsi/csi-attacher
15+
csiAttacherTagv0: v0.4.2
16+
csiAttacherTagv1: v1.1.0
17+
csiAttacherTagv2: v1.2.0
18+
csiDriverImage: hpestorage/csi-driver
19+
csiDriverTag: v1.1.0-beta
20+
csiProvisionerImage: quay.io/k8scsi/csi-provisioner
21+
csiProvisionerTagv0: v0.4.3
22+
csiProvisionerTagv1: v1.4.0
23+
csiResizerImage: quay.io/k8scsi/csi-resizer
24+
csiResizerTag: v0.3.0
25+
csiSnapshotterImage: quay.io/k8scsi/csi-snapshotter
26+
csiSnapshotterTagv0: v0.4.2
27+
csiSnapshotterTagv1: v1.2.2
28+
csiSnapshotterTagv2: v1.2.2
29+
cspImage: hpestorage/nimble-csp
30+
cspName: nimble-csp
31+
cspTag: v1.1.0-beta
32+
flavor: kubernetes
33+
fsType: xfs
34+
imagePullPolicy: Always
35+
logLevel: info
36+
nodeRegistrarImage: quay.io/k8scsi/csi-node-driver-registrar
37+
nodeRegistrarImagev0: quay.io/k8scsi/driver-registrar
38+
nodeRegistrarTagv0: v0.4.1
39+
nodeRegistrarTagv1: v1.1.0
40+
nodeRegistrarTagv2: v1.1.0
41+
password: admin
42+
postInstallImage: alpine
43+
postInstallTag: "3.3"
44+
secretName: nimble-secret
45+
serviceName: nimble-csp-svc
46+
servicePort: "8080"
47+
serviceWaitTime: "10"
48+
storageClass:
49+
create: true
50+
defaultClass: false
51+
name: hpe-standard
52+
username: admin
53+
volumeDescription: Volume created by the HPE CSI Driver for Kubernetes
54+

0 commit comments

Comments
 (0)