From 56c6887c0a8f71509060bec2d0464f2da2f8ebcc Mon Sep 17 00:00:00 2001 From: Anvith KS Date: Sun, 22 Aug 2021 14:28:08 +0530 Subject: [PATCH 1/7] Added Helm charts for deployment. --- helm/esdk/.helmignore | 23 +++ helm/esdk/Chart.yaml | 31 ++++ helm/esdk/templates/huawei-csi-configmap.yaml | 10 ++ .../esdk/templates/huawei-csi-controller.yaml | 117 +++++++++++++++ helm/esdk/templates/huawei-csi-node.yaml | 137 ++++++++++++++++++ helm/esdk/values.yaml | 19 +++ 6 files changed, 337 insertions(+) create mode 100644 helm/esdk/.helmignore create mode 100644 helm/esdk/Chart.yaml create mode 100644 helm/esdk/templates/huawei-csi-configmap.yaml create mode 100644 helm/esdk/templates/huawei-csi-controller.yaml create mode 100644 helm/esdk/templates/huawei-csi-node.yaml create mode 100644 helm/esdk/values.yaml diff --git a/helm/esdk/.helmignore b/helm/esdk/.helmignore new file mode 100644 index 00000000..0e8a0eb3 --- /dev/null +++ b/helm/esdk/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/helm/esdk/Chart.yaml b/helm/esdk/Chart.yaml new file mode 100644 index 00000000..6c4264fc --- /dev/null +++ b/helm/esdk/Chart.yaml @@ -0,0 +1,31 @@ +apiVersion: v2 +name: esdk +description: Huawei Container Storage Interface (CSI) Driver + +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 0.1.0 + +# compatible Kubernetes versions, helm installation fails fail if the cluster runs an unsupported Kubernetes version +# kubeVersion: >= 1.13.0 < 1.19.0 +# kubeVersion: 1.17.7 +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +# It is recommended to use it with quotes. +appVersion: "2.2.13" + +home: https://github.com/Huawei/eSDK_K8S_Plugin +sources: + - https://github.com/Huawei/eSDK_K8S_Plugin diff --git a/helm/esdk/templates/huawei-csi-configmap.yaml b/helm/esdk/templates/huawei-csi-configmap.yaml new file mode 100644 index 00000000..ecc9447a --- /dev/null +++ b/helm/esdk/templates/huawei-csi-configmap.yaml @@ -0,0 +1,10 @@ +kind: ConfigMap +apiVersion: v1 +metadata: + name: huawei-csi-configmap + namespace: kube-system +data: + csi.json: | + {{ $length := len .Values.backends }} {{ if gt $length 0 }} { {{ end }} + "backends": {{ .Values.backends | toJson }} + {{ $length := len .Values.backends }} {{ if gt $length 0 }} } {{ end }} \ No newline at end of file diff --git a/helm/esdk/templates/huawei-csi-controller.yaml b/helm/esdk/templates/huawei-csi-controller.yaml new file mode 100644 index 00000000..ce6b1fe9 --- /dev/null +++ b/helm/esdk/templates/huawei-csi-controller.yaml @@ -0,0 +1,117 @@ +--- +kind: ServiceAccount +apiVersion: v1 +metadata: + name: huawei-csi-controller + namespace: kube-system +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: huawei-csi-provisioner-runner +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "create", "delete", "patch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots"] + verbs: ["list", "watch", "create", "update", "patch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents"] + verbs: ["get", "list", "watch", "create", "delete", "patch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots/status"] + verbs: ["update"] + - apiGroups: ["storage.k8s.io"] + resources: ["csinodes"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] + +--- +kind: Deployment +apiVersion: apps/v1 +metadata: + name: huawei-csi-controller + namespace: kube-system +spec: + replicas: 1 + selector: + matchLabels: + app: huawei-csi-controller + template: + metadata: + labels: + app: huawei-csi-controller + spec: + serviceAccount: huawei-csi-controller + hostNetwork: true + containers: + - name: csi-provisioner + image: {{ .Values.images.sidecar.provisioner }} + args: + - "--csi-address=$(ADDRESS)" + - "--timeout=6h" + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + imagePullPolicy: {{ .Values.imagePullPolicy }} + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ + + - name: csi-attacher + image: {{ .Values.images.sidecar.attacher }} + args: + - "--csi-address=$(ADDRESS)" + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + imagePullPolicy: {{ .Values.imagePullPolicy }} + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ + - name: huawei-csi-driver + image: {{ required "Must provide the CSI controller service container image." .Values.images.huaweiCsiControllerService }} + args: + - "--endpoint=$(CSI_ENDPOINT)" + - "--controller" + - "--containerized" + - "--backend-update-interval=120" + - "--driver-name=csi.huawei.com" + env: + - name: CSI_ENDPOINT + value: /var/lib/csi/sockets/pluginproxy/csi.sock + imagePullPolicy: {{ .Values.imagePullPolicy }} + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ + - name: log + mountPath: /var/log + - name: config-map + mountPath: /etc/huawei + - name: secret + mountPath: /etc/huawei/secret + volumes: + - name: socket-dir + emptyDir: + - name: log + hostPath: + path: /var/log/ + type: Directory + - name: config-map + configMap: + name: huawei-csi-configmap + - name: secret + secret: + secretName: huawei-csi-secret \ No newline at end of file diff --git a/helm/esdk/templates/huawei-csi-node.yaml b/helm/esdk/templates/huawei-csi-node.yaml new file mode 100644 index 00000000..578e1c76 --- /dev/null +++ b/helm/esdk/templates/huawei-csi-node.yaml @@ -0,0 +1,137 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: huawei-csi-node + namespace: kube-system + +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: huawei-csi-driver-registrar-runner + namespace: kube-system +rules: + - apiGroups: [""] + resources: ["events"] + verbs: ["get", "list", "watch", "create", "update", "patch"] + +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: huawei-csi-driver-registrar-role +subjects: + - kind: ServiceAccount + name: huawei-csi-node + namespace: kube-system +roleRef: + kind: ClusterRole + name: huawei-csi-driver-registrar-runner + apiGroup: rbac.authorization.k8s.io +--- +kind: DaemonSet +apiVersion: apps/v1 +metadata: + name: huawei-csi-node + namespace: kube-system +spec: + selector: + matchLabels: + app: huawei-csi-node + template: + metadata: + labels: + app: huawei-csi-node + spec: + serviceAccountName: huawei-csi-node + hostPID: true + hostNetwork: true + containers: + - name: csi-node-driver-registrar + image: {{ .Values.images.sidecar.registrar }} + args: + - "--csi-address=/csi/csi.sock" + - "--kubelet-registration-path=/var/lib/kubelet/plugins/csi.huawei.com/csi.sock" + imagePullPolicy: "IfNotPresent" + volumeMounts: + - name: plugin-dir + mountPath: /csi + - name: registration-dir + mountPath: /registration + - name: huawei-csi-driver + image: {{ required "Must provide the CSI controller service node image." .Values.images.huaweiCsiNodeService }} + args: + - "--endpoint=/csi/csi.sock" + - "--containerized" + - "--driver-name=csi.huawei.com" + - "--volume-use-multipath=true" + securityContext: + privileged: true + capabilities: + add: ["SYS_ADMIN"] + allowPrivilegeEscalation: true + imagePullPolicy: "IfNotPresent" + lifecycle: + preStop: + exec: + command: ["/bin/sh", "-c", "rm -f /csi/csi.sock"] + env: + - name: CSI_ESDK_NODENAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + volumeMounts: + - name: plugin-dir + mountPath: /csi + - name: pods-dir + mountPath: /var/lib/kubelet + mountPropagation: "Bidirectional" + - name: etc-dir + mountPath: /etc + - name: log-dir + mountPath: /var/log + - name: dev-dir + mountPath: /dev + mountPropagation: "HostToContainer" + - name: iscsi-dir + mountPath: /var/lib/iscsi + - name: config-map + mountPath: /etc/huawei + - name: secret + mountPath: /etc/huawei/secret + volumes: + - name: plugin-dir + hostPath: + path: /var/lib/kubelet/plugins/csi.huawei.com + type: DirectoryOrCreate + - name: registration-dir + hostPath: + path: /var/lib/kubelet/plugins_registry + type: Directory + - name: pods-dir + hostPath: + path: /var/lib/kubelet + type: Directory + - name: etc-dir + hostPath: + path: /etc + type: Directory + - name: dev-dir + hostPath: + path: /dev + type: Directory + - name: iscsi-dir + hostPath: + path: /var/lib/iscsi + - name: log-dir + hostPath: + path: /var/log/ + type: Directory + - name: config-map + configMap: + name: huawei-csi-configmap + - name: secret + secret: + secretName: huawei-csi-secret \ No newline at end of file diff --git a/helm/esdk/values.yaml b/helm/esdk/values.yaml new file mode 100644 index 00000000..61851a3f --- /dev/null +++ b/helm/esdk/values.yaml @@ -0,0 +1,19 @@ +backends: + - storage: "fusionstorage-san" + name: "test-esdk-deployment" + urls: + - "https://1.2.3.4:28443" + pools: + - "StoragePool001" + parameters: + protocol: "iscsi" + portals: + - "1.2.3.4" +images: + sidecar: + attacher: quay.io/k8scsi/csi-attacher:v1.2.1 + provisioner: quay.io/k8scsi/csi-provisioner:v1.6.0 + registrar: quay.io/k8scsi/csi-node-driver-registrar:v2.0.1 + huaweiCsiControllerService: + huaweiCsiNodeService: +imagePullPolicy: "IfNotPresent" \ No newline at end of file From 13e1c7a9413213f67fd24996568c7f50df6dff80 Mon Sep 17 00:00:00 2001 From: Anvith KS Date: Sun, 22 Aug 2021 14:37:11 +0530 Subject: [PATCH 2/7] Updated the values file to remove the default value for driver image and tag --- helm/esdk/values.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/helm/esdk/values.yaml b/helm/esdk/values.yaml index 61851a3f..5d791f21 100644 --- a/helm/esdk/values.yaml +++ b/helm/esdk/values.yaml @@ -14,6 +14,6 @@ images: attacher: quay.io/k8scsi/csi-attacher:v1.2.1 provisioner: quay.io/k8scsi/csi-provisioner:v1.6.0 registrar: quay.io/k8scsi/csi-node-driver-registrar:v2.0.1 - huaweiCsiControllerService: - huaweiCsiNodeService: -imagePullPolicy: "IfNotPresent" \ No newline at end of file + huaweiCsiControllerService: huawei-csi:test + huaweiCsiNodeService: huawei-csi:test +imagePullPolicy: "IfNotPresent" From 54d845c4f1663075a6c0790682de4d5467be7c0f Mon Sep 17 00:00:00 2001 From: Anvith KS Date: Mon, 23 Aug 2021 11:33:37 +0530 Subject: [PATCH 3/7] Updated the configmap with prettyJson and nindent for json generation --- helm/esdk/templates/huawei-csi-configmap.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/helm/esdk/templates/huawei-csi-configmap.yaml b/helm/esdk/templates/huawei-csi-configmap.yaml index ecc9447a..e274bda2 100644 --- a/helm/esdk/templates/huawei-csi-configmap.yaml +++ b/helm/esdk/templates/huawei-csi-configmap.yaml @@ -6,5 +6,5 @@ metadata: data: csi.json: | {{ $length := len .Values.backends }} {{ if gt $length 0 }} { {{ end }} - "backends": {{ .Values.backends | toJson }} - {{ $length := len .Values.backends }} {{ if gt $length 0 }} } {{ end }} \ No newline at end of file + "backends": {{ .Values.backends | toPrettyJson | nindent 8 }} + {{ $length := len .Values.backends }} {{ if gt $length 0 }} } {{ end }} From 45fde8e098a9dd6720e87d164d682f1db8889478 Mon Sep 17 00:00:00 2001 From: Anvith KS Date: Mon, 23 Aug 2021 13:29:52 +0530 Subject: [PATCH 4/7] Updated the Helm chart with kubeVersion --- helm/esdk/Chart.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/helm/esdk/Chart.yaml b/helm/esdk/Chart.yaml index 6c4264fc..b4400abf 100644 --- a/helm/esdk/Chart.yaml +++ b/helm/esdk/Chart.yaml @@ -18,8 +18,7 @@ type: application version: 0.1.0 # compatible Kubernetes versions, helm installation fails fail if the cluster runs an unsupported Kubernetes version -# kubeVersion: >= 1.13.0 < 1.19.0 -# kubeVersion: 1.17.7 +kubeVersion: ">= 1.13.0 < 1.19.3" # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. From 339eb07cb1f1b226b3c5d903b17feead22526163 Mon Sep 17 00:00:00 2001 From: Anvith KS Date: Tue, 24 Aug 2021 11:48:09 +0530 Subject: [PATCH 5/7] Addressed review comments. Removed unused env var. Added comments. Add new line at eof --- helm/esdk/templates/huawei-csi-controller.yaml | 3 ++- helm/esdk/templates/huawei-csi-node.yaml | 14 +++++--------- helm/esdk/values.yaml | 6 ++++++ 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/helm/esdk/templates/huawei-csi-controller.yaml b/helm/esdk/templates/huawei-csi-controller.yaml index ce6b1fe9..916ec639 100644 --- a/helm/esdk/templates/huawei-csi-controller.yaml +++ b/helm/esdk/templates/huawei-csi-controller.yaml @@ -114,4 +114,5 @@ spec: name: huawei-csi-configmap - name: secret secret: - secretName: huawei-csi-secret \ No newline at end of file + secretName: huawei-csi-secret + diff --git a/helm/esdk/templates/huawei-csi-node.yaml b/helm/esdk/templates/huawei-csi-node.yaml index 578e1c76..ebedbc45 100644 --- a/helm/esdk/templates/huawei-csi-node.yaml +++ b/helm/esdk/templates/huawei-csi-node.yaml @@ -53,14 +53,14 @@ spec: args: - "--csi-address=/csi/csi.sock" - "--kubelet-registration-path=/var/lib/kubelet/plugins/csi.huawei.com/csi.sock" - imagePullPolicy: "IfNotPresent" + imagePullPolicy: {{ .Values.imagePullPolicy }} volumeMounts: - name: plugin-dir mountPath: /csi - name: registration-dir mountPath: /registration - name: huawei-csi-driver - image: {{ required "Must provide the CSI controller service node image." .Values.images.huaweiCsiNodeService }} + image: {{ required "Must provide the CSI node service node image." .Values.images.huaweiCsiNodeService }} args: - "--endpoint=/csi/csi.sock" - "--containerized" @@ -71,17 +71,12 @@ spec: capabilities: add: ["SYS_ADMIN"] allowPrivilegeEscalation: true - imagePullPolicy: "IfNotPresent" + imagePullPolicy: {{ .Values.imagePullPolicy }} lifecycle: preStop: exec: command: ["/bin/sh", "-c", "rm -f /csi/csi.sock"] env: - - name: CSI_ESDK_NODENAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: spec.nodeName volumeMounts: - name: plugin-dir mountPath: /csi @@ -134,4 +129,5 @@ spec: name: huawei-csi-configmap - name: secret secret: - secretName: huawei-csi-secret \ No newline at end of file + secretName: huawei-csi-secret + diff --git a/helm/esdk/values.yaml b/helm/esdk/values.yaml index 5d791f21..16731068 100644 --- a/helm/esdk/values.yaml +++ b/helm/esdk/values.yaml @@ -1,3 +1,4 @@ +# An array of storages with the access info backends: - storage: "fusionstorage-san" name: "test-esdk-deployment" @@ -10,10 +11,15 @@ backends: portals: - "1.2.3.4" images: + # The image name and tag for the attacher, provisioner and registrar sidecars. These must match the appropriate Kubernetes version. sidecar: attacher: quay.io/k8scsi/csi-attacher:v1.2.1 provisioner: quay.io/k8scsi/csi-provisioner:v1.6.0 registrar: quay.io/k8scsi/csi-node-driver-registrar:v2.0.1 + # The image name and tag for the Huawei CSI controller service container huaweiCsiControllerService: huawei-csi:test + # The image name and tag for the Huawei CSI node service container huaweiCsiNodeService: huawei-csi:test +# Default image pull policy for container images imagePullPolicy: "IfNotPresent" + From 3daf4ad9eb1cbde50e1937d601e06bbbf4d09304 Mon Sep 17 00:00:00 2001 From: Anvith KS Date: Tue, 24 Aug 2021 13:41:37 +0530 Subject: [PATCH 6/7] Removed the unused env section as per review comments --- helm/esdk/templates/huawei-csi-node.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/helm/esdk/templates/huawei-csi-node.yaml b/helm/esdk/templates/huawei-csi-node.yaml index ebedbc45..484ecca7 100644 --- a/helm/esdk/templates/huawei-csi-node.yaml +++ b/helm/esdk/templates/huawei-csi-node.yaml @@ -76,7 +76,6 @@ spec: preStop: exec: command: ["/bin/sh", "-c", "rm -f /csi/csi.sock"] - env: volumeMounts: - name: plugin-dir mountPath: /csi From e1fc4e6a30f7b1e3b9789cb3dbdb4cda7523b805 Mon Sep 17 00:00:00 2001 From: Anvith KS Date: Thu, 9 Sep 2021 15:38:53 +0530 Subject: [PATCH 7/7] Fixed controller cluster role binding. Added multipath flag --- .../esdk/templates/huawei-csi-controller.yaml | 47 +++++++++++++++++++ helm/esdk/templates/huawei-csi-node.yaml | 2 +- helm/esdk/values.yaml | 2 + 3 files changed, 50 insertions(+), 1 deletion(-) diff --git a/helm/esdk/templates/huawei-csi-controller.yaml b/helm/esdk/templates/huawei-csi-controller.yaml index 916ec639..6d51e0ed 100644 --- a/helm/esdk/templates/huawei-csi-controller.yaml +++ b/helm/esdk/templates/huawei-csi-controller.yaml @@ -38,6 +38,53 @@ rules: resources: ["nodes"] verbs: ["get", "list", "watch"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: huawei-csi-provisioner-role +subjects: + - kind: ServiceAccount + name: huawei-csi-controller + namespace: kube-system +roleRef: + kind: ClusterRole + name: huawei-csi-provisioner-runner + apiGroup: rbac.authorization.k8s.io + +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: huawei-csi-attacher-runner +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: ["csinodes"] + verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments"] + verbs: ["get", "list", "watch", "update"] + +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: huawei-csi-attacher-role +subjects: + - kind: ServiceAccount + name: huawei-csi-controller + namespace: kube-system +roleRef: + kind: ClusterRole + name: huawei-csi-attacher-runner + apiGroup: rbac.authorization.k8s.io + --- kind: Deployment apiVersion: apps/v1 diff --git a/helm/esdk/templates/huawei-csi-node.yaml b/helm/esdk/templates/huawei-csi-node.yaml index 484ecca7..c6955bee 100644 --- a/helm/esdk/templates/huawei-csi-node.yaml +++ b/helm/esdk/templates/huawei-csi-node.yaml @@ -65,7 +65,7 @@ spec: - "--endpoint=/csi/csi.sock" - "--containerized" - "--driver-name=csi.huawei.com" - - "--volume-use-multipath=true" + - "--volume-use-multipath={{ .Values.multipath.enable }}" securityContext: privileged: true capabilities: diff --git a/helm/esdk/values.yaml b/helm/esdk/values.yaml index 16731068..d52ef124 100644 --- a/helm/esdk/values.yaml +++ b/helm/esdk/values.yaml @@ -22,4 +22,6 @@ images: huaweiCsiNodeService: huawei-csi:test # Default image pull policy for container images imagePullPolicy: "IfNotPresent" +multipath: + enable: false