csiplugin_k8s_minikube.yaml

# The YAML script to deploy DatenLord CSI plugins to K8S
# Run the following cmd to generate YAML script for MicroK8s
# sed -e 's/\/var\/lib\/kubelet/\/var\/snap\/microk8s\/common\/var\/lib\/kubelet/g' csiplugin_k8s.yaml > microk8s.yaml

apiVersion: v1
kind: Namespace
metadata:
  name: csi-datenlord
  labels:
    name: csi-datenlord

---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: csi-nodeplugin-sa
  namespace: csi-datenlord

---
apiVersion: scheduling.k8s.io/v1beta1
kind: PriorityClass
metadata:
  name: csi-datenlord-critical
  namespace: csi-datenlord
value: 1000000000
globalDefault: false
description: "This priority class is used for csi pods."

---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: csi-nodeplugin-cr
  namespace: csi-datenlord
rules:
  - apiGroups: [""]
    resources: ["secrets"]
    verbs: ["get", "list"]
  - apiGroups: [""]
    resources: ["nodes"]
    verbs: ["get", "list", "update"]
  - apiGroups: [""]
    resources: ["namespaces"]
    verbs: ["get", "list"]
  - apiGroups: [""]
    resources: ["persistentvolumes"]
    verbs: ["get", "list", "watch", "update"]
  - apiGroups: ["storage.k8s.io"]
    resources: ["storageclasses"]
    verbs: ["get", "list", "watch"]

---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: csi-nodeplugin-crb
  namespace: csi-datenlord
subjects:
  - kind: ServiceAccount
    name: csi-nodeplugin-sa
    namespace: csi-datenlord
roleRef:
  kind: ClusterRole
  name: csi-nodeplugin-cr
  apiGroup: rbac.authorization.k8s.io

---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: csi-controller-sa
  namespace: csi-datenlord

---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: csi-attacher-cr
  namespace: csi-datenlord
rules:
  - apiGroups: [""] # "" Indicates the core API group
    resources: ["pods"]
    verbs: ["get", "watch", "list"]
  - apiGroups: [""]
    resources: ["secrets"]
    verbs: ["get", "list"]
  - apiGroups: [""]
    resources: ["events"]
    verbs: ["get", "list", "watch", "update", "patch"]
  - apiGroups: [""]
    resources: ["persistentvolumes"]
    verbs: ["get", "list", "watch", "update", "patch"]
  - apiGroups: [""]
    resources: ["persistentvolumeclaims"]
    verbs: ["get", "list", "watch"]
  - apiGroups: [""]
    resources: ["persistentvolumeclaims/status"]
    verbs: ["update", "patch"]
  - apiGroups: [""]
    resources: ["nodes"]
    verbs: ["get", "list", "watch"]
  - apiGroups: ["storage.k8s.io"]
    resources: ["volumeattachments"]
    verbs: ["get", "list", "watch", "update", "patch"]
  - apiGroups: ["storage.k8s.io"]
    resources: ["volumeattachments/status"]
    verbs: ["patch"]
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: csi-attacher-crb
  namespace: csi-datenlord
subjects:
  - kind: ServiceAccount
    name: csi-controller-sa
    namespace: csi-datenlord
roleRef:
  kind: ClusterRole
  name: csi-attacher-cr
  apiGroup: rbac.authorization.k8s.io

---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: csi-controller-cr
  namespace: csi-datenlord
rules:
  - apiGroups: [""]
    resources: ["secrets"]
    verbs: ["get", "list"]
  - apiGroups: [""]
    resources: ["persistentvolumes"]
    verbs: ["get", "list", "watch", "create", "delete"]
  - apiGroups: [""]
    resources: ["persistentvolumeclaims"]
    verbs: ["get", "list", "watch", "update"]
  - apiGroups: ["storage.k8s.io"]
    resources: ["storageclasses"]
    verbs: ["get", "list", "watch"]
  - apiGroups: [""]
    resources: ["events"]
    verbs: ["list", "watch", "update", "create", "patch"]
  - apiGroups: [""]
    resources: ["nodes"]
    verbs: ["get", "list", "update", "watch"]
  - apiGroups: ["snapshot.storage.k8s.io"]
    resources: ["volumesnapshotclasses"]
    verbs: ["get", "list", "watch"]
  - apiGroups: ["snapshot.storage.k8s.io"]
    resources: ["volumesnapshotcontents"]
    verbs: ["create", "get", "list", "watch", "update", "delete"]
  - apiGroups: ["snapshot.storage.k8s.io"]
    resources: ["volumesnapshotcontents/status"]
    verbs: ["update"]
  - apiGroups: ["snapshot.storage.k8s.io"]
    resources: ["volumesnapshots"]
    verbs: ["get", "list", "watch", "update"]
  - apiGroups: ["apiextensions.k8s.io"]
    resources: ["customresourcedefinitions"]
    verbs: ["create", "list", "watch", "delete"]
  - apiGroups: ["storage.k8s.io"]
    resources: ["csinodes"]
    verbs: ["get", "list", "watch"]

---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: csi-controller-crb
  namespace: csi-datenlord
subjects:
  - kind: ServiceAccount
    name: csi-controller-sa
    namespace: csi-datenlord
roleRef:
  kind: ClusterRole
  name: csi-controller-cr
  apiGroup: rbac.authorization.k8s.io

---
kind: DaemonSet
apiVersion: apps/v1
metadata:
  name: csi-nodeplugin-datenlord
  namespace: csi-datenlord
spec:
  selector:
    matchLabels:
      app: csi-nodeplugin-datenlord
  template:
    metadata:
      labels:
        app: csi-nodeplugin-datenlord
    spec:
      hostNetwork: false # TODO: should use k8s network?
      serviceAccount: csi-nodeplugin-sa
      priorityClassName: csi-datenlord-critical
      containers:
        - name: datenlord-node-plugin
          securityContext:
            privileged: true
            capabilities:
              add: ["SYS_ADMIN"]
            allowPrivilegeEscalation: true
          image: datenlord/csiplugin:e2e_test
          imagePullPolicy: "IfNotPresent"
          args :
            - "--endpoint=$(CSI_NODE_ENDPOINT)"
            - "--drivername=csi.datenlord.io"
            - "--nodeid=$(NODE_ID)"
            - "--nodeip=$(NODE_IP)"
            - "--workerport=50051"
            - "--runas=node"
            - "--datadir=$(DATA_DIR)"
            - "--etcd=http://csi-etcd-svc:2379"
          env:
            - name: NODE_ID
              valueFrom:
                fieldRef:
                  fieldPath: spec.nodeName
            - name: NODE_IP
              valueFrom:
                fieldRef:
                  fieldPath: status.podIP
            - name: CSI_NODE_ENDPOINT
              value: unix:///plugin/node.sock
            - name: RUST_LOG
              value: csi=debug
            - name: RUST_BACKTRACE
              value: full
            - name: DATA_DIR
              value: /tmp/datenlord-data
          ports:
            - containerPort: 50051
              hostPort: 50051
              name: workerport
            #- containerPort: 9898
            #  name: healthz
            #  protocol: TCP
          #livenessProbe:
          #  failureThreshold: 20
          #  httpGet:
          #    path: /healthz
          #    port: healthz
          #  initialDelaySeconds: 10
          #  timeoutSeconds: 3
          #  periodSeconds: 2
          volumeMounts:
            - name: socket-dir
              mountPath: /plugin
            - name: mountpoint-dir
              # This mount path must be the same inside and outside container
              # since DatenLord use symbolic link to publish volume here
              mountPath: /var/lib/kubelet/pods
              mountPropagation: Bidirectional
            #- name: plugins-dir
            #  mountPath: /var/lib/kubelet/plugins
            #  mountPropagation: Bidirectional
            - name: csi-data-dir
              # This mount path must be the same inside and outside container
              # since DatenLord use symbolic link to publish volume here
              mountPath: /tmp
            - name: dev-dir
              mountPath: /dev
        - name: csi-node-driver-registrar
          securityContext:
            privileged: true
          image: quay.io/k8scsi/csi-node-driver-registrar:v1.3.0
          imagePullPolicy: "IfNotPresent"
          args:
            - "--v=5"
            - "--csi-address=$(ADDRESS)"
            - "--kubelet-registration-path=/var/lib/kubelet/plugins/csi.datenlord.io/node.sock" # this path is outside container for microk8s
          lifecycle:
            preStop:
              exec:
                command: ["/bin/sh", "-c", "rm -rf /registration/csi.datenlord.io /registration/csi.datenlord.io-reg.sock"]
          env:
            - name: ADDRESS
              value: /plugin/node.sock
            - name: KUBE_NODE_NAME
              valueFrom:
                fieldRef:
                  fieldPath: spec.nodeName
          volumeMounts:
            - name: socket-dir
              mountPath: /plugin
            - name: registration-dir
              mountPath: /registration
        #- name: liveness-probe
        #  securityContext:
        #    privileged: true
        #  image: quay.io/k8scsi/livenessprobe:v2.0.0
        #  imagePullPolicy: "IfNotPresent"
        #  args:
        #    - "--v=5"
        #    - "--csi-address=$(ADDRESS)"
        #    - "--probe-timeout=60s"
        #    - "--health-port=9898"
        #  env:
        #    - name: ADDRESS
        #      value: /plugin/node.sock
        #  volumeMounts:
        #    - name: socket-dir
        #      mountPath: /plugin
      volumes:
        - name: socket-dir
          hostPath:
            path: /var/lib/kubelet/plugins/csi.datenlord.io
            type: DirectoryOrCreate
        - name: csi-data-dir
          hostPath:
            path: /tmp
            type: Directory
        - name: dev-dir
          hostPath:
            path: /dev
            type: Directory
        - name: registration-dir
          hostPath:
            path: /var/lib/kubelet/plugins_registry
            type: Directory
        - name: mountpoint-dir
          hostPath:
            path: /var/lib/kubelet/pods
            type: Directory
        #- name: plugins-dir
        #  hostPath:
        #    path: /var/lib/kubelet/plugins
        #    type: Directory

---
apiVersion: v1
kind: Service
metadata:
  name: csi-etcd-svc
  namespace: csi-datenlord
spec:
  selector:
    app: csi-etcd
  ports:
    - protocol: TCP
      port: 2379
      targetPort: 2379
---
kind: StatefulSet
apiVersion: apps/v1
metadata:
  name: csi-etcd
  namespace: csi-datenlord
  labels:
    app: csi-etcd
spec:
  selector:
    matchLabels:
      app: csi-etcd
  serviceName: "csi-etcd-svc"
  replicas: 1
  template:
    metadata:
      name: csi-etcd
      namespace: csi-datenlord
      labels:
        app: csi-etcd
    spec:
      containers:
        - name: csi-etcd
          image: gcr.io/etcd-development/etcd:v3.4.13
          imagePullPolicy: "IfNotPresent"
          ports:
          - containerPort: 2379
            name: client
          - containerPort: 2380
            name: peer
          command:
            - "/usr/local/bin/etcd"
          args:
            - "--name=$(NODE_ID)"
            - "--initial-advertise-peer-urls=http://$(NODE_IP):2380"
            - "--listen-peer-urls=http://0.0.0.0:2380"
            - "--advertise-client-urls=http://$(NODE_IP):2379"
            - "--listen-client-urls=http://0.0.0.0:2379"
            - "--initial-cluster=$(NODE_ID)=http://$(NODE_IP):2380"
            - "--listen-metrics-urls=http://0.0.0.0:2381"
            - "--data-dir=/var/lib/etcd/default.etcd"
          env:
            - name: NODE_ID
              valueFrom:
                fieldRef:
                  fieldPath: metadata.name
            - name: NODE_IP
              valueFrom:
                fieldRef:
                  fieldPath: status.podIP
          livenessProbe:
            httpGet:
              path: /health
              port: 2381
            initialDelaySeconds: 10
            periodSeconds: 5
          volumeMounts:
            - name: etcd-data-dir
              mountPath: /var/lib/etcd
      volumes:
        - name: etcd-data-dir
          hostPath:
            path: /var/lib/etcd
            type: DirectoryOrCreate

---
kind: StatefulSet
apiVersion: apps/v1
metadata:
  name: csi-controller-datenlord
  namespace: csi-datenlord
spec:
  selector:
    matchLabels:
      app: csi-controller-datenlord # Must match .spec.template.metadata.labels  
  serviceName: "datenlord-controller-svc"
  replicas: 1
  template:
    metadata:
      labels:
        app: csi-controller-datenlord
    spec:
      hostNetwork: false # TODO: should use k8s network?
      serviceAccount: csi-controller-sa
      priorityClassName: csi-datenlord-critical
      containers:
        - name: datenlord-controller-plugin
          securityContext:
            # This is necessary only for systems with SELinux, where
            # non-privileged sidecar containers cannot access unix domain socket
            # created by privileged CSI driver container.
            privileged: true
          image: datenlord/csiplugin:e2e_test
          imagePullPolicy: "IfNotPresent"
          args :
            - "--endpoint=$(CSI_CONTROLLER_ENDPOINT)"
            - "--drivername=csi.datenlord.io"
            - "--nodeid=$(NODE_ID)"
            - "--nodeip=$(NODE_IP)"
            - "--workerport=50051"
            - "--runas=controller"
            - "--datadir=$(DATA_DIR)"
            - "--etcd=http://csi-etcd-svc:2379"
          env:
            - name: NODE_ID
              valueFrom:
                fieldRef:
                  fieldPath: spec.nodeName
            - name: NODE_IP
              valueFrom:
                fieldRef:
                  fieldPath: status.podIP
            - name: CSI_CONTROLLER_ENDPOINT
              value: unix:///plugin/controller.sock
            - name: RUST_LOG
              value: csi=debug
            - name: RUST_BACKTRACE
              value: full
            - name: DATA_DIR
              value: /tmp/datenlord-data
          #ports:
          #  - containerPort: 50051
          #    hostPort: 50051
          #  - containerPort: 9898
          #    name: healthz
          #    protocol: TCP
          #livenessProbe:
          #  failureThreshold: 5
          #  httpGet:
          #    path: /healthz
          #    port: healthz
          #  initialDelaySeconds: 10
          #  timeoutSeconds: 3
          #  periodSeconds: 2
          volumeMounts:
            - name: socket-dir
              mountPath: /plugin
        - name: csi-attacher
          securityContext:
            # This is necessary only for systems with SELinux, where
            # non-privileged sidecar containers cannot access unix domain socket
            # created by privileged CSI driver container.
            privileged: true
          image: quay.io/k8scsi/csi-attacher:v2.2.0
          args:
            - "--v=5"
            - "--csi-address=$(ADDRESS)"
          env:
            - name: ADDRESS
              value: /plugin/controller.sock
          imagePullPolicy: "IfNotPresent"
          volumeMounts:
            - name: socket-dir
              mountPath: /plugin
        - name: csi-provisioner
          securityContext:
            # This is necessary only for systems with SELinux, where
            # non-privileged sidecar containers cannot access unix domain socket
            # created by privileged CSI driver container.
            privileged: true
          image: quay.io/k8scsi/csi-provisioner:v1.6.0
          args:
            - "--csi-address=$(ADDRESS)"
            - "--volume-name-prefix=datenlord-pv"
            - "--feature-gates=Topology=true"
            - "--strict-topology"
            - "--v=5"
          env:
            - name: ADDRESS
              value: /plugin/controller.sock
          imagePullPolicy: "IfNotPresent"
          volumeMounts:
            - name: socket-dir
              mountPath: /plugin
        - name: csi-snapshotter
          securityContext:
            # This is necessary only for systems with SELinux, where
            # non-privileged sidecar containers cannot access unix domain socket
            # created by privileged CSI driver container.
            privileged: true
          image: quay.io/k8scsi/csi-snapshotter:v2.1.1
          imagePullPolicy: "IfNotPresent"
          args:
            - "--csi-address=$(ADDRESS)"
            - "--snapshot-name-prefix=datenlord-snapshot"
            - "--v=5"
          env:
            - name: ADDRESS
              value: /plugin/controller.sock
          volumeMounts:
            - name: socket-dir
              mountPath: /plugin
        #- name: liveness-probe
        #  image: quay.io/k8scsi/livenessprobe:v2.0.0
        #  imagePullPolicy: "IfNotPresent"
        #  args:
        #    - "--v=5"
        #    - "--csi-address=$(ADDRESS)"
        #    - "--probe-timeout=60s"
        #    - "--health-port=9898"
        #  env:
        #    - name: ADDRESS
        #      value: /plugin/controller.sock
        #  volumeMounts:
        #    - name: socket-dir
        #      mountPath: /plugin
        - name: csi-resizer
          securityContext:
            # This is necessary only for systems with SELinux, where
            # non-privileged sidecar containers cannot access unix domain socket
            # created by privileged CSI driver container.
            privileged: true
          image: quay.io/k8scsi/csi-resizer:v0.5.0
          args:
            - "--v=5"
            - "--csi-address=$(ADDRESS)"
          env:
            - name: ADDRESS
              value: /plugin/controller.sock
          imagePullPolicy: "IfNotPresent"
          volumeMounts:
            - name: socket-dir
              mountPath: /plugin
      volumes:
        - name: socket-dir
          # emptyDir: {} # TODO: use emptyDir when release
          hostPath:
            path: /var/lib/kubelet/plugins/csi.datenlord.io
            type: DirectoryOrCreate

---
apiVersion: storage.k8s.io/v1
kind: CSIDriver
metadata:
  name: csi.datenlord.io
spec:
  attachRequired: false # No controller publish/unpublish
  # To determine at runtime which mode a volume uses, pod info and its
  # "csi.storage.k8s.io/ephemeral" entry are needed.
  # Informaion includes pod name, uid and namespace.
  podInfoOnMount: true
  # Supports persistent and ephemeral inline volumes, added in Kubernetes 1.16, this field is beta
  volumeLifecycleModes:
  - Persistent
  - Ephemeral

---
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
  name: csi-datenlord-sc
  namespace: csi-datenlord
provisioner: csi.datenlord.io
reclaimPolicy: Delete # Retain
volumeBindingMode: Immediate # WaitForFirstConsumer
allowVolumeExpansion: true

---
apiVersion: snapshot.storage.k8s.io/v1beta1
kind: VolumeSnapshotClass
metadata:
  name: csi-datenlord-snapclass
driver: csi.datenlord.io
deletionPolicy: Delete