From b4f00fddb8b6e4b33fa16a1bcdd7f155a9de7850 Mon Sep 17 00:00:00 2001
From: Eero Tamminen <eero.t.tamminen@intel.com>
Date: Thu, 25 Aug 2022 20:26:48 +0300
Subject: [PATCH] Use intel-gpu-plugin with intel-gpu-fakedev generated devices

Adding new kustomization (base) directory for this is (a bit simpler)
alternative than kustomizing existing gpu_plugin/ base, which requires
changing current initcontainer to a run-time one, and adding init
container for fake device generator.

See: https://github.com/intel/intel-device-plugins-for-kubernetes/pull/1118

Signed-off-by: Eero Tamminen <eero.t.tamminen@intel.com>
---
 .../gpu_fakedev/base/fakedev-config.json      |   8 ++
 .../gpu_fakedev/base/intel-gpu-plugin.yaml    | 107 ++++++++++++++++++
 .../gpu_fakedev/base/kustomization.yaml       |   6 +
 deployments/gpu_fakedev/kustomization.yaml    |   2 +
 4 files changed, 123 insertions(+)
 create mode 100644 deployments/gpu_fakedev/base/fakedev-config.json
 create mode 100644 deployments/gpu_fakedev/base/intel-gpu-plugin.yaml
 create mode 100644 deployments/gpu_fakedev/base/kustomization.yaml
 create mode 100644 deployments/gpu_fakedev/kustomization.yaml

diff --git a/deployments/gpu_fakedev/base/fakedev-config.json b/deployments/gpu_fakedev/base/fakedev-config.json
new file mode 100644
index 000000000..e9840b23f
--- /dev/null
+++ b/deployments/gpu_fakedev/base/fakedev-config.json
@@ -0,0 +1,8 @@
+{
+	"Info": "8x 4 GiB DG1 [Iris Xe MAX Graphics] GPUs",
+	"DevCount": 8,
+	"DevMemSize": 4294967296,
+	"Capabilities": {
+		"platform": "fake_DG1",
+	}
+}
diff --git a/deployments/gpu_fakedev/base/intel-gpu-plugin.yaml b/deployments/gpu_fakedev/base/intel-gpu-plugin.yaml
new file mode 100644
index 000000000..be34d5423
--- /dev/null
+++ b/deployments/gpu_fakedev/base/intel-gpu-plugin.yaml
@@ -0,0 +1,107 @@
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: intel-gpu-plugin-fake
+  labels:
+    app: intel-gpu-plugin-fake
+spec:
+  selector:
+    matchLabels:
+      app: intel-gpu-plugin-fake
+  template:
+    metadata:
+      labels:
+        app: intel-gpu-plugin-fake
+    spec:
+      initContainers:
+      - name: fakedev-generator
+        image: intel/intel-gpu-fakedev:devel
+        imagePullPolicy: IfNotPresent
+        securityContext:
+          readOnlyRootFilesystem: false
+          allowPrivilegeEscalation: false
+          capabilities:
+            drop: [ "ALL" ]
+            add:  [ "MKNOD" ]
+        # container runtime prevents writing to /sys & /dev,
+        # so volumes need to be mounted elsewhere
+        volumeMounts:
+        - name: devfs
+          mountPath: /tmp/fakedev/dev
+          readOnly: false
+        - name: sysfs
+          mountPath: /tmp/fakedev/sys
+          readOnly: false
+        - name: fake-conf
+          mountPath: /config
+          readOnly: true
+        # files are generated under CWD
+        workingDir: /tmp/fakedev
+        # generate fake sysfs / devfs files for GPU plugin based on config
+        command: ["/generator", "-json", "/config/fakedev.json", "-verbose"]
+      - name: intel-gpu-initcontainer
+        # convert generated sysfs content to NFD feature labels file
+        image: intel/intel-gpu-initcontainer:devel
+        imagePullPolicy: IfNotPresent
+        securityContext:
+          readOnlyRootFilesystem: true
+          allowPrivilegeEscalation: false
+          capabilities:
+            drop: [ "ALL" ]
+        # expects sysfs here
+        volumeMounts:
+        - name: sysfs
+          mountPath: /host-sys
+          readOnly: true
+        - name: nfd-features
+          mountPath: /nfd
+          readOnly: false
+        workingDir: /usr/local/bin/gpu-sw
+        # needed until GPU plugin drops NFD hook usage due to:
+        # https://github.com/kubernetes-sigs/node-feature-discovery/issues/856
+        command: ["sh", "-c", "./intel-gpu-nfdhook | tee /nfd/fake-gpu"]
+      containers:
+      - name: intel-gpu-plugin
+        env:
+          - name: NODE_NAME
+            valueFrom:
+              fieldRef:
+                fieldPath: spec.nodeName
+        image: intel/intel-gpu-plugin:devel
+        imagePullPolicy: IfNotPresent
+        securityContext:
+          readOnlyRootFilesystem: true
+          allowPrivilegeEscalation: false
+          capabilities:
+            drop: [ "ALL" ]
+        args: [ "-prefix=/tmp/fakedev" ]
+        # devfs host & container paths must match for everything to work
+        volumeMounts:
+        - name: devfs
+          mountPath: /tmp/fakedev/dev
+          readOnly: true
+        - name: sysfs
+          mountPath: /tmp/fakedev/sys
+          readOnly: true
+        - name: kubeletsockets
+          mountPath: /var/lib/kubelet/device-plugins
+      volumes:
+      - name: devfs
+        hostPath:
+          path: /tmp/fakedev/dev
+          type: DirectoryOrCreate
+      - name: sysfs
+        hostPath:
+        emptyDir: {}
+      - name: fake-conf
+        configMap:
+          name: fakedev-config
+      - name: kubeletsockets
+        hostPath:
+          path: /var/lib/kubelet/device-plugins
+      - name: nfd-features
+        hostPath:
+          path: /etc/kubernetes/node-feature-discovery/features.d/
+          type: DirectoryOrCreate
+      nodeSelector:
+        kubernetes.io/arch: amd64
diff --git a/deployments/gpu_fakedev/base/kustomization.yaml b/deployments/gpu_fakedev/base/kustomization.yaml
new file mode 100644
index 000000000..e8aebeb6e
--- /dev/null
+++ b/deployments/gpu_fakedev/base/kustomization.yaml
@@ -0,0 +1,6 @@
+configMapGenerator:
+- name: fakedev-config
+  files:
+  - fakedev-config.json
+resources:
+  - intel-gpu-plugin.yaml
diff --git a/deployments/gpu_fakedev/kustomization.yaml b/deployments/gpu_fakedev/kustomization.yaml
new file mode 100644
index 000000000..f191f3aae
--- /dev/null
+++ b/deployments/gpu_fakedev/kustomization.yaml
@@ -0,0 +1,2 @@
+bases:
+  - base