feat: MOFED subscription mounts (#946)

Add needed mounts for subscription in RHEL with non CRIO container
runtime.

manifests/state-ofed-driver/0050_ofed-driver-ds.yaml

@@ -237,12 +237,18 @@ spec:
             type: DirectoryOrCreate
         {{- range .AdditionalVolumeMounts.Volumes }}
         - name: {{ .Name }}
+          {{- if and .ConfigMap .ConfigMap.Items }}
           configMap:
             name: {{ .Name }}
             items:
             {{- range .ConfigMap.Items }}
             {{ . | yaml | nindentPrefix 14 "- " }}
             {{- end }}
+          {{- else if .HostPath }}
+          hostPath:
+            path: {{ .HostPath.Path }}
+            type: {{ .HostPath.Type }}
+          {{- end }}
         {{- end }}
         {{- if .RuntimeSpec.UseDtk }}
         - name: shared-doca-driver-toolkit

pkg/nodeinfo/attributes.go

@@ -35,3 +35,10 @@ const (
 	NodeLabelCudaVersionMajor = "nvidia.com/cuda.driver.major"
 	NodeLabelOSTreeVersion    = "feature.node.kubernetes.io/system-os_release.OSTREE_VERSION"
 )
+
+// Constants for Container Runtime
+const (
+	Docker     string = "docker"
+	Containerd string = "containerd"
+	CRIO       string = "cri-o"
+)

pkg/nodeinfo/node_info.go

@@ -22,6 +22,7 @@ package nodeinfo
 
 import (
 	"fmt"
+	"strings"
 
 	corev1 "k8s.io/api/core/v1"
 	"sigs.k8s.io/controller-runtime/pkg/client"
@@ -51,12 +52,13 @@ type provider struct {
 
 // NodePool represent a set of Nodes grouped by common attributes
 type NodePool struct {
-	Name         string
-	OsName       string
-	OsVersion    string
-	RhcosVersion string
-	Kernel       string
-	Arch         string
+	Name             string
+	OsName           string
+	OsVersion        string
+	RhcosVersion     string
+	Kernel           string
+	Arch             string
+	ContainerRuntime string
 }
 
 // GetNodePools partitions nodes into one or more node pools. The list of nodes to partition
@@ -113,6 +115,8 @@ func (p *provider) GetNodePools(filters ...Filter) []NodePool {
 		}
 		nodePool.Kernel = kernel
 
+		nodePool.ContainerRuntime = getContainerRuntime(node)
+
 		nodePool.Name = fmt.Sprintf("%s%s-%s", nodePool.OsName, nodePool.OsVersion, nodePool.Kernel)
 
 		if _, exists := nodePoolMap[nodePool.Name]; !exists {
@@ -128,3 +132,20 @@ func (p *provider) GetNodePools(filters ...Filter) []NodePool {
 
 	return nodePools
 }
+
+func getContainerRuntime(node *corev1.Node) string {
+	// runtimeVer string will look like <runtime>://<x.y.z>
+	runtimeVer := node.Status.NodeInfo.ContainerRuntimeVersion
+	var runtime string
+	switch {
+	case strings.HasPrefix(runtimeVer, "docker"):
+		runtime = Docker
+	case strings.HasPrefix(runtimeVer, "containerd"):
+		runtime = Containerd
+	case strings.HasPrefix(runtimeVer, "cri-o"):
+		runtime = CRIO
+	default:
+		runtime = ""
+	}
+	return runtime
+}

pkg/state/state_ofed.go

@@ -120,6 +120,42 @@ var RepoConfigPathMap = map[string]string{
 	"rhel":   "/etc/yum.repos.d",
 }
 
+// MountPathToVolumeSource maps a container mount path to a VolumeSource
+type MountPathToVolumeSource map[string]v1.VolumeSource
+
+// SubscriptionPathMap contains information on OS-specific paths
+// that provide entitlements/subscription details on the host.
+// These are used to enable Driver Container's access to packages controlled by
+// the distro through their subscription and support program.
+var SubscriptionPathMap = map[string]MountPathToVolumeSource{
+	"rhel": {
+		"/run/secrets/etc-pki-entitlement": v1.VolumeSource{
+			HostPath: &v1.HostPathVolumeSource{
+				Path: "/etc/pki/entitlement",
+				Type: newHostPathType(v1.HostPathDirectory),
+			},
+		},
+		"/run/secrets/redhat.repo": v1.VolumeSource{
+			HostPath: &v1.HostPathVolumeSource{
+				Path: "/etc/yum.repos.d/redhat.repo",
+				Type: newHostPathType(v1.HostPathFile),
+			},
+		},
+		"/run/secrets/rhsm": v1.VolumeSource{
+			HostPath: &v1.HostPathVolumeSource{
+				Path: "/etc/rhsm",
+				Type: newHostPathType(v1.HostPathDirectory),
+			},
+		},
+	},
+}
+
+func newHostPathType(pathType v1.HostPathType) *v1.HostPathType {
+	hostPathType := new(v1.HostPathType)
+	*hostPathType = pathType
+	return hostPathType
+}
+
 // ConfigMapKeysOverride contains static key override rules for ConfigMaps
 // now the only use-case is to override key name in the ConfigMap which automatically
 // populated by Openshift
@@ -492,6 +528,12 @@ func renderObjects(ctx context.Context, nodePool *nodeinfo.NodePool, useDtk bool
 		return nil, err
 	}
 
+	// set subscription volumes if needed
+	err = s.handleSubscriptionVolumes(ctx, osname, nodePool.ContainerRuntime, &additionalVolMounts)
+	if err != nil {
+		return nil, err
+	}
+
 	renderData := &ofedManifestRenderData{
 		CrSpec: cr.Spec.OFEDDriver,
 		RuntimeSpec: &ofedRuntimeSpec{
@@ -783,6 +825,40 @@ func (s *stateOFED) handleCertConfig(
 	return nil
 }
 
+// handleSubscriptionVolumes handles additional mounts required for subscriptions
+func (s *stateOFED) handleSubscriptionVolumes(
+	ctx context.Context, osname string, runtime string, mounts *additionalVolumeMounts) error {
+	reqLogger := log.FromContext(ctx)
+	if osname == "rhel" && runtime != nodeinfo.CRIO {
+		reqLogger.V(consts.LogLevelDebug).Info("Setting subscription mounts for RHEL with non CRIO container runtime")
+		pathToVolumeSource, ok := SubscriptionPathMap[osname]
+		if !ok {
+			return fmt.Errorf("failed to find subscription volumes definition for os: %v", osname)
+		}
+		// sort host path volumes to ensure ordering is preserved when adding to pod spec
+		mountPaths := make([]string, 0, len(pathToVolumeSource))
+		for k := range pathToVolumeSource {
+			mountPaths = append(mountPaths, k)
+		}
+		sort.Strings(mountPaths)
+
+		for num, mountPath := range mountPaths {
+			volMountSubscriptionName := fmt.Sprintf("subscription-config-%d", num)
+
+			volMountSubscription := v1.VolumeMount{
+				Name:      volMountSubscriptionName,
+				MountPath: mountPath,
+				ReadOnly:  true,
+			}
+			mounts.VolumeMounts = append(mounts.VolumeMounts, volMountSubscription)
+
+			subscriptionVol := v1.Volume{Name: volMountSubscriptionName, VolumeSource: pathToVolumeSource[mountPath]}
+			mounts.Volumes = append(mounts.Volumes, subscriptionVol)
+		}
+	}
+	return nil
+}
+
 // handleRepoConfig handles additional mounts required for custom repo if specified
 func (s *stateOFED) handleRepoConfig(
 	ctx context.Context, cr *mellanoxv1alpha1.NicClusterPolicy, osname string, mounts *additionalVolumeMounts) error {

pkg/state/state_ofed_test.go

@@ -326,6 +326,59 @@ var _ = Describe("MOFED state test", func() {
 				verifyPodAntiInfinity(ds.Spec.Template.Spec.Affinity)
 			}
 		})
+		It("Should Render subscription mounts for RHEL + containerd", func() {
+			client := mocks.ControllerRuntimeClient{}
+			manifestBaseDir := "../../manifests/state-ofed-driver"
+
+			files, err := utils.GetFilesWithSuffix(manifestBaseDir, render.ManifestFileSuffix...)
+			Expect(err).NotTo(HaveOccurred())
+			renderer := render.NewRenderer(files)
+
+			ofedState := stateOFED{
+				stateSkel: stateSkel{
+					name:        stateOFEDName,
+					description: stateOFEDDescription,
+					client:      &client,
+					renderer:    renderer,
+				},
+			}
+			cr := &v1alpha1.NicClusterPolicy{}
+			cr.Name = "nic-cluster-policy"
+			cr.Spec.OFEDDriver = &v1alpha1.OFEDDriverSpec{
+				ImageSpec: v1alpha1.ImageSpec{
+					Image:      "mofed",
+					Repository: "nvcr.io/mellanox",
+					Version:    "23.10-0.5.5.0",
+				},
+			}
+
+			By("Creating NodeProvider with 1 Nodes, RHEL with containerd")
+			node := getNode("node1", kernelFull1)
+			setContainerRuntime(node, "containerd://1.27.1")
+			node.Labels[nodeinfo.NodeLabelOSName] = "rhel"
+			infoProvider := nodeinfo.NewProvider([]*v1.Node{
+				node,
+			})
+			catalog := NewInfoCatalog()
+			catalog.Add(InfoTypeClusterType, &dummyProvider{})
+			catalog.Add(InfoTypeNodeInfo, infoProvider)
+			catalog.Add(InfoTypeDocaDriverImage, &dummyOfedImageProvider{tagExists: false})
+			objs, err := ofedState.GetManifestObjects(ctx, cr, catalog, testLogger)
+			Expect(err).NotTo(HaveOccurred())
+			// Expect 5 objects: 1 DS per pool, Service Account, Role, RoleBinding
+			Expect(len(objs)).To(Equal(4))
+			By("Verify Subscription mounts")
+			for _, obj := range objs {
+				if obj.GetKind() != "DaemonSet" {
+					continue
+				}
+				ds := appsv1.DaemonSet{}
+				err = runtime.DefaultUnstructuredConverter.FromUnstructured(obj.Object, &ds)
+				Expect(err).NotTo(HaveOccurred())
+				verifySubscriptionMounts(ds.Spec.Template.Spec.Containers[0].VolumeMounts)
+				verifySubscriptionVolumes(ds.Spec.Template.Spec.Volumes)
+			}
+		})
 	})
 	Context("Render Manifests DTK", func() {
 		It("Should Render DaemonSet with DTK and additional mounts", func() {
@@ -584,6 +637,88 @@ func verifyPodAntiInfinity(affinity *v1.Affinity) {
 	Expect(*affinity).To(BeEquivalentTo(expected))
 }
 
+func verifySubscriptionMounts(mounts []v1.VolumeMount) {
+	By("Verify Subscription Mounts")
+	sub0 := v1.VolumeMount{
+		Name:             "subscription-config-0",
+		ReadOnly:         true,
+		MountPath:        "/run/secrets/etc-pki-entitlement",
+		SubPath:          "",
+		MountPropagation: nil,
+		SubPathExpr:      "",
+	}
+	Expect(slices.Contains(mounts, sub0)).To(BeTrue())
+	sub1 := v1.VolumeMount{
+		Name:             "subscription-config-1",
+		ReadOnly:         true,
+		MountPath:        "/run/secrets/redhat.repo",
+		SubPath:          "",
+		MountPropagation: nil,
+		SubPathExpr:      "",
+	}
+	Expect(slices.Contains(mounts, sub1)).To(BeTrue())
+	sub2 := v1.VolumeMount{
+		Name:             "subscription-config-2",
+		ReadOnly:         true,
+		MountPath:        "/run/secrets/rhsm",
+		SubPath:          "",
+		MountPropagation: nil,
+		SubPathExpr:      "",
+	}
+	Expect(slices.Contains(mounts, sub2)).To(BeTrue())
+}
+
+func verifySubscriptionVolumes(volumes []v1.Volume) {
+	By("Verify Subscription Volumes")
+	sub0 := v1.Volume{
+		Name: "subscription-config-0",
+		VolumeSource: v1.VolumeSource{
+			HostPath: &v1.HostPathVolumeSource{
+				Path: "/etc/pki/entitlement",
+				Type: newHostPathType(v1.HostPathDirectory),
+			},
+		},
+	}
+	sub1 := v1.Volume{
+		Name: "subscription-config-1",
+		VolumeSource: v1.VolumeSource{
+			HostPath: &v1.HostPathVolumeSource{
+				Path: "/etc/yum.repos.d/redhat.repo",
+				Type: newHostPathType(v1.HostPathFile),
+			},
+		},
+	}
+	sub2 := v1.Volume{
+		Name: "subscription-config-2",
+		VolumeSource: v1.VolumeSource{
+			HostPath: &v1.HostPathVolumeSource{
+				Path: "/etc/rhsm",
+				Type: newHostPathType(v1.HostPathDirectory),
+			},
+		},
+	}
+	foundSub0 := false
+	foundSub1 := false
+	foundSub2 := false
+	for i := range volumes {
+		if volumes[i].Name == "subscription-config-0" {
+			Expect(volumes[i]).To(BeEquivalentTo(sub0))
+			foundSub0 = true
+		}
+		if volumes[i].Name == "subscription-config-1" {
+			Expect(volumes[i]).To(BeEquivalentTo(sub1))
+			foundSub1 = true
+		}
+		if volumes[i].Name == "subscription-config-2" {
+			Expect(volumes[i]).To(BeEquivalentTo(sub2))
+			foundSub2 = true
+		}
+	}
+	Expect(foundSub0).To(BeTrue())
+	Expect(foundSub1).To(BeTrue())
+	Expect(foundSub2).To(BeTrue())
+}
+
 func verifyAdditionalMounts(mounts []v1.VolumeMount) {
 	By("Verify Additional Mounts")
 	repo := v1.VolumeMount{
@@ -686,3 +821,11 @@ func getNode(name, kernelFull string) *v1.Node {
 		},
 	}
 }
+
+func setContainerRuntime(node *v1.Node, containerRuntime string) {
+	node.Status = v1.NodeStatus{
+		NodeInfo: v1.NodeSystemInfo{
+			ContainerRuntimeVersion: containerRuntime,
+		},
+	}
+}