Add RWX file volume support for vpc (#2960)

manifests/supervisorcluster/1.27/cns-csi.yaml

@@ -102,6 +102,9 @@ rules:
   - apiGroups: ["cns.vmware.com"]
     resources: ["cnsvolumeinfoes"]
     verbs: ["create", "get", "list", "watch", "delete", "patch"]
+  - apiGroups: ["crd.nsx.vmware.com"]
+    resources: ["networkinfos"]
+    verbs: ["get", "watch", "list"]
 ---
 kind: ClusterRoleBinding
 apiVersion: rbac.authorization.k8s.io/v1

manifests/supervisorcluster/1.28/cns-csi.yaml

@@ -102,6 +102,9 @@ rules:
   - apiGroups: ["cns.vmware.com"]
     resources: ["cnsvolumeinfoes"]
     verbs: ["create", "get", "list", "watch", "delete", "patch"]
+  - apiGroups: ["crd.nsx.vmware.com"]
+    resources: ["networkinfos"]
+    verbs: ["get", "watch", "list"]
 ---
 kind: ClusterRoleBinding
 apiVersion: rbac.authorization.k8s.io/v1

manifests/supervisorcluster/1.29/cns-csi.yaml

@@ -102,6 +102,9 @@ rules:
   - apiGroups: ["cns.vmware.com"]
     resources: ["cnsvolumeinfoes"]
     verbs: ["create", "get", "list", "watch", "delete", "patch"]
+  - apiGroups: ["crd.nsx.vmware.com"]
+    resources: ["networkinfos"]
+    verbs: ["get", "watch", "list"]
 ---
 kind: ClusterRoleBinding
 apiVersion: rbac.authorization.k8s.io/v1

manifests/supervisorcluster/1.30/cns-csi.yaml

@@ -102,6 +102,9 @@ rules:
   - apiGroups: ["cns.vmware.com"]
     resources: ["cnsvolumeinfoes"]
     verbs: ["create", "get", "list", "watch", "delete", "patch"]
+  - apiGroups: ["crd.nsx.vmware.com"]
+    resources: ["networkinfos"]
+    verbs: ["get", "watch", "list"]
 ---
 kind: ClusterRoleBinding
 apiVersion: rbac.authorization.k8s.io/v1

pkg/csi/service/common/constants.go

@@ -100,7 +100,7 @@ const (
 	// NfsV4FsType represents nfs4 mount type.
 	NfsV4FsType = "nfs4"
 
-	//NTFSFsType represents ntfs
+	// NTFSFsType represents ntfs
 	NTFSFsType = "ntfs"
 
 	// NfsFsType represents nfs mount type.
@@ -290,10 +290,10 @@ const (
 	// topology labels applied on the node by vSphere CSI driver.
 	TopologyLabelsDomain = "topology.csi.vmware.com"
 
-	//AnnGuestClusterRequestedTopology is the key for guest cluster requested topology
+	// AnnGuestClusterRequestedTopology is the key for guest cluster requested topology
 	AnnGuestClusterRequestedTopology = "csi.vsphere.volume-requested-topology"
 
-	//AnnVolumeAccessibleTopology is the annotation set by the supervisor cluster on PVC
+	// AnnVolumeAccessibleTopology is the annotation set by the supervisor cluster on PVC
 	AnnVolumeAccessibleTopology = "csi.vsphere.volume-accessible-topology"
 
 	// PVtoBackingDiskObjectIdSupportedVCenterMajor is the minimum major version of vCenter
@@ -429,10 +429,13 @@ const (
 	// WorkloadDomainIsolationFSS is FSS for Workload Domain isolation feature
 	// Used in PVCSI
 	WorkloadDomainIsolationFSS = "workload-domain-isolation"
+	// VPCCapabilitySupervisor is a supervisor capability indicating if VPC FSS is enabled
+	VPCCapabilitySupervisor = "VPC_Supported"
 )
 
 var WCPFeatureStates = map[string]struct{}{
 	PodVMOnStretchedSupervisor: {},
 	CSIDetachOnSupervisor:      {},
 	WorkloadDomainIsolation:    {},
+	VPCCapabilitySupervisor:    {},
 }

pkg/syncer/cnsoperator/controller/cnsfileaccessconfig/cnsfileaccessconfig_controller.go

@@ -41,6 +41,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/source"
 
 	vsanfstypes "github.com/vmware/govmomi/vsan/vsanfs/types"
+
 	cnsoperatorapis "sigs.k8s.io/vsphere-csi-driver/v3/pkg/apis/cnsoperator"
 	cnsfileaccessconfigv1alpha1 "sigs.k8s.io/vsphere-csi-driver/v3/pkg/apis/cnsoperator/cnsfileaccessconfig/v1alpha1"
 	volumes "sigs.k8s.io/vsphere-csi-driver/v3/pkg/common/cns-lib/volume"
@@ -506,20 +507,32 @@ func (r *ReconcileCnsFileAccessConfig) getVMExternalIP(ctx context.Context,
 	if err != nil {
 		return "", logger.LogNewErrorf(log, "Failed to identify the network provider. Error: %+v", err)
 	}
-	var nsxConfiguration bool
+
 	if networkProvider == "" {
 		return "", logger.LogNewError(log, "unable to find network provider information")
 	}
-	if networkProvider == cnsoperatorutil.NSXTNetworkProvider {
-		nsxConfiguration = true
-	} else if networkProvider == cnsoperatorutil.VDSNetworkProvider {
-		nsxConfiguration = false
-	} else {
+
+	networkTypes := []string{cnsoperatorutil.NSXTNetworkProvider, cnsoperatorutil.
+		VDSNetworkProvider}
+
+	if commonco.ContainerOrchestratorUtility.IsFSSEnabled(ctx, common.VPCCapabilitySupervisor) {
+		networkTypes = append(networkTypes, cnsoperatorutil.VPCNetworkProvider)
+	}
+
+	supported_found := false
+	for _, networkType := range networkTypes {
+		if networkType == networkProvider {
+			supported_found = true
+			break
+		}
+	}
+
+	if !supported_found {
 		return "", logger.LogNewErrorf(log, "Unknown network provider. Error: %+v", err)
 	}
 
 	tkgVMIP, err := cnsoperatorutil.GetTKGVMIP(ctx, r.vmOperatorClient,
-		r.dynamicClient, vm.Namespace, vm.Name, nsxConfiguration)
+		r.dynamicClient, vm.Namespace, vm.Name, networkProvider)
 	if err != nil {
 		return "", logger.LogNewErrorf(log, "Failed to get external facing IP address for VM %q/%q. Err: %+v",
 			vm.Namespace, vm.Name, err)

pkg/syncer/cnsoperator/util/util.go

@@ -23,6 +23,7 @@ import (
 	vmoperatortypes "github.com/vmware-tanzu/vm-operator/api/v1alpha1"
 	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	apitypes "k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/dynamic"
@@ -39,6 +40,12 @@ var virtualNetworkGVR = schema.GroupVersionResource{
 	Resource: "virtualnetworks",
 }
 
+var networkInfoGVR = schema.GroupVersionResource{
+	Group:    "crd.nsx.vmware.com",
+	Version:  "v1alpha1",
+	Resource: "networkinfos",
+}
+
 const (
 	snatIPAnnotation = "ncp/snat_ip"
 	// Namespace for system resources.
@@ -51,6 +58,9 @@ const (
 	NSXTNetworkProvider = "NSXT_CONTAINER_PLUGIN"
 	// VDSNetworkProvider holds the network provider name for VDS based setups.
 	VDSNetworkProvider = "VSPHERE_NETWORK"
+	// VPCNetworkProvider holds the network provider name for VPC based setups.
+	VPCNetworkProvider = "NSX_VPC"
+	vpcDefaultSnatIp   = "defaultSNATIP"
 )
 
 // GetVolumeID gets the volume ID from the PV that is bound to PVC by pvcName.
@@ -83,7 +93,7 @@ func GetVolumeID(ctx context.Context, client client.Client, pvcName string, name
 // given Supervisor Namespace based on the networking configuration (NSX-T or
 // VDS).
 func GetTKGVMIP(ctx context.Context, vmOperatorClient client.Client, dc dynamic.Interface,
-	vmNamespace, vmName string, nsxConfiguration bool) (string, error) {
+	vmNamespace, vmName string, network_provider_type string) (string, error) {
 	log := logger.GetLogger(ctx)
 	log.Infof("Determining external IP Address of VM: %s/%s", vmNamespace, vmName)
 	virtualMachineInstance := &vmoperatortypes.VirtualMachine{}
@@ -108,7 +118,7 @@ func GetTKGVMIP(ctx context.Context, vmOperatorClient client.Client, dc dynamic.
 
 	var ip string
 	var exists bool
-	if nsxConfiguration {
+	if network_provider_type == NSXTNetworkProvider {
 		virtualNetworkInstance, err := dc.Resource(virtualNetworkGVR).Namespace(vmNamespace).Get(ctx,
 			networkName, metav1.GetOptions{})
 		if err != nil {
@@ -120,11 +130,47 @@ func GetTKGVMIP(ctx context.Context, vmOperatorClient client.Client, dc dynamic.
 		if !exists {
 			return "", fmt.Errorf("failed to get SNAT IP annotation from VirtualMachine %s/%s", vmNamespace, vmName)
 		}
-	} else {
+	} else if network_provider_type == VDSNetworkProvider {
 		ip = virtualMachineInstance.Status.VmIp
 		if ip == "" {
 			return "", fmt.Errorf("vm.Status.VmIp is not populated for %s/%s", vmNamespace, vmName)
 		}
+	} else {
+		vpcName := vmNamespace
+		networkInfoInstance, err := dc.Resource(networkInfoGVR).Namespace(vmNamespace).Get(ctx,
+			vpcName, metav1.GetOptions{})
+		if err != nil {
+			return "", err
+		}
+		log.Debugf("Got NetworkInfo instance %s/%s", vmNamespace, networkInfoInstance.GetName())
+		vpcs, found, err := unstructured.NestedSlice(networkInfoInstance.Object, "vpcs")
+		if err != nil || !found || len(vpcs) == 0 {
+			return "", fmt.Errorf("failed to get vpcs from networkinfo %s/%s with error: %v",
+				vmNamespace, vmName, err)
+		}
+
+		vpc, ok := vpcs[0].(map[string]interface{})
+		if !ok {
+			return "", fmt.Errorf("failed to assert vpc to map[string]interface{} %s/%s",
+				vmNamespace, vmName)
+		}
+
+		for key, value := range vpc {
+			if key == vpcDefaultSnatIp {
+				ip, ok = value.(string)
+				if !ok {
+					return "", fmt.Errorf("failed to cast key %s value to string", key)
+				}
+				break
+			}
+
+		}
+
+		if ip == "" {
+			return "", fmt.Errorf("spec.vpc.defaultSNATIP is not populated for "+
+				"networkinfo %s/%s", vmNamespace, vmName)
+		}
+
 	}
 	log.Infof("Found external IP Address %s for VirtualMachine %s/%s", ip, vmNamespace, vmName)
 	return ip, nil