From 0b6d80bbe0dae63acd8a5155bb99a1ab6bd9167d Mon Sep 17 00:00:00 2001 From: Chethan Venkatesh Date: Fri, 27 Sep 2024 15:54:06 -0700 Subject: [PATCH] Add tkc cluster annotation on Supervisor PVC objects --- pkg/csi/service/common/constants.go | 4 ++ pkg/csi/service/wcpguest/controller.go | 28 ++++++++++- pkg/syncer/pvcsi_fullsync.go | 65 ++++++++++++++++++++++++++ 3 files changed, 95 insertions(+), 2 deletions(-) diff --git a/pkg/csi/service/common/constants.go b/pkg/csi/service/common/constants.go index a99303871b..47e31f52e8 100644 --- a/pkg/csi/service/common/constants.go +++ b/pkg/csi/service/common/constants.go @@ -296,6 +296,10 @@ const ( // AnnVolumeAccessibleTopology is the annotation set by the supervisor cluster on PVC AnnVolumeAccessibleTopology = "csi.vsphere.volume-accessible-topology" + // AnnTanzuGuestClusterOwner is the annotation key to set TanzuClusterID as value + // on the PVC in supervisor cluster + AnnTanzuGuestClusterOwner = "csi.vsphere.tanzu-kubernetes-cluster" + // PVtoBackingDiskObjectIdSupportedVCenterMajor is the minimum major version of vCenter // on which PV to BackingDiskObjectId mapping feature is supported. PVtoBackingDiskObjectIdSupportedVCenterMajor int = 7 diff --git a/pkg/csi/service/wcpguest/controller.go b/pkg/csi/service/wcpguest/controller.go index d6836601bc..b91b3ead9b 100644 --- a/pkg/csi/service/wcpguest/controller.go +++ b/pkg/csi/service/wcpguest/controller.go @@ -81,6 +81,7 @@ type controller struct { vmWatcher *cache.ListWatch supervisorNamespace string tanzukubernetesClusterUID string + tanzukubernetesClusterName string } // New creates a CNS controller @@ -99,6 +100,7 @@ func (c *controller) Init(config *commonconfig.Config, version string) error { return err } c.tanzukubernetesClusterUID = config.GC.TanzuKubernetesClusterUID + c.tanzukubernetesClusterName = config.GC.TanzuKubernetesClusterName c.restClientConfig = k8s.GetRestClientConfigForSupervisor(ctx, config.GC.Endpoint, config.GC.Port) c.supervisorClient, err = k8s.NewSupervisorClient(ctx, c.restClientConfig) if err != nil { @@ -311,13 +313,35 @@ func (c *controller) CreateVolume(ctx context.Context, req *csi.CreateVolumeRequ if err != nil { if errors.IsNotFound(err) { diskSize := strconv.FormatInt(volSizeMB, 10) + "Mi" - var annotations map[string]string + annotations := make(map[string]string) + // + // Function to generate annotation key with an index if it already exists + addAnnotationWithIndex := func(prefix, uid string, annotations map[string]string) { + index := 1 + key := fmt.Sprintf("%s-%d", prefix, index) + + // Check if key with current index exists, and increment until it's unique + for { + if _, exists := annotations[key]; !exists { + break + } + index++ + key = fmt.Sprintf("%s-%d", prefix, index) + } + + // Assign the value to the unique key + annotations[key] = c.tanzukubernetesClusterUID + } + + // Adding the annotation with index check + annotationKey := c.tanzukubernetesClusterName + c.tanzukubernetesClusterUID + addAnnotationWithIndex(common.AnnTanzuGuestClusterOwner, annotationKey, annotations) + if commonco.ContainerOrchestratorUtility.IsFSSEnabled(ctx, common.TKGsHA) && req.AccessibilityRequirements != nil && (commonco.ContainerOrchestratorUtility.IsFSSEnabled(ctx, common.WorkloadDomainIsolation) || !isFileVolumeRequest) { // Generate volume topology requirement annotation. - annotations = make(map[string]string) topologyAnnotation, err := generateGuestClusterRequestedTopologyJSON(req.AccessibilityRequirements.Preferred) if err != nil { msg := fmt.Sprintf("failed to generate accessibility topology for pvc with name: %s "+ diff --git a/pkg/syncer/pvcsi_fullsync.go b/pkg/syncer/pvcsi_fullsync.go index 369b37e240..d49553d8a4 100644 --- a/pkg/syncer/pvcsi_fullsync.go +++ b/pkg/syncer/pvcsi_fullsync.go @@ -19,6 +19,7 @@ package syncer import ( "context" "encoding/json" + "fmt" "reflect" "time" @@ -150,6 +151,14 @@ func PvcsiFullSync(ctx context.Context, metadataSyncer *metadataSyncInformer) er } } + // Set csi.vsphere.tanzu-kubernetes-cluster annotation on the Supervisor PVC which is + // requested from TKC Cluster + err = setTanzuClusterIDOnSupervisorPVC(ctx, metadataSyncer, supervisorNamespace) + if err != nil { + log.Errorf("FullSync: Failed to create CnsVolumeMetadataList from guest cluster. Err: %v", err) + return err + } + log.Infof("FullSync: End") return nil } @@ -355,6 +364,62 @@ func GenerateVolumeNodeAffinity(accessibleTopology []*csi.Topology) *v1.VolumeNo } } +func setTanzuClusterIDOnSupervisorPVC(ctx context.Context, metadataSyncer *metadataSyncInformer, + supervisorNamespace string) error { + log := logger.GetLogger(ctx) + log.Debugf("FullSync: Querying guest cluster API server for all PV objects.") + pvList, err := getPVsInBoundAvailableOrReleased(ctx, metadataSyncer) + if err != nil { + log.Errorf("FullSync: Failed to get PVs from guest cluster. Err: %v", err) + return err + } + for _, pv := range pvList { + if pv.Spec.ClaimRef != nil && pv.Status.Phase == v1.VolumeBound { + svPVC, err := metadataSyncer.supervisorClient.CoreV1().PersistentVolumeClaims(supervisorNamespace). + Get(ctx, pv.Spec.CSI.VolumeHandle, metav1.GetOptions{}) + if err != nil { + msg := fmt.Sprintf("failed to retrieve supervisor PVC %q in %q namespace. Error: %+v", + pv.Spec.CSI.VolumeHandle, supervisorNamespace, err) + log.Error(msg) + continue + } + if svPVC.Annotations[common.AnnTanzuGuestClusterOwner] != "" { + // Function to generate annotation key with an index if it already exists + addAnnotationWithIndex := func(prefix, uid string, annotations map[string]string) { + index := 1 + key := fmt.Sprintf("%s-%d", prefix, index) + + // Check if key with current index exists, and increment until it's unique + for { + if _, exists := annotations[key]; !exists { + break + } + index++ + key = fmt.Sprintf("%s-%d", prefix, index) + } + + // Assign the value to the unique key + annotations[key] = metadataSyncer.configInfo.Cfg.GC.TanzuKubernetesClusterUID + } + // Adding the annotation with index check + annotationKey := metadataSyncer.configInfo.Cfg.GC.TanzuKubernetesClusterName + + metadataSyncer.configInfo.Cfg.GC.TanzuKubernetesClusterUID + + addAnnotationWithIndex(common.AnnTanzuGuestClusterOwner, annotationKey, svPVC.Annotations) + _, err = metadataSyncer.supervisorClient.CoreV1().PersistentVolumeClaims(supervisorNamespace).Update( + ctx, svPVC, metav1.UpdateOptions{}) + if err != nil { + msg := fmt.Sprintf("failed to update supervisor PVC: %q with annoation %q in %q namespace. Error: %+v", + pv.Spec.CSI.VolumeHandle, common.AnnTanzuGuestClusterOwner, supervisorNamespace, err) + log.Error(msg) + continue + } + } + } + } + return nil +} + // compareCnsVolumeMetadatas compares input cnsvolumemetadata objects // and returns false if their labels are not deeply equal. func compareCnsVolumeMetadatas(guestObject *cnsvolumemetadatav1alpha1.CnsVolumeMetadataSpec,