Fix labeling virtual machines

controllers/elfmachine_controller.go

@@ -448,10 +448,8 @@ func (r *ElfMachineReconciler) reconcileNormal(ctx *context.MachineContext) (rec
 	}
 
 	// Reconcile the ElfMachine's Labels using the cluster info
-	if len(vm.Labels) == 0 {
-		if ok, err := r.reconcileLabels(ctx, vm); !ok {
-			return reconcile.Result{}, errors.Wrapf(err, "failed to reconcile labels")
-		}
+	if ok, err := r.reconcileLabels(ctx, vm); !ok {
+		return reconcile.Result{}, errors.Wrapf(err, "failed to reconcile labels")
 	}
 
 	// Reconcile the ElfMachine's providerID using the VM's UUID.
@@ -1188,10 +1186,26 @@ func (r *ElfMachineReconciler) getBootstrapData(ctx *context.MachineContext) (st
 }
 
 func (r *ElfMachineReconciler) reconcileLabels(ctx *context.MachineContext, vm *models.VM) (bool, error) {
-	creatorLabel, err := ctx.VMService.UpsertLabel(towerresources.GetVMLabelManaged(), "true")
-	if err != nil {
-		return false, errors.Wrapf(err, "failed to upsert label "+towerresources.GetVMLabelManaged())
+	managedLabelKey := towerresources.GetVMLabelManaged()
+	managedLabel := getLabelFromCache(managedLabelKey)
+	if managedLabel == nil {
+		var err error
+		managedLabel, err = ctx.VMService.UpsertLabel(managedLabelKey, "true")
+		if err != nil {
+			return false, errors.Wrapf(err, "failed to upsert label "+towerresources.GetVMLabelManaged())
+		}
+
+		setLabelCache(managedLabel)
 	}
+
+	// If the virtual machine has been labeled with managed label,
+	// it is considered that all labels have been labeled.
+	for i := 0; i < len(vm.Labels); i++ {
+		if *vm.Labels[i].ID == *managedLabel.ID {
+			return true, nil
+		}
+	}
+
 	namespaceLabel, err := ctx.VMService.UpsertLabel(towerresources.GetVMLabelNamespace(), ctx.ElfMachine.Namespace)
 	if err != nil {
 		return false, errors.Wrapf(err, "failed to upsert label "+towerresources.GetVMLabelNamespace())
@@ -1209,7 +1223,7 @@ func (r *ElfMachineReconciler) reconcileLabels(ctx *context.MachineContext, vm *
 		}
 	}
 
-	labelIDs := []string{*namespaceLabel.ID, *clusterNameLabel.ID, *creatorLabel.ID}
+	labelIDs := []string{*namespaceLabel.ID, *clusterNameLabel.ID, *managedLabel.ID}
 	if machineutil.IsControlPlaneMachine(ctx.ElfMachine) {
 		labelIDs = append(labelIDs, *vipLabel.ID)
 	}

controllers/elfmachine_controller_test.go

@@ -3447,6 +3447,48 @@ var _ = Describe("ElfMachineReconciler", func() {
 			Expect(err).ToNot(HaveOccurred())
 		})
 	})
+
+	Context("reconcileLabels", func() {
+		It("should add labels to the VM", func() {
+			managedLabel := &models.Label{
+				ID:    service.TowerString("managed-label"),
+				Key:   service.TowerString(towerresources.GetVMLabelManaged()),
+				Value: service.TowerString("true"),
+			}
+			namespaceLabel := &models.Label{
+				ID:    service.TowerString("namespace-label"),
+				Key:   service.TowerString(towerresources.GetVMLabelNamespace()),
+				Value: service.TowerString(elfMachine.Namespace),
+			}
+
+			clusterNameLabel := &models.Label{
+				ID:    service.TowerString("cluster-label"),
+				Key:   service.TowerString(towerresources.GetVMLabelClusterName()),
+				Value: service.TowerString(elfCluster.Name),
+			}
+
+			vm := fake.NewTowerVMFromElfMachine(elfMachine)
+			ctrlContext := newCtrlContexts(elfCluster, cluster, elfMachine, machine, secret, md)
+			machineContext := newMachineContext(ctrlContext, elfCluster, cluster, elfMachine, machine, mockVMService)
+			machineContext.VMService = mockVMService
+			mockVMService.EXPECT().UpsertLabel(*managedLabel.Key, *managedLabel.Value).Return(managedLabel, nil)
+			mockVMService.EXPECT().UpsertLabel(*namespaceLabel.Key, *namespaceLabel.Value).Return(namespaceLabel, nil)
+			mockVMService.EXPECT().UpsertLabel(*clusterNameLabel.Key, *clusterNameLabel.Value).Return(clusterNameLabel, nil)
+			mockVMService.EXPECT().AddLabelsToVM(*vm.ID, gomock.InAnyOrder([]string{*managedLabel.ID, *namespaceLabel.ID, *clusterNameLabel.ID})).Times(1)
+
+			reconciler := &ElfMachineReconciler{ControllerContext: ctrlContext, NewVMService: mockNewVMService}
+			ok, err := reconciler.reconcileLabels(machineContext, vm)
+			Expect(ok).To(BeTrue())
+			Expect(err).ToNot(HaveOccurred())
+			Expect(getLabelFromCache(*managedLabel.Key)).To(Equal(managedLabel))
+
+			vm.Labels = []*models.NestedLabel{{ID: managedLabel.ID}}
+			reconciler = &ElfMachineReconciler{ControllerContext: ctrlContext, NewVMService: mockNewVMService}
+			ok, err = reconciler.reconcileLabels(machineContext, vm)
+			Expect(ok).To(BeTrue())
+			Expect(err).ToNot(HaveOccurred())
+		})
+	})
 })
 
 func waitStaticIPAllocationSpec(mockNewVMService func(ctx goctx.Context, auth infrav1.Tower, logger logr.Logger) (service.VMService, error),

controllers/tower_cache.go

@@ -176,7 +176,7 @@ func getKeyForPGCache(pgName string) string {
 // setPGCache saves the specified placement group to the memory,
 // which can reduce access to the Tower service.
 func setPGCache(pg *models.VMPlacementGroup) {
-	inMemoryCache.Set(getKeyForPGCache(*pg.Name), *pg, gpuCacheDuration)
+	inMemoryCache.Set(getKeyForPGCache(*pg.Name), *pg, pgCacheDuration)
 }
 
 // delPGCaches deletes the specified placement group caches.
@@ -200,6 +200,33 @@ func getPGFromCache(pgName string) *models.VMPlacementGroup {
 	return nil
 }
 
+// labelCacheDuration is the lifespan of label cache.
+const labelCacheDuration = 30 * time.Minute
+
+func getKeyForLabelCache(labelKey string) string {
+	return fmt.Sprintf("label:%s:cache", labelKey)
+}
+
+// setLabelCache saves the specified label to the memory,
+// which can reduce access to the Tower service.
+func setLabelCache(label *models.Label) {
+	inMemoryCache.Set(getKeyForLabelCache(*label.Key), *label, labelCacheDuration)
+}
+
+// getLabelFromCache gets the specified label from the memory.
+func getLabelFromCache(labelKey string) *models.Label {
+	key := getKeyForLabelCache(labelKey)
+	if val, found := inMemoryCache.Get(key); found {
+		if label, ok := val.(models.Label); ok {
+			return &label
+		}
+		// Delete unexpected data.
+		inMemoryCache.Delete(key)
+	}
+
+	return nil
+}
+
 /* GPU */
 
 // gpuCacheDuration is the lifespan of gpu cache.

controllers/tower_cache_test.go

@@ -186,6 +186,20 @@ var _ = Describe("TowerCache", func() {
 		Expect(getPGFromCache(pgName)).To(BeNil())
 	})
 
+	It("Label Cache", func() {
+		label := &models.Label{
+			ID:    service.TowerString("label-id"),
+			Key:   service.TowerString("label-key"),
+			Value: service.TowerString("label-name"),
+		}
+
+		Expect(getPGFromCache(*label.Key)).To(BeNil())
+		setLabelCache(label)
+		Expect(getLabelFromCache(*label.Key)).To(Equal(label))
+		resetMemoryCache()
+		Expect(getLabelFromCache(*label.Key)).To(BeNil())
+	})
+
 	It("GPU Cache", func() {
 		gpuID := "gpu"
 		gpuVMInfo := models.GpuVMInfo{ID: service.TowerString(gpuID)}