Convert fleet-agent to controller-runtime

go.mod

@@ -70,6 +70,7 @@ require (
 	k8s.io/client-go v12.0.0+incompatible
 	k8s.io/klog/v2 v2.110.1
 	k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00
+	k8s.io/kubectl v0.27.3
 	k8s.io/kubernetes v1.28.4
 	sigs.k8s.io/cli-utils v0.34.0
 	sigs.k8s.io/controller-runtime v0.16.3
@@ -243,7 +244,6 @@ require (
 	k8s.io/code-generator v0.28.3 // indirect
 	k8s.io/component-base v0.28.3 // indirect
 	k8s.io/gengo v0.0.0-20230829151522-9cce18d56c01 // indirect
-	k8s.io/kubectl v0.27.3 // indirect
 	k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect
 	oras.land/oras-go v1.2.4 // indirect
 	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect

integrationtests/agent/bundle_deployment_status_test.go

@@ -1,38 +1,38 @@
 package agent
 
 import (
+	"context"
 	"os"
 
+	"github.com/rancher/fleet/integrationtests/utils"
 	"github.com/rancher/fleet/pkg/apis/fleet.cattle.io/v1alpha1"
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/intstr"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
-func orphanBundeResources() map[string][]v1alpha1.BundleResource {
-	v1, err := os.ReadFile(assetsPath + "/deployment-v1.yaml")
-	Expect(err).NotTo(HaveOccurred())
-	v2, err := os.ReadFile(assetsPath + "/deployment-v2.yaml")
-	Expect(err).NotTo(HaveOccurred())
-
-	return map[string][]v1alpha1.BundleResource{
-		"v1": {
-			{
-				Name:     "deployment-v1.yaml",
-				Content:  string(v1),
-				Encoding: "",
-			},
-		}, "v2": {
-			{
-				Name:     "deployment-v2.yaml",
-				Content:  string(v2),
-				Encoding: "",
-			},
+func init() {
+	v1, _ := os.ReadFile(assetsPath + "/deployment-v1.yaml")
+	v2, _ := os.ReadFile(assetsPath + "/deployment-v2.yaml")
+
+	resources["v1"] = []v1alpha1.BundleResource{
+		{
+			Name:     "deployment-v1.yaml",
+			Content:  string(v1),
+			Encoding: "",
+		},
+	}
+	resources["v2"] = []v1alpha1.BundleResource{
+		{
+			Name:     "deployment-v2.yaml",
+			Content:  string(v2),
+			Encoding: "",
 		},
 	}
 }
@@ -45,48 +45,216 @@ var _ = Describe("BundleDeployment status", Ordered, func() {
 	)
 
 	var (
-		env       *specEnv
 		namespace string
 		name      string
+		env       *specEnv
 	)
 
 	createBundleDeploymentV1 := func(name string) {
 		bundled := v1alpha1.BundleDeployment{
 			ObjectMeta: metav1.ObjectMeta{
 				Name:      name,
-				Namespace: namespace,
+				Namespace: clusterNS,
 			},
 			Spec: v1alpha1.BundleDeploymentSpec{
 				DeploymentID: "v1",
+				Options: v1alpha1.BundleDeploymentOptions{
+					DefaultNamespace: namespace,
+				},
 			},
 		}
 
-		b, err := env.controller.Create(&bundled)
+		err := k8sClient.Create(context.TODO(), &bundled)
 		Expect(err).To(BeNil())
-		Expect(b).To(Not(BeNil()))
+		Expect(bundled).To(Not(BeNil()))
 	}
 
-	BeforeAll(func() {
-		env = specEnvs["orphanbundle"]
-		name = "orphanbundle"
-		namespace = env.namespace
-		DeferCleanup(func() {
-			Expect(k8sClient.Delete(ctx, &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: namespace}})).ToNot(HaveOccurred())
-		})
-	})
+	createNamespace := func() string {
+		namespace, err := utils.NewNamespaceName()
+		Expect(err).ToNot(HaveOccurred())
+
+		ns := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: namespace}}
+		Expect(k8sClient.Create(context.Background(), ns)).ToNot(HaveOccurred())
+
+		return namespace
+	}
 
 	When("New bundle deployment is created", func() {
 		BeforeAll(func() {
+			name = "orphanbundletest1"
+			namespace = createNamespace()
+			DeferCleanup(func() {
+				Expect(k8sClient.Delete(ctx, &corev1.Namespace{
+					ObjectMeta: metav1.ObjectMeta{Name: namespace}})).ToNot(HaveOccurred())
+			})
+			env = &specEnv{namespace: namespace}
+
 			// this BundleDeployment will create a deployment with the resources from assets/deployment-v1.yaml
 			createBundleDeploymentV1(name)
+			DeferCleanup(func() {
+				Expect(k8sClient.Delete(context.TODO(), &v1alpha1.BundleDeployment{
+					ObjectMeta: metav1.ObjectMeta{Namespace: clusterNS, Name: name},
+				})).ToNot(HaveOccurred())
+			})
+		})
+
+		It("Detects the BundleDeployment as not ready", func() {
+			bd := &v1alpha1.BundleDeployment{}
+			err := k8sClient.Get(ctx, types.NamespacedName{Namespace: clusterNS, Name: name}, bd)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(bd.Status.Ready).To(BeFalse())
+		})
+
+		It("Eventually updates the BundleDeployment to make it ready and non modified", func() {
+			Eventually(env.isBundleDeploymentReadyAndNotModified).WithArguments(name).Should(BeTrue())
+		})
+
+		It("Deploys resources from BundleDeployment to the cluster", func() {
+			svc, err := env.getService(svcName)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(svc.Name).NotTo(BeEmpty())
+		})
+
+		It("Lists deployed resources in the status", func() {
+			bd := &v1alpha1.BundleDeployment{}
+			err := k8sClient.Get(ctx, types.NamespacedName{Namespace: clusterNS, Name: name}, bd)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(bd.Status.Resources).To(HaveLen(3))
+			ts := bd.Status.Resources[0].CreatedAt
+			Expect(ts.Time).ToNot(BeZero())
+			Expect(bd.Status.Resources).To(ContainElement(v1alpha1.BundleDeploymentResource{
+				Kind:       "Service",
+				APIVersion: "v1",
+				Namespace:  namespace,
+				Name:       "svc-test",
+				CreatedAt:  ts,
+			}))
+		})
+
+		Context("A release resource is modified", func() {
+			It("Modify service", func() {
+				svc, err := env.getService(svcName)
+				Expect(err).NotTo(HaveOccurred())
+				patch := svc.DeepCopy()
+				patch.Spec.Selector = map[string]string{"foo": "bar"}
+				Expect(k8sClient.Patch(ctx, patch, client.MergeFrom(&svc))).NotTo(HaveOccurred())
+			})
+
+			It("BundleDeployment status will not be Ready, and will contain the error message", func() {
+				Eventually(func() bool {
+					modifiedStatus := v1alpha1.ModifiedStatus{
+						Kind:       "Service",
+						APIVersion: "v1",
+						Namespace:  namespace,
+						Name:       "svc-test",
+						Create:     false,
+						Delete:     false,
+						Patch:      "{\"spec\":{\"selector\":{\"app.kubernetes.io/name\":\"MyApp\"}}}",
+					}
+					return env.isNotReadyAndModified(name, modifiedStatus, "service.v1 "+namespace+"/svc-test modified {\"spec\":{\"selector\":{\"app.kubernetes.io/name\":\"MyApp\"}}}")
+				}).Should(BeTrue(), "BundleDeployment status does not contain modified message")
+			})
+
+			It("Modify service to have its original value", func() {
+				svc, err := env.getService(svcName)
+				Expect(err).NotTo(HaveOccurred())
+				patch := svc.DeepCopy()
+				patch.Spec.Selector = map[string]string{"app.kubernetes.io/name": "MyApp"}
+				Expect(k8sClient.Patch(ctx, patch, client.MergeFrom(&svc))).To(BeNil())
+			})
+
+			It("BundleDeployment will eventually be ready and non modified", func() {
+				Eventually(env.isBundleDeploymentReadyAndNotModified).WithArguments(name).Should(BeTrue())
+			})
 		})
 
-		AfterAll(func() {
-			Expect(env.controller.Delete(namespace, name, nil)).NotTo(HaveOccurred())
+		Context("Upgrading to a release that will leave an orphan resource", func() {
+			It("Upgrade BundleDeployment to a release that deletes the svc with a finalizer", func() {
+				Eventually(func() bool {
+					bd := &v1alpha1.BundleDeployment{}
+					err := k8sClient.Get(ctx, types.NamespacedName{Namespace: clusterNS, Name: name}, bd)
+					Expect(err).To(BeNil())
+					bd.Spec.DeploymentID = "v2"
+					err = k8sClient.Update(ctx, bd)
+					return err == nil && bd != nil
+				}).Should(BeTrue())
+
+			})
+
+			It("BundleDeployment status will eventually be extra", func() {
+				Eventually(func() bool {
+					modifiedStatus := v1alpha1.ModifiedStatus{
+						Kind:       "Service",
+						APIVersion: "v1",
+						Namespace:  namespace,
+						Name:       "svc-finalizer",
+						Create:     false,
+						Delete:     true,
+						Patch:      "",
+					}
+					return env.isNotReadyAndModified(name, modifiedStatus, "service.v1 "+namespace+"/svc-finalizer extra")
+				}).Should(BeTrue())
+			})
+
+			It("Remove finalizer", func() {
+				svc, err := env.getService(svcFinalizerName)
+				Expect(err).NotTo(HaveOccurred())
+				patch := svc.DeepCopy()
+				patch.Finalizers = nil
+				Expect(k8sClient.Patch(ctx, patch, client.MergeFrom(&svc))).To(BeNil())
+			})
+
+			It("BundleDeployment will eventually be ready and non modified", func() {
+				Eventually(env.isBundleDeploymentReadyAndNotModified).WithArguments(name).Should(BeTrue())
+			})
+		})
+
+		Context("Delete a resource in the release", func() {
+			It("Delete service", func() {
+				svc, err := env.getService(svcName)
+				Expect(err).NotTo(HaveOccurred())
+				err = k8sClient.Delete(ctx, &svc)
+				Expect(err).NotTo(HaveOccurred())
+			})
+
+			It("BundleDeployment status will eventually be missing", func() {
+				Eventually(func() bool {
+					modifiedStatus := v1alpha1.ModifiedStatus{
+						Kind:       "Service",
+						APIVersion: "v1",
+						Namespace:  namespace,
+						Name:       "svc-test",
+						Create:     true,
+						Delete:     false,
+						Patch:      "",
+					}
+					return env.isNotReadyAndModified(name, modifiedStatus, "service.v1 "+namespace+"/svc-test missing")
+				}).Should(BeTrue())
+			})
+		})
+	})
+
+	When("Simulating how another operator modifies a dynamic resource", func() {
+		BeforeAll(func() {
+			namespace = createNamespace()
+			DeferCleanup(func() {
+				Expect(k8sClient.Delete(ctx, &corev1.Namespace{
+					ObjectMeta: metav1.ObjectMeta{Name: namespace}})).ToNot(HaveOccurred())
+			})
+			env = &specEnv{namespace: namespace}
+
+			name = "orphanbundletest2"
+			createBundleDeploymentV1(name)
+			DeferCleanup(func() {
+				Expect(k8sClient.Delete(context.TODO(), &v1alpha1.BundleDeployment{
+					ObjectMeta: metav1.ObjectMeta{Namespace: clusterNS, Name: name},
+				})).ToNot(HaveOccurred())
+			})
 		})
 
 		It("BundleDeployment is not ready", func() {
-			bd, err := env.controller.Get(namespace, name, metav1.GetOptions{})
+			bd := &v1alpha1.BundleDeployment{}
+			err := k8sClient.Get(ctx, types.NamespacedName{Namespace: clusterNS, Name: name}, bd)
 			Expect(err).NotTo(HaveOccurred())
 			Expect(bd.Status.Ready).To(BeFalse())
 		})
@@ -98,11 +266,12 @@ var _ = Describe("BundleDeployment status", Ordered, func() {
 		It("Resources from BundleDeployment are present in the cluster", func() {
 			svc, err := env.getService(svcName)
 			Expect(err).NotTo(HaveOccurred())
-			Expect(svc).To(Not(BeNil()))
+			Expect(svc.Name).NotTo(BeEmpty())
 		})
 
 		It("Lists deployed resources in the status", func() {
-			bd, err := env.controller.Get(namespace, name, metav1.GetOptions{})
+			bd := &v1alpha1.BundleDeployment{}
+			err := k8sClient.Get(ctx, types.NamespacedName{Namespace: clusterNS, Name: name}, bd)
 			Expect(err).NotTo(HaveOccurred())
 			Expect(bd.Status.Resources).To(HaveLen(3))
 			ts := bd.Status.Resources[0].CreatedAt
@@ -137,7 +306,7 @@ var _ = Describe("BundleDeployment status", Ordered, func() {
 						Patch:      "{\"spec\":{\"selector\":{\"app.kubernetes.io/name\":\"MyApp\"}}}",
 					}
 					return env.isNotReadyAndModified(name, modifiedStatus, "service.v1 "+namespace+"/svc-test modified {\"spec\":{\"selector\":{\"app.kubernetes.io/name\":\"MyApp\"}}}")
-				}).Should(BeTrue())
+				}).Should(BeTrue(), "BundleDeployment status does not contain modified message")
 			})
 
 			It("Modify service to have its original value", func() {
@@ -156,11 +325,12 @@ var _ = Describe("BundleDeployment status", Ordered, func() {
 		Context("Upgrading to a release that will leave an orphan resource", func() {
 			It("Upgrade BundleDeployment to a release that deletes the svc with a finalizer", func() {
 				Eventually(func() bool {
-					bd, err := env.controller.Get(namespace, name, metav1.GetOptions{})
+					bd := &v1alpha1.BundleDeployment{}
+					err := k8sClient.Get(ctx, types.NamespacedName{Namespace: clusterNS, Name: name}, bd)
 					Expect(err).To(BeNil())
 					bd.Spec.DeploymentID = "v2"
-					b, err := env.controller.Update(bd)
-					return err == nil && b != nil
+					err = k8sClient.Update(ctx, bd)
+					return err == nil && bd != nil
 				}).Should(BeTrue())
 
 			})
@@ -220,7 +390,21 @@ var _ = Describe("BundleDeployment status", Ordered, func() {
 
 	When("Simulating how another operator creates a dynamic resource", func() {
 		BeforeAll(func() {
+			namespace = createNamespace()
+			DeferCleanup(func() {
+				Expect(k8sClient.Delete(ctx, &corev1.Namespace{
+					ObjectMeta: metav1.ObjectMeta{Name: namespace}})).ToNot(HaveOccurred())
+			})
+			env = &specEnv{namespace: namespace}
+
+			name = "orphanbundletest2"
 			createBundleDeploymentV1(name)
+			DeferCleanup(func() {
+				Expect(k8sClient.Delete(context.TODO(), &v1alpha1.BundleDeployment{
+					ObjectMeta: metav1.ObjectMeta{Namespace: clusterNS, Name: name},
+				})).ToNot(HaveOccurred())
+			})
+
 			// It is possible that some operators copy the objectset.rio.cattle.io/hash label into a dynamically created objects.
 			// https://github.com/rancher/fleet/issues/1141
 			By("Simulating orphan resource creation", func() {
@@ -242,18 +426,14 @@ var _ = Describe("BundleDeployment status", Ordered, func() {
 			})
 		})
 
-		AfterAll(func() {
-			Expect(env.controller.Delete(namespace, name, nil)).NotTo(HaveOccurred())
-		})
-
 		It("BundleDeployment will eventually be ready and non modified", func() {
 			Eventually(env.isBundleDeploymentReadyAndNotModified).WithArguments(name).Should(BeTrue())
 		})
 
 		It("Resources from BundleDeployment are present in the cluster", func() {
 			svc, err := env.getService(svcName)
 			Expect(err).NotTo(HaveOccurred())
-			Expect(svc).To(Not(BeNil()))
+			Expect(svc.Name).NotTo(BeEmpty())
 		})
 	})
 })