From 868f241145d83fa4574d2311ae15822944137b15 Mon Sep 17 00:00:00 2001 From: ashnamehrotra Date: Wed, 27 Sep 2023 18:13:20 -0700 Subject: [PATCH] fix: clean up imagejob when eraser-manager pod is deleted (#875) --- Makefile | 10 ++- controllers/imagejob/imagejob_controller.go | 6 +- test/e2e/test-data/Dockerfile.dummyCollector | 3 + .../collector_delete_manager/eraser_test.go | 90 +++++++++++++++++++ .../collector_delete_manager/main_test.go | 59 ++++++++++++ test/e2e/util/utils.go | 21 +++-- 6 files changed, 178 insertions(+), 11 deletions(-) create mode 100644 test/e2e/test-data/Dockerfile.dummyCollector create mode 100644 test/e2e/tests/collector_delete_manager/eraser_test.go create mode 100644 test/e2e/tests/collector_delete_manager/main_test.go diff --git a/Makefile b/Makefile index fc1ecf586c..b8d3e10334 100644 --- a/Makefile +++ b/Makefile @@ -156,6 +156,13 @@ busybox-img: --build-arg IMG=$(BUSYBOX_BASE_IMG) test/e2e/test-data BUSYBOX_IMG=busybox-e2e-test:latest +collector-dummy-img: + docker build -t $(COLLECTOR_REPO):dummy \ + -f test/e2e/test-data/Dockerfile.dummyCollector \ + test/e2e/test-data +COLLECTOR_IMAGE_DUMMY=$(COLLECTOR_REPO):dummy + + vulnerable-img: docker pull $(VULNERABLE_IMG) @@ -171,7 +178,7 @@ non-vulnerable-img: -t ${NON_VULNERABLE_IMG} \ --target non-vulnerable . -e2e-test: vulnerable-img eol-img non-vulnerable-img busybox-img +e2e-test: vulnerable-img eol-img non-vulnerable-img busybox-img collector-dummy-img for test in $(E2E_TESTS); do \ CGO_ENABLED=0 \ PROJECT_ABSOLUTE_PATH=$(CURDIR) \ @@ -185,6 +192,7 @@ e2e-test: vulnerable-img eol-img non-vulnerable-img busybox-img COLLECTOR_IMAGE=${COLLECTOR_IMG} \ SCANNER_IMAGE=${TRIVY_SCANNER_IMG} \ BUSYBOX_IMAGE=${BUSYBOX_IMG} \ + COLLECTOR_IMAGE_DUMMY=${COLLECTOR_IMAGE_DUMMY} \ VULNERABLE_IMAGE=${VULNERABLE_IMG} \ NON_VULNERABLE_IMAGE=${NON_VULNERABLE_IMG} \ EOL_IMAGE=${EOL_IMG} \ diff --git a/controllers/imagejob/imagejob_controller.go b/controllers/imagejob/imagejob_controller.go index b627c5e823..403bfb4675 100644 --- a/controllers/imagejob/imagejob_controller.go +++ b/controllers/imagejob/imagejob_controller.go @@ -255,7 +255,11 @@ func (r *Reconciler) handleRunningJob(ctx context.Context, imageJob *eraserv1.Im Namespace: namespace, }, &template) if err != nil { - return err + imageJob.Status = eraserv1.ImageJobStatus{ + Phase: eraserv1.PhaseFailed, + DeleteAfter: controllerUtils.After(time.Now(), 1), + } + return r.updateJobStatus(ctx, imageJob) } listOpts := podListOptions(&template) diff --git a/test/e2e/test-data/Dockerfile.dummyCollector b/test/e2e/test-data/Dockerfile.dummyCollector new file mode 100644 index 0000000000..a37a8b8f5d --- /dev/null +++ b/test/e2e/test-data/Dockerfile.dummyCollector @@ -0,0 +1,3 @@ +FROM busybox:latest + +ENTRYPOINT ["yes"] diff --git a/test/e2e/tests/collector_delete_manager/eraser_test.go b/test/e2e/tests/collector_delete_manager/eraser_test.go new file mode 100644 index 0000000000..aa486a7b42 --- /dev/null +++ b/test/e2e/tests/collector_delete_manager/eraser_test.go @@ -0,0 +1,90 @@ +//go:build e2e +// +build e2e + +package e2e + +import ( + "context" + "testing" + "time" + + "github.com/eraser-dev/eraser/test/e2e/util" + + eraserv1alpha1 "github.com/eraser-dev/eraser/api/v1alpha1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "sigs.k8s.io/e2e-framework/klient/wait" + "sigs.k8s.io/e2e-framework/klient/wait/conditions" + "sigs.k8s.io/e2e-framework/pkg/envconf" + "sigs.k8s.io/e2e-framework/pkg/features" +) + +func TestDeleteManager(t *testing.T) { + deleteManagerFeat := features.New("Deleting manager pod while current ImageJob is running should delete ImageJob and restart"). + Assess("Wait for eraser pods running", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { + c, err := cfg.NewClient() + if err != nil { + t.Fatal("Failed to create new client", err) + } + + err = wait.For( + util.NumPodsPresentForLabel(ctx, c, 3, util.ImageJobTypeLabelKey+"="+util.CollectorLabel), + wait.WithTimeout(time.Minute*2), + wait.WithInterval(time.Millisecond*500), + ) + if err != nil { + t.Fatal(err) + } + + return ctx + }). + Assess("Delete controller-manager pod", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { + c, err := cfg.NewClient() + if err != nil { + t.Fatal("Failed to create new client", err) + } + + // get manager pod + var podList corev1.PodList + err = c.Resources().List(ctx, &podList, func(o *metav1.ListOptions) { + o.LabelSelector = labels.SelectorFromSet(map[string]string{util.ManagerLabelKey: util.ManagerLabelValue}).String() + }) + if err != nil { + t.Errorf("could not list manager pods: %v", err) + } + + if len(podList.Items) != 1 { + t.Error("incorrect number of manager pods: ", len(podList.Items)) + } + + // get current ImageJob before deleting manager pod + var jobList eraserv1alpha1.ImageJobList + err = c.Resources().List(ctx, &jobList) + if err != nil { + t.Errorf("could not list ImageJob: %v", err) + } + + t.Log("job", jobList.Items[0], "name", jobList.Items[0].Name) + + if len(jobList.Items) != 1 { + t.Error("incorrect number of ImageJobs: ", len(jobList.Items)) + } + + // delete manager pod + if err := util.KubectlDelete(cfg.KubeconfigFile(), util.TestNamespace, []string{"pod", podList.Items[0].Name}); err != nil { + t.Error("unable to delete eraser-controller-manager pod") + } + + // wait for deletion of ImageJob + err = wait.For(conditions.New(c.Resources()).ResourcesDeleted(&jobList), wait.WithTimeout(util.Timeout)) + if err != nil { + t.Errorf("error waiting for ImageJob to be deleted: %v", err) + } + + return ctx + }). + Feature() + + util.Testenv.Test(t, deleteManagerFeat) +} diff --git a/test/e2e/tests/collector_delete_manager/main_test.go b/test/e2e/tests/collector_delete_manager/main_test.go new file mode 100644 index 0000000000..f5387839b8 --- /dev/null +++ b/test/e2e/tests/collector_delete_manager/main_test.go @@ -0,0 +1,59 @@ +//go:build e2e +// +build e2e + +package e2e + +import ( + "os" + "testing" + + eraserv1alpha1 "github.com/eraser-dev/eraser/api/v1alpha1" + "github.com/eraser-dev/eraser/test/e2e/util" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/client-go/kubernetes/scheme" + "sigs.k8s.io/e2e-framework/pkg/env" + "sigs.k8s.io/e2e-framework/pkg/envconf" + "sigs.k8s.io/e2e-framework/pkg/envfuncs" +) + +func TestMain(m *testing.M) { + utilruntime.Must(eraserv1alpha1.AddToScheme(scheme.Scheme)) + + removerImage := util.ParsedImages.RemoverImage + managerImage := util.ParsedImages.ManagerImage + collectorImage := util.ParsedImages.CollectorImage + + util.Testenv = env.NewWithConfig(envconf.New()) + // Create KinD Cluster + util.Testenv.Setup( + envfuncs.CreateKindClusterWithConfig(util.KindClusterName, util.NodeVersion, util.KindConfigPath), + envfuncs.CreateNamespace(util.TestNamespace), + util.LoadImageToCluster(util.KindClusterName, util.ManagerImage, util.ManagerTarballPath), + util.LoadImageToCluster(util.KindClusterName, util.RemoverImage, util.RemoverTarballPath), + util.LoadImageToCluster(util.KindClusterName, util.RemoverImage, util.RemoverTarballPath), + util.LoadImageToCluster(util.KindClusterName, util.CollectorDummyImage, ""), + util.LoadImageToCluster(util.KindClusterName, util.NonVulnerableImage, ""), + util.HelmDeployLatestEraserRelease(util.TestNamespace, + "--set", util.ScannerEnable.Set("false"), + "--set", util.CollectorEnable.Set("false"), + "--set", util.RemoverImageRepo.Set(removerImage.Repo), + "--set", util.RemoverImageTag.Set(removerImage.Tag), + "--set", util.ManagerImageRepo.Set(managerImage.Repo), + "--set", util.ManagerImageTag.Set(managerImage.Tag), + ), + util.UpgradeEraserHelm(util.TestNamespace, + "--set", util.ScannerEnable.Set("false"), + "--set", util.RemoverImageRepo.Set(removerImage.Repo), + "--set", util.RemoverImageTag.Set(removerImage.Tag), + "--set", util.CollectorEnable.Set("true"), + "--set", util.CollectorImageRepo.Set(collectorImage.Repo), + "--set", util.CollectorImageTag.Set("dummy"), + "--set", util.ManagerImageRepo.Set(managerImage.Repo), + "--set", util.ManagerImageTag.Set(managerImage.Tag), + "--set", util.CleanupOnSuccessDelay.Set("2m"), + ), + ).Finish( + envfuncs.DestroyKindCluster(util.KindClusterName), + ) + os.Exit(util.Testenv.Run(m)) +} diff --git a/test/e2e/util/utils.go b/test/e2e/util/utils.go index 0c3616eddf..09b77d734a 100644 --- a/test/e2e/util/utils.go +++ b/test/e2e/util/utils.go @@ -82,18 +82,21 @@ const ( CollectorLabel = "collector" ManualLabel = "manual" ImageJobTypeLabelKey = "eraser.sh/type" + ManagerLabelKey = "control-plane" + ManagerLabelValue = "controller-manager" ) var ( - Testenv env.Environment - RemoverImage = os.Getenv("REMOVER_IMAGE") - ManagerImage = os.Getenv("MANAGER_IMAGE") - CollectorImage = os.Getenv("COLLECTOR_IMAGE") - ScannerImage = os.Getenv("SCANNER_IMAGE") - VulnerableImage = os.Getenv("VULNERABLE_IMAGE") - NonVulnerableImage = os.Getenv("NON_VULNERABLE_IMAGE") - EOLImage = os.Getenv("EOL_IMAGE") - BusyboxImage = os.Getenv("BUSYBOX_IMAGE") + Testenv env.Environment + RemoverImage = os.Getenv("REMOVER_IMAGE") + ManagerImage = os.Getenv("MANAGER_IMAGE") + CollectorImage = os.Getenv("COLLECTOR_IMAGE") + ScannerImage = os.Getenv("SCANNER_IMAGE") + VulnerableImage = os.Getenv("VULNERABLE_IMAGE") + NonVulnerableImage = os.Getenv("NON_VULNERABLE_IMAGE") + EOLImage = os.Getenv("EOL_IMAGE") + BusyboxImage = os.Getenv("BUSYBOX_IMAGE") + CollectorDummyImage = os.Getenv("COLLECTOR_IMAGE_DUMMY") RemoverTarballPath = os.Getenv("REMOVER_TARBALL_PATH") ManagerTarballPath = os.Getenv("MANAGER_TARBALL_PATH")