From 02e1b9dc8a5f7d3e6016b4e4fa4984c3afb0606d Mon Sep 17 00:00:00 2001 From: Chih-Sheng Huang Date: Wed, 4 Dec 2024 10:44:32 -0800 Subject: [PATCH] Recreate missing resource slice (#255) Resolve #228 Add self healing resource slice controller to re-create the missing resource slice --- cmd/eno-controller/main.go | 24 +- .../reconciliation/helpers_test.go | 2 + internal/controllers/selfhealing/slice.go | 191 +++++++ .../controllers/selfhealing/slice_test.go | 510 ++++++++++++++++++ 4 files changed, 719 insertions(+), 8 deletions(-) create mode 100644 internal/controllers/selfhealing/slice.go create mode 100644 internal/controllers/selfhealing/slice_test.go diff --git a/cmd/eno-controller/main.go b/cmd/eno-controller/main.go index 44200154..58671de5 100644 --- a/cmd/eno-controller/main.go +++ b/cmd/eno-controller/main.go @@ -22,6 +22,7 @@ import ( "github.com/Azure/eno/internal/controllers/flowcontrol" "github.com/Azure/eno/internal/controllers/replication" "github.com/Azure/eno/internal/controllers/rollout" + "github.com/Azure/eno/internal/controllers/selfhealing" "github.com/Azure/eno/internal/controllers/synthesis" "github.com/Azure/eno/internal/controllers/watch" "github.com/Azure/eno/internal/controllers/watchdog" @@ -48,14 +49,15 @@ func main() { func runController() error { ctx := ctrl.SetupSignalHandler() var ( - debugLogging bool - watchdogThres time.Duration - rolloutCooldown time.Duration - dispatchCooldown time.Duration - taintToleration string - nodeAffinity string - concurrencyLimit int - synconf = &synthesis.Config{} + debugLogging bool + watchdogThres time.Duration + rolloutCooldown time.Duration + dispatchCooldown time.Duration + selfHealingGracePeriod time.Duration + taintToleration string + nodeAffinity string + concurrencyLimit int + synconf = &synthesis.Config{} mgrOpts = &manager.Options{ Rest: ctrl.GetConfigOrDie(), @@ -73,6 +75,7 @@ func runController() error { flag.StringVar(&taintToleration, "taint-toleration", "", "Node NoSchedule taint to be tolerated by synthesizer pods e.g. taintKey=taintValue to match on value, just taintKey to match on presence of the taint") flag.StringVar(&nodeAffinity, "node-affinity", "", "Synthesizer pods will be created with this required node affinity expression e.g. labelKey=labelValue to match on value, just labelKey to match on presence of the label") flag.IntVar(&concurrencyLimit, "concurrency-limit", 10, "Upper bound on active syntheses. This effectively limits the number of running synthesizer pods spawned by Eno.") + flag.DurationVar(&selfHealingGracePeriod, "self-healing-grace-period", time.Minute*5, "How long before the self-healing controllers are allowed to start the resynthesis process.") mgrOpts.Bind(flag.CommandLine) flag.Parse() @@ -113,6 +116,11 @@ func runController() error { return fmt.Errorf("constructing rollout controller: %w", err) } + err = selfhealing.NewSliceController(mgr, selfHealingGracePeriod) + if err != nil { + return fmt.Errorf("constructing self healing resource slice controller: %w", err) + } + err = synthesis.NewPodLifecycleController(mgr, synconf) if err != nil { return fmt.Errorf("constructing pod lifecycle controller: %w", err) diff --git a/internal/controllers/reconciliation/helpers_test.go b/internal/controllers/reconciliation/helpers_test.go index 75a43c70..2d11f3ee 100644 --- a/internal/controllers/reconciliation/helpers_test.go +++ b/internal/controllers/reconciliation/helpers_test.go @@ -11,6 +11,7 @@ import ( "github.com/Azure/eno/internal/controllers/liveness" "github.com/Azure/eno/internal/controllers/replication" "github.com/Azure/eno/internal/controllers/rollout" + "github.com/Azure/eno/internal/controllers/selfhealing" "github.com/Azure/eno/internal/controllers/synthesis" "github.com/Azure/eno/internal/controllers/watch" "github.com/Azure/eno/internal/controllers/watchdog" @@ -32,6 +33,7 @@ func registerControllers(t *testing.T, mgr *testutil.Manager) { require.NoError(t, flowcontrol.NewSynthesisConcurrencyLimiter(mgr.Manager, 10, 0)) require.NoError(t, liveness.NewNamespaceController(mgr.Manager, 3, time.Second)) require.NoError(t, watch.NewController(mgr.Manager)) + require.NoError(t, selfhealing.NewSliceController(mgr.Manager, time.Minute*5)) } func writeGenericComposition(t *testing.T, client client.Client) (*apiv1.Synthesizer, *apiv1.Composition) { diff --git a/internal/controllers/selfhealing/slice.go b/internal/controllers/selfhealing/slice.go new file mode 100644 index 00000000..08bcb0b1 --- /dev/null +++ b/internal/controllers/selfhealing/slice.go @@ -0,0 +1,191 @@ +package selfhealing + +import ( + "context" + "fmt" + "time" + + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/util/workqueue" + "k8s.io/utils/ptr" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + apiv1 "github.com/Azure/eno/api/v1" + "github.com/Azure/eno/internal/manager" + "github.com/go-logr/logr" +) + +// sliceController check if the resource slice is deleted but it is still present in the composition current synthesis status. +// If yes, it will update the composition PendingResynthesis status to trigger re-synthesis process. +type sliceController struct { + client client.Client + noCacheReader client.Reader + selfHealingGracePeriod time.Duration +} + +func NewSliceController(mgr ctrl.Manager, selfHealingGracePeriod time.Duration) error { + s := &sliceController{ + client: mgr.GetClient(), + noCacheReader: mgr.GetAPIReader(), + selfHealingGracePeriod: selfHealingGracePeriod, + } + return ctrl.NewControllerManagedBy(mgr). + Named("selfHealingSliceController"). + Watches(&apiv1.Composition{}, newCompositionHandler()). + Watches(&apiv1.ResourceSlice{}, newSliceHandler()). + WithLogConstructor(manager.NewLogConstructor(mgr, "selfHealingSliceController")). + Complete(s) +} + +func (s *sliceController) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + logger := logr.FromContextOrDiscard(ctx) + + comp := &apiv1.Composition{} + err := s.client.Get(ctx, req.NamespacedName, comp) + if err != nil { + return ctrl.Result{}, client.IgnoreNotFound(fmt.Errorf("gettting composition: %w", err)) + } + + syn := &apiv1.Synthesizer{} + syn.Name = comp.Spec.Synthesizer.Name + err = s.client.Get(ctx, client.ObjectKeyFromObject(syn), syn) + if err != nil { + return ctrl.Result{}, client.IgnoreNotFound(fmt.Errorf("gettting synthesizer: %w", err)) + } + + logger = logger.WithValues("compositionGeneration", comp.Generation, + "compositionName", comp.Name, + "compositionNamespace", comp.Namespace, + "synthesisID", comp.Status.GetCurrentSynthesisUUID()) + + // Skip if the composition is not eligible for resynthesis, and check the synthesis result later + if notEligibleForResynthesis(comp) { + logger.V(1).Info("not eligible for resynthesis when checking the missing resource slice") + // Use default grace period if the time since last synthesized is exceeds than the grace period + if comp.Status.CurrentSynthesis == nil || + comp.Status.CurrentSynthesis.Synthesized == nil || + (s.selfHealingGracePeriod-time.Since(comp.Status.CurrentSynthesis.Synthesized.Time)) <= 0 { + return ctrl.Result{Requeue: true, RequeueAfter: s.selfHealingGracePeriod}, nil + } + + // Use the remaining grace period if the time since the last synthesized is less than the grace period + return ctrl.Result{Requeue: true, RequeueAfter: s.selfHealingGracePeriod - time.Since(comp.Status.CurrentSynthesis.Synthesized.Time)}, nil + } + + // Check if any resource slice referenced by the composition is deleted. + for _, ref := range comp.Status.CurrentSynthesis.ResourceSlices { + slice := &apiv1.ResourceSlice{} + slice.Name = ref.Name + slice.Namespace = comp.Namespace + err := s.client.Get(ctx, client.ObjectKeyFromObject(slice), slice) + if errors.IsNotFound(err) { + // Ensure the resource slice is missing by checking the resource from api-server + isMissing, err := s.isSliceMissing(ctx, slice) + if err != nil { + return ctrl.Result{}, err + } + if !isMissing { + continue + } + + // The resource slice should not be deleted if it is still referenced by the composition. + // Update the composition status to trigger re-synthesis process. + logger.V(1).Info("found missing resource slice and start resynthesis", "compositionName", comp.Name, "resourceSliceName", ref.Name) + comp.Status.PendingResynthesis = ptr.To(metav1.Now()) + err = s.client.Status().Update(ctx, comp) + if err != nil { + return ctrl.Result{}, fmt.Errorf("updating composition pending resynthesis: %w", err) + } + return ctrl.Result{}, nil + } + if err != nil { + return ctrl.Result{}, fmt.Errorf("getting resource slice: %w", err) + } + } + + return ctrl.Result{}, nil +} + +func (s *sliceController) isSliceMissing(ctx context.Context, slice *apiv1.ResourceSlice) (bool, error) { + err := s.noCacheReader.Get(ctx, client.ObjectKeyFromObject(slice), slice) + if errors.IsNotFound(err) { + return true, nil + } + if err != nil { + return false, fmt.Errorf("getting resource slice from non cache reader: %w", err) + } + + return false, nil +} + +// Compositions aren't eligible to trigger resynthesis when: +// - They haven't ever been synthesized (they'll use the latest inputs anyway) +// - They are currently being synthesized or deleted +// - They are already pending resynthesis +// +// Composition should be resynthesized when the referenced resource slice is deleted +func notEligibleForResynthesis(comp *apiv1.Composition) bool { + return comp.Status.CurrentSynthesis == nil || + comp.Status.CurrentSynthesis.Synthesized == nil || + comp.DeletionTimestamp != nil || + comp.Status.PendingResynthesis != nil +} + +func newCompositionHandler() handler.EventHandler { + apply := func(ctx context.Context, rli workqueue.RateLimitingInterface, obj client.Object) { + comp, ok := obj.(*apiv1.Composition) + if !ok { + logr.FromContextOrDiscard(ctx).V(0).Info("unexpected type given to newCompositionHandler") + return + } + + rli.Add(reconcile.Request{NamespacedName: types.NamespacedName{Namespace: comp.Namespace, Name: comp.Name}}) + } + return &handler.Funcs{ + CreateFunc: func(ctx context.Context, ce event.CreateEvent, rli workqueue.RateLimitingInterface) { + // No need to handle composition creation event for now + }, + UpdateFunc: func(ctx context.Context, ue event.UpdateEvent, rli workqueue.RateLimitingInterface) { + // Check the updated composition only + apply(ctx, rli, ue.ObjectNew) + }, + DeleteFunc: func(ctx context.Context, de event.DeleteEvent, rli workqueue.RateLimitingInterface) { + // No need to handle composition deletion event for now + }, + } +} + +func newSliceHandler() handler.EventHandler { + apply := func(rli workqueue.RateLimitingInterface, obj client.Object) { + owner := metav1.GetControllerOf(obj) + if owner == nil { + // No need to check the deleted resource slice which doesn't have an owner + return + } + // Pass the composition name to the request to check missing resource slice + rli.Add(reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: owner.Name, + Namespace: obj.GetNamespace(), + }, + }) + } + + return &handler.Funcs{ + CreateFunc: func(ctx context.Context, ce event.CreateEvent, rli workqueue.RateLimitingInterface) { + // No need to hanlde creation event for now + }, + UpdateFunc: func(ctx context.Context, ue event.UpdateEvent, rli workqueue.RateLimitingInterface) { + // No need to handle update event for now + }, + DeleteFunc: func(ctx context.Context, de event.DeleteEvent, rli workqueue.RateLimitingInterface) { + apply(rli, de.Object) + }, + } +} diff --git a/internal/controllers/selfhealing/slice_test.go b/internal/controllers/selfhealing/slice_test.go new file mode 100644 index 00000000..e9dac4ac --- /dev/null +++ b/internal/controllers/selfhealing/slice_test.go @@ -0,0 +1,510 @@ +package selfhealing + +import ( + "context" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/util/retry" + "k8s.io/utils/ptr" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + + apiv1 "github.com/Azure/eno/api/v1" + "github.com/Azure/eno/internal/controllers/flowcontrol" + "github.com/Azure/eno/internal/controllers/rollout" + "github.com/Azure/eno/internal/controllers/synthesis" + "github.com/Azure/eno/internal/testutil" + krmv1 "github.com/Azure/eno/pkg/krm/functions/api/v1" +) + +var testSynthesisConfig = &synthesis.Config{ + SliceCreationQPS: 15, + PodNamespace: "default", + ExecutorImage: "test-image", +} + +func registerControllers(t *testing.T, mgr *testutil.Manager) { + require.NoError(t, NewSliceController(mgr.Manager, time.Minute*5)) + require.NoError(t, rollout.NewController(mgr.Manager, time.Microsecond*10)) + require.NoError(t, synthesis.NewPodLifecycleController(mgr.Manager, testSynthesisConfig)) + require.NoError(t, flowcontrol.NewSynthesisConcurrencyLimiter(mgr.Manager, 1, time.Microsecond*10)) +} + +func TestDeleteSliceRecreation(t *testing.T) { + ctx := testutil.NewContext(t) + mgr := testutil.NewManager(t) + registerControllers(t, mgr) + + testNS := "default" + testutil.WithFakeExecutor(t, mgr, func(ctx context.Context, s *apiv1.Synthesizer, input *krmv1.ResourceList) (*krmv1.ResourceList, error) { + cm := &unstructured.Unstructured{ + Object: map[string]any{ + "apiVersion": "v1", + "kind": "ConfigMap", + "metadata": map[string]any{ + "name": "test-cm", + "namespace": testNS, + }, + }, + } + output := &krmv1.ResourceList{Items: []*unstructured.Unstructured{cm}} + return output, nil + }) + mgr.Start(t) + + // Create synthesizer + syn := &apiv1.Synthesizer{} + syn.Name = "test-syn" + syn.Spec.Image = "test-image" + require.NoError(t, mgr.GetClient().Create(ctx, syn)) + + // Create composition + comp := &apiv1.Composition{} + comp.Name = "test-comp" + comp.Namespace = testNS + comp.Spec.Synthesizer = apiv1.SynthesizerRef{Name: syn.Name} + require.NoError(t, mgr.GetClient().Create(ctx, comp)) + + // Get the composition for resource slice owner ref + testutil.Eventually(t, func() bool { + err := mgr.GetClient().Get(ctx, client.ObjectKeyFromObject(comp), comp) + return err == nil + }) + + // Create resource slice + readyTime := metav1.Now() + slice := &apiv1.ResourceSlice{} + slice.Name = "test-slice" + slice.Namespace = testNS + slice.Spec.Resources = []apiv1.Manifest{{Manifest: "{}"}} + slice.Status.Resources = []apiv1.ResourceState{{Ready: &readyTime, Reconciled: true}} + ownerRef := metav1.OwnerReference{ + APIVersion: comp.GetObjectKind().GroupVersionKind().Version, + Kind: comp.GetObjectKind().GroupVersionKind().Kind, + Name: comp.GetName(), + UID: comp.GetUID(), + Controller: ptr.To(true), + } + slice.SetOwnerReferences(append(slice.GetOwnerReferences(), ownerRef)) + require.NoError(t, mgr.GetClient().Create(ctx, slice)) + + // Synthesis has completed with resource slice ref + err := retry.RetryOnConflict(retry.DefaultRetry, func() error { + err := mgr.GetClient().Get(ctx, client.ObjectKeyFromObject(comp), comp) + if err != nil { + return err + } + comp.Status.CurrentSynthesis = &apiv1.Synthesis{ + Synthesized: ptr.To(metav1.Now()), + ObservedCompositionGeneration: comp.Generation, + ResourceSlices: []*apiv1.ResourceSliceRef{{Name: "test-slice"}}, + UUID: "test-uuid", + } + return mgr.GetClient().Status().Update(ctx, comp) + }) + require.NoError(t, err) + + // Check resource slice is existed before deletion + testutil.Eventually(t, func() bool { + err := mgr.GetClient().Get(ctx, client.ObjectKeyFromObject(slice), slice) + return err == nil + }) + // Remove the finalizer and delete the resource slice + slice.SetAnnotations(map[string]string{}) + require.NoError(t, mgr.GetClient().Update(ctx, slice)) + require.NoError(t, mgr.GetClient().Delete(ctx, slice)) + + // Check the the resource slice referenced by composition is missing + testutil.Eventually(t, func() bool { + for _, ref := range comp.Status.CurrentSynthesis.ResourceSlices { + slice := &apiv1.ResourceSlice{} + slice.Name = ref.Name + slice.Namespace = comp.Namespace + err := mgr.GetClient().Get(ctx, client.ObjectKeyFromObject(slice), slice) + if errors.IsNotFound(err) { + return true + } + } + return false + }) + + // Wait for the composition is re-synthesized + testutil.Eventually(t, func() bool { + err := mgr.GetClient().Get(ctx, client.ObjectKeyFromObject(comp), comp) + if err != nil { + return false + } + + return comp.Status.PendingResynthesis == nil && + comp.Status.CurrentSynthesis != nil && + comp.Status.CurrentSynthesis.Synthesized != nil && + comp.Status.CurrentSynthesis.ResourceSlices != nil + }) + + // Check there is no resource slice referenced by composition missing + testutil.Eventually(t, func() bool { + err := mgr.GetClient().Get(ctx, client.ObjectKeyFromObject(comp), comp) + if err != nil { + return false + } + // Must have at least one resource slice ref + if comp.Status.CurrentSynthesis != nil && len(comp.Status.CurrentSynthesis.ResourceSlices) == 0 { + return false + } + for _, ref := range comp.Status.CurrentSynthesis.ResourceSlices { + rs := &apiv1.ResourceSlice{} + rs.Name = ref.Name + rs.Namespace = comp.Namespace + err := mgr.GetClient().Get(ctx, client.ObjectKeyFromObject(rs), rs) + if err != nil { + return false + } + // The re-creation resource slice's name prefix is composition name by design. + if !strings.HasPrefix(rs.Name, comp.Name) { + return false + } + } + return true + }) +} + +func TestUpdateCompositionSliceRecreation(t *testing.T) { + ctx := testutil.NewContext(t) + mgr := testutil.NewManager(t) + registerControllers(t, mgr) + + testNS := "default" + testutil.WithFakeExecutor(t, mgr, func(ctx context.Context, s *apiv1.Synthesizer, input *krmv1.ResourceList) (*krmv1.ResourceList, error) { + cm := &unstructured.Unstructured{ + Object: map[string]any{ + "apiVersion": "v1", + "kind": "ConfigMap", + "metadata": map[string]any{ + "name": "test-cm", + "namespace": testNS, + }, + }, + } + output := &krmv1.ResourceList{Items: []*unstructured.Unstructured{cm}} + return output, nil + }) + mgr.Start(t) + + // Create synthesizer + syn := &apiv1.Synthesizer{} + syn.Name = "test-syn" + syn.Spec.Image = "test-image" + require.NoError(t, mgr.GetClient().Create(ctx, syn)) + + // Create composition with resource slice ref + comp := &apiv1.Composition{} + comp.Name = "test-comp" + comp.Namespace = testNS + comp.Spec.Synthesizer = apiv1.SynthesizerRef{Name: syn.Name} + require.NoError(t, mgr.GetClient().Create(ctx, comp)) + + // Ensure both composition and synthesizer are created + testutil.Eventually(t, func() bool { + err := mgr.GetClient().Get(ctx, client.ObjectKeyFromObject(comp), comp) + if err != nil { + return false + } + err = mgr.GetClient().Get(ctx, client.ObjectKeyFromObject(syn), syn) + if err != nil { + return false + } + return true + }) + + // Don't create resource slice, update composition status to trigger re-synthesis for missing resource slice + err := retry.RetryOnConflict(retry.DefaultRetry, func() error { + err := mgr.GetClient().Get(ctx, client.ObjectKeyFromObject(comp), comp) + if err != nil { + return err + } + comp.Status.CurrentSynthesis = &apiv1.Synthesis{ + Synthesized: ptr.To(metav1.Now()), + ObservedCompositionGeneration: comp.Generation, + ResourceSlices: []*apiv1.ResourceSliceRef{{Name: "test-slice"}}, + UUID: "test-uuid", + } + return mgr.GetClient().Status().Update(ctx, comp) + }) + require.NoError(t, err) + + // Check the the resource slice referenced by composition is existed + testutil.Eventually(t, func() bool { + err := mgr.GetClient().Get(ctx, client.ObjectKeyFromObject(comp), comp) + if err != nil { + return false + } + // Must have at least one resource slice ref + if comp.Status.CurrentSynthesis == nil || len(comp.Status.CurrentSynthesis.ResourceSlices) == 0 { + return false + } + for _, ref := range comp.Status.CurrentSynthesis.ResourceSlices { + slice := &apiv1.ResourceSlice{} + slice.Name = ref.Name + slice.Namespace = comp.Namespace + err := mgr.GetClient().Get(ctx, client.ObjectKeyFromObject(slice), slice) + if err != nil { + return false + } + } + return true + }) +} + +func TestRequeueForNotEligibleResynthesis(t *testing.T) { + ctx := testutil.NewContext(t) + mgr := testutil.NewManager(t) + mgr.Start(t) + + testNS := "default" + // Create synthesizer + syn := &apiv1.Synthesizer{} + syn.Name = "test-syn" + syn.Spec.Image = "test-image" + require.NoError(t, mgr.GetClient().Create(ctx, syn)) + + // Create composition + comp := &apiv1.Composition{} + comp.Name = "test-comp" + comp.Namespace = testNS + comp.Spec.Synthesizer = apiv1.SynthesizerRef{Name: syn.Name} + require.NoError(t, mgr.GetClient().Create(ctx, comp)) + + // Create resource slice + readyTime := metav1.Now() + slice := &apiv1.ResourceSlice{} + slice.Name = "test-slice" + slice.Namespace = testNS + slice.Spec.Resources = []apiv1.Manifest{{Manifest: "{}"}} + slice.Status.Resources = []apiv1.ResourceState{{Ready: &readyTime, Reconciled: true}} + require.NoError(t, mgr.GetClient().Create(ctx, slice)) + + // Check the both composition and synthesizer are existed before reconciliation + testutil.Eventually(t, func() bool { + err := mgr.GetClient().Get(ctx, client.ObjectKeyFromObject(comp), comp) + if err != nil { + return false + } + err = mgr.GetClient().Get(ctx, client.ObjectKeyFromObject(syn), syn) + if err != nil { + return false + } + return true + }) + + // Reconcile the resource slice controller to re-create the missing resource slice + s := &sliceController{client: mgr.GetClient(), selfHealingGracePeriod: time.Minute * 5} + req := ctrl.Request{NamespacedName: types.NamespacedName{Namespace: comp.Namespace, Name: comp.Name}} + res, err := s.Reconcile(ctx, req) + require.NoError(t, err) + // Request should be requeue due to composition CurrentSynthesis is emtpy and not eligible for resynthesis + assert.True(t, res.Requeue) + assert.Equal(t, s.selfHealingGracePeriod, res.RequeueAfter) + + // Update composition with synthesize time and pending synthesis time for re-queue + synthesized := time.Now().Add(-5 * time.Hour) + err = retry.RetryOnConflict(retry.DefaultRetry, func() error { + err := mgr.GetClient().Get(ctx, client.ObjectKeyFromObject(comp), comp) + if err != nil { + return err + } + comp.Status.CurrentSynthesis = &apiv1.Synthesis{Synthesized: ptr.To(metav1.NewTime(synthesized))} + comp.Status.PendingResynthesis = ptr.To(metav1.Now()) + return mgr.GetClient().Status().Update(ctx, comp) + }) + res, err = s.Reconcile(ctx, req) + require.NoError(t, err) + // Should re-quque after grace period time, due to (gracePeriod - time.Since(synthesized)) < 0 + assert.True(t, res.Requeue) + assert.Equal(t, s.selfHealingGracePeriod, res.RequeueAfter) + + // Update composition with synthesize time for re-queue + synthesized = time.Now().Add(-time.Minute) + err = retry.RetryOnConflict(retry.DefaultRetry, func() error { + err := mgr.GetClient().Get(ctx, client.ObjectKeyFromObject(comp), comp) + if err != nil { + return err + } + comp.Status.CurrentSynthesis = &apiv1.Synthesis{Synthesized: ptr.To(metav1.NewTime(synthesized))} + comp.Status.PendingResynthesis = ptr.To(metav1.Now()) + return mgr.GetClient().Status().Update(ctx, comp) + }) + res, err = s.Reconcile(ctx, req) + require.NoError(t, err) + assert.True(t, res.Requeue) + assert.True(t, res.RequeueAfter.Seconds() > 0) + // Re-queue after (gracePeriod - time.Since(synthesized)) + assert.True(t, s.selfHealingGracePeriod > res.RequeueAfter) + +} + +func TestNotEligibleForResynthesis(t *testing.T) { + tests := []struct { + name string + comp *apiv1.Composition + expected bool + }{ + { + name: "CurrentSynthesis is nil", + comp: &apiv1.Composition{}, + expected: true, + }, + { + name: "CurrentSynthesis Synthesized is nil", + comp: &apiv1.Composition{ + Status: apiv1.CompositionStatus{CurrentSynthesis: &apiv1.Synthesis{}}, + }, + expected: true, + }, + { + name: "PendingResynthesis is not nil", + comp: &apiv1.Composition{ + Status: apiv1.CompositionStatus{PendingResynthesis: ptr.To(metav1.Now())}, + }, + expected: true, + }, + { + name: "composition deletion time stamp is not nil", + comp: &apiv1.Composition{ + ObjectMeta: metav1.ObjectMeta{ + DeletionTimestamp: ptr.To(metav1.Now()), + }, + }, + expected: true, + }, + { + name: "composition deletion time stamp is not nil and PendingResynthesis is not nil", + comp: &apiv1.Composition{ + ObjectMeta: metav1.ObjectMeta{ + DeletionTimestamp: ptr.To(metav1.Now()), + }, + Status: apiv1.CompositionStatus{PendingResynthesis: ptr.To(metav1.Now())}, + }, + expected: true, + }, + { + name: "CurrentSynthesis is nil and PendingResynthesis is not nil", + comp: &apiv1.Composition{ + Status: apiv1.CompositionStatus{ + CurrentSynthesis: nil, + PendingResynthesis: ptr.To(metav1.Now()), + }, + }, + expected: true, + }, + { + name: "CurrentSynthesis Synthesized is nil and PendingResynthesis is not nil", + comp: &apiv1.Composition{ + Status: apiv1.CompositionStatus{ + CurrentSynthesis: &apiv1.Synthesis{Synthesized: nil}, + PendingResynthesis: ptr.To(metav1.Now()), + }, + }, + expected: true, + }, + { + name: "CurrentSynthesis is nil and deletion time stamp is not nil", + comp: &apiv1.Composition{ + ObjectMeta: metav1.ObjectMeta{ + DeletionTimestamp: ptr.To(metav1.Now()), + }, + Status: apiv1.CompositionStatus{CurrentSynthesis: nil}, + }, + expected: true, + }, + { + name: "CurrentSynthesis Synthesized is nil and deletion time stamp is not nil", + comp: &apiv1.Composition{ + ObjectMeta: metav1.ObjectMeta{ + DeletionTimestamp: ptr.To(metav1.Now()), + }, + Status: apiv1.CompositionStatus{CurrentSynthesis: &apiv1.Synthesis{Synthesized: nil}}, + }, + expected: true, + }, + { + name: "composition is synthesized", + comp: &apiv1.Composition{ + Status: apiv1.CompositionStatus{ + CurrentSynthesis: &apiv1.Synthesis{ + Synthesized: ptr.To(metav1.Now()), + }, + }, + }, + expected: false, + }, + { + name: "composition is synthesized and PendingResynthesis is not nil", + comp: &apiv1.Composition{ + Status: apiv1.CompositionStatus{ + CurrentSynthesis: &apiv1.Synthesis{ + Synthesized: ptr.To(metav1.Now()), + }, + PendingResynthesis: ptr.To(metav1.Now()), + }, + }, + expected: true, + }, + { + name: "composition is synthesized and deletion time stamp is not nil", + comp: &apiv1.Composition{ + ObjectMeta: metav1.ObjectMeta{ + DeletionTimestamp: ptr.To(metav1.Now()), + }, + Status: apiv1.CompositionStatus{ + CurrentSynthesis: &apiv1.Synthesis{ + Synthesized: ptr.To(metav1.Now()), + }, + }, + }, + expected: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + res := notEligibleForResynthesis(tt.comp) + assert.Equal(t, tt.expected, res) + }) + } +} + +func TestIsSliceMissing(t *testing.T) { + ctx := testutil.NewContext(t) + mgr := testutil.NewManager(t) + mgr.Start(t) + + s := &sliceController{client: mgr.GetClient(), noCacheReader: mgr.GetAPIReader(), selfHealingGracePeriod: time.Minute} + + slice := &apiv1.ResourceSlice{} + slice.Name = "test-slice" + slice.Namespace = "default" + + isMissing, err := s.isSliceMissing(ctx, slice) + require.NoError(t, err) + require.True(t, isMissing) + + // Create resource slice + require.NoError(t, mgr.GetClient().Create(ctx, slice)) + testutil.Eventually(t, func() bool { + err := mgr.GetClient().Get(ctx, client.ObjectKeyFromObject(slice), slice) + return err == nil + }) + + // Check resource slice is existed + isMissing, err = s.isSliceMissing(ctx, slice) + require.NoError(t, err) + require.False(t, isMissing) +}