diff --git a/test/envtest/deploy_resources.go b/test/envtest/deploy_resources.go index 00a17c055..782ac1639 100644 --- a/test/envtest/deploy_resources.go +++ b/test/envtest/deploy_resources.go @@ -20,6 +20,20 @@ import ( konnectv1alpha1 "github.com/kong/kubernetes-configuration/api/konnect/v1alpha1" ) +type objOption func(obj client.Object) + +// WithAnnotation returns an objOption that sets the given key-value pair as an annotation on the object. +func WithAnnotation(key, value string) objOption { + return func(obj client.Object) { + annotations := obj.GetAnnotations() + if annotations == nil { + annotations = make(map[string]string) + } + annotations[key] = value + obj.SetAnnotations(annotations) + } +} + // deployKonnectAPIAuthConfiguration deploys a KonnectAPIAuthConfiguration resource // and returns the resource. func deployKonnectAPIAuthConfiguration( @@ -40,7 +54,7 @@ func deployKonnectAPIAuthConfiguration( }, } require.NoError(t, cl.Create(ctx, apiAuth)) - t.Logf("deployed %s KonnectAPIAuthConfiguration resource", client.ObjectKeyFromObject(apiAuth)) + t.Logf("deployed new %s KonnectAPIAuthConfiguration", client.ObjectKeyFromObject(apiAuth)) return apiAuth } @@ -93,7 +107,7 @@ func deployKonnectGatewayControlPlane( }, } require.NoError(t, cl.Create(ctx, cp)) - t.Logf("deployed %s KonnectGatewayControlPlane resource", client.ObjectKeyFromObject(cp)) + t.Logf("deployed new %s KonnectGatewayControlPlane", client.ObjectKeyFromObject(cp)) return cp } @@ -125,22 +139,77 @@ func deployKonnectGatewayControlPlaneWithID( return cp } -// deployKongService deploys a KongService resource and returns the resource. -func deployKongService( +// deployKongServiceAttachedToCP deploys a KongService resource and returns the resource. +func deployKongServiceAttachedToCP( t *testing.T, ctx context.Context, cl client.Client, - kongService *configurationv1alpha1.KongService, + cp *konnectv1alpha1.KonnectGatewayControlPlane, + opts ...objOption, ) *configurationv1alpha1.KongService { t.Helper() name := "kongservice-" + uuid.NewString()[:8] - kongService.Name = name - kongService.Spec.Name = lo.ToPtr(name) - require.NoError(t, cl.Create(ctx, kongService)) - t.Logf("deployed %s KongService resource", client.ObjectKeyFromObject(kongService)) + kongService := configurationv1alpha1.KongService{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + }, + Spec: configurationv1alpha1.KongServiceSpec{ + KongServiceAPISpec: configurationv1alpha1.KongServiceAPISpec{ + Name: lo.ToPtr(name), + }, + ControlPlaneRef: &configurationv1alpha1.ControlPlaneRef{ + Type: configurationv1alpha1.ControlPlaneRefKonnectNamespacedRef, + KonnectNamespacedRef: &configurationv1alpha1.KonnectNamespacedRef{ + Name: cp.Name, + }, + }, + }, + } + + for _, opt := range opts { + opt(&kongService) + } + require.NoError(t, cl.Create(ctx, &kongService)) + t.Logf("deployed new %s KongService", client.ObjectKeyFromObject(&kongService)) - return kongService + return &kongService +} + +// deployKongRouteAttachedToService deploys a KongRoute resource and returns the resource. +func deployKongRouteAttachedToService( + t *testing.T, + ctx context.Context, + cl client.Client, + kongService *configurationv1alpha1.KongService, + opts ...objOption, +) *configurationv1alpha1.KongRoute { + t.Helper() + + name := "kongroute-" + uuid.NewString()[:8] + kongRoute := configurationv1alpha1.KongRoute{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + }, + Spec: configurationv1alpha1.KongRouteSpec{ + KongRouteAPISpec: configurationv1alpha1.KongRouteAPISpec{ + Name: lo.ToPtr(name), + }, + ServiceRef: &configurationv1alpha1.ServiceRef{ + Type: configurationv1alpha1.ServiceRefNamespacedRef, + NamespacedRef: &configurationv1alpha1.NamespacedServiceRef{ + Name: kongService.Name, + }, + }, + }, + } + for _, opt := range opts { + opt(&kongRoute) + } + require.NoError(t, cl.Create(ctx, &kongRoute)) + t.Logf("deployed new %s KongRoute", client.ObjectKeyFromObject(&kongRoute)) + + return &kongRoute } // deployKongConsumerWithProgrammed deploys a KongConsumer resource and returns the resource. @@ -171,7 +240,6 @@ func deployKongConsumerWithProgrammed( } // deployKongPluginBinding deploys a KongPluginBinding resource and returns the resource. -// The caller can also specify the status which will be updated on the resource. func deployKongPluginBinding( t *testing.T, ctx context.Context, @@ -184,7 +252,6 @@ func deployKongPluginBinding( require.NoError(t, cl.Create(ctx, kpb)) t.Logf("deployed new unmanaged KongPluginBinding %s", client.ObjectKeyFromObject(kpb)) - require.NoError(t, cl.Status().Update(ctx, kpb)) return kpb } @@ -412,6 +479,28 @@ func deployKongKeyAttachedToCP( } require.NoError(t, cl.Create(ctx, key)) t.Logf("deployed new KongKey %s", client.ObjectKeyFromObject(key)) - return key } + +// deployProxyCachePlugin deploys the proxy-cache KongPlugin resource and returns the resource. +// The provided client should be namespaced, i.e. created with `client.NewNamespacedClient(client, ns)` +func deployProxyCachePlugin( + t *testing.T, + ctx context.Context, + cl client.Client, +) *configurationv1.KongPlugin { + t.Helper() + + plugin := &configurationv1.KongPlugin{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "proxy-cache-kp-", + }, + PluginName: "proxy-cache", + Config: apiextensionsv1.JSON{ + Raw: []byte(`{"response_code": [200], "request_method": ["GET", "HEAD"], "content_type": ["text/plain; charset=utf-8"], "cache_ttl": 300, "strategy": "memory"}`), + }, + } + require.NoError(t, cl.Create(ctx, plugin)) + t.Logf("deployed new %s KongPlugin (%s)", client.ObjectKeyFromObject(plugin), plugin.PluginName) + return plugin +} diff --git a/test/envtest/kongpluginbinding_managed_test.go b/test/envtest/kongpluginbinding_managed_test.go index 689b94431..e09c9229f 100644 --- a/test/envtest/kongpluginbinding_managed_test.go +++ b/test/envtest/kongpluginbinding_managed_test.go @@ -29,6 +29,12 @@ import ( ) func TestKongPluginBindingManaged(t *testing.T) { + // NOTE: Since this test checks the behavior of creating KongPluginBindings + // based on annotations/ on objects that can have plugins bound to them, + // need to delete these at the end of each respective subtest to prevent them + // from being picked up in other tests and cause reconciler to create/update/delete + // KongPluginBindings. + t.Parallel() ctx, cancel := Context(t, context.Background()) defer cancel() @@ -48,63 +54,18 @@ func TestKongPluginBindingManaged(t *testing.T) { cp := deployKonnectGatewayControlPlaneWithID(t, ctx, clientNamespaced, apiAuth) factory := ops.NewMockSDKFactory(t) - serviceID := uuid.NewString() - factory.SDK.ServicesSDK.EXPECT(). - CreateService(mock.Anything, cp.GetKonnectStatus().GetKonnectID(), mock.Anything). - Return( - &sdkkonnectops.CreateServiceResponse{ - Service: &sdkkonnectcomp.Service{ - ID: lo.ToPtr(serviceID), - }, - }, - nil, - ) - factory.SDK.ServicesSDK.EXPECT(). - UpsertService(mock.Anything, mock.Anything, mock.Anything).Maybe(). - Return( - &sdkkonnectops.UpsertServiceResponse{ - Service: &sdkkonnectcomp.Service{ - ID: lo.ToPtr(serviceID), - }, - }, - nil, - ) + sdk := factory.SDK require.NoError(t, manager.SetupCacheIndicesForKonnectTypes(ctx, mgr, false)) reconcilers := []Reconciler{ konnect.NewKongPluginReconciler(false, mgr.GetClient()), konnect.NewKonnectEntityReconciler(factory, false, mgr.GetClient(), - konnect.WithKonnectEntitySyncPeriod[configurationv1alpha1.KongPluginBinding](konnectSyncTime), - ), - konnect.NewKonnectEntityReconciler(factory, false, mgr.GetClient(), - konnect.WithKonnectEntitySyncPeriod[configurationv1alpha1.KongService](konnectSyncTime), + konnect.WithKonnectEntitySyncPeriod[configurationv1alpha1.KongPluginBinding](konnectInfiniteSyncTime), ), } StartReconcilers(ctx, t, mgr, logs, reconcilers...) - pluginID := uuid.NewString() - factory.SDK.PluginSDK.EXPECT(). - CreatePlugin(mock.Anything, cp.GetKonnectStatus().GetKonnectID(), mock.Anything). - Return( - &sdkkonnectops.CreatePluginResponse{ - Plugin: &sdkkonnectcomp.Plugin{ - ID: lo.ToPtr(pluginID), - }, - }, - nil, - ) - factory.SDK.PluginSDK.EXPECT(). - UpsertPlugin(mock.Anything, mock.Anything, mock.Anything).Maybe(). - Return( - &sdkkonnectops.UpsertPluginResponse{ - Plugin: &sdkkonnectcomp.Plugin{ - ID: lo.ToPtr(pluginID), - }, - }, - nil, - ) - rateLimitingkongPlugin := &configurationv1.KongPlugin{ ObjectMeta: metav1.ObjectMeta{ GenerateName: "rate-limiting-kp-", @@ -117,113 +78,324 @@ func TestKongPluginBindingManaged(t *testing.T) { require.NoError(t, clientNamespaced.Create(ctx, rateLimitingkongPlugin)) t.Logf("deployed %s KongPlugin (%s) resource", client.ObjectKeyFromObject(rateLimitingkongPlugin), rateLimitingkongPlugin.PluginName) - wKongPluginBinding := setupWatch[configurationv1alpha1.KongPluginBindingList](t, ctx, clientWithWatch, client.InNamespace(ns.Name)) - wKongPlugin := setupWatch[configurationv1.KongPluginList](t, ctx, clientWithWatch, client.InNamespace(ns.Name)) - kongService := deployKongService(t, ctx, clientNamespaced, - &configurationv1alpha1.KongService{ - ObjectMeta: metav1.ObjectMeta{ - Annotations: map[string]string{ - consts.PluginsAnnotationKey: rateLimitingkongPlugin.Name, + t.Run("binding to KongService", func(t *testing.T) { + serviceID := uuid.NewString() + + pluginID := uuid.NewString() + sdk.PluginSDK.EXPECT(). + CreatePlugin(mock.Anything, cp.GetKonnectStatus().GetKonnectID(), mock.Anything). + Return( + &sdkkonnectops.CreatePluginResponse{ + Plugin: &sdkkonnectcomp.Plugin{ + ID: lo.ToPtr(pluginID), + }, }, + nil, + ) + + wKongPluginBinding := setupWatch[configurationv1alpha1.KongPluginBindingList](t, ctx, clientWithWatch, client.InNamespace(ns.Name)) + wKongPlugin := setupWatch[configurationv1.KongPluginList](t, ctx, clientWithWatch, client.InNamespace(ns.Name)) + + kongService := deployKongServiceAttachedToCP(t, ctx, clientNamespaced, cp, + WithAnnotation(consts.PluginsAnnotationKey, rateLimitingkongPlugin.Name), + ) + t.Cleanup(func() { + require.NoError(t, clientNamespaced.Delete(ctx, kongService)) + }) + updateKongServiceStatusWithProgrammed(t, ctx, clientNamespaced, kongService, serviceID, cp.GetKonnectStatus().GetKonnectID()) + + t.Logf("waiting for KongPluginBinding to be created") + kongPluginBinding := watchFor(t, ctx, wKongPluginBinding, watch.Added, + func(kpb *configurationv1alpha1.KongPluginBinding) bool { + return kpb.Spec.Targets.ServiceReference != nil && + kpb.Spec.Targets.ServiceReference.Name == kongService.Name && + kpb.Spec.PluginReference.Name == rateLimitingkongPlugin.Name }, - Spec: configurationv1alpha1.KongServiceSpec{ - ControlPlaneRef: &configurationv1alpha1.ControlPlaneRef{ - Type: configurationv1alpha1.ControlPlaneRefKonnectNamespacedRef, - KonnectNamespacedRef: &configurationv1alpha1.KonnectNamespacedRef{ - Name: cp.Name, - }, + "KongPluginBinding wasn't created", + ) + t.Logf( + "checking that managed KongPlugin %s gets plugin-in-use finalizer added", + client.ObjectKeyFromObject(rateLimitingkongPlugin), + ) + _ = watchFor(t, ctx, wKongPlugin, watch.Modified, + func(kp *configurationv1.KongPlugin) bool { + return kp.Name == rateLimitingkongPlugin.Name && + controllerutil.ContainsFinalizer(kp, consts.PluginInUseFinalizer) + }, + "KongPlugin wasn't updated to get plugin-in-use finalizer added", + ) + + t.Logf("delete managed kongPluginBinding %s, then check it gets recreated", client.ObjectKeyFromObject(kongPluginBinding)) + require.NoError(t, clientNamespaced.Delete(ctx, kongPluginBinding)) + kongPluginBinding = watchFor(t, ctx, wKongPluginBinding, watch.Added, + func(kpb *configurationv1alpha1.KongPluginBinding) bool { + return kpb.Spec.Targets.ServiceReference != nil && + kpb.Spec.Targets.ServiceReference.Name == kongService.Name && + kpb.Spec.PluginReference.Name == rateLimitingkongPlugin.Name + }, + "KongPluginBinding wasn't recreated", + ) + + assert.EventuallyWithT(t, func(c *assert.CollectT) { + assert.True(c, sdk.PluginSDK.AssertExpectations(t)) + }, waitTime, tickTime) + + t.Logf( + "remove annotation from KongService %s and check that managed KongPluginBinding %s gets deleted, "+ + "then check that gateway.konghq.com/plugin-in-use finalizer gets removed from %s KongPlugin", + client.ObjectKeyFromObject(kongService), + client.ObjectKeyFromObject(kongPluginBinding), + client.ObjectKeyFromObject(rateLimitingkongPlugin), + ) + + sdk.PluginSDK.EXPECT(). + DeletePlugin(mock.Anything, cp.GetKonnectStatus().GetKonnectID(), mock.Anything). + Return( + &sdkkonnectops.DeletePluginResponse{ + StatusCode: 200, }, - KongServiceAPISpec: configurationv1alpha1.KongServiceAPISpec{ - URL: lo.ToPtr("http://example.com"), + nil, + ) + + t.Logf( + "checking that managed KongPlugin %s gets plugin-in-use finalizer removed", + client.ObjectKeyFromObject(rateLimitingkongPlugin), + ) + wKongPlugin = setupWatch[configurationv1.KongPluginList](t, ctx, clientWithWatch, client.InNamespace(ns.Name)) + delete(kongService.Annotations, consts.PluginsAnnotationKey) + require.NoError(t, clientNamespaced.Update(ctx, kongService)) + _ = watchFor(t, ctx, wKongPlugin, watch.Modified, + func(kp *configurationv1.KongPlugin) bool { + return kp.Name == rateLimitingkongPlugin.Name && + !controllerutil.ContainsFinalizer(kp, consts.PluginInUseFinalizer) + }, + "KongPlugin wasn't updated to get plugin-in-use finalizer removed", + ) + + t.Logf( + "checking that managed KongPluginBinding %s gets deleted", + client.ObjectKeyFromObject(kongPluginBinding), + ) + assert.EventuallyWithT(t, + func(c *assert.CollectT) { + assert.True(c, k8serrors.IsNotFound( + clientNamespaced.Get(ctx, client.ObjectKeyFromObject(kongPluginBinding), kongPluginBinding), + )) + }, waitTime, tickTime, + "KongPluginBinding wasn't deleted after removing annotation from KongService", + ) + + assert.EventuallyWithT(t, func(c *assert.CollectT) { + assert.True(c, sdk.PluginSDK.AssertExpectations(t)) + }, waitTime, tickTime) + }) + + t.Run("binding to KongRoute", func(t *testing.T) { + serviceID := uuid.NewString() + routeID := uuid.NewString() + + wKongPluginBinding := setupWatch[configurationv1alpha1.KongPluginBindingList](t, ctx, clientWithWatch, client.InNamespace(ns.Name)) + wKongPlugin := setupWatch[configurationv1.KongPluginList](t, ctx, clientWithWatch, client.InNamespace(ns.Name)) + kongService := deployKongServiceAttachedToCP(t, ctx, clientNamespaced, cp) + t.Cleanup(func() { + require.NoError(t, clientNamespaced.Delete(ctx, kongService)) + }) + updateKongServiceStatusWithProgrammed(t, ctx, clientNamespaced, kongService, serviceID, cp.GetKonnectStatus().GetKonnectID()) + kongRoute := deployKongRouteAttachedToService(t, ctx, clientNamespaced, kongService, + WithAnnotation(consts.PluginsAnnotationKey, rateLimitingkongPlugin.Name), + ) + t.Cleanup(func() { + require.NoError(t, clientNamespaced.Delete(ctx, kongRoute)) + }) + updateKongRouteStatusWithProgrammed(t, ctx, clientNamespaced, kongRoute, routeID, cp.GetKonnectStatus().GetKonnectID(), serviceID) + + t.Logf("waiting for KongPluginBinding to be created") + kongPluginBinding := watchFor(t, ctx, wKongPluginBinding, watch.Added, + func(kpb *configurationv1alpha1.KongPluginBinding) bool { + return kpb.Spec.Targets.RouteReference != nil && + kpb.Spec.Targets.RouteReference.Name == kongRoute.Name && + kpb.Spec.PluginReference.Name == rateLimitingkongPlugin.Name + }, + "KongPluginBinding wasn't created", + ) + t.Logf( + "checking that managed KongPlugin %s gets plugin-in-use finalizer added", + client.ObjectKeyFromObject(rateLimitingkongPlugin), + ) + _ = watchFor(t, ctx, wKongPlugin, watch.Modified, + func(kp *configurationv1.KongPlugin) bool { + return kp.Name == rateLimitingkongPlugin.Name && + controllerutil.ContainsFinalizer(kp, consts.PluginInUseFinalizer) + }, + "KongPlugin wasn't updated to get plugin-in-use finalizer added", + ) + + t.Logf("delete managed kongPluginBinding %s, then check it gets recreated", client.ObjectKeyFromObject(kongPluginBinding)) + require.NoError(t, clientNamespaced.Delete(ctx, kongPluginBinding)) + kongPluginBinding = watchFor(t, ctx, wKongPluginBinding, watch.Added, + func(kpb *configurationv1alpha1.KongPluginBinding) bool { + return kpb.Spec.Targets.RouteReference != nil && + kpb.Spec.Targets.RouteReference.Name == kongRoute.Name && + kpb.Spec.PluginReference.Name == rateLimitingkongPlugin.Name + }, + "KongPluginBinding wasn't recreated", + ) + + assert.EventuallyWithT(t, func(c *assert.CollectT) { + assert.True(c, sdk.PluginSDK.AssertExpectations(t)) + }, waitTime, tickTime) + + t.Logf( + "remove annotation from KongRoute %s and check that managed KongPluginBinding %s gets deleted, "+ + "then check that gateway.konghq.com/plugin-in-use finalizer gets removed from %s KongPlugin", + client.ObjectKeyFromObject(kongRoute), + client.ObjectKeyFromObject(kongPluginBinding), + client.ObjectKeyFromObject(rateLimitingkongPlugin), + ) + + sdk.PluginSDK.EXPECT(). + DeletePlugin(mock.Anything, cp.GetKonnectStatus().GetKonnectID(), mock.Anything). + Return( + &sdkkonnectops.DeletePluginResponse{ + StatusCode: 200, }, + nil, + ) + + t.Logf( + "checking that managed KongPlugin %s gets plugin-in-use finalizer removed", + client.ObjectKeyFromObject(rateLimitingkongPlugin), + ) + delete(kongRoute.Annotations, consts.PluginsAnnotationKey) + require.NoError(t, clientNamespaced.Update(ctx, kongRoute)) + _ = watchFor(t, ctx, wKongPlugin, watch.Modified, + func(kp *configurationv1.KongPlugin) bool { + return kp.Name == rateLimitingkongPlugin.Name && + !controllerutil.ContainsFinalizer(kp, consts.PluginInUseFinalizer) }, - }, - ) - - t.Logf("waiting for KongPluginBinding to be created") - kongPluginBinding := watchFor(t, ctx, wKongPluginBinding, watch.Added, - func(kpb *configurationv1alpha1.KongPluginBinding) bool { - return kpb.Spec.Targets.ServiceReference != nil && - kpb.Spec.Targets.ServiceReference.Name == kongService.Name && - kpb.Spec.PluginReference.Name == rateLimitingkongPlugin.Name - }, - "KongPluginBinding wasn't created", - ) - t.Logf( - "checking that managed KongPlugin %s gets plugin-in-use finalizer added", - client.ObjectKeyFromObject(rateLimitingkongPlugin), - ) - _ = watchFor(t, ctx, wKongPlugin, watch.Modified, - func(kp *configurationv1.KongPlugin) bool { - return kp.Name == rateLimitingkongPlugin.Name && - controllerutil.ContainsFinalizer(kp, consts.PluginInUseFinalizer) - }, - "KongPlugin wasn't updated to get plugin-in-use finalizer added", - ) - - t.Logf("delete managed kongPluginBinding %s, then check it gets recreated", client.ObjectKeyFromObject(kongPluginBinding)) - require.NoError(t, clientNamespaced.Delete(ctx, kongPluginBinding)) - kongPluginBinding = watchFor(t, ctx, wKongPluginBinding, watch.Added, - func(kpb *configurationv1alpha1.KongPluginBinding) bool { - return kpb.Spec.Targets.ServiceReference != nil && - kpb.Spec.Targets.ServiceReference.Name == kongService.Name && - kpb.Spec.PluginReference.Name == rateLimitingkongPlugin.Name - }, - "KongPluginBinding wasn't recreated", - ) - - assert.EventuallyWithT(t, func(c *assert.CollectT) { - assert.True(c, factory.SDK.ServicesSDK.AssertExpectations(t)) - assert.True(c, factory.SDK.PluginSDK.AssertExpectations(t)) - }, waitTime, tickTime) - - t.Logf( - "remove annotation from KongService %s and check that managed KongPluginBinding %s gets deleted, "+ - "then check that gateway.konghq.com/plugin-in-use finalizer gets removed from %s KongPlugin", - client.ObjectKeyFromObject(kongService), - client.ObjectKeyFromObject(kongPluginBinding), - client.ObjectKeyFromObject(rateLimitingkongPlugin), - ) - - factory.SDK.PluginSDK.EXPECT(). - DeletePlugin(mock.Anything, cp.GetKonnectStatus().GetKonnectID(), mock.Anything). - Return( - &sdkkonnectops.DeletePluginResponse{ - StatusCode: 200, + "KongPlugin wasn't updated to get plugin-in-use finalizer removed", + ) + + t.Logf( + "checking that managed KongPluginBinding %s gets deleted", + client.ObjectKeyFromObject(kongPluginBinding), + ) + assert.EventuallyWithT(t, + func(c *assert.CollectT) { + assert.True(c, k8serrors.IsNotFound( + clientNamespaced.Get(ctx, client.ObjectKeyFromObject(kongPluginBinding), kongPluginBinding), + )) + }, waitTime, tickTime, + "KongPluginBinding wasn't deleted after removing annotation from KongRoute", + ) + + assert.EventuallyWithT(t, func(c *assert.CollectT) { + assert.True(c, sdk.PluginSDK.AssertExpectations(t)) + }, waitTime, tickTime) + }) + + t.Run("binding to KongService and KongRoute", func(t *testing.T) { + serviceID := uuid.NewString() + routeID := uuid.NewString() + + wKongPluginBinding := setupWatch[configurationv1alpha1.KongPluginBindingList](t, ctx, clientWithWatch, client.InNamespace(ns.Name)) + wKongPlugin := setupWatch[configurationv1.KongPluginList](t, ctx, clientWithWatch, client.InNamespace(ns.Name)) + kongService := deployKongServiceAttachedToCP(t, ctx, clientNamespaced, cp, + WithAnnotation(consts.PluginsAnnotationKey, rateLimitingkongPlugin.Name), + ) + t.Cleanup(func() { + require.NoError(t, clientNamespaced.Delete(ctx, kongService)) + }) + updateKongServiceStatusWithProgrammed(t, ctx, clientNamespaced, kongService, serviceID, cp.GetKonnectStatus().GetKonnectID()) + kongRoute := deployKongRouteAttachedToService(t, ctx, clientNamespaced, kongService, + WithAnnotation(consts.PluginsAnnotationKey, rateLimitingkongPlugin.Name), + ) + t.Cleanup(func() { + require.NoError(t, clientNamespaced.Delete(ctx, kongRoute)) + }) + updateKongRouteStatusWithProgrammed(t, ctx, clientNamespaced, kongRoute, routeID, cp.GetKonnectStatus().GetKonnectID(), serviceID) + + t.Logf("waiting for KongPluginBinding to be created") + kongPluginBinding := watchFor(t, ctx, wKongPluginBinding, watch.Added, + func(kpb *configurationv1alpha1.KongPluginBinding) bool { + return kpb.Spec.Targets.RouteReference != nil && + kpb.Spec.Targets.RouteReference.Name == kongRoute.Name && + kpb.Spec.Targets.ServiceReference != nil && + kpb.Spec.Targets.ServiceReference.Name == kongService.Name && + kpb.Spec.PluginReference.Name == rateLimitingkongPlugin.Name }, - nil, - ) - - t.Logf( - "checking that managed KongPlugin %s gets plugin-in-use finalizer removed", - client.ObjectKeyFromObject(rateLimitingkongPlugin), - ) - wKongPlugin = setupWatch[configurationv1.KongPluginList](t, ctx, clientWithWatch, client.InNamespace(ns.Name)) - kongServiceToPatch := kongService.DeepCopy() - delete(kongServiceToPatch.Annotations, consts.PluginsAnnotationKey) - require.NoError(t, clientNamespaced.Patch(ctx, kongServiceToPatch, client.MergeFrom(kongService))) - _ = watchFor(t, ctx, wKongPlugin, watch.Modified, - func(kp *configurationv1.KongPlugin) bool { - return kp.Name == rateLimitingkongPlugin.Name && - !controllerutil.ContainsFinalizer(kp, consts.PluginInUseFinalizer) - }, - "KongPlugin wasn't updated to get plugin-in-use finalizer removed", - ) - - t.Logf( - "checking that managed KongPluginBinding %s gets deleted", - client.ObjectKeyFromObject(kongPluginBinding), - ) - assert.EventuallyWithT(t, - func(c *assert.CollectT) { - assert.True(c, k8serrors.IsNotFound( - clientNamespaced.Get(ctx, client.ObjectKeyFromObject(kongPluginBinding), kongPluginBinding), - )) - }, waitTime, tickTime, - "KongPluginBinding wasn't deleted after removing annotation from KongService", - ) - - assert.EventuallyWithT(t, func(c *assert.CollectT) { - assert.True(c, factory.SDK.PluginSDK.AssertExpectations(t)) - }, waitTime, tickTime) + "KongPluginBinding wasn't created", + ) + t.Logf( + "checking that managed KongPlugin %s gets plugin-in-use finalizer added", + client.ObjectKeyFromObject(rateLimitingkongPlugin), + ) + _ = watchFor(t, ctx, wKongPlugin, watch.Modified, + func(kp *configurationv1.KongPlugin) bool { + return kp.Name == rateLimitingkongPlugin.Name && + controllerutil.ContainsFinalizer(kp, consts.PluginInUseFinalizer) + }, + "KongPlugin wasn't updated to get plugin-in-use finalizer added", + ) + + t.Logf("delete managed kongPluginBinding %s, then check it gets recreated", client.ObjectKeyFromObject(kongPluginBinding)) + require.NoError(t, clientNamespaced.Delete(ctx, kongPluginBinding)) + _ = watchFor(t, ctx, wKongPluginBinding, watch.Added, + func(kpb *configurationv1alpha1.KongPluginBinding) bool { + return kpb.Spec.Targets.RouteReference != nil && + kpb.Spec.Targets.RouteReference.Name == kongRoute.Name && + kpb.Spec.Targets.ServiceReference != nil && + kpb.Spec.Targets.ServiceReference.Name == kongService.Name && + kpb.Spec.PluginReference.Name == rateLimitingkongPlugin.Name + }, + "KongPluginBinding wasn't recreated", + ) + + t.Logf( + "remove annotation from KongRoute %s and check that there exists (is created) "+ + "a managed KongPluginBinding with only KongService in its targets", + client.ObjectKeyFromObject(kongRoute), + ) + + delete(kongRoute.Annotations, consts.PluginsAnnotationKey) + require.NoError(t, clientNamespaced.Update(ctx, kongRoute)) + kongPluginBinding = watchFor(t, ctx, wKongPluginBinding, watch.Added, + func(kpb *configurationv1alpha1.KongPluginBinding) bool { + return kpb.Spec.Targets.RouteReference == nil && + kpb.Spec.Targets.ServiceReference != nil && + kpb.Spec.Targets.ServiceReference.Name == kongService.Name && + kpb.Spec.PluginReference.Name == rateLimitingkongPlugin.Name + }, + "KongPluginBinding wasn't created", + ) + + t.Logf( + "remove annotation from KongService %s and check a managed KongPluginBinding (%s) gets deleted", + client.ObjectKeyFromObject(kongService), + client.ObjectKeyFromObject(kongPluginBinding), + ) + sdk.PluginSDK.EXPECT(). + DeletePlugin(mock.Anything, cp.GetKonnectStatus().GetKonnectID(), mock.Anything). + Return( + &sdkkonnectops.DeletePluginResponse{ + StatusCode: 200, + }, + nil, + ) + delete(kongService.Annotations, consts.PluginsAnnotationKey) + require.NoError(t, clientNamespaced.Update(ctx, kongService)) + + assert.EventuallyWithT(t, + func(c *assert.CollectT) { + assert.True(c, k8serrors.IsNotFound( + clientNamespaced.Get(ctx, client.ObjectKeyFromObject(kongPluginBinding), kongPluginBinding), + )) + }, waitTime, tickTime, + "KongPluginBinding wasn't deleted after removing annotation from KongRoute", + ) + + assert.EventuallyWithT(t, func(c *assert.CollectT) { + assert.True(c, sdk.PluginSDK.AssertExpectations(t)) + }, waitTime, tickTime) + }) } diff --git a/test/envtest/kongpluginbinding_unmanaged_test.go b/test/envtest/kongpluginbinding_unmanaged_test.go index be7354e5f..978d9cc24 100644 --- a/test/envtest/kongpluginbinding_unmanaged_test.go +++ b/test/envtest/kongpluginbinding_unmanaged_test.go @@ -11,9 +11,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" - apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/watch" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" @@ -48,183 +46,352 @@ func TestKongPluginBindingUnmanaged(t *testing.T) { cp := deployKonnectGatewayControlPlaneWithID(t, ctx, clientNamespaced, apiAuth) factory := ops.NewMockSDKFactory(t) - serviceID := uuid.NewString() - factory.SDK.ServicesSDK.EXPECT(). - CreateService(mock.Anything, cp.GetKonnectStatus().GetKonnectID(), mock.Anything). - Return( - &sdkkonnectops.CreateServiceResponse{ - Service: &sdkkonnectcomp.Service{ - ID: lo.ToPtr(serviceID), - }, - }, - nil, - ) - factory.SDK.ServicesSDK.EXPECT(). - UpsertService(mock.Anything, mock.Anything, mock.Anything).Maybe(). - Return( - &sdkkonnectops.UpsertServiceResponse{ - Service: &sdkkonnectcomp.Service{ - ID: lo.ToPtr(serviceID), - }, - }, - nil, - ) + sdk := factory.SDK require.NoError(t, manager.SetupCacheIndicesForKonnectTypes(ctx, mgr, false)) reconcilers := []Reconciler{ konnect.NewKongPluginReconciler(false, mgr.GetClient()), konnect.NewKonnectEntityReconciler(factory, false, mgr.GetClient(), - konnect.WithKonnectEntitySyncPeriod[configurationv1alpha1.KongPluginBinding](konnectSyncTime), - ), - konnect.NewKonnectEntityReconciler(factory, false, mgr.GetClient(), - konnect.WithKonnectEntitySyncPeriod[configurationv1alpha1.KongService](konnectSyncTime), + konnect.WithKonnectEntitySyncPeriod[configurationv1alpha1.KongPluginBinding](konnectInfiniteSyncTime), ), } StartReconcilers(ctx, t, mgr, logs, reconcilers...) - proxyCacheKongPlugin := &configurationv1.KongPlugin{ - ObjectMeta: metav1.ObjectMeta{ - GenerateName: "proxy-cache-kp-", - }, - PluginName: "proxy-cache", - Config: apiextensionsv1.JSON{ - Raw: []byte(`{"response_code": [200], "request_method": ["GET", "HEAD"], "content_type": ["text/plain; charset=utf-8"], "cache_ttl": 300, "strategy": "memory"}`), - }, - } - require.NoError(t, clientNamespaced.Create(ctx, proxyCacheKongPlugin)) - t.Logf("deployed %s KongPlugin (%s) resource", client.ObjectKeyFromObject(proxyCacheKongPlugin), proxyCacheKongPlugin.PluginName) - - kongService := deployKongService(t, ctx, clientNamespaced, - &configurationv1alpha1.KongService{ - Spec: configurationv1alpha1.KongServiceSpec{ - ControlPlaneRef: &configurationv1alpha1.ControlPlaneRef{ - Type: configurationv1alpha1.ControlPlaneRefKonnectNamespacedRef, - KonnectNamespacedRef: &configurationv1alpha1.KonnectNamespacedRef{ - Name: cp.Name, + t.Run("binding to KongService", func(t *testing.T) { + proxyCacheKongPlugin := deployProxyCachePlugin(t, ctx, clientNamespaced) + + serviceID := uuid.NewString() + pluginID := uuid.NewString() + + createCall := sdk.PluginSDK.EXPECT(). + CreatePlugin(mock.Anything, cp.GetKonnectStatus().GetKonnectID(), mock.Anything). + Return( + &sdkkonnectops.CreatePluginResponse{ + Plugin: &sdkkonnectcomp.Plugin{ + ID: lo.ToPtr(pluginID), }, }, - KongServiceAPISpec: configurationv1alpha1.KongServiceAPISpec{ - URL: lo.ToPtr("http://example.com"), + nil, + ) + defer createCall.Unset() + + kongService := deployKongServiceAttachedToCP(t, ctx, clientNamespaced, cp) + updateKongServiceStatusWithProgrammed(t, ctx, clientNamespaced, kongService, serviceID, cp.GetKonnectStatus().GetKonnectID()) + + wKongPlugin := setupWatch[configurationv1.KongPluginList](t, ctx, clientWithWatch, client.InNamespace(ns.Name)) + kpb := deployKongPluginBinding(t, ctx, clientNamespaced, + &configurationv1alpha1.KongPluginBinding{ + Spec: configurationv1alpha1.KongPluginBindingSpec{ + ControlPlaneRef: &configurationv1alpha1.ControlPlaneRef{ + Type: configurationv1alpha1.ControlPlaneRefKonnectNamespacedRef, + KonnectNamespacedRef: &configurationv1alpha1.KonnectNamespacedRef{ + Name: cp.Name, + }, + }, + PluginReference: configurationv1alpha1.PluginRef{ + Name: proxyCacheKongPlugin.Name, + }, + Targets: configurationv1alpha1.KongPluginBindingTargets{ + ServiceReference: &configurationv1alpha1.TargetRefWithGroupKind{ + Group: configurationv1alpha1.GroupVersion.Group, + Kind: "KongService", + Name: kongService.Name, + }, + }, }, }, - }, - ) - - pluginID := uuid.NewString() - factory.SDK.PluginSDK.EXPECT(). - CreatePlugin(mock.Anything, cp.GetKonnectStatus().GetKonnectID(), mock.Anything). - Return( - &sdkkonnectops.CreatePluginResponse{ - Plugin: &sdkkonnectcomp.Plugin{ - ID: lo.ToPtr(pluginID), + ) + t.Logf( + "wait for the controller to pick the new unmanaged KongPluginBinding %s and put a %s finalizer on the referenced plugin %s", + client.ObjectKeyFromObject(kpb), + consts.PluginInUseFinalizer, + client.ObjectKeyFromObject(proxyCacheKongPlugin), + ) + _ = watchFor(t, ctx, wKongPlugin, watch.Modified, + func(kp *configurationv1.KongPlugin) bool { + return kp.Name == proxyCacheKongPlugin.Name && + controllerutil.ContainsFinalizer(kp, consts.PluginInUseFinalizer) + }, + "KongPlugin wasn't updated to get the plugin-in-use finalizer", + ) + assert.EventuallyWithT(t, func(c *assert.CollectT) { + assert.True(c, sdk.PluginSDK.AssertExpectations(t)) + }, waitTime, tickTime) + + sdk.PluginSDK.EXPECT(). + DeletePlugin(mock.Anything, cp.GetKonnectStatus().GetKonnectID(), mock.Anything). + Return( + &sdkkonnectops.DeletePluginResponse{ + StatusCode: 200, + }, + nil, + ) + + t.Logf("delete the KongPlugin %s, then check it does not get collected", client.ObjectKeyFromObject(proxyCacheKongPlugin)) + require.NoError(t, clientNamespaced.Delete(ctx, proxyCacheKongPlugin)) + assert.EventuallyWithT(t, func(c *assert.CollectT) { + assert.False(c, k8serrors.IsNotFound( + clientNamespaced.Get(ctx, client.ObjectKeyFromObject(proxyCacheKongPlugin), proxyCacheKongPlugin), + )) + assert.True(c, proxyCacheKongPlugin.DeletionTimestamp != nil) + assert.True(c, controllerutil.ContainsFinalizer(proxyCacheKongPlugin, consts.PluginInUseFinalizer)) + }, waitTime, tickTime) + + t.Logf("delete the unmanaged KongPluginBinding %s, then check the proxy-cache KongPlugin %s gets collected", + client.ObjectKeyFromObject(kpb), + client.ObjectKeyFromObject(proxyCacheKongPlugin), + ) + require.NoError(t, clientNamespaced.Delete(ctx, kpb)) + assert.EventuallyWithT(t, func(c *assert.CollectT) { + assert.True(c, k8serrors.IsNotFound( + clientNamespaced.Get(ctx, client.ObjectKeyFromObject(proxyCacheKongPlugin), proxyCacheKongPlugin), + )) + }, waitTime, tickTime, "KongPlugin did not get deleted but should have") + + t.Logf( + "delete the KongService %s and check it gets collected, as the KongPluginBinding finalizer should have been removed", + client.ObjectKeyFromObject(kongService), + ) + require.NoError(t, clientNamespaced.Delete(ctx, kongService)) + assert.EventuallyWithT(t, func(c *assert.CollectT) { + assert.True(c, k8serrors.IsNotFound( + clientNamespaced.Get(ctx, client.ObjectKeyFromObject(kongService), kongService), + )) + }, waitTime, tickTime) + + assert.EventuallyWithT(t, func(c *assert.CollectT) { + assert.True(c, sdk.PluginSDK.AssertExpectations(t)) + }, waitTime, tickTime) + }) + t.Run("binding to KongRoute", func(t *testing.T) { + proxyCacheKongPlugin := deployProxyCachePlugin(t, ctx, clientNamespaced) + + serviceID := uuid.NewString() + routeID := uuid.NewString() + pluginID := uuid.NewString() + + createCall := sdk.PluginSDK.EXPECT(). + CreatePlugin(mock.Anything, cp.GetKonnectStatus().GetKonnectID(), mock.Anything). + Return( + &sdkkonnectops.CreatePluginResponse{ + Plugin: &sdkkonnectcomp.Plugin{ + ID: lo.ToPtr(pluginID), + }, + }, + nil, + ) + defer createCall.Unset() + + kongService := deployKongServiceAttachedToCP(t, ctx, clientNamespaced, cp) + updateKongServiceStatusWithProgrammed(t, ctx, clientNamespaced, kongService, serviceID, cp.GetKonnectStatus().GetKonnectID()) + kongRoute := deployKongRouteAttachedToService(t, ctx, clientNamespaced, kongService) + updateKongRouteStatusWithProgrammed(t, ctx, clientNamespaced, kongRoute, routeID, cp.GetKonnectStatus().GetKonnectID(), serviceID) + + wKongPlugin := setupWatch[configurationv1.KongPluginList](t, ctx, clientWithWatch, client.InNamespace(ns.Name)) + kpb := deployKongPluginBinding(t, ctx, clientNamespaced, + &configurationv1alpha1.KongPluginBinding{ + Spec: configurationv1alpha1.KongPluginBindingSpec{ + ControlPlaneRef: &configurationv1alpha1.ControlPlaneRef{ + Type: configurationv1alpha1.ControlPlaneRefKonnectNamespacedRef, + KonnectNamespacedRef: &configurationv1alpha1.KonnectNamespacedRef{ + Name: cp.Name, + }, + }, + PluginReference: configurationv1alpha1.PluginRef{ + Name: proxyCacheKongPlugin.Name, + }, + Targets: configurationv1alpha1.KongPluginBindingTargets{ + RouteReference: &configurationv1alpha1.TargetRefWithGroupKind{ + Group: configurationv1alpha1.GroupVersion.Group, + Kind: "KongRoute", + Name: kongRoute.Name, + }, + }, }, }, - nil, - ) - factory.SDK.PluginSDK.EXPECT(). - UpsertPlugin(mock.Anything, mock.Anything, mock.Anything).Maybe(). - Return( - &sdkkonnectops.UpsertPluginResponse{ - Plugin: &sdkkonnectcomp.Plugin{ - ID: lo.ToPtr(pluginID), + ) + t.Logf( + "wait for the controller to pick the new unmanaged KongPluginBinding %s and put a %s finalizer on the referenced plugin %s", + client.ObjectKeyFromObject(kpb), + consts.PluginInUseFinalizer, + client.ObjectKeyFromObject(proxyCacheKongPlugin), + ) + _ = watchFor(t, ctx, wKongPlugin, watch.Modified, + func(kp *configurationv1.KongPlugin) bool { + return kp.Name == proxyCacheKongPlugin.Name && + controllerutil.ContainsFinalizer(kp, consts.PluginInUseFinalizer) + }, + "KongPlugin wasn't updated to get the plugin-in-use finalizer", + ) + assert.EventuallyWithT(t, func(c *assert.CollectT) { + assert.True(c, sdk.PluginSDK.AssertExpectations(t)) + }, waitTime, tickTime) + + sdk.PluginSDK.EXPECT(). + DeletePlugin(mock.Anything, cp.GetKonnectStatus().GetKonnectID(), mock.Anything). + Return( + &sdkkonnectops.DeletePluginResponse{ + StatusCode: 200, }, + nil, + ) + + t.Logf("delete the KongPlugin %s, then check it does not get collected", client.ObjectKeyFromObject(proxyCacheKongPlugin)) + require.NoError(t, clientNamespaced.Delete(ctx, proxyCacheKongPlugin)) + assert.EventuallyWithT(t, func(c *assert.CollectT) { + assert.False(c, k8serrors.IsNotFound( + clientNamespaced.Get(ctx, client.ObjectKeyFromObject(proxyCacheKongPlugin), proxyCacheKongPlugin), + )) + assert.True(c, proxyCacheKongPlugin.DeletionTimestamp != nil) + assert.True(c, controllerutil.ContainsFinalizer(proxyCacheKongPlugin, consts.PluginInUseFinalizer)) + }, waitTime, tickTime) + + t.Logf("delete the unmanaged KongPluginBinding %s, then check the proxy-cache KongPlugin %s gets collected", + client.ObjectKeyFromObject(kpb), + client.ObjectKeyFromObject(proxyCacheKongPlugin), + ) + require.NoError(t, clientNamespaced.Delete(ctx, kpb)) + _ = watchFor(t, ctx, wKongPlugin, watch.Deleted, + func(kp *configurationv1.KongPlugin) bool { + return kp.Name == proxyCacheKongPlugin.Name }, - nil, + "KongPlugin did not got deleted but shouldn't have", + ) + + t.Logf( + "delete the KongRoute %s and check it gets collected, as the KongPluginBinding finalizer should have been removed", + client.ObjectKeyFromObject(kongRoute), ) + require.NoError(t, clientNamespaced.Delete(ctx, kongRoute)) + assert.EventuallyWithT(t, func(c *assert.CollectT) { + assert.True(c, k8serrors.IsNotFound( + clientNamespaced.Get(ctx, client.ObjectKeyFromObject(kongRoute), kongRoute), + )) + }, waitTime, tickTime) + + assert.EventuallyWithT(t, func(c *assert.CollectT) { + assert.True(c, sdk.PluginSDK.AssertExpectations(t)) + }, waitTime, tickTime) + }) + + t.Run("binding to KongService and KongRoute", func(t *testing.T) { + proxyCacheKongPlugin := deployProxyCachePlugin(t, ctx, clientNamespaced) + + serviceID := uuid.NewString() + routeID := uuid.NewString() + pluginID := uuid.NewString() - wKongPlugin := setupWatch[configurationv1.KongPluginList](t, ctx, clientWithWatch, client.InNamespace(ns.Name)) - kpb := deployKongPluginBinding(t, ctx, clientNamespaced, - &configurationv1alpha1.KongPluginBinding{ - Spec: configurationv1alpha1.KongPluginBindingSpec{ - ControlPlaneRef: &configurationv1alpha1.ControlPlaneRef{ - Type: configurationv1alpha1.ControlPlaneRefKonnectNamespacedRef, - KonnectNamespacedRef: &configurationv1alpha1.KonnectNamespacedRef{ - Name: cp.Name, + kongService := deployKongServiceAttachedToCP(t, ctx, clientNamespaced, cp) + updateKongServiceStatusWithProgrammed(t, ctx, clientNamespaced, kongService, serviceID, cp.GetKonnectStatus().GetKonnectID()) + kongRoute := deployKongRouteAttachedToService(t, ctx, clientNamespaced, kongService) + updateKongRouteStatusWithProgrammed(t, ctx, clientNamespaced, kongRoute, routeID, cp.GetKonnectStatus().GetKonnectID(), serviceID) + + wKongPlugin := setupWatch[configurationv1.KongPluginList](t, ctx, clientWithWatch, client.InNamespace(ns.Name)) + sdk.PluginSDK.EXPECT(). + CreatePlugin( + mock.Anything, + cp.GetKonnectStatus().GetKonnectID(), + mock.MatchedBy(func(pi sdkkonnectcomp.PluginInput) bool { + return pi.Route != nil && pi.Route.ID != nil && *pi.Route.ID == routeID && + pi.Service != nil && pi.Service.ID != nil && *pi.Service.ID == serviceID + })). + Return( + &sdkkonnectops.CreatePluginResponse{ + Plugin: &sdkkonnectcomp.Plugin{ + ID: lo.ToPtr(pluginID), }, }, - PluginReference: configurationv1alpha1.PluginRef{ - Name: proxyCacheKongPlugin.Name, - }, - Targets: configurationv1alpha1.KongPluginBindingTargets{ - ServiceReference: &configurationv1alpha1.TargetRefWithGroupKind{ - Group: configurationv1alpha1.GroupVersion.Group, - Kind: "KongService", - Name: kongService.Name, + nil, + ) + kpb := deployKongPluginBinding(t, ctx, clientNamespaced, + &configurationv1alpha1.KongPluginBinding{ + Spec: configurationv1alpha1.KongPluginBindingSpec{ + ControlPlaneRef: &configurationv1alpha1.ControlPlaneRef{ + Type: configurationv1alpha1.ControlPlaneRefKonnectNamespacedRef, + KonnectNamespacedRef: &configurationv1alpha1.KonnectNamespacedRef{ + Name: cp.Name, + }, + }, + PluginReference: configurationv1alpha1.PluginRef{ + Name: proxyCacheKongPlugin.Name, + }, + Targets: configurationv1alpha1.KongPluginBindingTargets{ + RouteReference: &configurationv1alpha1.TargetRefWithGroupKind{ + Group: configurationv1alpha1.GroupVersion.Group, + Kind: "KongRoute", + Name: kongRoute.Name, + }, + ServiceReference: &configurationv1alpha1.TargetRefWithGroupKind{ + Group: configurationv1alpha1.GroupVersion.Group, + Kind: "KongService", + Name: kongService.Name, + }, }, }, }, - }, - ) - t.Logf( - "wait for the controller to pick the new unmanaged KongPluginBinding %s and put a %s finalizer on the referenced plugin %s", - client.ObjectKeyFromObject(kpb), - consts.PluginInUseFinalizer, - client.ObjectKeyFromObject(proxyCacheKongPlugin), - ) - _ = watchFor(t, ctx, wKongPlugin, watch.Modified, - func(kp *configurationv1.KongPlugin) bool { - return kp.Name == proxyCacheKongPlugin.Name && - controllerutil.ContainsFinalizer(kp, consts.PluginInUseFinalizer) - }, - "KongPlugin wasn't updated to get the plugin-in-use finalizer", - ) - assert.EventuallyWithT(t, func(c *assert.CollectT) { - assert.True(c, factory.SDK.PluginSDK.AssertExpectations(t)) - }, waitTime, tickTime) - - factory.SDK.PluginSDK.EXPECT(). - DeletePlugin(mock.Anything, cp.GetKonnectStatus().GetKonnectID(), mock.Anything). - Return( - &sdkkonnectops.DeletePluginResponse{ - StatusCode: 200, + ) + t.Logf( + "wait for the controller to pick the new unmanaged KongPluginBinding %s and put a %s finalizer on the referenced plugin %s", + client.ObjectKeyFromObject(kpb), + consts.PluginInUseFinalizer, + client.ObjectKeyFromObject(proxyCacheKongPlugin), + ) + _ = watchFor(t, ctx, wKongPlugin, watch.Modified, + func(kp *configurationv1.KongPlugin) bool { + return kp.Name == proxyCacheKongPlugin.Name && + controllerutil.ContainsFinalizer(kp, consts.PluginInUseFinalizer) }, - nil, - ) - - t.Logf("delete the KongPlugin %s, then check it does not get collected", client.ObjectKeyFromObject(proxyCacheKongPlugin)) - require.NoError(t, clientNamespaced.Delete(ctx, proxyCacheKongPlugin)) - assert.EventuallyWithT(t, func(c *assert.CollectT) { - assert.False(c, k8serrors.IsNotFound( - clientNamespaced.Get(ctx, client.ObjectKeyFromObject(proxyCacheKongPlugin), proxyCacheKongPlugin), - )) - assert.True(c, proxyCacheKongPlugin.DeletionTimestamp != nil) - assert.True(c, controllerutil.ContainsFinalizer(proxyCacheKongPlugin, consts.PluginInUseFinalizer)) - }, waitTime, tickTime) - - t.Logf("delete the unmanaged KongPluginBinding %s, then check the proxy-cache KongPlugin %s gets collected", - client.ObjectKeyFromObject(kpb), - client.ObjectKeyFromObject(proxyCacheKongPlugin), - ) - require.NoError(t, clientNamespaced.Delete(ctx, kpb)) - _ = watchFor(t, ctx, wKongPlugin, watch.Deleted, - func(kp *configurationv1.KongPlugin) bool { - return kp.Name == proxyCacheKongPlugin.Name - }, - "KongPlugin did not got deleted but shouldn't have", - ) - - t.Logf( - "delete the KongService %s and check it gets collected, as the KongPluginBinding finalizer should have been removed", - client.ObjectKeyFromObject(kongService), - ) - factory.SDK.ServicesSDK.EXPECT(). - DeleteService(mock.Anything, cp.GetKonnectStatus().GetKonnectID(), mock.Anything). - Return( - &sdkkonnectops.DeleteServiceResponse{ - StatusCode: 200, + "KongPlugin wasn't updated to get the plugin-in-use finalizer", + ) + assert.EventuallyWithT(t, func(c *assert.CollectT) { + assert.True(c, sdk.PluginSDK.AssertExpectations(t)) + }, waitTime, tickTime) + + sdk.PluginSDK.EXPECT(). + DeletePlugin(mock.Anything, cp.GetKonnectStatus().GetKonnectID(), mock.Anything). + Return( + &sdkkonnectops.DeletePluginResponse{ + StatusCode: 200, + }, + nil, + ) + + t.Logf("delete the KongPlugin %s, then check it does not get collected", client.ObjectKeyFromObject(proxyCacheKongPlugin)) + require.NoError(t, clientNamespaced.Delete(ctx, proxyCacheKongPlugin)) + assert.EventuallyWithT(t, func(c *assert.CollectT) { + assert.False(c, k8serrors.IsNotFound( + clientNamespaced.Get(ctx, client.ObjectKeyFromObject(proxyCacheKongPlugin), proxyCacheKongPlugin), + )) + assert.True(c, proxyCacheKongPlugin.DeletionTimestamp != nil) + assert.True(c, controllerutil.ContainsFinalizer(proxyCacheKongPlugin, consts.PluginInUseFinalizer)) + }, waitTime, tickTime) + + t.Logf("delete the unmanaged KongPluginBinding %s, then check the proxy-cache KongPlugin %s gets collected", + client.ObjectKeyFromObject(kpb), + client.ObjectKeyFromObject(proxyCacheKongPlugin), + ) + require.NoError(t, clientNamespaced.Delete(ctx, kpb)) + _ = watchFor(t, ctx, wKongPlugin, watch.Deleted, + func(kp *configurationv1.KongPlugin) bool { + return kp.Name == proxyCacheKongPlugin.Name }, - nil, - ) - require.NoError(t, clientNamespaced.Delete(ctx, kongService)) - assert.EventuallyWithT(t, func(c *assert.CollectT) { - assert.True(c, k8serrors.IsNotFound( - clientNamespaced.Get(ctx, client.ObjectKeyFromObject(kongService), kongService), - )) - }, waitTime, tickTime) - - assert.EventuallyWithT(t, func(c *assert.CollectT) { - assert.True(c, factory.SDK.ServicesSDK.AssertExpectations(t)) - assert.True(c, factory.SDK.PluginSDK.AssertExpectations(t)) - }, waitTime, tickTime) + "KongPlugin did not got deleted but shouldn't have", + ) + + t.Logf( + "delete the KongRoute %s and check it gets collected, as the KongPluginBinding finalizer should have been removed", + client.ObjectKeyFromObject(kongRoute), + ) + require.NoError(t, clientNamespaced.Delete(ctx, kongRoute)) + assert.EventuallyWithT(t, func(c *assert.CollectT) { + assert.True(c, k8serrors.IsNotFound( + clientNamespaced.Get(ctx, client.ObjectKeyFromObject(kongRoute), kongRoute), + )) + }, waitTime, tickTime) + + assert.EventuallyWithT(t, func(c *assert.CollectT) { + assert.True(c, sdk.PluginSDK.AssertExpectations(t)) + }, waitTime, tickTime) + }) } diff --git a/test/envtest/kongplugincleanupfinalizer_test.go b/test/envtest/kongplugincleanupfinalizer_test.go index 022a5983c..45b0589f8 100644 --- a/test/envtest/kongplugincleanupfinalizer_test.go +++ b/test/envtest/kongplugincleanupfinalizer_test.go @@ -6,7 +6,6 @@ import ( "slices" "testing" - "github.com/samber/lo" "github.com/stretchr/testify/require" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -60,23 +59,8 @@ func TestKongPluginFinalizer(t *testing.T) { require.NoError(t, clientNamespaced.Create(ctx, rateLimitingkongPlugin)) t.Logf("deployed %s KongPlugin (%s) resource", client.ObjectKeyFromObject(rateLimitingkongPlugin), rateLimitingkongPlugin.PluginName) - kongService := deployKongService(t, ctx, clientNamespaced, - &configurationv1alpha1.KongService{ - Spec: configurationv1alpha1.KongServiceSpec{ - ControlPlaneRef: &configurationv1alpha1.ControlPlaneRef{ - Type: configurationv1alpha1.ControlPlaneRefKonnectNamespacedRef, - KonnectNamespacedRef: &configurationv1alpha1.KonnectNamespacedRef{ - Name: cp.Name, - }, - }, - KongServiceAPISpec: configurationv1alpha1.KongServiceAPISpec{ - URL: lo.ToPtr("http://example.com"), - }, - }, - }, - ) - wKongService := setupWatch[configurationv1alpha1.KongServiceList](t, ctx, clientWithWatch, client.InNamespace(ns.Name)) + kongService := deployKongServiceAttachedToCP(t, ctx, clientNamespaced, cp) kpb := deployKongPluginBinding(t, ctx, clientNamespaced, &configurationv1alpha1.KongPluginBinding{ Spec: configurationv1alpha1.KongPluginBindingSpec{ diff --git a/test/envtest/update_status.go b/test/envtest/update_status.go new file mode 100644 index 000000000..5cb1dc7f1 --- /dev/null +++ b/test/envtest/update_status.go @@ -0,0 +1,74 @@ +package envtest + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/kong/gateway-operator/controller/konnect/conditions" + k8sutils "github.com/kong/gateway-operator/pkg/utils/kubernetes" + + configurationv1alpha1 "github.com/kong/kubernetes-configuration/api/configuration/v1alpha1" + konnectv1alpha1 "github.com/kong/kubernetes-configuration/api/konnect/v1alpha1" +) + +func updateKongServiceStatusWithProgrammed( + t *testing.T, + ctx context.Context, + cl client.Client, + obj *configurationv1alpha1.KongService, + id string, + cpID string, +) { + obj.Status.Konnect = &konnectv1alpha1.KonnectEntityStatusWithControlPlaneRef{ + ControlPlaneID: cpID, + KonnectEntityStatus: konnectEntityStatus(id), + } + obj.Status.Conditions = []metav1.Condition{ + programmedCondition(obj.GetGeneration()), + } + + require.NoError(t, cl.Status().Update(ctx, obj)) +} + +func updateKongRouteStatusWithProgrammed( + t *testing.T, + ctx context.Context, + cl client.Client, + obj *configurationv1alpha1.KongRoute, + id string, + cpID string, + serviceID string, +) { + obj.Status.Konnect = &konnectv1alpha1.KonnectEntityStatusWithControlPlaneAndServiceRefs{ + ServiceID: serviceID, + ControlPlaneID: cpID, + KonnectEntityStatus: konnectEntityStatus(id), + } + obj.Status.Conditions = []metav1.Condition{ + programmedCondition(obj.GetGeneration()), + } + + require.NoError(t, cl.Status().Update(ctx, obj)) +} + +func konnectEntityStatus(id string) konnectv1alpha1.KonnectEntityStatus { + return konnectv1alpha1.KonnectEntityStatus{ + ID: id, + ServerURL: "https://api.konghq.com", + OrgID: "org-id", + } +} + +func programmedCondition(generation int64) metav1.Condition { + return k8sutils.NewConditionWithGeneration( + conditions.KonnectEntityProgrammedConditionType, + metav1.ConditionTrue, + conditions.KonnectEntityProgrammedReasonProgrammed, + "", + generation, + ) +} diff --git a/test/envtest/watch.go b/test/envtest/watch.go index 6b3d84acd..8633f2732 100644 --- a/test/envtest/watch.go +++ b/test/envtest/watch.go @@ -5,6 +5,7 @@ import ( "testing" "time" + "github.com/kr/pretty" "github.com/stretchr/testify/require" "k8s.io/apimachinery/pkg/watch" "sigs.k8s.io/controller-runtime/pkg/client" @@ -56,25 +57,33 @@ func watchFor[ ctx, cancel := context.WithTimeout(ctx, clientWatchTimeout) defer cancel() - var ret T + var ( + obj T + receivedAtLeastOneObj bool + ) for found := false; !found; { select { case <-ctx.Done(): - require.Fail(t, failMsg) + if receivedAtLeastOneObj { + require.Failf(t, failMsg, "Last object received:\n%v", pretty.Sprint(obj)) + } else { + require.Fail(t, failMsg) + } case e := <-w.ResultChan(): if e.Type != eventType { continue } - obj, ok := e.Object.(T) + var ok bool + obj, ok = e.Object.(T) if !ok { continue } + receivedAtLeastOneObj = true if !predicate(obj) { continue } found = true - ret = obj } } - return ret + return obj }