diff --git a/pkg/controller/sync/controller.go b/pkg/controller/sync/controller.go index 8627875174..a6284cca1f 100644 --- a/pkg/controller/sync/controller.go +++ b/pkg/controller/sync/controller.go @@ -268,7 +268,19 @@ func (s *KubeFedSyncController) reconcile(qualifiedName util.QualifiedName) util apiResource := s.typeConfig.GetTargetType() gvk := apiResourceToGVK(&apiResource) klog.V(2).Infof("Ensuring the removal of the label %q from %s %q in member clusters.", util.ManagedByKubeFedLabelKey, gvk.Kind, qualifiedName) - err = s.removeManagedLabel(gvk, qualifiedName) + // We can't compute resource placement, therefore we try to + // remove it from all member clusters. + clusters, err := s.informer.GetClusters() + if err != nil { + wrappedErr := errors.Wrap(err, "failed to get member clusters") + runtime.HandleError(wrappedErr) + return util.StatusError + } + clusterNames := sets.NewString() + for _, cluster := range clusters { + clusterNames = clusterNames.Insert(cluster.Name) + } + err = s.removeManagedLabel(gvk, qualifiedName, clusterNames) if err != nil { wrappedErr := errors.Wrapf(err, "failed to remove the label %q from %s %q in member clusters", util.ManagedByKubeFedLabelKey, gvk.Kind, qualifiedName) runtime.HandleError(wrappedErr) @@ -501,7 +513,19 @@ func (s *KubeFedSyncController) ensureDeletion(fedResource FederatedResource) ut return util.StatusError } klog.V(2).Infof("Initiating the removal of the label %q from resources previously managed by %s %q.", util.ManagedByKubeFedLabelKey, kind, key) - err = s.removeManagedLabel(fedResource.TargetGVK(), fedResource.TargetName()) + clusters, err := s.informer.GetClusters() + if err != nil { + wrappedErr := errors.Wrap(err, "failed to get member clusters") + runtime.HandleError(wrappedErr) + return util.StatusError + } + targetClusters, err := fedResource.ComputePlacement(clusters) + if err != nil { + wrappedErr := errors.Wrapf(err, "failed to compute placement for %s %q", kind, key) + runtime.HandleError(wrappedErr) + return util.StatusError + } + err = s.removeManagedLabel(fedResource.TargetGVK(), fedResource.TargetName(), targetClusters) if err != nil { wrappedErr := errors.Wrapf(err, "failed to remove the label %q from all resources previously managed by %s %q", util.ManagedByKubeFedLabelKey, kind, key) runtime.HandleError(wrappedErr) @@ -533,8 +557,8 @@ func (s *KubeFedSyncController) ensureDeletion(fedResource FederatedResource) ut // removeManagedLabel attempts to remove the managed label from // resources with the given name in member clusters. -func (s *KubeFedSyncController) removeManagedLabel(gvk schema.GroupVersionKind, qualifiedName util.QualifiedName) error { - ok, err := s.handleDeletionInClusters(gvk, qualifiedName, func(dispatcher dispatch.UnmanagedDispatcher, clusterName string, clusterObj *unstructured.Unstructured) { +func (s *KubeFedSyncController) removeManagedLabel(gvk schema.GroupVersionKind, qualifiedName util.QualifiedName, clusters sets.String) error { + ok, err := s.handleDeletionInClusters(gvk, qualifiedName, clusters, func(dispatcher dispatch.UnmanagedDispatcher, clusterName string, clusterObj *unstructured.Unstructured) { if clusterObj.GetDeletionTimestamp() != nil { return } @@ -554,8 +578,17 @@ func (s *KubeFedSyncController) deleteFromClusters(fedResource FederatedResource gvk := fedResource.TargetGVK() qualifiedName := fedResource.TargetName() + clusters, err := s.informer.GetClusters() + if err != nil { + return false, err + } + targetClusters, err := fedResource.ComputePlacement(clusters) + if err != nil { + return false, err + } + remainingClusters := []string{} - ok, err := s.handleDeletionInClusters(gvk, qualifiedName, func(dispatcher dispatch.UnmanagedDispatcher, clusterName string, clusterObj *unstructured.Unstructured) { + ok, err := s.handleDeletionInClusters(gvk, qualifiedName, targetClusters, func(dispatcher dispatch.UnmanagedDispatcher, clusterName string, clusterObj *unstructured.Unstructured) { // If the containing namespace of a FederatedNamespace is // marked for deletion, it is impossible to require the // removal of the namespace in advance of removal of the sync @@ -615,9 +648,17 @@ func (s *KubeFedSyncController) ensureRemovedOrUnmanaged(fedResource FederatedRe return errors.Wrap(err, "failed to get a list of clusters") } + targetClusters, err := fedResource.ComputePlacement(clusters) + if err != nil { + return errors.Wrapf(err, "failed to compute placement for %s %q", fedResource.FederatedKind(), fedResource.FederatedName().Name) + } + dispatcher := dispatch.NewCheckUnmanagedDispatcher(s.informer.GetClientForCluster, fedResource.TargetGVK(), fedResource.TargetName()) unreadyClusters := []string{} for _, cluster := range clusters { + if !targetClusters.Has(cluster.Name) { + continue + } if !util.IsClusterReady(&cluster.Status) { unreadyClusters = append(unreadyClusters, cluster.Name) continue @@ -639,9 +680,9 @@ func (s *KubeFedSyncController) ensureRemovedOrUnmanaged(fedResource FederatedRe // handleDeletionInClusters invokes the provided deletion handler for // each managed resource in member clusters. -func (s *KubeFedSyncController) handleDeletionInClusters(gvk schema.GroupVersionKind, qualifiedName util.QualifiedName, +func (s *KubeFedSyncController) handleDeletionInClusters(gvk schema.GroupVersionKind, qualifiedName util.QualifiedName, clusters sets.String, deletionFunc func(dispatcher dispatch.UnmanagedDispatcher, clusterName string, clusterObj *unstructured.Unstructured)) (bool, error) { - clusters, err := s.informer.GetClusters() + memberClusters, err := s.informer.GetClusters() if err != nil { return false, errors.Wrap(err, "failed to get a list of clusters") } @@ -649,8 +690,11 @@ func (s *KubeFedSyncController) handleDeletionInClusters(gvk schema.GroupVersion dispatcher := dispatch.NewUnmanagedDispatcher(s.informer.GetClientForCluster, gvk, qualifiedName) retrievalFailureClusters := []string{} unreadyClusters := []string{} - for _, cluster := range clusters { + for _, cluster := range memberClusters { clusterName := cluster.Name + if !clusters.Has(clusterName) { + continue + } if !util.IsClusterReady(&cluster.Status) { unreadyClusters = append(unreadyClusters, clusterName) diff --git a/scripts/pre-commit.sh b/scripts/pre-commit.sh index dfc6fd390e..5378382bf1 100755 --- a/scripts/pre-commit.sh +++ b/scripts/pre-commit.sh @@ -105,6 +105,19 @@ function run-e2e-tests-with-in-memory-controllers() { ${IN_MEMORY_E2E_TEST_CMD} } +function run-e2e-tests-with-not-ready-clusters() { + # Run the tests without any verbosity. The unhealthy nodes generate + # too much logs. + go test -timeout 900s ./test/e2e \ + -args -kubeconfig=${HOME}/.kube/config \ + -single-call-timeout=2m \ + -ginkgo.randomizeAllSpecs \ + -limited-scope=true \ + -in-memory-controllers=true \ + -simulate-federation=true \ + -ginkgo.focus='\[NOT_READY\]' +} + function run-namespaced-e2e-tests() { local namespaced_e2e_test_cmd="${E2E_TEST_CMD} -kubefed-namespace=foo -limited-scope=true" # Run the placement test separately to avoid crud failures if @@ -200,6 +213,9 @@ kubectl scale deployments kubefed-controller-manager -n kube-federation-system - echo "Running e2e tests with race detector against cluster-scoped kubefed with in-memory controllers" run-e2e-tests-with-in-memory-controllers +echo "Running e2e tests with not-ready clusters" +run-e2e-tests-with-not-ready-clusters + # FederatedTypeConfig controller is needed to remove finalizers from # FederatedTypeConfigs in order to successfully delete the KubeFed # control plane in the next step. diff --git a/test/common/crudtester.go b/test/common/crudtester.go index d4a2693859..4903e026c8 100644 --- a/test/common/crudtester.go +++ b/test/common/crudtester.go @@ -41,6 +41,7 @@ import ( "sigs.k8s.io/kubefed/pkg/apis/core/common" "sigs.k8s.io/kubefed/pkg/apis/core/typeconfig" fedv1a1 "sigs.k8s.io/kubefed/pkg/apis/core/v1alpha1" + "sigs.k8s.io/kubefed/pkg/apis/core/v1beta1" genericclient "sigs.k8s.io/kubefed/pkg/client/generic" "sigs.k8s.io/kubefed/pkg/controller/sync" "sigs.k8s.io/kubefed/pkg/controller/sync/status" @@ -65,6 +66,7 @@ type FederatedTypeCrudTester struct { // operation that involves member clusters may take longer due to // propagation latency. clusterWaitTimeout time.Duration + clustersNamespace string } type TestClusterConfig struct { @@ -77,7 +79,7 @@ type TestCluster struct { Client util.ResourceClient } -func NewFederatedTypeCrudTester(testLogger TestLogger, typeConfig typeconfig.Interface, kubeConfig *rest.Config, testClusters map[string]TestCluster, waitInterval, clusterWaitTimeout time.Duration) (*FederatedTypeCrudTester, error) { +func NewFederatedTypeCrudTester(testLogger TestLogger, typeConfig typeconfig.Interface, kubeConfig *rest.Config, testClusters map[string]TestCluster, clustersNamespace string, waitInterval, clusterWaitTimeout time.Duration) (*FederatedTypeCrudTester, error) { return &FederatedTypeCrudTester{ tl: testLogger, typeConfig: typeConfig, @@ -87,11 +89,12 @@ func NewFederatedTypeCrudTester(testLogger TestLogger, typeConfig typeconfig.Int testClusters: testClusters, waitInterval: waitInterval, clusterWaitTimeout: clusterWaitTimeout, + clustersNamespace: clustersNamespace, }, nil } -func (c *FederatedTypeCrudTester) CheckLifecycle(targetObject *unstructured.Unstructured, overrides []interface{}) { - fedObject := c.CheckCreate(targetObject, overrides) +func (c *FederatedTypeCrudTester) CheckLifecycle(targetObject *unstructured.Unstructured, overrides []interface{}, selectors map[string]string) { + fedObject := c.CheckCreate(targetObject, overrides, selectors) c.CheckStatusCreated(util.NewQualifiedName(fedObject)) @@ -104,7 +107,7 @@ func (c *FederatedTypeCrudTester) CheckLifecycle(targetObject *unstructured.Unst c.CheckDelete(fedObject, false) } -func (c *FederatedTypeCrudTester) Create(targetObject *unstructured.Unstructured, overrides []interface{}) *unstructured.Unstructured { +func (c *FederatedTypeCrudTester) Create(targetObject *unstructured.Unstructured, overrides []interface{}, selectors map[string]string) *unstructured.Unstructured { qualifiedName := util.NewQualifiedName(targetObject) kind := c.typeConfig.GetTargetType().Kind fedKind := c.typeConfig.GetFederatedType().Kind @@ -113,10 +116,7 @@ func (c *FederatedTypeCrudTester) Create(targetObject *unstructured.Unstructured c.tl.Fatalf("Error obtaining %s from %s %q: %v", fedKind, kind, qualifiedName, err) } - fedObject, err = c.setAdditionalTestData(fedObject, overrides, targetObject.GetGenerateName()) - if err != nil { - c.tl.Fatalf("Error setting overrides and placement on %s %q: %v", fedKind, qualifiedName, err) - } + fedObject = c.setAdditionalTestData(fedObject, overrides, selectors, targetObject.GetGenerateName()) return c.createResource(c.typeConfig.GetFederatedType(), fedObject) } @@ -141,15 +141,15 @@ func (c *FederatedTypeCrudTester) resourceClient(apiResource metav1.APIResource) return client } -func (c *FederatedTypeCrudTester) CheckCreate(targetObject *unstructured.Unstructured, overrides []interface{}) *unstructured.Unstructured { - fedObject := c.Create(targetObject, overrides) +func (c *FederatedTypeCrudTester) CheckCreate(targetObject *unstructured.Unstructured, overrides []interface{}, selectors map[string]string) *unstructured.Unstructured { + fedObject := c.Create(targetObject, overrides, selectors) c.CheckPropagation(fedObject) return fedObject } // AdditionalTestData additionally sets fixture overrides and placement clusternames into federated object -func (c *FederatedTypeCrudTester) setAdditionalTestData(fedObject *unstructured.Unstructured, overrides []interface{}, generateName string) (*unstructured.Unstructured, error) { +func (c *FederatedTypeCrudTester) setAdditionalTestData(fedObject *unstructured.Unstructured, overrides []interface{}, selectors map[string]string, generateName string) *unstructured.Unstructured { fedKind := c.typeConfig.GetFederatedType().Kind qualifiedName := util.NewQualifiedName(fedObject) @@ -159,17 +159,23 @@ func (c *FederatedTypeCrudTester) setAdditionalTestData(fedObject *unstructured. c.tl.Fatalf("Error updating overrides in %s %q: %v", fedKind, qualifiedName, err) } } - clusterNames := []string{} - for name := range c.testClusters { - clusterNames = append(clusterNames, name) - } - err := util.SetClusterNames(fedObject, clusterNames) - if err != nil { - c.tl.Fatalf("Error setting cluster names in %s %q: %v", fedKind, qualifiedName, err) + if selectors != nil { + if err := util.SetClusterSelector(fedObject, selectors); err != nil { + c.tl.Fatalf("Error setting cluster selectors for %s/%s: %v", fedObject.GetKind(), fedObject.GetName(), err) + } + } else { + clusterNames := []string{} + for name := range c.testClusters { + clusterNames = append(clusterNames, name) + } + err := util.SetClusterNames(fedObject, clusterNames) + if err != nil { + c.tl.Fatalf("Error setting cluster names in %s %q: %v", fedKind, qualifiedName, err) + } } fedObject.SetGenerateName(generateName) - return fedObject, err + return fedObject } func (c *FederatedTypeCrudTester) CheckUpdate(fedObject *unstructured.Unstructured) { @@ -334,7 +340,14 @@ func (c *FederatedTypeCrudTester) CheckDelete(fedObject *unstructured.Unstructur if deletingInCluster { stateMsg = "not present" } + clusters, err := util.ComputePlacement(fedObject, c.getClusters(), false) + if err != nil { + c.tl.Fatalf("Couldn't retrieve clusters for %s/%s: %v", federatedKind, name, err) + } for clusterName, testCluster := range c.testClusters { + if !clusters.Has(clusterName) { + continue + } namespace = util.QualifiedNameForCluster(clusterName, qualifiedName).Namespace err = wait.PollImmediate(c.waitInterval, waitTimeout, func() (bool, error) { obj, err := testCluster.Client.Resources(namespace).Get(context.Background(), name, metav1.GetOptions{}) @@ -425,16 +438,33 @@ func (c *FederatedTypeCrudTester) CheckReplicaSet(fedObject *unstructured.Unstru } } +func (c *FederatedTypeCrudTester) getClusters() []*v1beta1.KubeFedCluster { + client, err := genericclient.New(c.kubeConfig) + if err != nil { + c.tl.Fatalf("Failed to get kubefed clientset: %v", err) + } + + fedClusters := []*v1beta1.KubeFedCluster{} + for cluster := range c.testClusters { + clusterResource := &v1beta1.KubeFedCluster{} + err = client.Get(context.Background(), clusterResource, c.clustersNamespace, cluster) + if err != nil { + c.tl.Fatalf("Cannot get cluster %s: %v", cluster, err) + } + fedClusters = append(fedClusters, clusterResource) + } + return fedClusters +} + // CheckPropagation checks propagation for the crud tester's clients func (c *FederatedTypeCrudTester) CheckPropagation(fedObject *unstructured.Unstructured) { federatedKind := c.typeConfig.GetFederatedType().Kind qualifiedName := util.NewQualifiedName(fedObject) - clusterNames, err := util.GetClusterNames(fedObject) + selectedClusters, err := util.ComputePlacement(fedObject, c.getClusters(), false) if err != nil { c.tl.Fatalf("Error retrieving cluster names for %s %q: %v", federatedKind, qualifiedName, err) } - selectedClusters := sets.NewString(clusterNames...) templateVersion, err := sync.GetTemplateHash(fedObject.Object) if err != nil { diff --git a/test/e2e/crd.go b/test/e2e/crd.go index 38d7c8c8d0..c114a9dd66 100644 --- a/test/e2e/crd.go +++ b/test/e2e/crd.go @@ -215,10 +215,10 @@ overrides: return targetObj, overrides, nil } - crudTester, targetObject, overrides := initCrudTest(f, tl, typeConfig, testObjectsFunc) + crudTester, targetObject, overrides := initCrudTest(f, tl, f.KubeFedSystemNamespace(), typeConfig, testObjectsFunc) // Make a copy for use in the orphan check. deletionTargetObject := targetObject.DeepCopy() - crudTester.CheckLifecycle(targetObject, overrides) + crudTester.CheckLifecycle(targetObject, overrides, nil) if namespaced { // This check should not fail so long as the main test loop @@ -228,7 +228,7 @@ overrides: tl.Fatalf("Test of orphaned deletion assumes deletion of the containing namespace") } // Perform a check of orphan deletion. - fedObject := crudTester.CheckCreate(deletionTargetObject, nil) + fedObject := crudTester.CheckCreate(deletionTargetObject, nil, nil) orphanDeletion := true crudTester.CheckDelete(fedObject, orphanDeletion) } diff --git a/test/e2e/crud.go b/test/e2e/crud.go index b76129888e..b8746977f0 100644 --- a/test/e2e/crud.go +++ b/test/e2e/crud.go @@ -61,8 +61,8 @@ var _ = Describe("Federated", func() { Describe(fmt.Sprintf("%q", typeConfigName), func() { It("should be created, read, updated and deleted successfully", func() { typeConfig, testObjectsFunc := getCrudTestInput(f, tl, typeConfigName, fixture) - crudTester, targetObject, overrides := initCrudTest(f, tl, typeConfig, testObjectsFunc) - crudTester.CheckLifecycle(targetObject, overrides) + crudTester, targetObject, overrides := initCrudTest(f, tl, f.KubeFedSystemNamespace(), typeConfig, testObjectsFunc) + crudTester.CheckLifecycle(targetObject, overrides, nil) }) for _, remoteStatusTypeName := range containedTypeNames { @@ -78,8 +78,8 @@ var _ = Describe("Federated", func() { tl.Logf("Show the content of the kubefedconfig file: '%v'", kubeFedConfig) typeConfig, testObjectsFunc := getCrudTestInput(f, tl, typeConfigName, fixture) - crudTester, targetObject, overrides := initCrudTest(f, tl, typeConfig, testObjectsFunc) - fedObject := crudTester.CheckCreate(targetObject, overrides) + crudTester, targetObject, overrides := initCrudTest(f, tl, f.KubeFedSystemNamespace(), typeConfig, testObjectsFunc) + fedObject := crudTester.CheckCreate(targetObject, overrides, nil) By("Checking the remote status filled for each federated resource for every cluster") tl.Logf("Checking the existence of a remote status for each fedObj in every cluster: %v", fedObject) @@ -105,12 +105,12 @@ var _ = Describe("Federated", func() { typeConfig, testObjectsFunc := getCrudTestInput(f, tl, typeConfigName, fixture) // Initialize the test without creating a federated namespace. - crudTester, targetObject, overrides := initCrudTestWithPropagation(f, tl, typeConfig, testObjectsFunc, false) + crudTester, targetObject, overrides := initCrudTestWithPropagation(f, tl, f.KubeFedSystemNamespace(), typeConfig, testObjectsFunc, false) kind := typeConfig.GetFederatedType().Kind By(fmt.Sprintf("Creating a %s whose containing namespace is not federated", kind)) - fedObject := crudTester.Create(targetObject, overrides) + fedObject := crudTester.Create(targetObject, overrides, nil) qualifiedName := util.NewQualifiedName(fedObject) @@ -143,7 +143,7 @@ var _ = Describe("Federated", func() { It("should have the managed label removed if not managed", func() { typeConfig, testObjectsFunc := getCrudTestInput(f, tl, typeConfigName, fixture) - crudTester, targetObject, _ := initCrudTest(f, tl, typeConfig, testObjectsFunc) + crudTester, targetObject, _ := initCrudTest(f, tl, f.KubeFedSystemNamespace(), typeConfig, testObjectsFunc) testClusters := crudTester.TestClusters() @@ -192,7 +192,7 @@ var _ = Describe("Federated", func() { It("should not be deleted if unlabeled", func() { typeConfig, testObjectsFunc := getCrudTestInput(f, tl, typeConfigName, fixture) - crudTester, targetObject, _ := initCrudTest(f, tl, typeConfig, testObjectsFunc) + crudTester, targetObject, _ := initCrudTest(f, tl, f.KubeFedSystemNamespace(), typeConfig, testObjectsFunc) testClusters := crudTester.TestClusters() @@ -315,13 +315,13 @@ func getCrudTestInput(f framework.KubeFedFramework, tl common.TestLogger, return typeConfig, testObjectsFunc } -func initCrudTest(f framework.KubeFedFramework, tl common.TestLogger, +func initCrudTest(f framework.KubeFedFramework, tl common.TestLogger, clustersNamespace string, typeConfig typeconfig.Interface, testObjectsFunc testObjectsAccessor) ( *common.FederatedTypeCrudTester, *unstructured.Unstructured, []interface{}) { - return initCrudTestWithPropagation(f, tl, typeConfig, testObjectsFunc, true) + return initCrudTestWithPropagation(f, tl, clustersNamespace, typeConfig, testObjectsFunc, true) } -func initCrudTestWithPropagation(f framework.KubeFedFramework, tl common.TestLogger, +func initCrudTestWithPropagation(f framework.KubeFedFramework, tl common.TestLogger, clustersNamespace string, typeConfig typeconfig.Interface, testObjectsFunc testObjectsAccessor, ensureNamespacePropagation bool) ( *common.FederatedTypeCrudTester, *unstructured.Unstructured, []interface{}) { @@ -343,7 +343,7 @@ func initCrudTestWithPropagation(f framework.KubeFedFramework, tl common.TestLog targetAPIResource := typeConfig.GetTargetType() testClusters := f.ClusterDynamicClients(&targetAPIResource, userAgent) - crudTester, err := common.NewFederatedTypeCrudTester(tl, typeConfig, kubeConfig, testClusters, framework.PollInterval, framework.TestContext.SingleCallTimeout) + crudTester, err := common.NewFederatedTypeCrudTester(tl, typeConfig, kubeConfig, testClusters, clustersNamespace, framework.PollInterval, framework.TestContext.SingleCallTimeout) if err != nil { tl.Fatalf("Error creating crudtester for %q: %v", federatedKind, err) } diff --git a/test/e2e/deleteoptions.go b/test/e2e/deleteoptions.go index 9abd7cb6bc..98c0c9ca70 100644 --- a/test/e2e/deleteoptions.go +++ b/test/e2e/deleteoptions.go @@ -39,8 +39,8 @@ var _ = Describe("DeleteOptions", func() { It("Deployment should be created and deleted successfully, but ReplicaSet that created by Deployment won't be deleted", func() { typeConfig, testObjectsFunc := getCrudTestInput(f, tl, typeConfigName, fixture) - crudTester, targetObject, overrides := initCrudTest(f, tl, typeConfig, testObjectsFunc) - fedObject := crudTester.CheckCreate(targetObject, overrides) + crudTester, targetObject, overrides := initCrudTest(f, tl, f.KubeFedSystemNamespace(), typeConfig, testObjectsFunc) + fedObject := crudTester.CheckCreate(targetObject, overrides, nil) By("Set PropagationPolicy property as 'Orphan' on the DeleteOptions for Federated Deployment") orphan := metav1.DeletePropagationOrphan diff --git a/test/e2e/e2e.go b/test/e2e/e2e.go index ac762ace64..9b7f103fdf 100644 --- a/test/e2e/e2e.go +++ b/test/e2e/e2e.go @@ -49,9 +49,9 @@ func RunE2ETests(t *testing.T) { var _ = ginkgo.SynchronizedBeforeSuite(func() []byte { // Run only on Ginkgo node 1 - if framework.TestContext.ScaleTest { - // Scale testing will initialize an in-memory control plane - // after the creation of a simulated federation. + // Some tests require simulated federation and will initialize an + // in-memory control plane. + if framework.TestContext.SimulateFederation { return nil } diff --git a/test/e2e/framework/test_context.go b/test/e2e/framework/test_context.go index 8ab5676d67..74bbd2cb02 100644 --- a/test/e2e/framework/test_context.go +++ b/test/e2e/framework/test_context.go @@ -38,6 +38,7 @@ type TestContextType struct { WaitForFinalization bool ScaleTest bool ScaleClusterCount int + SimulateFederation bool } func (t *TestContextType) RunControllers() bool { @@ -72,6 +73,7 @@ func registerFlags(t *TestContextType) { flag.BoolVar(&t.WaitForFinalization, "wait-for-finalization", true, "Whether the test suite should wait for finalization before stopping fixtures or exiting. Setting this to false will speed up test execution but likely result in wedged namespaces and is only recommended for disposeable clusters.") flag.BoolVar(&t.ScaleTest, "scale-test", false, "Whether the test suite should be configured for scale testing. Not compatible with most tests.") + flag.BoolVar(&t.SimulateFederation, "simulate-federation", false, "Whether the tests require a simulated federation.") flag.IntVar(&t.ScaleClusterCount, "scale-cluster-count", 1, "How many member clusters to simulate when scale testing.") } @@ -83,6 +85,9 @@ func validateFlags(t *TestContextType) { if t.ScaleTest { t.InMemoryControllers = true t.LimitedScope = true + // Scale testing will initialize an in-memory control plane + // after the creation of a simulated federation. + t.SimulateFederation = true // Scale testing will create a namespace per simulated cluster // and for large numbers of such namespaces the finalization // wait could be considerable. diff --git a/test/e2e/not_ready.go b/test/e2e/not_ready.go new file mode 100644 index 0000000000..1be30c890f --- /dev/null +++ b/test/e2e/not_ready.go @@ -0,0 +1,201 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "context" + "time" + + apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/wait" + runtimeclient "sigs.k8s.io/controller-runtime/pkg/client" + + fedv1b1 "sigs.k8s.io/kubefed/pkg/apis/core/v1beta1" + genericclient "sigs.k8s.io/kubefed/pkg/client/generic" + "sigs.k8s.io/kubefed/pkg/controller/util" + "sigs.k8s.io/kubefed/pkg/kubefedctl" + "sigs.k8s.io/kubefed/test/common" + "sigs.k8s.io/kubefed/test/e2e/framework" + + . "github.com/onsi/ginkgo" //nolint:stylecheck +) + +// WARNING This test modifies the runtime behavior of the sync +// controller. Running it concurrently with other tests that use the +// sync controller is likely to result in unexpected behavior. + +// This test is intended to validate CRUD operations even in the +// presence of not ready federated clusters if they are not the target +// of the operations. The test creates multiple self joins to simulate +// healthy and unhealthy clusters. +// +// Usually joining a cluster creates a namespace with the same name as +// the kubefed system namespace in the host cluster. To support +// multiple self-joins, the kubefed namespace in member clusters needs +// to vary by join. +// +// This test needs to run namespaced controllers since each cluster +// will be simulated by a single namespace in the host cluster. The +// name of each member cluster's namespace should match the name of +// the member cluster. +// +var _ = Describe("[NOT_READY] Simulated not-ready nodes", func() { + baseName := "unhealthy-test" + f := framework.NewKubeFedFramework(baseName) + + tl := framework.NewE2ELogger() + + typeConfigFixtures := common.TypeConfigFixturesOrDie(tl) + + It("should simulate unhealthy clusters", func() { + if !framework.TestContext.LimitedScope { + framework.Skipf("Simulated scale testing is not compatible with cluster-scoped federation.") + } + if !framework.TestContext.InMemoryControllers { + framework.Skipf("Simulated scale testing requires in-process controllers.") + } + + client := f.KubeClient(baseName) + + // Create the host cluster namespace + generateName := "unhealthy-host-" + hostNamespace, err := framework.CreateNamespace(client, generateName) + if err != nil { + tl.Fatalf("Error creating namespace: %v", err) + } + defer framework.DeleteNamespace(client, hostNamespace) + + // Reconfigure the test context to ensure the fixture setup + // will work correctly with the simulated federation. + framework.TestContext.KubeFedSystemNamespace = hostNamespace + hostConfig := f.KubeConfig() + + unhealthyCluster := "unhealthy" + _, err = kubefedctl.TestOnlyJoinClusterForNamespace(hostConfig, hostConfig, hostNamespace, unhealthyCluster, hostNamespace, unhealthyCluster, "", apiextv1.NamespaceScoped, false, false) + if err != nil { + tl.Fatalf("Error joining unhealthy cluster: %v", err) + } + + healthyCluster := "healthy" + _, err = kubefedctl.TestOnlyJoinClusterForNamespace(hostConfig, hostConfig, hostNamespace, healthyCluster, hostNamespace, healthyCluster, "", apiextv1.NamespaceScoped, false, false) + if err != nil { + tl.Fatalf("Error joining healthy cluster: %v", err) + } + hostClient, err := genericclient.New(hostConfig) + if err != nil { + tl.Fatalf("Failed to get kubefed clientset: %v", err) + } + healthyFedCluster := &unstructured.Unstructured{} + healthyFedCluster.SetGroupVersionKind(schema.GroupVersionKind{ + Kind: "KubeFedCluster", + Group: fedv1b1.SchemeGroupVersion.Group, + Version: fedv1b1.SchemeGroupVersion.Version, + }) + + err = hostClient.Get(context.Background(), healthyFedCluster, hostNamespace, healthyCluster) + if err != nil { + tl.Fatalf("Cannot get healthyCluster: %v", err) + } + addLabel(healthyFedCluster, "healthy", "true") + err = hostClient.Update(context.TODO(), healthyFedCluster) + if err != nil { + tl.Fatalf("Error updating label for healthy cluster: %v", err) + } + + // Override naming methods to allow the sync controller to + // work with a simulated federation environment. + oldNamespaceForCluster := util.NamespaceForCluster + util.NamespaceForCluster = func(clusterName, namespace string) string { + return clusterName + } + defer func() { + util.NamespaceForCluster = oldNamespaceForCluster + }() + + oldNamespaceForResource := util.NamespaceForResource + util.NamespaceForResource = func(resourceNamespace, fedNamespace string) string { + return fedNamespace + } + defer func() { + util.NamespaceForResource = oldNamespaceForResource + }() + oldQualifiedNameForCluster := util.QualifiedNameForCluster + util.QualifiedNameForCluster = func(clusterName string, qualifiedName util.QualifiedName) util.QualifiedName { + return util.QualifiedName{ + Name: qualifiedName.Name, + Namespace: clusterName, + } + } + defer func() { + util.QualifiedNameForCluster = oldQualifiedNameForCluster + }() + + // Ensure that the cluster controller is able to successfully + // health check the simulated clusters. + framework.SetUpControlPlane() + framework.WaitForUnmanagedClusterReadiness() + + fedCluster := &fedv1b1.KubeFedCluster{ + ObjectMeta: v1.ObjectMeta{ + Name: unhealthyCluster, + Namespace: hostNamespace, + }, + } + err = hostClient.Patch(context.Background(), fedCluster, runtimeclient.RawPatch(types.MergePatchType, []byte(`{"spec": {"apiEndpoint": "http://invalid_adress"}}`))) + if err != nil { + tl.Fatalf("Failed to patch kubefed cluster: %v", err) + } + + err = wait.Poll(time.Second*5, time.Second*30, func() (bool, error) { + cluster := &fedv1b1.KubeFedCluster{} + err := hostClient.Get(context.TODO(), cluster, hostNamespace, unhealthyCluster) + if err != nil { + tl.Fatalf("Failed to retrieve unhealthy cluster: %v", err) + } + return !util.IsClusterReady(&cluster.Status), nil + }) + if err != nil { + tl.Fatalf("Error waiting for unhealthy cluster: %v", err) + } + + // Enable federation of namespaces to ensure that the sync + // controller for a namespaced type will be able to start. + enableTypeFederation(tl, hostConfig, hostNamespace, "namespaces") + + // Enable federation of a namespaced type. + targetType := "secrets" + typeConfig := enableTypeFederation(tl, hostConfig, hostNamespace, targetType) + // Perform crud testing for the type + testObjectsFunc := func(namespace string, clusterNames []string) (*unstructured.Unstructured, []interface{}, error) { + fixture := typeConfigFixtures[targetType] + targetObject, err := common.NewTestTargetObject(typeConfig, namespace, fixture) + if err != nil { + return nil, nil, err + } + return targetObject, nil, err + } + crudTester, targetObject, overrides := initCrudTestWithPropagation(f, tl, hostNamespace, typeConfig, testObjectsFunc, false) + fedObject := crudTester.CheckCreate(targetObject, overrides, map[string]string{"healthy": "true"}) + crudTester.CheckStatusCreated(util.NewQualifiedName(fedObject)) + crudTester.CheckUpdate(fedObject) + crudTester.CheckDelete(fedObject, false) + }) +}) diff --git a/test/e2e/placement.go b/test/e2e/placement.go index 0ad650a86c..af0965ff65 100644 --- a/test/e2e/placement.go +++ b/test/e2e/placement.go @@ -88,8 +88,8 @@ var _ = Describe("Placement", func() { } return targetObject, nil, err } - crudTester, desiredTargetObject, _ := initCrudTest(f, tl, selectedTypeConfig, testObjectsFunc) - fedObject := crudTester.CheckCreate(desiredTargetObject, nil) + crudTester, desiredTargetObject, _ := initCrudTest(f, tl, f.KubeFedSystemNamespace(), selectedTypeConfig, testObjectsFunc) + fedObject := crudTester.CheckCreate(desiredTargetObject, nil, nil) defer func() { crudTester.CheckDelete(fedObject, false) }() diff --git a/test/e2e/scale.go b/test/e2e/scale.go index 6432cd573e..dbab8bde85 100644 --- a/test/e2e/scale.go +++ b/test/e2e/scale.go @@ -280,8 +280,8 @@ var _ = Describe("Simulated Scale", func() { } return targetObject, nil, err } - crudTester, targetObject, overrides := initCrudTestWithPropagation(f, tl, typeConfig, testObjectsFunc, false) - crudTester.CheckLifecycle(targetObject, overrides) + crudTester, targetObject, overrides := initCrudTestWithPropagation(f, tl, f.KubeFedSystemNamespace(), typeConfig, testObjectsFunc, false) + crudTester.CheckLifecycle(targetObject, overrides, nil) // Delete clusters to minimize errors logged by the cluster // controller.