From 182a01e278fa1f1fcfb0d36eecf8fa216f9e274a Mon Sep 17 00:00:00 2001 From: Philippe Scorsolini Date: Sun, 16 Jul 2023 17:30:29 +0200 Subject: [PATCH 1/7] tests(e2e): introduce E2EConfig and presets Signed-off-by: Philippe Scorsolini --- .github/workflows/ci.yml | 5 +- go.mod | 2 +- go.sum | 4 +- test/e2e/README.md | 61 +++++- test/e2e/apiextensions_test.go | 57 +---- test/e2e/compSchemaValidation_test.go | 85 ++++++++ test/e2e/config/config.go | 301 ++++++++++++++++++++++++++ test/e2e/consts.go | 42 ++++ test/e2e/funcs/env.go | 2 +- test/e2e/funcs/feature.go | 2 +- test/e2e/install_test.go | 5 +- test/e2e/main_test.go | 161 ++++++-------- test/e2e/pkg_test.go | 4 + test/e2e/xfn_test.go | 79 +++++-- 14 files changed, 626 insertions(+), 184 deletions(-) create mode 100644 test/e2e/compSchemaValidation_test.go create mode 100644 test/e2e/config/config.go create mode 100644 test/e2e/consts.go diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 22ee86fcb42..32f04c017e9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -239,8 +239,9 @@ jobs: needs: detect-noop if: needs.detect-noop.outputs.noop != 'true' strategy: + fail-fast: false matrix: - area: [lifecycle, pkg, apiextensions, xfn] + test-suite: [base, composition-webhook-schema-validation, composition-functions] steps: - name: Setup QEMU @@ -297,7 +298,7 @@ jobs: BUILD_ARGS: "--load" - name: Run E2E Tests - run: make e2e E2E_TEST_FLAGS="-test.v -labels area=${{ matrix.area }}" + run: make e2e E2E_TEST_FLAGS="-test.v --test-suite ${{ matrix.test-suite }}" publish-artifacts: runs-on: ubuntu-22.04 diff --git a/go.mod b/go.mod index 16c9799a747..b00e8724e3f 100644 --- a/go.mod +++ b/go.mod @@ -33,7 +33,7 @@ require ( kernel.org/pub/linux/libs/security/libcap/cap v1.2.69 sigs.k8s.io/controller-runtime v0.15.0 sigs.k8s.io/controller-tools v0.12.1 - sigs.k8s.io/e2e-framework v0.2.0 + sigs.k8s.io/e2e-framework v0.2.1-0.20230716064705-49e8554b536f sigs.k8s.io/kind v0.20.0 sigs.k8s.io/yaml v1.3.0 ) diff --git a/go.sum b/go.sum index ff32c864ec3..27a988f669a 100644 --- a/go.sum +++ b/go.sum @@ -969,8 +969,8 @@ sigs.k8s.io/controller-runtime v0.15.0 h1:ML+5Adt3qZnMSYxZ7gAverBLNPSMQEibtzAgp0 sigs.k8s.io/controller-runtime v0.15.0/go.mod h1:7ngYvp1MLT+9GeZ+6lH3LOlcHkp/+tzA/fmHa4iq9kk= sigs.k8s.io/controller-tools v0.12.1 h1:GyQqxzH5wksa4n3YDIJdJJOopztR5VDM+7qsyg5yE4U= sigs.k8s.io/controller-tools v0.12.1/go.mod h1:rXlpTfFHZMpZA8aGq9ejArgZiieHd+fkk/fTatY8A2M= -sigs.k8s.io/e2e-framework v0.2.0 h1:gD6AWWAHFcHibI69E9TgkNFhh0mVwWtRCHy2RU057jQ= -sigs.k8s.io/e2e-framework v0.2.0/go.mod h1:E6JXj/V4PIlb95jsn2WrNKG+Shb45xaaI7C0+BH4PL8= +sigs.k8s.io/e2e-framework v0.2.1-0.20230716064705-49e8554b536f h1:BN6JOYAOMYCC8FPSfALNFvH9f6Sf4k+fM8OwuZfHL4g= +sigs.k8s.io/e2e-framework v0.2.1-0.20230716064705-49e8554b536f/go.mod h1:7k84BFZzTqYNO1qxF4gDmQRxEoSw62lSBDXSAf43e2A= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= sigs.k8s.io/kind v0.20.0 h1:f0sc3v9mQbGnjBUaqSFST1dwIuiikKVGgoTwpoP33a8= diff --git a/test/e2e/README.md b/test/e2e/README.md index 7d95362463d..25d32ad807c 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -45,12 +45,12 @@ E2E_TEST_FLAGS="-test.v -test.failfast -destroy-kind-cluster=false" # Use an existing Kubernetes cluster. Note that the E2E tests can't deploy your # local build of Crossplane in this scenario, so you'll have to do it yourself. -E2E_TEST_FLAGS="-create-kind-cluster=false -destroy-kind-cluster=false -kubeconfig=$HOME/.kube/config" +E2E_TEST_FLAGS="-create-kind-cluster=false -destroy-kind-cluster=false -kubeconfig=$HOME/.kube/config" make e2e # Run the CrossplaneUpgrade feature, against an existing kind cluster named # "kind" (or creating it if it doesn't exist), # without installing Crossplane # first, as the feature expects the cluster to be empty, but still loading the -# images to # it. Setting the tests to fail fast and not destroying the cluster +# images to it. Setting the tests to fail fast and not destroying the cluster # afterward in order to allow debugging it. E2E_TEST_FLAGS="-test.v -v 4 -test.failfast \ -destroy-kind-cluster=false \ @@ -58,13 +58,21 @@ E2E_TEST_FLAGS="-test.v -v 4 -test.failfast \ -install-crossplane=false \ -feature=CrossplaneUpgrade" make e2e -# Run the all tests not installing or upgrading Crossplane against the currently +# Run all the tests not installing or upgrading Crossplane against the currently # selected cluster where Crossplane has already been installed. E2E_TEST_FLAGS="-test.v -v 4 -test.failfast \ -kubeconfig=$HOME/.kube/config \ -skip-labels modify-crossplane-installation=true \ -create-kind-cluster=false \ -install-crossplane=false" make go.build e2e-run-tests + +# Run the composition-webhook-schema-validation suite of tests, which will +# result in all tests marked as "test-suite=base" or +# "test-suite=composition-webhook-schema-validation" being run against a kind +# cluster with Crossplane installed with composition-webhook-schema-validation +# enabled +E2E_TEST_FLAGS="-test.v -v 4 -test.failfast \ + -test-suite=composition-webhook-schema-validation " make e2e ``` ## Test Parallelism @@ -76,9 +84,49 @@ and less error-prone to write tests when you don't have to worry about one test potentially conflicting with another - for example by installing the same provider another test would install. -In order to achieve some parallelism at the CI level all tests are labelled with -an area (e.g. `pkg`, `install`, `apiextensions`, etc). The [CI GitHub workflow] -uses a matrix strategy to invoke each area as its own job, running in parallel. +The [CI GitHub workflow] uses a matrix strategy to run multiple jobs in parallel, +each running a test suite, see the dedicated section for more details. + +We are currently splitting the tests to be able to run all basic tests against +the default installation of Crossplane, and for each alpha feature covered we +run all basic tests plus the feature specific ones against a Crossplane +installation with the feature enabled. + +In the future we could think of improving parallelism by, for example, adding +the area to the matrix or spinning multiple kind clusters for each job. + +## Test Suite + +In order to be able to run specific subsets of tests, we introduced the concept +of test suites. To run a specific test suite use the `-test-suite` flag. + +A test suite is currently defined by: +- A key to be used as value of the `-test-suite` flag. +- A set of labels that will be used to filter the tests to run, which will be + added to the user-provided ones, preserving the latter ones in case of key + conflicts. +- A list of Helm install options defining the initial Crossplane installation + for the given test suite, which will be used to install Crossplane before + running the tests. E.g. adding flags to enable alpha features and feature + specific flags. +- A list of additional setup steps to be run before installing Crossplane and + running the tests. E.g. Loading additional images into the cluster. +- Whether the suite should include the default suite or not, meaning that + install options will be added to the default ones and setup + +Test suites enable use cases such as installing Crossplane with a specific +alpha feature enabled and running all the basic tests, plus the ones specific to +that feature, to make sure the feature is not breaking any default behavior. + +In case a test needs a specific Crossplane configuration, it must still take +care of upgrading the installation to the desired configuration, but should then +use `E2EConfig.GetSelectedSuiteInstallOpts` to retrieve at runtime the baseline +installation options to be sure to restore the previous state. This allows tests +to run against any suite if needed. + +Test suites can be combined with labels to run a subset of tests, e.g. +splitting by area, or with the usual Go `-run ` flag to run only +specific tests, in such case, suite labels will be ignored altogether. ## Adding a Test @@ -157,6 +205,7 @@ func TestSomeFeature(t *testing.T) { features.New("ConfigurationWithDependency"). WithLabel(LabelArea, ...). WithLabel(LabelSize, ...). + WithLabel(config.LabelTestSuite, config.TestSuiteDefault). // ... WithSetup("ReadyPrerequisites", ... ). // ... other setup steps ... diff --git a/test/e2e/apiextensions_test.go b/test/e2e/apiextensions_test.go index d6da6a5d5de..4f4595f032f 100644 --- a/test/e2e/apiextensions_test.go +++ b/test/e2e/apiextensions_test.go @@ -21,12 +21,11 @@ import ( "time" "sigs.k8s.io/e2e-framework/pkg/features" - "sigs.k8s.io/e2e-framework/third_party/helm" xpv1 "github.com/crossplane/crossplane-runtime/apis/common/v1" apiextensionsv1 "github.com/crossplane/crossplane/apis/apiextensions/v1" - pkgv1 "github.com/crossplane/crossplane/apis/pkg/v1" + "github.com/crossplane/crossplane/test/e2e/config" "github.com/crossplane/crossplane/test/e2e/funcs" ) @@ -45,6 +44,7 @@ func TestCompositionMinimal(t *testing.T) { features.New("CompositionMinimal"). WithLabel(LabelArea, LabelAreaAPIExtensions). WithLabel(LabelSize, LabelSizeSmall). + WithLabel(config.LabelTestSuite, config.TestSuiteDefault). WithSetup("PrerequisitesAreCreated", funcs.AllOf( funcs.ApplyResources(FieldManager, manifests, "setup/*.yaml"), funcs.ResourcesCreatedWithin(30*time.Second, manifests, "setup/*.yaml"), @@ -78,6 +78,7 @@ func TestCompositionPatchAndTransform(t *testing.T) { features.New("CompositionPatchAndTransform"). WithLabel(LabelArea, LabelAreaAPIExtensions). WithLabel(LabelSize, LabelSizeSmall). + WithLabel(config.LabelTestSuite, config.TestSuiteDefault). WithSetup("CreatePrerequisites", funcs.AllOf( funcs.ApplyResources(FieldManager, manifests, "setup/*.yaml"), funcs.ResourcesCreatedWithin(30*time.Second, manifests, "setup/*.yaml"), @@ -104,55 +105,3 @@ func TestCompositionPatchAndTransform(t *testing.T) { ) } - -func TestCompositionValidation(t *testing.T) { - manifests := "test/e2e/manifests/apiextensions/composition/validation" - - cases := features.Table{ - { - // A valid Composition should be created when validated in strict mode. - Name: "ValidCompositionIsAccepted", - Assessment: funcs.AllOf( - funcs.ApplyResources(FieldManager, manifests, "composition-valid.yaml"), - funcs.ResourcesCreatedWithin(30*time.Second, manifests, "composition-valid.yaml"), - ), - }, - { - // An invalid Composition should be rejected when validated in strict mode. - Name: "InvalidCompositionIsRejected", - Assessment: funcs.ResourcesFailToApply(FieldManager, manifests, "composition-invalid.yaml"), - }, - } - environment.Test(t, - cases.Build("CompositionValidation"). - WithLabel(LabelStage, LabelStageAlpha). - WithLabel(LabelArea, LabelAreaAPIExtensions). - WithLabel(LabelSize, LabelSizeSmall). - WithLabel(LabelModifyCrossplaneInstallation, LabelModifyCrossplaneInstallationTrue). - // Enable our feature flag. - WithSetup("EnableAlphaCompositionValidation", funcs.AllOf( - funcs.AsFeaturesFunc(funcs.HelmUpgrade(HelmOptions(helm.WithArgs("--set args={--debug,--enable-composition-webhook-schema-validation}"))...)), - funcs.ReadyToTestWithin(1*time.Minute, namespace), - )). - WithSetup("CreatePrerequisites", funcs.AllOf( - funcs.ApplyResources(FieldManager, manifests, "setup/*.yaml"), - funcs.ResourcesCreatedWithin(30*time.Second, manifests, "setup/*.yaml"), - funcs.ResourcesHaveConditionWithin(1*time.Minute, manifests, "setup/definition.yaml", apiextensionsv1.WatchingComposite()), - funcs.ResourcesHaveConditionWithin(1*time.Minute, manifests, "setup/provider.yaml", pkgv1.Healthy(), pkgv1.Active()), - )). - WithTeardown("DeleteValidComposition", funcs.AllOf( - funcs.DeleteResources(manifests, "*-valid.yaml"), - funcs.ResourcesDeletedWithin(30*time.Second, manifests, "*-valid.yaml"), - )). - WithTeardown("DeletePrerequisites", funcs.AllOf( - funcs.DeleteResources(manifests, "setup/*.yaml"), - funcs.ResourcesDeletedWithin(3*time.Minute, manifests, "setup/*.yaml"), - )). - // Disable our feature flag. - WithTeardown("DisableAlphaCompositionValidation", funcs.AllOf( - funcs.AsFeaturesFunc(funcs.HelmUpgrade(HelmOptions()...)), - funcs.ReadyToTestWithin(1*time.Minute, namespace), - )). - Feature(), - ) -} diff --git a/test/e2e/compSchemaValidation_test.go b/test/e2e/compSchemaValidation_test.go new file mode 100644 index 00000000000..e63118c9351 --- /dev/null +++ b/test/e2e/compSchemaValidation_test.go @@ -0,0 +1,85 @@ +package e2e + +import ( + "testing" + "time" + + "sigs.k8s.io/e2e-framework/pkg/features" + "sigs.k8s.io/e2e-framework/third_party/helm" + + apiextensionsv1 "github.com/crossplane/crossplane/apis/apiextensions/v1" + pkgv1 "github.com/crossplane/crossplane/apis/pkg/v1" + "github.com/crossplane/crossplane/test/e2e/config" + "github.com/crossplane/crossplane/test/e2e/funcs" +) + +const ( + // SuiteCompositionWebhookSchemaValidation is the value for the + // config.LabelTestSuite label to be assigned to tests that should be part + // of the Composition Webhook Schema Validation test suite. + SuiteCompositionWebhookSchemaValidation = "composition-webhook-schema-validation" +) + +func init() { + E2EConfig.AddTestSuite(SuiteCompositionWebhookSchemaValidation, + config.WithHelmInstallOpts( + helm.WithArgs("--set args={--debug,--enable-composition-webhook-schema-validation}"), + ), + config.WithLabelsToSelect(features.Labels{ + config.LabelTestSuite: []string{SuiteCompositionWebhookSchemaValidation, config.TestSuiteDefault}, + }), + ) +} + +func TestCompositionValidation(t *testing.T) { + manifests := "test/e2e/manifests/apiextensions/composition/validation" + + cases := features.Table{ + { + // A valid Composition should be created when validated in strict mode. + Name: "ValidCompositionIsAccepted", + Assessment: funcs.AllOf( + funcs.ApplyResources(FieldManager, manifests, "composition-valid.yaml"), + funcs.ResourcesCreatedWithin(30*time.Second, manifests, "composition-valid.yaml"), + ), + }, + { + // An invalid Composition should be rejected when validated in strict mode. + Name: "InvalidCompositionIsRejected", + Assessment: funcs.ResourcesFailToApply(FieldManager, manifests, "composition-invalid.yaml"), + }, + } + environment.Test(t, + cases.Build("CompositionValidation"). + WithLabel(LabelStage, LabelStageAlpha). + WithLabel(LabelArea, LabelAreaAPIExtensions). + WithLabel(LabelSize, LabelSizeSmall). + WithLabel(LabelModifyCrossplaneInstallation, LabelModifyCrossplaneInstallationTrue). + WithLabel(config.LabelTestSuite, SuiteCompositionWebhookSchemaValidation). + // Enable our feature flag. + WithSetup("EnableAlphaCompositionValidation", funcs.AllOf( + funcs.AsFeaturesFunc(funcs.HelmUpgrade(E2EConfig.GetSuiteInstallOpts(SuiteCompositionWebhookSchemaValidation)...)), + funcs.ReadyToTestWithin(1*time.Minute, namespace), + )). + WithSetup("CreatePrerequisites", funcs.AllOf( + funcs.ApplyResources(FieldManager, manifests, "setup/*.yaml"), + funcs.ResourcesCreatedWithin(30*time.Second, manifests, "setup/*.yaml"), + funcs.ResourcesHaveConditionWithin(1*time.Minute, manifests, "setup/definition.yaml", apiextensionsv1.WatchingComposite()), + funcs.ResourcesHaveConditionWithin(1*time.Minute, manifests, "setup/provider.yaml", pkgv1.Healthy(), pkgv1.Active()), + )). + WithTeardown("DeleteValidComposition", funcs.AllOf( + funcs.DeleteResources(manifests, "*-valid.yaml"), + funcs.ResourcesDeletedWithin(30*time.Second, manifests, "*-valid.yaml"), + )). + WithTeardown("DeletePrerequisites", funcs.AllOf( + funcs.DeleteResources(manifests, "setup/*.yaml"), + funcs.ResourcesDeletedWithin(3*time.Minute, manifests, "setup/*.yaml"), + )). + // Disable our feature flag. + WithTeardown("DisableAlphaCompositionValidation", funcs.AllOf( + funcs.AsFeaturesFunc(funcs.HelmUpgrade(E2EConfig.GetSelectedSuiteInstallOpts()...)), + funcs.ReadyToTestWithin(1*time.Minute, namespace), + )). + Feature(), + ) +} diff --git a/test/e2e/config/config.go b/test/e2e/config/config.go new file mode 100644 index 00000000000..f374f3b8660 --- /dev/null +++ b/test/e2e/config/config.go @@ -0,0 +1,301 @@ +/* +Copyright 2023 The Crossplane Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package config contains the e2e test configuration. +package config + +import ( + "flag" + "fmt" + "os" + "sort" + + "k8s.io/utils/pointer" + "sigs.k8s.io/e2e-framework/pkg/env" + "sigs.k8s.io/e2e-framework/pkg/envconf" + "sigs.k8s.io/e2e-framework/pkg/features" + "sigs.k8s.io/e2e-framework/third_party/helm" +) + +// LabelTestSuite is used to define the suite each test should be part of. +const LabelTestSuite = "test-suite" + +// TestSuiteDefault is the default suite's key and value for LabelTestSuite. +const TestSuiteDefault = "base" + +const testSuiteFlag = "test-suite" + +// E2EConfig is these e2e test configuration. +type E2EConfig struct { + createKindCluster *bool + destroyKindCluster *bool + preinstallCrossplane *bool + loadImagesKindCluster *bool + kindClusterName *string + + selectedTestSuite *selectedTestSuite + + specificTestSelected *bool + suites map[string]testSuite +} + +type selectedTestSuite struct { + name string + set bool +} + +func (s *selectedTestSuite) String() string { + if !s.set { + fmt.Println("HERE: No test suite selected, using default") + return TestSuiteDefault + } + return s.name +} + +func (s *selectedTestSuite) Set(v string) error { + fmt.Printf("HERE: Setting test suite to %s\n", v) + s.name = v + s.set = true + return nil +} + +// testSuite is a test suite, allows to specify a set of options to be used +// for a suite, by default all options will include the base suite +// "SuiteDefault". +type testSuite struct { + excludeBaseSuite bool + helmInstallOpts []helm.Option + additionalSetupFuncs []conditionalSetupFunc + labelsToSelect features.Labels +} + +type conditionalSetupFunc struct { + condition func() bool + f []env.Func +} + +// NewFromFlags creates a new e2e test configuration, setting up the flags, but +// not parsing them yet, which is left to the caller to do. +func NewFromFlags() E2EConfig { + c := E2EConfig{ + suites: map[string]testSuite{}, + } + c.kindClusterName = flag.String("kind-cluster-name", "", "name of the kind cluster to use") + c.createKindCluster = flag.Bool("create-kind-cluster", true, "create a kind cluster (and deploy Crossplane) before running tests, if the cluster does not already exist with the same name") + c.destroyKindCluster = flag.Bool("destroy-kind-cluster", true, "destroy the kind cluster when tests complete") + c.preinstallCrossplane = flag.Bool("preinstall-crossplane", true, "install Crossplane before running tests") + c.loadImagesKindCluster = flag.Bool("load-images-kind-cluster", true, "load Crossplane images into the kind cluster before running tests") + c.selectedTestSuite = &selectedTestSuite{} + flag.Var(c.selectedTestSuite, testSuiteFlag, "test suite defining environment setup and tests to run") + // Need to override the default usage message to allow setting the available + // suites at runtime. + flag.Usage = func() { + if f := flag.Lookup(testSuiteFlag); f != nil { + f.Usage = fmt.Sprintf("%s. Available options: %+v", f.Usage, c.getAvailableSuitesOptions()) + } + _, _ = fmt.Fprintf(flag.CommandLine.Output(), "Usage of %s:\n", os.Args[0]) + flag.PrintDefaults() + } + return c +} + +func (e *E2EConfig) getAvailableSuitesOptions() (opts []string) { + for s := range e.suites { + opts = append(opts, s) + } + sort.Strings(opts) + return +} + +// GetKindClusterName returns the name of the kind cluster, returns empty string +// if it's not a kind cluster. +func (e *E2EConfig) GetKindClusterName() string { + if !e.IsKindCluster() { + return "" + } + if *e.kindClusterName == "" { + name := envconf.RandomName("crossplane-e2e", 32) + e.kindClusterName = &name + } + return *e.kindClusterName +} + +// IsKindCluster returns true if the test is running against a kind cluster. +func (e *E2EConfig) IsKindCluster() bool { + return *e.createKindCluster || *e.kindClusterName != "" +} + +// ShouldLoadImages returns true if the test should load images into the kind +// cluster. +func (e *E2EConfig) ShouldLoadImages() bool { + return *e.loadImagesKindCluster && e.IsKindCluster() +} + +// GetSuiteInstallOpts returns the helm install options for the specified +// suite, appending additional specified ones +func (e *E2EConfig) GetSuiteInstallOpts(suite string, extra ...helm.Option) []helm.Option { + p, ok := e.suites[suite] + if !ok { + panic(fmt.Sprintf("The selected suite %q does not exist", suite)) + } + opts := p.helmInstallOpts + if !p.excludeBaseSuite { + opts = append(e.suites[TestSuiteDefault].helmInstallOpts, opts...) + } + return append(opts, extra...) +} + +// GetSelectedSuiteInstallOpts returns the helm install options for the +// selected suite, appending additional specified ones. +func (e *E2EConfig) GetSelectedSuiteInstallOpts(extra ...helm.Option) []helm.Option { + return e.GetSuiteInstallOpts(e.selectedTestSuite.String(), extra...) +} + +// AddTestSuite adds a new test suite, panics if already defined. +func (e *E2EConfig) AddTestSuite(name string, opts ...TestSuiteOpt) { + if _, ok := e.suites[name]; ok { + panic(fmt.Sprintf("suite already defined: %s", name)) + } + + o := testSuite{} + for _, opt := range opts { + opt(&o) + } + e.suites[name] = o +} + +// AddDefaultTestSuite adds the default suite, panics if already defined. +func (e *E2EConfig) AddDefaultTestSuite(opts ...TestSuiteOpt) { + e.AddTestSuite(TestSuiteDefault, append([]TestSuiteOpt{WithoutBaseDefaultTestSuite()}, opts...)...) +} + +// TestSuiteOpt is an option to midify a testSuite. +type TestSuiteOpt func(*testSuite) + +// WithoutBaseDefaultTestSuite sets the provided testSuite to not include the base +// one. +func WithoutBaseDefaultTestSuite() TestSuiteOpt { + return func(suite *testSuite) { + suite.excludeBaseSuite = true + } +} + +// WithLabelsToSelect sets the provided testSuite to include the provided +// labels, if not already specified by the user +func WithLabelsToSelect(labels features.Labels) TestSuiteOpt { + return func(suite *testSuite) { + suite.labelsToSelect = labels + } +} + +// WithHelmInstallOpts sets the provided testSuite to include the provided +// helm install options. +func WithHelmInstallOpts(opts ...helm.Option) TestSuiteOpt { + return func(suite *testSuite) { + suite.helmInstallOpts = append(suite.helmInstallOpts, opts...) + } +} + +// WithConditionalEnvSetupFuncs sets the provided testSuite to include the +// provided env setup funcs, if the condition is true, when evaluated. +func WithConditionalEnvSetupFuncs(condition func() bool, funcs ...env.Func) TestSuiteOpt { + return func(suite *testSuite) { + suite.additionalSetupFuncs = append(suite.additionalSetupFuncs, conditionalSetupFunc{condition, funcs}) + } +} + +// HelmOptions valid for installing and upgrading the Crossplane Helm chart. +// Used to install Crossplane before any test starts, but some tests also use +// these options - for example to reinstall Crossplane with a feature flag +// enabled. +func (e *E2EConfig) HelmOptions(extra ...helm.Option) []helm.Option { + return append(e.GetSelectedSuiteInstallOpts(), extra...) +} + +// HelmOptionsForSuite returns the Helm options for the specified suite, +// appending additional specified ones. +func (e *E2EConfig) HelmOptionsForSuite(suite string, extra ...helm.Option) []helm.Option { + return append(e.GetSuiteInstallOpts(suite), extra...) +} + +// ShouldInstallCrossplane returns true if the test should install Crossplane +// before starting. +func (e *E2EConfig) ShouldInstallCrossplane() bool { + return *e.preinstallCrossplane +} + +// ShouldDestroyKindCluster returns true if the test should destroy the kind +// cluster after finishing. +func (e *E2EConfig) ShouldDestroyKindCluster() bool { + return *e.destroyKindCluster && e.IsKindCluster() +} + +// GetSelectedSuiteLabels returns the labels to select for the selected suite. +func (e *E2EConfig) getSelectedSuiteLabels() features.Labels { + if !e.selectedTestSuite.set { + return nil + } + return e.suites[e.selectedTestSuite.String()].labelsToSelect +} + +// GetSelectedSuiteAdditionalEnvSetup returns the additional env setup funcs +// for the selected suite, to be run before installing Crossplane, if required. +func (e *E2EConfig) GetSelectedSuiteAdditionalEnvSetup() (out []env.Func) { + selectedTestSuite := e.selectedTestSuite.String() + for _, s := range e.suites[selectedTestSuite].additionalSetupFuncs { + if s.condition() { + out = append(out, s.f...) + } + } + if selectedTestSuite == TestSuiteDefault { + for name, suite := range e.suites { + if name == TestSuiteDefault { + continue + } + for _, setupFunc := range suite.additionalSetupFuncs { + if setupFunc.condition() { + out = append(out, setupFunc.f...) + } + } + } + } + return out +} + +// EnrichLabels returns the provided labels enriched with the selected suite +// labels, preserving user-specified ones in case of key conflicts. +func (e *E2EConfig) EnrichLabels(labels features.Labels) features.Labels { + if e.isSelectingTests() { + return labels + } + if labels == nil { + labels = make(features.Labels) + } + for k, v := range e.getSelectedSuiteLabels() { + if _, ok := labels[k]; ok { + continue + } + labels[k] = v + } + return labels +} + +func (e *E2EConfig) isSelectingTests() bool { + if e.specificTestSelected == nil { + f := flag.Lookup("test.run") + e.specificTestSelected = pointer.Bool(f != nil && f.Value.String() != "") + } + return *e.specificTestSelected +} diff --git a/test/e2e/consts.go b/test/e2e/consts.go new file mode 100644 index 00000000000..c29388ded71 --- /dev/null +++ b/test/e2e/consts.go @@ -0,0 +1,42 @@ +package e2e + +// LabelArea represents the 'area' of a feature. For example 'apiextensions', +// 'pkg', etc. Assessments roll up to features, which roll up to feature areas. +// Features within an area may be split across different test functions. +const LabelArea = "area" + +// LabelModifyCrossplaneInstallation is used to mark tests that are going to +// modify Crossplane's installation, e.g. installing, uninstalling or upgrading +// it. +const LabelModifyCrossplaneInstallation = "modify-crossplane-installation" + +// LabelModifyCrossplaneInstallationTrue is used to mark tests that are going to +// modify Crossplane's installation. +const LabelModifyCrossplaneInstallationTrue = "true" + +// LabelStage represents the 'stage' of a feature - alpha, beta, etc. Generally +// available features have no stage label. +const LabelStage = "stage" + +const ( + // LabelStageAlpha is used for tests of alpha features. + LabelStageAlpha = "alpha" + + // LabelStageBeta is used for tests of beta features. + LabelStageBeta = "beta" +) + +// LabelSize represents the 'size' (i.e. duration) of a test. +const LabelSize = "size" + +const ( + // LabelSizeSmall is used for tests that usually complete in a minute. + LabelSizeSmall = "small" + + // LabelSizeLarge is used for test that usually complete in over a minute. + LabelSizeLarge = "large" +) + +// FieldManager is the server-side apply field manager used when applying +// manifests. +const FieldManager = "crossplane-e2e-tests" diff --git a/test/e2e/funcs/env.go b/test/e2e/funcs/env.go index 46bc7f987f1..6360460c432 100644 --- a/test/e2e/funcs/env.go +++ b/test/e2e/funcs/env.go @@ -117,7 +117,7 @@ func EnvFuncs(fns ...env.Func) env.Func { // The configuration is placed in test context afterward func CreateKindClusterWithConfig(clusterName, configFilePath string) env.Func { return EnvFuncs( - envfuncs.CreateKindClusterWithConfig(clusterName, "\"\"", configFilePath), + envfuncs.CreateKindClusterWithConfig(clusterName, "", configFilePath), func(ctx context.Context, config *envconf.Config) (context.Context, error) { b, err := os.ReadFile(filepath.Clean(configFilePath)) if err != nil { diff --git a/test/e2e/funcs/feature.go b/test/e2e/funcs/feature.go index 4e78f2d4ef4..27d2087a412 100644 --- a/test/e2e/funcs/feature.go +++ b/test/e2e/funcs/feature.go @@ -412,7 +412,7 @@ func CopyImageToRegistry(clusterName, ns, sName, image string, timeout time.Dura } i := strings.Split(srcRef.String(), "/") - err = wait.For(func() (done bool, err error) { + err = wait.For(func(_ context.Context) (done bool, err error) { err = crane.Push(src, fmt.Sprintf("%s/%s", reg, i[1]), crane.Insecure) if err != nil { return false, nil //nolint:nilerr // we want to retry and to throw error diff --git a/test/e2e/install_test.go b/test/e2e/install_test.go index 648d40b5db8..2c209e520c6 100644 --- a/test/e2e/install_test.go +++ b/test/e2e/install_test.go @@ -29,6 +29,7 @@ import ( xpv1 "github.com/crossplane/crossplane-runtime/apis/common/v1" apiextensionsv1 "github.com/crossplane/crossplane/apis/apiextensions/v1" + "github.com/crossplane/crossplane/test/e2e/config" "github.com/crossplane/crossplane/test/e2e/funcs" ) @@ -54,6 +55,7 @@ func TestCrossplaneLifecycle(t *testing.T) { WithLabel(LabelArea, LabelAreaLifecycle). WithLabel(LabelSize, LabelSizeSmall). WithLabel(LabelModifyCrossplaneInstallation, LabelModifyCrossplaneInstallationTrue). + WithLabel(config.LabelTestSuite, config.TestSuiteDefault). WithSetup("CreatePrerequisites", funcs.AllOf( funcs.ApplyResources(FieldManager, manifests, "setup/*.yaml"), funcs.ResourcesCreatedWithin(30*time.Second, manifests, "setup/*.yaml"), @@ -96,6 +98,7 @@ func TestCrossplaneLifecycle(t *testing.T) { features.New("CrossplaneUpgrade"). WithLabel(LabelArea, LabelAreaLifecycle). WithLabel(LabelSize, LabelSizeSmall). + WithLabel(config.LabelTestSuite, config.TestSuiteDefault). // We expect Crossplane to have been uninstalled first Assess("CrossplaneIsNotInstalled", funcs.AllOf( funcs.ResourceDeletedWithin(1*time.Minute, &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: namespace}}), @@ -125,7 +128,7 @@ func TestCrossplaneLifecycle(t *testing.T) { )). Assess("ClaimIsAvailable", funcs.ResourcesHaveConditionWithin(2*time.Minute, manifests, "claim.yaml", xpv1.Available())). Assess("UpgradeCrossplane", funcs.AllOf( - funcs.AsFeaturesFunc(funcs.HelmUpgrade(HelmOptions()...)), + funcs.AsFeaturesFunc(funcs.HelmUpgrade(E2EConfig.GetSelectedSuiteInstallOpts()...)), funcs.ReadyToTestWithin(1*time.Minute, namespace), )). Assess("CoreDeploymentIsAvailable", funcs.DeploymentBecomesAvailableWithin(1*time.Minute, namespace, "crossplane")). diff --git a/test/e2e/main_test.go b/test/e2e/main_test.go index a9464975ffd..fec4ceab8a0 100644 --- a/test/e2e/main_test.go +++ b/test/e2e/main_test.go @@ -17,7 +17,8 @@ limitations under the License. package e2e import ( - "flag" + "context" + "fmt" "os" "path/filepath" "strings" @@ -29,93 +30,34 @@ import ( "sigs.k8s.io/e2e-framework/pkg/env" "sigs.k8s.io/e2e-framework/pkg/envconf" "sigs.k8s.io/e2e-framework/pkg/envfuncs" + "sigs.k8s.io/e2e-framework/pkg/features" "sigs.k8s.io/e2e-framework/third_party/helm" + "github.com/crossplane/crossplane/test/e2e/config" "github.com/crossplane/crossplane/test/e2e/funcs" ) -// LabelArea represents the 'area' of a feature. For example 'apiextensions', -// 'pkg', etc. Assessments roll up to features, which roll up to feature areas. -// Features within an area may be split across different test functions. -const LabelArea = "area" - -// LabelModifyCrossplaneInstallation is used to mark tests that are going to -// modify Crossplane's installation, e.g. installing, uninstalling or upgrading -// it. -const LabelModifyCrossplaneInstallation = "modify-crossplane-installation" - -// LabelModifyCrossplaneInstallationTrue is used to mark tests that are going to -// modify Crossplane's installation. -const LabelModifyCrossplaneInstallationTrue = "true" - -// LabelStage represents the 'stage' of a feature - alpha, beta, etc. Generally -// available features have no stage label. -const LabelStage = "stage" - -const ( - // LabelStageAlpha is used for tests of alpha features. - LabelStageAlpha = "alpha" - - // LabelStageBeta is used for tests of beta features. - LabelStageBeta = "beta" -) - -// LabelSize represents the 'size' (i.e. duration) of a test. -const LabelSize = "size" - -const ( - // LabelSizeSmall is used for tests that usually complete in a minute. - LabelSizeSmall = "small" - - // LabelSizeLarge is used for test that usually complete in over a minute. - LabelSizeLarge = "large" -) - +// TODO(phisco): make it configurable const namespace = "crossplane-system" +// TODO(phisco): make it configurable const crdsDir = "cluster/crds" -// The caller (e.g. make e2e) must ensure these exists. +// The caller (e.g. make e2e) must ensure these exist. // Run `make build e2e-tag-images` to produce them const ( + // TODO(phisco): make it configurable imgcore = "crossplane-e2e/crossplane:latest" - imgxfn = "crossplane-e2e/xfn:latest" ) const ( - helmChartDir = "cluster/charts/crossplane" + // TODO(phisco): make it configurable + helmChartDir = "cluster/charts/crossplane" + // TODO(phisco): make it configurable helmReleaseName = "crossplane" ) -// FieldManager is the server-side apply field manager used when applying -// manifests. -const FieldManager = "crossplane-e2e-tests" - -// HelmOptions valid for installing and upgrading the Crossplane Helm chart. -// Used to install Crossplane before any test starts, but some tests also use -// these options - for example to reinstall Crossplane with a feature flag -// enabled. -func HelmOptions(extra ...helm.Option) []helm.Option { - o := []helm.Option{ - helm.WithName(helmReleaseName), - helm.WithNamespace(namespace), - helm.WithChart(helmChartDir), - // wait for the deployment to be ready for up to 5 minutes before returning - helm.WithWait(), - helm.WithTimeout("5m"), - helm.WithArgs( - // Run with debug logging to ensure all log statements are run. - "--set args={--debug}", - "--set image.repository="+strings.Split(imgcore, ":")[0], - "--set image.tag="+strings.Split(imgcore, ":")[1], - - "--set xfn.args={--debug}", - "--set xfn.image.repository="+strings.Split(imgxfn, ":")[0], - "--set xfn.image.tag="+strings.Split(imgxfn, ":")[1], - ), - } - return append(o, extra...) -} +var E2EConfig = config.NewFromFlags() var ( // The test environment, shared by all E2E test functions. @@ -123,35 +65,51 @@ var ( clusterName string ) +func init() { + // Set the default suite, to be used as base for all the other suites. + E2EConfig.AddDefaultTestSuite( + config.WithoutBaseDefaultTestSuite(), + config.WithHelmInstallOpts( + helm.WithName(helmReleaseName), + helm.WithNamespace(namespace), + helm.WithChart(helmChartDir), + // wait for the deployment to be ready for up to 5 minutes before returning + helm.WithWait(), + helm.WithTimeout("5m"), + helm.WithArgs( + // Run with debug logging to ensure all log statements are run. + "--set args={--debug}", + "--set image.repository="+strings.Split(imgcore, ":")[0], + "--set image.tag="+strings.Split(imgcore, ":")[1], + ), + ), + config.WithLabelsToSelect(features.Labels{ + config.LabelTestSuite: []string{config.TestSuiteDefault}, + }), + ) +} + func TestMain(m *testing.M) { // TODO(negz): Global loggers are dumb and klog is dumb. Remove this when // e2e-framework is running controller-runtime v0.15.x per // https://github.com/kubernetes-sigs/e2e-framework/issues/270 log.SetLogger(klog.NewKlogr()) - kindClusterName := flag.String("kind-cluster-name", "", "name of the kind cluster to use") - create := flag.Bool("create-kind-cluster", true, "create a kind cluster (and deploy Crossplane) before running tests, if the cluster does not already exist with the same name") - destroy := flag.Bool("destroy-kind-cluster", true, "destroy the kind cluster when tests complete") - install := flag.Bool("install-crossplane", true, "install Crossplane before running tests") - load := flag.Bool("load-images-kind-cluster", true, "load Crossplane images into the kind cluster before running tests") + cfg, err := envconf.NewFromFlags() + if err != nil { + panic(err) + } var setup []env.Func var finish []env.Func - cfg, _ := envconf.NewFromFlags() - - clusterName = envconf.RandomName("crossplane-e2e", 32) - if *kindClusterName != "" { - clusterName = *kindClusterName - } - + // Parse flags, populating E2EConfig too. // we want to create the cluster if it doesn't exist, but only if we're - isKindCluster := *create || *kindClusterName != "" - if isKindCluster { + if E2EConfig.IsKindCluster() { + clusterName := E2EConfig.GetKindClusterName() kindCfg, err := filepath.Abs(filepath.Join("test", "e2e", "testdata", "kindConfig.yaml")) if err != nil { - log.Log.Error(err, "error getting kind config file") - os.Exit(1) + panic(fmt.Sprintf("error getting kind config file: %s", err.Error())) } setup = []env.Func{ funcs.CreateKindClusterWithConfig(clusterName, kindCfg), @@ -159,18 +117,29 @@ func TestMain(m *testing.M) { } else { cfg.WithKubeconfigFile(conf.ResolveKubeConfigFile()) } + + // Enrich the selected labels with the ones from the suite. + // Not replacing the user provided ones if any. + cfg.WithLabels(E2EConfig.EnrichLabels(cfg.Labels())) + environment = env.NewWithConfig(cfg) - if *load && isKindCluster { + if E2EConfig.ShouldLoadImages() { + clusterName := E2EConfig.GetKindClusterName() setup = append(setup, envfuncs.LoadDockerImageToCluster(clusterName, imgcore), - envfuncs.LoadDockerImageToCluster(clusterName, imgxfn), ) } - if *install { + + // Add the setup functions defined by the suite being used + setup = append(setup, + E2EConfig.GetSelectedSuiteAdditionalEnvSetup()..., + ) + + if E2EConfig.ShouldInstallCrossplane() { setup = append(setup, envfuncs.CreateNamespace(namespace), - funcs.HelmInstall(HelmOptions()...), + funcs.HelmInstall(E2EConfig.GetSelectedSuiteInstallOpts()...), ) } @@ -179,10 +148,18 @@ func TestMain(m *testing.M) { // We want to destroy the cluster if we created it, but only if we created it, // otherwise the random name will be meaningless. - if *destroy && isKindCluster { - finish = []env.Func{envfuncs.DestroyKindCluster(clusterName)} + if E2EConfig.ShouldDestroyKindCluster() { + finish = []env.Func{envfuncs.DestroyKindCluster(E2EConfig.GetKindClusterName())} } + // Check that all features are specifying a suite they belong to via LabelTestSuite. + environment.BeforeEachFeature(func(ctx context.Context, _ *envconf.Config, t *testing.T, feature features.Feature) (context.Context, error) { + if _, exists := feature.Labels()[config.LabelTestSuite]; !exists { + t.Fatalf("Feature %s does not have a %s label, setting it to %s", feature.Name(), config.LabelTestSuite, config.TestSuiteDefault) + } + return ctx, nil + }) + environment.Setup(setup...) environment.Finish(finish...) os.Exit(environment.Run(m)) diff --git a/test/e2e/pkg_test.go b/test/e2e/pkg_test.go index 4c26269bb8e..cc40b35cb21 100644 --- a/test/e2e/pkg_test.go +++ b/test/e2e/pkg_test.go @@ -25,6 +25,7 @@ import ( xpv1 "github.com/crossplane/crossplane-runtime/apis/common/v1" pkgv1 "github.com/crossplane/crossplane/apis/pkg/v1" + "github.com/crossplane/crossplane/test/e2e/config" "github.com/crossplane/crossplane/test/e2e/funcs" ) @@ -41,6 +42,7 @@ func TestConfigurationPullFromPrivateRegistry(t *testing.T) { features.New("ConfigurationPullFromPrivateRegistry"). WithLabel(LabelArea, LabelAreaPkg). WithLabel(LabelSize, LabelSizeSmall). + WithLabel(config.LabelTestSuite, config.TestSuiteDefault). WithSetup("CreateConfiguration", funcs.AllOf( funcs.ApplyResources(FieldManager, manifests, "*.yaml"), funcs.ResourcesCreatedWithin(1*time.Minute, manifests, "*.yaml"), @@ -62,6 +64,7 @@ func TestConfigurationWithDependency(t *testing.T) { features.New("ConfigurationWithDependency"). WithLabel(LabelArea, LabelAreaPkg). WithLabel(LabelSize, LabelSizeSmall). + WithLabel(config.LabelTestSuite, config.TestSuiteDefault). WithSetup("ApplyConfiguration", funcs.AllOf( funcs.ApplyResources(FieldManager, manifests, "configuration.yaml"), funcs.ResourcesCreatedWithin(1*time.Minute, manifests, "configuration.yaml"), @@ -91,6 +94,7 @@ func TestProviderUpgrade(t *testing.T) { features.New("ProviderUpgrade"). WithLabel(LabelArea, LabelAreaPkg). WithLabel(LabelSize, LabelSizeSmall). + WithLabel(config.LabelTestSuite, config.TestSuiteDefault). WithSetup("ApplyInitialProvider", funcs.AllOf( funcs.ApplyResources(FieldManager, manifests, "provider-initial.yaml"), funcs.ResourcesCreatedWithin(1*time.Minute, manifests, "provider-initial.yaml"), diff --git a/test/e2e/xfn_test.go b/test/e2e/xfn_test.go index be22147f68d..d757fe4f910 100644 --- a/test/e2e/xfn_test.go +++ b/test/e2e/xfn_test.go @@ -16,6 +16,7 @@ package e2e import ( "context" + "strings" "testing" "time" @@ -30,23 +31,64 @@ import ( xpv1 "github.com/crossplane/crossplane-runtime/apis/common/v1" v1 "github.com/crossplane/crossplane/apis/apiextensions/v1" + "github.com/crossplane/crossplane/test/e2e/config" "github.com/crossplane/crossplane/test/e2e/funcs" "github.com/crossplane/crossplane/test/e2e/utils" ) const ( + + // LabelAreaXFN is the label used to select tests that are part of the XFN + // area. + LabelAreaXFN = "xfn" + + // SuiteCompositionFunctions is the value for the + // config.LabelTestSuite label to be assigned to tests that should be part + // of the Composition functions test suite. + SuiteCompositionFunctions = "composition-functions" + + // The caller (e.g. make e2e) must ensure these exist. + // Run `make build e2e-tag-images` to produce them + // TODO(phisco): make it configurable + imgxfn = "crossplane-e2e/xfn:latest" + registryNs = "xfn-registry" timeoutFive = 5 * time.Minute timeoutOne = 1 * time.Minute ) -func TestXfnRunnerImagePull(t *testing.T) { +func init() { + E2EConfig.AddTestSuite(SuiteCompositionFunctions, + config.WithHelmInstallOpts( + helm.WithArgs( + "--set args={--debug,--enable-composition-functions}", + "--set xfn.args={--debug}", + "--set xfn.enabled=true", + "--set xfn.image.repository="+strings.Split(imgxfn, ":")[0], + "--set xfn.image.tag="+strings.Split(imgxfn, ":")[1], + "--set xfn.resources.limits.cpu=100m", + "--set xfn.resources.requests.cpu=100m", + ), + ), + config.WithLabelsToSelect(features.Labels{ + config.LabelTestSuite: []string{SuiteCompositionFunctions, config.TestSuiteDefault}, + }), + config.WithConditionalEnvSetupFuncs( + E2EConfig.ShouldLoadImages, envfuncs.LoadDockerImageToCluster(E2EConfig.GetKindClusterName(), imgxfn), + ), + ) +} +func TestXfnRunnerImagePull(t *testing.T) { manifests := "test/e2e/manifests/xfnrunner/private-registry/pull" environment.Test(t, features.New("PullFnImageFromPrivateRegistryWithCustomCert"). - WithLabel(LabelArea, "xfn"). + WithLabel(LabelArea, LabelAreaXFN). + WithLabel(LabelStage, LabelStageAlpha). + WithLabel(LabelSize, LabelSizeLarge). + WithLabel(LabelModifyCrossplaneInstallation, LabelModifyCrossplaneInstallationTrue). + WithLabel(config.LabelTestSuite, SuiteCompositionFunctions). WithSetup("InstallRegistryWithCustomTlsCertificate", funcs.AllOf( funcs.AsFeaturesFunc(envfuncs.CreateNamespace(registryNs)), @@ -111,17 +153,11 @@ func TestXfnRunnerImagePull(t *testing.T) { funcs.CopyImageToRegistry(clusterName, registryNs, "private-docker-registry", "crossplane-e2e/fn-labelizer:latest", timeoutOne)). WithSetup("CrossplaneDeployedWithFunctionsEnabled", funcs.AllOf( funcs.AsFeaturesFunc(funcs.HelmUpgrade( - HelmOptions( + E2EConfig.GetSuiteInstallOpts(SuiteCompositionFunctions, helm.WithArgs( - "--set args={--debug,--enable-composition-functions}", - "--set xfn.enabled=true", - "--set xfn.args={--debug}", - "--set registryCaBundleConfig.name=reg-ca", "--set registryCaBundleConfig.key=domain.crt", - "--set xfn.resources.requests.cpu=100m", - "--set xfn.resources.limits.cpu=100m", - ), - helm.WithWait())...)), + "--set registryCaBundleConfig.name=reg-ca", + ))...)), funcs.ReadyToTestWithin(1*time.Minute, namespace), )). WithSetup("ProviderNopDeployed", funcs.AllOf( @@ -173,7 +209,7 @@ func TestXfnRunnerImagePull(t *testing.T) { }, )). WithTeardown("CrossplaneDeployedWithoutFunctionsEnabled", funcs.AllOf( - funcs.AsFeaturesFunc(funcs.HelmUpgrade(HelmOptions()...)), + funcs.AsFeaturesFunc(funcs.HelmUpgrade(E2EConfig.GetSelectedSuiteInstallOpts()...)), funcs.ReadyToTestWithin(1*time.Minute, namespace), )). Feature(), @@ -184,7 +220,11 @@ func TestXfnRunnerWriteToTmp(t *testing.T) { manifests := "test/e2e/manifests/xfnrunner/tmp-writer" environment.Test(t, features.New("CreateAFileInTmpFolder"). - WithLabel(LabelArea, "xfn"). + WithLabel(LabelArea, LabelAreaXFN). + WithLabel(LabelStage, LabelStageAlpha). + WithLabel(LabelSize, LabelSizeLarge). + WithLabel(LabelModifyCrossplaneInstallation, LabelModifyCrossplaneInstallationTrue). + WithLabel(config.LabelTestSuite, SuiteCompositionFunctions). WithSetup("InstallRegistry", funcs.AllOf( funcs.AsFeaturesFunc(envfuncs.CreateNamespace(registryNs)), @@ -210,16 +250,7 @@ func TestXfnRunnerWriteToTmp(t *testing.T) { WithSetup("CopyFnImageToRegistry", funcs.CopyImageToRegistry(clusterName, registryNs, "public-docker-registry", "crossplane-e2e/fn-tmp-writer:latest", timeoutOne)). WithSetup("CrossplaneDeployedWithFunctionsEnabled", funcs.AllOf( - funcs.AsFeaturesFunc(funcs.HelmUpgrade( - HelmOptions( - helm.WithArgs( - "--set args={--debug,--enable-composition-functions}", - "--set xfn.enabled=true", - "--set xfn.args={--debug}", - "--set xfn.resources.requests.cpu=100m", - "--set xfn.resources.limits.cpu=100m", - ), - helm.WithWait())...)), + funcs.AsFeaturesFunc(funcs.HelmUpgrade(E2EConfig.GetSuiteInstallOpts(SuiteCompositionFunctions)...)), funcs.ReadyToTestWithin(1*time.Minute, namespace), )). WithSetup("ProviderNopDeployed", funcs.AllOf( @@ -258,7 +289,7 @@ func TestXfnRunnerWriteToTmp(t *testing.T) { )). WithTeardown("RemoveRegistry", funcs.AsFeaturesFunc(envfuncs.DeleteNamespace(registryNs))). WithTeardown("CrossplaneDeployedWithoutFunctionsEnabled", funcs.AllOf( - funcs.AsFeaturesFunc(funcs.HelmUpgrade(HelmOptions()...)), + funcs.AsFeaturesFunc(funcs.HelmUpgrade(E2EConfig.GetSelectedSuiteInstallOpts()...)), funcs.ReadyToTestWithin(1*time.Minute, namespace), )). Feature(), From 241dd6377785d9e8076f8b4afcb426219f985a0c Mon Sep 17 00:00:00 2001 From: Philippe Scorsolini Date: Fri, 21 Jul 2023 12:12:43 +0200 Subject: [PATCH 2/7] tests(e2e): rename schema aware validation test cases Signed-off-by: Philippe Scorsolini --- test/e2e/compSchemaValidation_test.go | 5 +++-- .../validation/configuration-platform-ref-aws-valid.yaml | 7 +++++++ 2 files changed, 10 insertions(+), 2 deletions(-) create mode 100644 test/e2e/manifests/apiextensions/composition/validation/configuration-platform-ref-aws-valid.yaml diff --git a/test/e2e/compSchemaValidation_test.go b/test/e2e/compSchemaValidation_test.go index e63118c9351..8c5541426b8 100644 --- a/test/e2e/compSchemaValidation_test.go +++ b/test/e2e/compSchemaValidation_test.go @@ -37,7 +37,8 @@ func TestCompositionValidation(t *testing.T) { cases := features.Table{ { // A valid Composition should be created when validated in strict mode. - Name: "ValidCompositionIsAccepted", + Name: "ValidCompositionIsAcceptedStrictMode", + Description: "A valid Composition should be created when validated in strict mode.", Assessment: funcs.AllOf( funcs.ApplyResources(FieldManager, manifests, "composition-valid.yaml"), funcs.ResourcesCreatedWithin(30*time.Second, manifests, "composition-valid.yaml"), @@ -45,7 +46,7 @@ func TestCompositionValidation(t *testing.T) { }, { // An invalid Composition should be rejected when validated in strict mode. - Name: "InvalidCompositionIsRejected", + Name: "InvalidCompositionIsRejectedStrictMode", Assessment: funcs.ResourcesFailToApply(FieldManager, manifests, "composition-invalid.yaml"), }, } diff --git a/test/e2e/manifests/apiextensions/composition/validation/configuration-platform-ref-aws-valid.yaml b/test/e2e/manifests/apiextensions/composition/validation/configuration-platform-ref-aws-valid.yaml new file mode 100644 index 00000000000..a5d73403ff0 --- /dev/null +++ b/test/e2e/manifests/apiextensions/composition/validation/configuration-platform-ref-aws-valid.yaml @@ -0,0 +1,7 @@ +--- +apiVersion: pkg.crossplane.io/v1 +kind: Configuration +metadata: + name: platform-ref-aws +spec: + package: xpkg.upbound.io/upbound/platform-ref-aws:v0.6.0 \ No newline at end of file From 9015f02e91466635a5477666a09178644c458798 Mon Sep 17 00:00:00 2001 From: Philippe Scorsolini Date: Mon, 24 Jul 2023 11:34:55 +0200 Subject: [PATCH 3/7] chore: cleanup Signed-off-by: Philippe Scorsolini --- test/e2e/config/config.go | 3 +-- .../validation/configuration-platform-ref-aws-valid.yaml | 7 ------- 2 files changed, 1 insertion(+), 9 deletions(-) delete mode 100644 test/e2e/manifests/apiextensions/composition/validation/configuration-platform-ref-aws-valid.yaml diff --git a/test/e2e/config/config.go b/test/e2e/config/config.go index f374f3b8660..71bd10a8ec6 100644 --- a/test/e2e/config/config.go +++ b/test/e2e/config/config.go @@ -58,14 +58,13 @@ type selectedTestSuite struct { func (s *selectedTestSuite) String() string { if !s.set { - fmt.Println("HERE: No test suite selected, using default") return TestSuiteDefault } return s.name } func (s *selectedTestSuite) Set(v string) error { - fmt.Printf("HERE: Setting test suite to %s\n", v) + fmt.Printf("Setting test suite to %s\n", v) s.name = v s.set = true return nil diff --git a/test/e2e/manifests/apiextensions/composition/validation/configuration-platform-ref-aws-valid.yaml b/test/e2e/manifests/apiextensions/composition/validation/configuration-platform-ref-aws-valid.yaml deleted file mode 100644 index a5d73403ff0..00000000000 --- a/test/e2e/manifests/apiextensions/composition/validation/configuration-platform-ref-aws-valid.yaml +++ /dev/null @@ -1,7 +0,0 @@ ---- -apiVersion: pkg.crossplane.io/v1 -kind: Configuration -metadata: - name: platform-ref-aws -spec: - package: xpkg.upbound.io/upbound/platform-ref-aws:v0.6.0 \ No newline at end of file From 642f2a17e613fe97e423e112387deb5569cedb80 Mon Sep 17 00:00:00 2001 From: Philippe Scorsolini Date: Wed, 2 Aug 2023 18:22:51 +0200 Subject: [PATCH 4/7] e2e: fix error message Signed-off-by: Philippe Scorsolini --- test/e2e/main_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e/main_test.go b/test/e2e/main_test.go index fec4ceab8a0..f00bbc71fc0 100644 --- a/test/e2e/main_test.go +++ b/test/e2e/main_test.go @@ -155,7 +155,7 @@ func TestMain(m *testing.M) { // Check that all features are specifying a suite they belong to via LabelTestSuite. environment.BeforeEachFeature(func(ctx context.Context, _ *envconf.Config, t *testing.T, feature features.Feature) (context.Context, error) { if _, exists := feature.Labels()[config.LabelTestSuite]; !exists { - t.Fatalf("Feature %s does not have a %s label, setting it to %s", feature.Name(), config.LabelTestSuite, config.TestSuiteDefault) + t.Fatalf("Feature %q does not have the required %q label set", feature.Name(), config.LabelTestSuite) } return ctx, nil }) From b352ac01eb60a4c3323d325e07b686996aec580e Mon Sep 17 00:00:00 2001 From: Philippe Scorsolini Date: Tue, 8 Aug 2023 19:36:33 +0200 Subject: [PATCH 5/7] chore: address review comments Signed-off-by: Philippe Scorsolini --- test/e2e/README.md | 9 ++- test/e2e/apiextensions_test.go | 4 +- ...test.go => comp_schema_validation_test.go} | 8 +- test/e2e/config/config.go | 78 +++++++++++++------ test/e2e/install_test.go | 4 +- test/e2e/main_test.go | 53 ++++++------- test/e2e/pkg_test.go | 6 +- test/e2e/xfn_test.go | 25 +++--- 8 files changed, 105 insertions(+), 82 deletions(-) rename test/e2e/{compSchemaValidation_test.go => comp_schema_validation_test.go} (91%) diff --git a/test/e2e/README.md b/test/e2e/README.md index 25d32ad807c..57383beeb88 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -112,7 +112,8 @@ A test suite is currently defined by: - A list of additional setup steps to be run before installing Crossplane and running the tests. E.g. Loading additional images into the cluster. - Whether the suite should include the default suite or not, meaning that - install options will be added to the default ones and setup + install options will be added to the default ones if not explicitly specified + not to do so. Test suites enable use cases such as installing Crossplane with a specific alpha feature enabled and running all the basic tests, plus the ones specific to @@ -205,10 +206,10 @@ func TestSomeFeature(t *testing.T) { features.New("ConfigurationWithDependency"). WithLabel(LabelArea, ...). WithLabel(LabelSize, ...). - WithLabel(config.LabelTestSuite, config.TestSuiteDefault). - // ... + WithLabel(config.LabelTestSuite, config.TestSuiteDefault). + // ... WithSetup("ReadyPrerequisites", ... ). - // ... other setup steps ... + // ... other setup steps ... Assess("DoSomething", ... ). Assess("SomethingElseIsInSomeState", ... ). // ... other assess steps ... diff --git a/test/e2e/apiextensions_test.go b/test/e2e/apiextensions_test.go index 4f4595f032f..b91565a4c47 100644 --- a/test/e2e/apiextensions_test.go +++ b/test/e2e/apiextensions_test.go @@ -40,7 +40,7 @@ const LabelAreaAPIExtensions = "apiextensions" func TestCompositionMinimal(t *testing.T) { manifests := "test/e2e/manifests/apiextensions/composition/minimal" - environment.Test(t, + e2eConfig.Test(t, features.New("CompositionMinimal"). WithLabel(LabelArea, LabelAreaAPIExtensions). WithLabel(LabelSize, LabelSizeSmall). @@ -74,7 +74,7 @@ func TestCompositionMinimal(t *testing.T) { func TestCompositionPatchAndTransform(t *testing.T) { manifests := "test/e2e/manifests/apiextensions/composition/patch-and-transform" - environment.Test(t, + e2eConfig.Test(t, features.New("CompositionPatchAndTransform"). WithLabel(LabelArea, LabelAreaAPIExtensions). WithLabel(LabelSize, LabelSizeSmall). diff --git a/test/e2e/compSchemaValidation_test.go b/test/e2e/comp_schema_validation_test.go similarity index 91% rename from test/e2e/compSchemaValidation_test.go rename to test/e2e/comp_schema_validation_test.go index 8c5541426b8..f2112c16915 100644 --- a/test/e2e/compSchemaValidation_test.go +++ b/test/e2e/comp_schema_validation_test.go @@ -21,7 +21,7 @@ const ( ) func init() { - E2EConfig.AddTestSuite(SuiteCompositionWebhookSchemaValidation, + e2eConfig.AddTestSuite(SuiteCompositionWebhookSchemaValidation, config.WithHelmInstallOpts( helm.WithArgs("--set args={--debug,--enable-composition-webhook-schema-validation}"), ), @@ -50,7 +50,7 @@ func TestCompositionValidation(t *testing.T) { Assessment: funcs.ResourcesFailToApply(FieldManager, manifests, "composition-invalid.yaml"), }, } - environment.Test(t, + e2eConfig.Test(t, cases.Build("CompositionValidation"). WithLabel(LabelStage, LabelStageAlpha). WithLabel(LabelArea, LabelAreaAPIExtensions). @@ -59,7 +59,7 @@ func TestCompositionValidation(t *testing.T) { WithLabel(config.LabelTestSuite, SuiteCompositionWebhookSchemaValidation). // Enable our feature flag. WithSetup("EnableAlphaCompositionValidation", funcs.AllOf( - funcs.AsFeaturesFunc(funcs.HelmUpgrade(E2EConfig.GetSuiteInstallOpts(SuiteCompositionWebhookSchemaValidation)...)), + funcs.AsFeaturesFunc(e2eConfig.HelmUpgradeCrossplaneToSuite(SuiteCompositionWebhookSchemaValidation)), funcs.ReadyToTestWithin(1*time.Minute, namespace), )). WithSetup("CreatePrerequisites", funcs.AllOf( @@ -78,7 +78,7 @@ func TestCompositionValidation(t *testing.T) { )). // Disable our feature flag. WithTeardown("DisableAlphaCompositionValidation", funcs.AllOf( - funcs.AsFeaturesFunc(funcs.HelmUpgrade(E2EConfig.GetSelectedSuiteInstallOpts()...)), + funcs.AsFeaturesFunc(e2eConfig.HelmUpgradeCrossplaneToBase()), funcs.ReadyToTestWithin(1*time.Minute, namespace), )). Feature(), diff --git a/test/e2e/config/config.go b/test/e2e/config/config.go index 71bd10a8ec6..2e6be36da9f 100644 --- a/test/e2e/config/config.go +++ b/test/e2e/config/config.go @@ -23,10 +23,13 @@ import ( "sort" "k8s.io/utils/pointer" + "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/e2e-framework/pkg/env" "sigs.k8s.io/e2e-framework/pkg/envconf" "sigs.k8s.io/e2e-framework/pkg/features" "sigs.k8s.io/e2e-framework/third_party/helm" + + "github.com/crossplane/crossplane/test/e2e/funcs" ) // LabelTestSuite is used to define the suite each test should be part of. @@ -37,8 +40,8 @@ const TestSuiteDefault = "base" const testSuiteFlag = "test-suite" -// E2EConfig is these e2e test configuration. -type E2EConfig struct { +// Config is these e2e test configuration. +type Config struct { createKindCluster *bool destroyKindCluster *bool preinstallCrossplane *bool @@ -49,6 +52,8 @@ type E2EConfig struct { specificTestSelected *bool suites map[string]testSuite + + env.Environment } type selectedTestSuite struct { @@ -64,7 +69,7 @@ func (s *selectedTestSuite) String() string { } func (s *selectedTestSuite) Set(v string) error { - fmt.Printf("Setting test suite to %s\n", v) + log.Log.Info("Setting test suite", "value", v) s.name = v s.set = true return nil @@ -87,8 +92,8 @@ type conditionalSetupFunc struct { // NewFromFlags creates a new e2e test configuration, setting up the flags, but // not parsing them yet, which is left to the caller to do. -func NewFromFlags() E2EConfig { - c := E2EConfig{ +func NewFromFlags() Config { + c := Config{ suites: map[string]testSuite{}, } c.kindClusterName = flag.String("kind-cluster-name", "", "name of the kind cluster to use") @@ -110,7 +115,7 @@ func NewFromFlags() E2EConfig { return c } -func (e *E2EConfig) getAvailableSuitesOptions() (opts []string) { +func (e *Config) getAvailableSuitesOptions() (opts []string) { for s := range e.suites { opts = append(opts, s) } @@ -120,7 +125,7 @@ func (e *E2EConfig) getAvailableSuitesOptions() (opts []string) { // GetKindClusterName returns the name of the kind cluster, returns empty string // if it's not a kind cluster. -func (e *E2EConfig) GetKindClusterName() string { +func (e *Config) GetKindClusterName() string { if !e.IsKindCluster() { return "" } @@ -131,20 +136,43 @@ func (e *E2EConfig) GetKindClusterName() string { return *e.kindClusterName } +// SetEnvironment sets the environment to be used by the e2e test configuration. +func (e *Config) SetEnvironment(env env.Environment) { + e.Environment = env +} + // IsKindCluster returns true if the test is running against a kind cluster. -func (e *E2EConfig) IsKindCluster() bool { +func (e *Config) IsKindCluster() bool { return *e.createKindCluster || *e.kindClusterName != "" } // ShouldLoadImages returns true if the test should load images into the kind // cluster. -func (e *E2EConfig) ShouldLoadImages() bool { +func (e *Config) ShouldLoadImages() bool { return *e.loadImagesKindCluster && e.IsKindCluster() } -// GetSuiteInstallOpts returns the helm install options for the specified +// HelmUpgradeCrossplaneToSuite returns a features.Func that upgrades crossplane using +// the specified suite's helm install options. +func (e *Config) HelmUpgradeCrossplaneToSuite(suite string, extra ...helm.Option) env.Func { + return funcs.HelmUpgrade(e.getSuiteInstallOpts(suite, extra...)...) +} + +// HelmUpgradeCrossplaneToBase returns a features.Func that upgrades crossplane using +// the specified suite's helm install options. +func (e *Config) HelmUpgradeCrossplaneToBase() env.Func { + return e.HelmUpgradeCrossplaneToSuite(e.selectedTestSuite.String()) +} + +// HelmInstallBaseCrossplane returns a features.Func that installs crossplane using +// the default suite's helm install options. +func (e *Config) HelmInstallBaseCrossplane() env.Func { + return funcs.HelmInstall(e.getSuiteInstallOpts(e.selectedTestSuite.String())...) +} + +// getSuiteInstallOpts returns the helm install options for the specified // suite, appending additional specified ones -func (e *E2EConfig) GetSuiteInstallOpts(suite string, extra ...helm.Option) []helm.Option { +func (e *Config) getSuiteInstallOpts(suite string, extra ...helm.Option) []helm.Option { p, ok := e.suites[suite] if !ok { panic(fmt.Sprintf("The selected suite %q does not exist", suite)) @@ -158,12 +186,12 @@ func (e *E2EConfig) GetSuiteInstallOpts(suite string, extra ...helm.Option) []he // GetSelectedSuiteInstallOpts returns the helm install options for the // selected suite, appending additional specified ones. -func (e *E2EConfig) GetSelectedSuiteInstallOpts(extra ...helm.Option) []helm.Option { - return e.GetSuiteInstallOpts(e.selectedTestSuite.String(), extra...) +func (e *Config) GetSelectedSuiteInstallOpts(extra ...helm.Option) []helm.Option { + return e.getSuiteInstallOpts(e.selectedTestSuite.String(), extra...) } // AddTestSuite adds a new test suite, panics if already defined. -func (e *E2EConfig) AddTestSuite(name string, opts ...TestSuiteOpt) { +func (e *Config) AddTestSuite(name string, opts ...TestSuiteOpt) { if _, ok := e.suites[name]; ok { panic(fmt.Sprintf("suite already defined: %s", name)) } @@ -176,7 +204,7 @@ func (e *E2EConfig) AddTestSuite(name string, opts ...TestSuiteOpt) { } // AddDefaultTestSuite adds the default suite, panics if already defined. -func (e *E2EConfig) AddDefaultTestSuite(opts ...TestSuiteOpt) { +func (e *Config) AddDefaultTestSuite(opts ...TestSuiteOpt) { e.AddTestSuite(TestSuiteDefault, append([]TestSuiteOpt{WithoutBaseDefaultTestSuite()}, opts...)...) } @@ -219,30 +247,30 @@ func WithConditionalEnvSetupFuncs(condition func() bool, funcs ...env.Func) Test // Used to install Crossplane before any test starts, but some tests also use // these options - for example to reinstall Crossplane with a feature flag // enabled. -func (e *E2EConfig) HelmOptions(extra ...helm.Option) []helm.Option { +func (e *Config) HelmOptions(extra ...helm.Option) []helm.Option { return append(e.GetSelectedSuiteInstallOpts(), extra...) } -// HelmOptionsForSuite returns the Helm options for the specified suite, +// HelmOptionsToSuite returns the Helm options for the specified suite, // appending additional specified ones. -func (e *E2EConfig) HelmOptionsForSuite(suite string, extra ...helm.Option) []helm.Option { - return append(e.GetSuiteInstallOpts(suite), extra...) +func (e *Config) HelmOptionsToSuite(suite string, extra ...helm.Option) []helm.Option { + return append(e.getSuiteInstallOpts(suite), extra...) } // ShouldInstallCrossplane returns true if the test should install Crossplane // before starting. -func (e *E2EConfig) ShouldInstallCrossplane() bool { +func (e *Config) ShouldInstallCrossplane() bool { return *e.preinstallCrossplane } // ShouldDestroyKindCluster returns true if the test should destroy the kind // cluster after finishing. -func (e *E2EConfig) ShouldDestroyKindCluster() bool { +func (e *Config) ShouldDestroyKindCluster() bool { return *e.destroyKindCluster && e.IsKindCluster() } // GetSelectedSuiteLabels returns the labels to select for the selected suite. -func (e *E2EConfig) getSelectedSuiteLabels() features.Labels { +func (e *Config) getSelectedSuiteLabels() features.Labels { if !e.selectedTestSuite.set { return nil } @@ -251,7 +279,7 @@ func (e *E2EConfig) getSelectedSuiteLabels() features.Labels { // GetSelectedSuiteAdditionalEnvSetup returns the additional env setup funcs // for the selected suite, to be run before installing Crossplane, if required. -func (e *E2EConfig) GetSelectedSuiteAdditionalEnvSetup() (out []env.Func) { +func (e *Config) GetSelectedSuiteAdditionalEnvSetup() (out []env.Func) { selectedTestSuite := e.selectedTestSuite.String() for _, s := range e.suites[selectedTestSuite].additionalSetupFuncs { if s.condition() { @@ -275,7 +303,7 @@ func (e *E2EConfig) GetSelectedSuiteAdditionalEnvSetup() (out []env.Func) { // EnrichLabels returns the provided labels enriched with the selected suite // labels, preserving user-specified ones in case of key conflicts. -func (e *E2EConfig) EnrichLabels(labels features.Labels) features.Labels { +func (e *Config) EnrichLabels(labels features.Labels) features.Labels { if e.isSelectingTests() { return labels } @@ -291,7 +319,7 @@ func (e *E2EConfig) EnrichLabels(labels features.Labels) features.Labels { return labels } -func (e *E2EConfig) isSelectingTests() bool { +func (e *Config) isSelectingTests() bool { if e.specificTestSelected == nil { f := flag.Lookup("test.run") e.specificTestSelected = pointer.Bool(f != nil && f.Value.String() != "") diff --git a/test/e2e/install_test.go b/test/e2e/install_test.go index 2c209e520c6..a905fb870bf 100644 --- a/test/e2e/install_test.go +++ b/test/e2e/install_test.go @@ -48,7 +48,7 @@ const LabelAreaLifecycle = "lifecycle" // if not disabled explicitly. func TestCrossplaneLifecycle(t *testing.T) { manifests := "test/e2e/manifests/lifecycle/upgrade" - environment.Test(t, + e2eConfig.Test(t, // Test that it's possible to cleanly uninstall Crossplane, even after // having created and deleted a claim. features.New("CrossplaneUninstall"). @@ -128,7 +128,7 @@ func TestCrossplaneLifecycle(t *testing.T) { )). Assess("ClaimIsAvailable", funcs.ResourcesHaveConditionWithin(2*time.Minute, manifests, "claim.yaml", xpv1.Available())). Assess("UpgradeCrossplane", funcs.AllOf( - funcs.AsFeaturesFunc(funcs.HelmUpgrade(E2EConfig.GetSelectedSuiteInstallOpts()...)), + funcs.AsFeaturesFunc(e2eConfig.HelmUpgradeCrossplaneToBase()), funcs.ReadyToTestWithin(1*time.Minute, namespace), )). Assess("CoreDeploymentIsAvailable", funcs.DeploymentBecomesAvailableWithin(1*time.Minute, namespace, "crossplane")). diff --git a/test/e2e/main_test.go b/test/e2e/main_test.go index f00bbc71fc0..0acbca01d5f 100644 --- a/test/e2e/main_test.go +++ b/test/e2e/main_test.go @@ -57,17 +57,19 @@ const ( helmReleaseName = "crossplane" ) -var E2EConfig = config.NewFromFlags() - var ( - // The test environment, shared by all E2E test functions. - environment env.Environment + e2eConfig = config.NewFromFlags() clusterName string ) -func init() { +func TestMain(m *testing.M) { + // TODO(negz): Global loggers are dumb and klog is dumb. Remove this when + // e2e-framework is running controller-runtime v0.15.x per + // https://github.com/kubernetes-sigs/e2e-framework/issues/270 + log.SetLogger(klog.NewKlogr()) + // Set the default suite, to be used as base for all the other suites. - E2EConfig.AddDefaultTestSuite( + e2eConfig.AddDefaultTestSuite( config.WithoutBaseDefaultTestSuite(), config.WithHelmInstallOpts( helm.WithName(helmReleaseName), @@ -87,13 +89,6 @@ func init() { config.LabelTestSuite: []string{config.TestSuiteDefault}, }), ) -} - -func TestMain(m *testing.M) { - // TODO(negz): Global loggers are dumb and klog is dumb. Remove this when - // e2e-framework is running controller-runtime v0.15.x per - // https://github.com/kubernetes-sigs/e2e-framework/issues/270 - log.SetLogger(klog.NewKlogr()) cfg, err := envconf.NewFromFlags() if err != nil { @@ -103,10 +98,10 @@ func TestMain(m *testing.M) { var setup []env.Func var finish []env.Func - // Parse flags, populating E2EConfig too. + // Parse flags, populating Config too. // we want to create the cluster if it doesn't exist, but only if we're - if E2EConfig.IsKindCluster() { - clusterName := E2EConfig.GetKindClusterName() + if e2eConfig.IsKindCluster() { + clusterName := e2eConfig.GetKindClusterName() kindCfg, err := filepath.Abs(filepath.Join("test", "e2e", "testdata", "kindConfig.yaml")) if err != nil { panic(fmt.Sprintf("error getting kind config file: %s", err.Error())) @@ -120,12 +115,12 @@ func TestMain(m *testing.M) { // Enrich the selected labels with the ones from the suite. // Not replacing the user provided ones if any. - cfg.WithLabels(E2EConfig.EnrichLabels(cfg.Labels())) + cfg.WithLabels(e2eConfig.EnrichLabels(cfg.Labels())) - environment = env.NewWithConfig(cfg) + e2eConfig.SetEnvironment(env.NewWithConfig(cfg)) - if E2EConfig.ShouldLoadImages() { - clusterName := E2EConfig.GetKindClusterName() + if e2eConfig.ShouldLoadImages() { + clusterName := e2eConfig.GetKindClusterName() setup = append(setup, envfuncs.LoadDockerImageToCluster(clusterName, imgcore), ) @@ -133,13 +128,13 @@ func TestMain(m *testing.M) { // Add the setup functions defined by the suite being used setup = append(setup, - E2EConfig.GetSelectedSuiteAdditionalEnvSetup()..., + e2eConfig.GetSelectedSuiteAdditionalEnvSetup()..., ) - if E2EConfig.ShouldInstallCrossplane() { + if e2eConfig.ShouldInstallCrossplane() { setup = append(setup, envfuncs.CreateNamespace(namespace), - funcs.HelmInstall(E2EConfig.GetSelectedSuiteInstallOpts()...), + e2eConfig.HelmInstallBaseCrossplane(), ) } @@ -148,19 +143,19 @@ func TestMain(m *testing.M) { // We want to destroy the cluster if we created it, but only if we created it, // otherwise the random name will be meaningless. - if E2EConfig.ShouldDestroyKindCluster() { - finish = []env.Func{envfuncs.DestroyKindCluster(E2EConfig.GetKindClusterName())} + if e2eConfig.ShouldDestroyKindCluster() { + finish = []env.Func{envfuncs.DestroyKindCluster(e2eConfig.GetKindClusterName())} } // Check that all features are specifying a suite they belong to via LabelTestSuite. - environment.BeforeEachFeature(func(ctx context.Context, _ *envconf.Config, t *testing.T, feature features.Feature) (context.Context, error) { + e2eConfig.BeforeEachFeature(func(ctx context.Context, _ *envconf.Config, t *testing.T, feature features.Feature) (context.Context, error) { if _, exists := feature.Labels()[config.LabelTestSuite]; !exists { t.Fatalf("Feature %q does not have the required %q label set", feature.Name(), config.LabelTestSuite) } return ctx, nil }) - environment.Setup(setup...) - environment.Finish(finish...) - os.Exit(environment.Run(m)) + e2eConfig.Setup(setup...) + e2eConfig.Finish(finish...) + os.Exit(e2eConfig.Run(m)) } diff --git a/test/e2e/pkg_test.go b/test/e2e/pkg_test.go index cc40b35cb21..bda8bc73337 100644 --- a/test/e2e/pkg_test.go +++ b/test/e2e/pkg_test.go @@ -38,7 +38,7 @@ const LabelAreaPkg = "pkg" func TestConfigurationPullFromPrivateRegistry(t *testing.T) { manifests := "test/e2e/manifests/pkg/configuration/private" - environment.Test(t, + e2eConfig.Test(t, features.New("ConfigurationPullFromPrivateRegistry"). WithLabel(LabelArea, LabelAreaPkg). WithLabel(LabelSize, LabelSizeSmall). @@ -60,7 +60,7 @@ func TestConfigurationPullFromPrivateRegistry(t *testing.T) { func TestConfigurationWithDependency(t *testing.T) { manifests := "test/e2e/manifests/pkg/configuration/dependency" - environment.Test(t, + e2eConfig.Test(t, features.New("ConfigurationWithDependency"). WithLabel(LabelArea, LabelAreaPkg). WithLabel(LabelSize, LabelSizeSmall). @@ -90,7 +90,7 @@ func TestProviderUpgrade(t *testing.T) { // resource has been created. manifests := "test/e2e/manifests/pkg/provider" - environment.Test(t, + e2eConfig.Test(t, features.New("ProviderUpgrade"). WithLabel(LabelArea, LabelAreaPkg). WithLabel(LabelSize, LabelSizeSmall). diff --git a/test/e2e/xfn_test.go b/test/e2e/xfn_test.go index d757fe4f910..33fba3fd09b 100644 --- a/test/e2e/xfn_test.go +++ b/test/e2e/xfn_test.go @@ -59,7 +59,7 @@ const ( ) func init() { - E2EConfig.AddTestSuite(SuiteCompositionFunctions, + e2eConfig.AddTestSuite(SuiteCompositionFunctions, config.WithHelmInstallOpts( helm.WithArgs( "--set args={--debug,--enable-composition-functions}", @@ -75,14 +75,14 @@ func init() { config.LabelTestSuite: []string{SuiteCompositionFunctions, config.TestSuiteDefault}, }), config.WithConditionalEnvSetupFuncs( - E2EConfig.ShouldLoadImages, envfuncs.LoadDockerImageToCluster(E2EConfig.GetKindClusterName(), imgxfn), + e2eConfig.ShouldLoadImages, envfuncs.LoadDockerImageToCluster(e2eConfig.GetKindClusterName(), imgxfn), ), ) } func TestXfnRunnerImagePull(t *testing.T) { manifests := "test/e2e/manifests/xfnrunner/private-registry/pull" - environment.Test(t, + e2eConfig.Test(t, features.New("PullFnImageFromPrivateRegistryWithCustomCert"). WithLabel(LabelArea, LabelAreaXFN). WithLabel(LabelStage, LabelStageAlpha). @@ -152,12 +152,11 @@ func TestXfnRunnerImagePull(t *testing.T) { WithSetup("CopyFnImageToRegistry", funcs.CopyImageToRegistry(clusterName, registryNs, "private-docker-registry", "crossplane-e2e/fn-labelizer:latest", timeoutOne)). WithSetup("CrossplaneDeployedWithFunctionsEnabled", funcs.AllOf( - funcs.AsFeaturesFunc(funcs.HelmUpgrade( - E2EConfig.GetSuiteInstallOpts(SuiteCompositionFunctions, - helm.WithArgs( - "--set registryCaBundleConfig.key=domain.crt", - "--set registryCaBundleConfig.name=reg-ca", - ))...)), + funcs.AsFeaturesFunc(e2eConfig.HelmUpgradeCrossplaneToSuite(SuiteCompositionFunctions, + helm.WithArgs( + "--set registryCaBundleConfig.key=domain.crt", + "--set registryCaBundleConfig.name=reg-ca", + ))), funcs.ReadyToTestWithin(1*time.Minute, namespace), )). WithSetup("ProviderNopDeployed", funcs.AllOf( @@ -209,7 +208,7 @@ func TestXfnRunnerImagePull(t *testing.T) { }, )). WithTeardown("CrossplaneDeployedWithoutFunctionsEnabled", funcs.AllOf( - funcs.AsFeaturesFunc(funcs.HelmUpgrade(E2EConfig.GetSelectedSuiteInstallOpts()...)), + funcs.AsFeaturesFunc(e2eConfig.HelmUpgradeCrossplaneToBase()), funcs.ReadyToTestWithin(1*time.Minute, namespace), )). Feature(), @@ -218,7 +217,7 @@ func TestXfnRunnerImagePull(t *testing.T) { func TestXfnRunnerWriteToTmp(t *testing.T) { manifests := "test/e2e/manifests/xfnrunner/tmp-writer" - environment.Test(t, + e2eConfig.Test(t, features.New("CreateAFileInTmpFolder"). WithLabel(LabelArea, LabelAreaXFN). WithLabel(LabelStage, LabelStageAlpha). @@ -250,7 +249,7 @@ func TestXfnRunnerWriteToTmp(t *testing.T) { WithSetup("CopyFnImageToRegistry", funcs.CopyImageToRegistry(clusterName, registryNs, "public-docker-registry", "crossplane-e2e/fn-tmp-writer:latest", timeoutOne)). WithSetup("CrossplaneDeployedWithFunctionsEnabled", funcs.AllOf( - funcs.AsFeaturesFunc(funcs.HelmUpgrade(E2EConfig.GetSuiteInstallOpts(SuiteCompositionFunctions)...)), + funcs.AsFeaturesFunc(e2eConfig.HelmUpgradeCrossplaneToSuite(SuiteCompositionFunctions)), funcs.ReadyToTestWithin(1*time.Minute, namespace), )). WithSetup("ProviderNopDeployed", funcs.AllOf( @@ -289,7 +288,7 @@ func TestXfnRunnerWriteToTmp(t *testing.T) { )). WithTeardown("RemoveRegistry", funcs.AsFeaturesFunc(envfuncs.DeleteNamespace(registryNs))). WithTeardown("CrossplaneDeployedWithoutFunctionsEnabled", funcs.AllOf( - funcs.AsFeaturesFunc(funcs.HelmUpgrade(E2EConfig.GetSelectedSuiteInstallOpts()...)), + funcs.AsFeaturesFunc(e2eConfig.HelmUpgradeCrossplaneToBase()), funcs.ReadyToTestWithin(1*time.Minute, namespace), )). Feature(), From 26529274785abf38c01acfbd525072678dd71227 Mon Sep 17 00:00:00 2001 From: Philippe Scorsolini Date: Wed, 9 Aug 2023 10:08:02 +0200 Subject: [PATCH 6/7] chore: wrap environment and switch to t.Name Signed-off-by: Philippe Scorsolini --- test/e2e/README.md | 2 +- test/e2e/apiextensions_test.go | 8 +-- test/e2e/comp_schema_validation_test.go | 10 ++-- test/e2e/config/{config.go => environment.go} | 53 ++++++++++--------- test/e2e/install_test.go | 8 +-- test/e2e/main_test.go | 36 ++++++------- test/e2e/pkg_test.go | 12 ++--- test/e2e/xfn_test.go | 22 ++++---- 8 files changed, 78 insertions(+), 73 deletions(-) rename test/e2e/config/{config.go => environment.go} (83%) diff --git a/test/e2e/README.md b/test/e2e/README.md index 57383beeb88..80b48da844b 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -203,7 +203,7 @@ func TestSomeFeature(t *testing.T) { // ... other variables or constants ... environment.Test(t, - features.New("ConfigurationWithDependency"). + features.New(t.Name()). WithLabel(LabelArea, ...). WithLabel(LabelSize, ...). WithLabel(config.LabelTestSuite, config.TestSuiteDefault). diff --git a/test/e2e/apiextensions_test.go b/test/e2e/apiextensions_test.go index b91565a4c47..3e6ec40db2c 100644 --- a/test/e2e/apiextensions_test.go +++ b/test/e2e/apiextensions_test.go @@ -40,8 +40,8 @@ const LabelAreaAPIExtensions = "apiextensions" func TestCompositionMinimal(t *testing.T) { manifests := "test/e2e/manifests/apiextensions/composition/minimal" - e2eConfig.Test(t, - features.New("CompositionMinimal"). + environment.Test(t, + features.New(t.Name()). WithLabel(LabelArea, LabelAreaAPIExtensions). WithLabel(LabelSize, LabelSizeSmall). WithLabel(config.LabelTestSuite, config.TestSuiteDefault). @@ -74,8 +74,8 @@ func TestCompositionMinimal(t *testing.T) { func TestCompositionPatchAndTransform(t *testing.T) { manifests := "test/e2e/manifests/apiextensions/composition/patch-and-transform" - e2eConfig.Test(t, - features.New("CompositionPatchAndTransform"). + environment.Test(t, + features.New(t.Name()). WithLabel(LabelArea, LabelAreaAPIExtensions). WithLabel(LabelSize, LabelSizeSmall). WithLabel(config.LabelTestSuite, config.TestSuiteDefault). diff --git a/test/e2e/comp_schema_validation_test.go b/test/e2e/comp_schema_validation_test.go index f2112c16915..9b80c82c264 100644 --- a/test/e2e/comp_schema_validation_test.go +++ b/test/e2e/comp_schema_validation_test.go @@ -21,7 +21,7 @@ const ( ) func init() { - e2eConfig.AddTestSuite(SuiteCompositionWebhookSchemaValidation, + environment.AddTestSuite(SuiteCompositionWebhookSchemaValidation, config.WithHelmInstallOpts( helm.WithArgs("--set args={--debug,--enable-composition-webhook-schema-validation}"), ), @@ -50,8 +50,8 @@ func TestCompositionValidation(t *testing.T) { Assessment: funcs.ResourcesFailToApply(FieldManager, manifests, "composition-invalid.yaml"), }, } - e2eConfig.Test(t, - cases.Build("CompositionValidation"). + environment.Test(t, + cases.Build(t.Name()). WithLabel(LabelStage, LabelStageAlpha). WithLabel(LabelArea, LabelAreaAPIExtensions). WithLabel(LabelSize, LabelSizeSmall). @@ -59,7 +59,7 @@ func TestCompositionValidation(t *testing.T) { WithLabel(config.LabelTestSuite, SuiteCompositionWebhookSchemaValidation). // Enable our feature flag. WithSetup("EnableAlphaCompositionValidation", funcs.AllOf( - funcs.AsFeaturesFunc(e2eConfig.HelmUpgradeCrossplaneToSuite(SuiteCompositionWebhookSchemaValidation)), + funcs.AsFeaturesFunc(environment.HelmUpgradeCrossplaneToSuite(SuiteCompositionWebhookSchemaValidation)), funcs.ReadyToTestWithin(1*time.Minute, namespace), )). WithSetup("CreatePrerequisites", funcs.AllOf( @@ -78,7 +78,7 @@ func TestCompositionValidation(t *testing.T) { )). // Disable our feature flag. WithTeardown("DisableAlphaCompositionValidation", funcs.AllOf( - funcs.AsFeaturesFunc(e2eConfig.HelmUpgradeCrossplaneToBase()), + funcs.AsFeaturesFunc(environment.HelmUpgradeCrossplaneToBase()), funcs.ReadyToTestWithin(1*time.Minute, namespace), )). Feature(), diff --git a/test/e2e/config/config.go b/test/e2e/config/environment.go similarity index 83% rename from test/e2e/config/config.go rename to test/e2e/config/environment.go index 2e6be36da9f..0a4f7d20592 100644 --- a/test/e2e/config/config.go +++ b/test/e2e/config/environment.go @@ -40,8 +40,9 @@ const TestSuiteDefault = "base" const testSuiteFlag = "test-suite" -// Config is these e2e test configuration. -type Config struct { +// Environment is these e2e test configuration, wraps the e2e-framework +// environment. +type Environment struct { createKindCluster *bool destroyKindCluster *bool preinstallCrossplane *bool @@ -56,6 +57,8 @@ type Config struct { env.Environment } +// selectedTestSuite implements the flag.Value interface. To be able to +// distinguish between empty string and an unset value. type selectedTestSuite struct { name string set bool @@ -85,6 +88,8 @@ type testSuite struct { labelsToSelect features.Labels } +// conditionalSetupFunc wraps a list of env.Func and a condition that will be +// evaluated to decide whether the provided functions should be used or not. type conditionalSetupFunc struct { condition func() bool f []env.Func @@ -92,8 +97,8 @@ type conditionalSetupFunc struct { // NewFromFlags creates a new e2e test configuration, setting up the flags, but // not parsing them yet, which is left to the caller to do. -func NewFromFlags() Config { - c := Config{ +func NewFromFlags() Environment { + c := Environment{ suites: map[string]testSuite{}, } c.kindClusterName = flag.String("kind-cluster-name", "", "name of the kind cluster to use") @@ -115,7 +120,7 @@ func NewFromFlags() Config { return c } -func (e *Config) getAvailableSuitesOptions() (opts []string) { +func (e *Environment) getAvailableSuitesOptions() (opts []string) { for s := range e.suites { opts = append(opts, s) } @@ -125,7 +130,7 @@ func (e *Config) getAvailableSuitesOptions() (opts []string) { // GetKindClusterName returns the name of the kind cluster, returns empty string // if it's not a kind cluster. -func (e *Config) GetKindClusterName() string { +func (e *Environment) GetKindClusterName() string { if !e.IsKindCluster() { return "" } @@ -137,42 +142,42 @@ func (e *Config) GetKindClusterName() string { } // SetEnvironment sets the environment to be used by the e2e test configuration. -func (e *Config) SetEnvironment(env env.Environment) { +func (e *Environment) SetEnvironment(env env.Environment) { e.Environment = env } // IsKindCluster returns true if the test is running against a kind cluster. -func (e *Config) IsKindCluster() bool { +func (e *Environment) IsKindCluster() bool { return *e.createKindCluster || *e.kindClusterName != "" } // ShouldLoadImages returns true if the test should load images into the kind // cluster. -func (e *Config) ShouldLoadImages() bool { +func (e *Environment) ShouldLoadImages() bool { return *e.loadImagesKindCluster && e.IsKindCluster() } // HelmUpgradeCrossplaneToSuite returns a features.Func that upgrades crossplane using // the specified suite's helm install options. -func (e *Config) HelmUpgradeCrossplaneToSuite(suite string, extra ...helm.Option) env.Func { +func (e *Environment) HelmUpgradeCrossplaneToSuite(suite string, extra ...helm.Option) env.Func { return funcs.HelmUpgrade(e.getSuiteInstallOpts(suite, extra...)...) } // HelmUpgradeCrossplaneToBase returns a features.Func that upgrades crossplane using // the specified suite's helm install options. -func (e *Config) HelmUpgradeCrossplaneToBase() env.Func { +func (e *Environment) HelmUpgradeCrossplaneToBase() env.Func { return e.HelmUpgradeCrossplaneToSuite(e.selectedTestSuite.String()) } // HelmInstallBaseCrossplane returns a features.Func that installs crossplane using // the default suite's helm install options. -func (e *Config) HelmInstallBaseCrossplane() env.Func { +func (e *Environment) HelmInstallBaseCrossplane() env.Func { return funcs.HelmInstall(e.getSuiteInstallOpts(e.selectedTestSuite.String())...) } // getSuiteInstallOpts returns the helm install options for the specified // suite, appending additional specified ones -func (e *Config) getSuiteInstallOpts(suite string, extra ...helm.Option) []helm.Option { +func (e *Environment) getSuiteInstallOpts(suite string, extra ...helm.Option) []helm.Option { p, ok := e.suites[suite] if !ok { panic(fmt.Sprintf("The selected suite %q does not exist", suite)) @@ -186,12 +191,12 @@ func (e *Config) getSuiteInstallOpts(suite string, extra ...helm.Option) []helm. // GetSelectedSuiteInstallOpts returns the helm install options for the // selected suite, appending additional specified ones. -func (e *Config) GetSelectedSuiteInstallOpts(extra ...helm.Option) []helm.Option { +func (e *Environment) GetSelectedSuiteInstallOpts(extra ...helm.Option) []helm.Option { return e.getSuiteInstallOpts(e.selectedTestSuite.String(), extra...) } // AddTestSuite adds a new test suite, panics if already defined. -func (e *Config) AddTestSuite(name string, opts ...TestSuiteOpt) { +func (e *Environment) AddTestSuite(name string, opts ...TestSuiteOpt) { if _, ok := e.suites[name]; ok { panic(fmt.Sprintf("suite already defined: %s", name)) } @@ -204,7 +209,7 @@ func (e *Config) AddTestSuite(name string, opts ...TestSuiteOpt) { } // AddDefaultTestSuite adds the default suite, panics if already defined. -func (e *Config) AddDefaultTestSuite(opts ...TestSuiteOpt) { +func (e *Environment) AddDefaultTestSuite(opts ...TestSuiteOpt) { e.AddTestSuite(TestSuiteDefault, append([]TestSuiteOpt{WithoutBaseDefaultTestSuite()}, opts...)...) } @@ -247,30 +252,30 @@ func WithConditionalEnvSetupFuncs(condition func() bool, funcs ...env.Func) Test // Used to install Crossplane before any test starts, but some tests also use // these options - for example to reinstall Crossplane with a feature flag // enabled. -func (e *Config) HelmOptions(extra ...helm.Option) []helm.Option { +func (e *Environment) HelmOptions(extra ...helm.Option) []helm.Option { return append(e.GetSelectedSuiteInstallOpts(), extra...) } // HelmOptionsToSuite returns the Helm options for the specified suite, // appending additional specified ones. -func (e *Config) HelmOptionsToSuite(suite string, extra ...helm.Option) []helm.Option { +func (e *Environment) HelmOptionsToSuite(suite string, extra ...helm.Option) []helm.Option { return append(e.getSuiteInstallOpts(suite), extra...) } // ShouldInstallCrossplane returns true if the test should install Crossplane // before starting. -func (e *Config) ShouldInstallCrossplane() bool { +func (e *Environment) ShouldInstallCrossplane() bool { return *e.preinstallCrossplane } // ShouldDestroyKindCluster returns true if the test should destroy the kind // cluster after finishing. -func (e *Config) ShouldDestroyKindCluster() bool { +func (e *Environment) ShouldDestroyKindCluster() bool { return *e.destroyKindCluster && e.IsKindCluster() } // GetSelectedSuiteLabels returns the labels to select for the selected suite. -func (e *Config) getSelectedSuiteLabels() features.Labels { +func (e *Environment) getSelectedSuiteLabels() features.Labels { if !e.selectedTestSuite.set { return nil } @@ -279,7 +284,7 @@ func (e *Config) getSelectedSuiteLabels() features.Labels { // GetSelectedSuiteAdditionalEnvSetup returns the additional env setup funcs // for the selected suite, to be run before installing Crossplane, if required. -func (e *Config) GetSelectedSuiteAdditionalEnvSetup() (out []env.Func) { +func (e *Environment) GetSelectedSuiteAdditionalEnvSetup() (out []env.Func) { selectedTestSuite := e.selectedTestSuite.String() for _, s := range e.suites[selectedTestSuite].additionalSetupFuncs { if s.condition() { @@ -303,7 +308,7 @@ func (e *Config) GetSelectedSuiteAdditionalEnvSetup() (out []env.Func) { // EnrichLabels returns the provided labels enriched with the selected suite // labels, preserving user-specified ones in case of key conflicts. -func (e *Config) EnrichLabels(labels features.Labels) features.Labels { +func (e *Environment) EnrichLabels(labels features.Labels) features.Labels { if e.isSelectingTests() { return labels } @@ -319,7 +324,7 @@ func (e *Config) EnrichLabels(labels features.Labels) features.Labels { return labels } -func (e *Config) isSelectingTests() bool { +func (e *Environment) isSelectingTests() bool { if e.specificTestSelected == nil { f := flag.Lookup("test.run") e.specificTestSelected = pointer.Bool(f != nil && f.Value.String() != "") diff --git a/test/e2e/install_test.go b/test/e2e/install_test.go index a905fb870bf..e6770611ac4 100644 --- a/test/e2e/install_test.go +++ b/test/e2e/install_test.go @@ -48,10 +48,10 @@ const LabelAreaLifecycle = "lifecycle" // if not disabled explicitly. func TestCrossplaneLifecycle(t *testing.T) { manifests := "test/e2e/manifests/lifecycle/upgrade" - e2eConfig.Test(t, + environment.Test(t, // Test that it's possible to cleanly uninstall Crossplane, even after // having created and deleted a claim. - features.New("CrossplaneUninstall"). + features.New(t.Name()+"Uninstall"). WithLabel(LabelArea, LabelAreaLifecycle). WithLabel(LabelSize, LabelSizeSmall). WithLabel(LabelModifyCrossplaneInstallation, LabelModifyCrossplaneInstallationTrue). @@ -95,7 +95,7 @@ func TestCrossplaneLifecycle(t *testing.T) { funcs.ResourceDeletedWithin(3*time.Minute, &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: namespace}}), )). Feature(), - features.New("CrossplaneUpgrade"). + features.New(t.Name()+"Upgrade"). WithLabel(LabelArea, LabelAreaLifecycle). WithLabel(LabelSize, LabelSizeSmall). WithLabel(config.LabelTestSuite, config.TestSuiteDefault). @@ -128,7 +128,7 @@ func TestCrossplaneLifecycle(t *testing.T) { )). Assess("ClaimIsAvailable", funcs.ResourcesHaveConditionWithin(2*time.Minute, manifests, "claim.yaml", xpv1.Available())). Assess("UpgradeCrossplane", funcs.AllOf( - funcs.AsFeaturesFunc(e2eConfig.HelmUpgradeCrossplaneToBase()), + funcs.AsFeaturesFunc(environment.HelmUpgradeCrossplaneToBase()), funcs.ReadyToTestWithin(1*time.Minute, namespace), )). Assess("CoreDeploymentIsAvailable", funcs.DeploymentBecomesAvailableWithin(1*time.Minute, namespace, "crossplane")). diff --git a/test/e2e/main_test.go b/test/e2e/main_test.go index 0acbca01d5f..a19a77a333f 100644 --- a/test/e2e/main_test.go +++ b/test/e2e/main_test.go @@ -58,7 +58,7 @@ const ( ) var ( - e2eConfig = config.NewFromFlags() + environment = config.NewFromFlags() clusterName string ) @@ -69,7 +69,7 @@ func TestMain(m *testing.M) { log.SetLogger(klog.NewKlogr()) // Set the default suite, to be used as base for all the other suites. - e2eConfig.AddDefaultTestSuite( + environment.AddDefaultTestSuite( config.WithoutBaseDefaultTestSuite(), config.WithHelmInstallOpts( helm.WithName(helmReleaseName), @@ -98,10 +98,10 @@ func TestMain(m *testing.M) { var setup []env.Func var finish []env.Func - // Parse flags, populating Config too. + // Parse flags, populating Environment too. // we want to create the cluster if it doesn't exist, but only if we're - if e2eConfig.IsKindCluster() { - clusterName := e2eConfig.GetKindClusterName() + if environment.IsKindCluster() { + clusterName := environment.GetKindClusterName() kindCfg, err := filepath.Abs(filepath.Join("test", "e2e", "testdata", "kindConfig.yaml")) if err != nil { panic(fmt.Sprintf("error getting kind config file: %s", err.Error())) @@ -115,12 +115,12 @@ func TestMain(m *testing.M) { // Enrich the selected labels with the ones from the suite. // Not replacing the user provided ones if any. - cfg.WithLabels(e2eConfig.EnrichLabels(cfg.Labels())) + cfg.WithLabels(environment.EnrichLabels(cfg.Labels())) - e2eConfig.SetEnvironment(env.NewWithConfig(cfg)) + environment.SetEnvironment(env.NewWithConfig(cfg)) - if e2eConfig.ShouldLoadImages() { - clusterName := e2eConfig.GetKindClusterName() + if environment.ShouldLoadImages() { + clusterName := environment.GetKindClusterName() setup = append(setup, envfuncs.LoadDockerImageToCluster(clusterName, imgcore), ) @@ -128,13 +128,13 @@ func TestMain(m *testing.M) { // Add the setup functions defined by the suite being used setup = append(setup, - e2eConfig.GetSelectedSuiteAdditionalEnvSetup()..., + environment.GetSelectedSuiteAdditionalEnvSetup()..., ) - if e2eConfig.ShouldInstallCrossplane() { + if environment.ShouldInstallCrossplane() { setup = append(setup, envfuncs.CreateNamespace(namespace), - e2eConfig.HelmInstallBaseCrossplane(), + environment.HelmInstallBaseCrossplane(), ) } @@ -143,19 +143,19 @@ func TestMain(m *testing.M) { // We want to destroy the cluster if we created it, but only if we created it, // otherwise the random name will be meaningless. - if e2eConfig.ShouldDestroyKindCluster() { - finish = []env.Func{envfuncs.DestroyKindCluster(e2eConfig.GetKindClusterName())} + if environment.ShouldDestroyKindCluster() { + finish = []env.Func{envfuncs.DestroyKindCluster(environment.GetKindClusterName())} } // Check that all features are specifying a suite they belong to via LabelTestSuite. - e2eConfig.BeforeEachFeature(func(ctx context.Context, _ *envconf.Config, t *testing.T, feature features.Feature) (context.Context, error) { + environment.BeforeEachFeature(func(ctx context.Context, _ *envconf.Config, t *testing.T, feature features.Feature) (context.Context, error) { if _, exists := feature.Labels()[config.LabelTestSuite]; !exists { t.Fatalf("Feature %q does not have the required %q label set", feature.Name(), config.LabelTestSuite) } return ctx, nil }) - e2eConfig.Setup(setup...) - e2eConfig.Finish(finish...) - os.Exit(e2eConfig.Run(m)) + environment.Setup(setup...) + environment.Finish(finish...) + os.Exit(environment.Run(m)) } diff --git a/test/e2e/pkg_test.go b/test/e2e/pkg_test.go index bda8bc73337..f684263572f 100644 --- a/test/e2e/pkg_test.go +++ b/test/e2e/pkg_test.go @@ -38,8 +38,8 @@ const LabelAreaPkg = "pkg" func TestConfigurationPullFromPrivateRegistry(t *testing.T) { manifests := "test/e2e/manifests/pkg/configuration/private" - e2eConfig.Test(t, - features.New("ConfigurationPullFromPrivateRegistry"). + environment.Test(t, + features.New(t.Name()). WithLabel(LabelArea, LabelAreaPkg). WithLabel(LabelSize, LabelSizeSmall). WithLabel(config.LabelTestSuite, config.TestSuiteDefault). @@ -60,8 +60,8 @@ func TestConfigurationPullFromPrivateRegistry(t *testing.T) { func TestConfigurationWithDependency(t *testing.T) { manifests := "test/e2e/manifests/pkg/configuration/dependency" - e2eConfig.Test(t, - features.New("ConfigurationWithDependency"). + environment.Test(t, + features.New(t.Name()). WithLabel(LabelArea, LabelAreaPkg). WithLabel(LabelSize, LabelSizeSmall). WithLabel(config.LabelTestSuite, config.TestSuiteDefault). @@ -90,8 +90,8 @@ func TestProviderUpgrade(t *testing.T) { // resource has been created. manifests := "test/e2e/manifests/pkg/provider" - e2eConfig.Test(t, - features.New("ProviderUpgrade"). + environment.Test(t, + features.New(t.Name()). WithLabel(LabelArea, LabelAreaPkg). WithLabel(LabelSize, LabelSizeSmall). WithLabel(config.LabelTestSuite, config.TestSuiteDefault). diff --git a/test/e2e/xfn_test.go b/test/e2e/xfn_test.go index 33fba3fd09b..bda513ba37f 100644 --- a/test/e2e/xfn_test.go +++ b/test/e2e/xfn_test.go @@ -59,7 +59,7 @@ const ( ) func init() { - e2eConfig.AddTestSuite(SuiteCompositionFunctions, + environment.AddTestSuite(SuiteCompositionFunctions, config.WithHelmInstallOpts( helm.WithArgs( "--set args={--debug,--enable-composition-functions}", @@ -75,15 +75,15 @@ func init() { config.LabelTestSuite: []string{SuiteCompositionFunctions, config.TestSuiteDefault}, }), config.WithConditionalEnvSetupFuncs( - e2eConfig.ShouldLoadImages, envfuncs.LoadDockerImageToCluster(e2eConfig.GetKindClusterName(), imgxfn), + environment.ShouldLoadImages, envfuncs.LoadDockerImageToCluster(environment.GetKindClusterName(), imgxfn), ), ) } -func TestXfnRunnerImagePull(t *testing.T) { +func TestXfnRunnerImagePullFromPrivateRegistryWithCustomCert(t *testing.T) { manifests := "test/e2e/manifests/xfnrunner/private-registry/pull" - e2eConfig.Test(t, - features.New("PullFnImageFromPrivateRegistryWithCustomCert"). + environment.Test(t, + features.New(t.Name()). WithLabel(LabelArea, LabelAreaXFN). WithLabel(LabelStage, LabelStageAlpha). WithLabel(LabelSize, LabelSizeLarge). @@ -152,7 +152,7 @@ func TestXfnRunnerImagePull(t *testing.T) { WithSetup("CopyFnImageToRegistry", funcs.CopyImageToRegistry(clusterName, registryNs, "private-docker-registry", "crossplane-e2e/fn-labelizer:latest", timeoutOne)). WithSetup("CrossplaneDeployedWithFunctionsEnabled", funcs.AllOf( - funcs.AsFeaturesFunc(e2eConfig.HelmUpgradeCrossplaneToSuite(SuiteCompositionFunctions, + funcs.AsFeaturesFunc(environment.HelmUpgradeCrossplaneToSuite(SuiteCompositionFunctions, helm.WithArgs( "--set registryCaBundleConfig.key=domain.crt", "--set registryCaBundleConfig.name=reg-ca", @@ -208,7 +208,7 @@ func TestXfnRunnerImagePull(t *testing.T) { }, )). WithTeardown("CrossplaneDeployedWithoutFunctionsEnabled", funcs.AllOf( - funcs.AsFeaturesFunc(e2eConfig.HelmUpgradeCrossplaneToBase()), + funcs.AsFeaturesFunc(environment.HelmUpgradeCrossplaneToBase()), funcs.ReadyToTestWithin(1*time.Minute, namespace), )). Feature(), @@ -217,8 +217,8 @@ func TestXfnRunnerImagePull(t *testing.T) { func TestXfnRunnerWriteToTmp(t *testing.T) { manifests := "test/e2e/manifests/xfnrunner/tmp-writer" - e2eConfig.Test(t, - features.New("CreateAFileInTmpFolder"). + environment.Test(t, + features.New(t.Name()). WithLabel(LabelArea, LabelAreaXFN). WithLabel(LabelStage, LabelStageAlpha). WithLabel(LabelSize, LabelSizeLarge). @@ -249,7 +249,7 @@ func TestXfnRunnerWriteToTmp(t *testing.T) { WithSetup("CopyFnImageToRegistry", funcs.CopyImageToRegistry(clusterName, registryNs, "public-docker-registry", "crossplane-e2e/fn-tmp-writer:latest", timeoutOne)). WithSetup("CrossplaneDeployedWithFunctionsEnabled", funcs.AllOf( - funcs.AsFeaturesFunc(e2eConfig.HelmUpgradeCrossplaneToSuite(SuiteCompositionFunctions)), + funcs.AsFeaturesFunc(environment.HelmUpgradeCrossplaneToSuite(SuiteCompositionFunctions)), funcs.ReadyToTestWithin(1*time.Minute, namespace), )). WithSetup("ProviderNopDeployed", funcs.AllOf( @@ -288,7 +288,7 @@ func TestXfnRunnerWriteToTmp(t *testing.T) { )). WithTeardown("RemoveRegistry", funcs.AsFeaturesFunc(envfuncs.DeleteNamespace(registryNs))). WithTeardown("CrossplaneDeployedWithoutFunctionsEnabled", funcs.AllOf( - funcs.AsFeaturesFunc(e2eConfig.HelmUpgradeCrossplaneToBase()), + funcs.AsFeaturesFunc(environment.HelmUpgradeCrossplaneToBase()), funcs.ReadyToTestWithin(1*time.Minute, namespace), )). Feature(), From c64c6797de381c58ef5650c12313fa15dec621af Mon Sep 17 00:00:00 2001 From: Philippe Scorsolini Date: Fri, 11 Aug 2023 09:23:10 +0200 Subject: [PATCH 7/7] chore: address review comments Signed-off-by: Philippe Scorsolini --- test/e2e/README.md | 2 +- test/e2e/config/environment.go | 4 ++-- test/e2e/main_test.go | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/test/e2e/README.md b/test/e2e/README.md index 80b48da844b..dc337a9f6a0 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -121,7 +121,7 @@ that feature, to make sure the feature is not breaking any default behavior. In case a test needs a specific Crossplane configuration, it must still take care of upgrading the installation to the desired configuration, but should then -use `E2EConfig.GetSelectedSuiteInstallOpts` to retrieve at runtime the baseline +use `environment.GetSelectedSuiteInstallOpts` to retrieve at runtime the baseline installation options to be sure to restore the previous state. This allows tests to run against any suite if needed. diff --git a/test/e2e/config/environment.go b/test/e2e/config/environment.go index 0a4f7d20592..466589ade65 100644 --- a/test/e2e/config/environment.go +++ b/test/e2e/config/environment.go @@ -95,9 +95,9 @@ type conditionalSetupFunc struct { f []env.Func } -// NewFromFlags creates a new e2e test configuration, setting up the flags, but +// NewEnvironmentFromFlags creates a new e2e test configuration, setting up the flags, but // not parsing them yet, which is left to the caller to do. -func NewFromFlags() Environment { +func NewEnvironmentFromFlags() Environment { c := Environment{ suites: map[string]testSuite{}, } diff --git a/test/e2e/main_test.go b/test/e2e/main_test.go index a19a77a333f..e578b02e926 100644 --- a/test/e2e/main_test.go +++ b/test/e2e/main_test.go @@ -58,7 +58,7 @@ const ( ) var ( - environment = config.NewFromFlags() + environment = config.NewEnvironmentFromFlags() clusterName string )