Merge pull request #24 from turkenh/composite-support

Add composite/claim support to Uptest
upbound · Oct 13, 2022 · 65442e1 · 65442e1
2 parents 62992a4 + 276e4a5
commit 65442e1
Show file tree

Hide file tree

Showing 16 changed files with 933 additions and 110 deletions.
diff --git a/README.md b/README.md
@@ -2,9 +2,70 @@
 
 The end to end integration testing tool for Crossplane providers and configurations.
 
+Uptest comes as a binary which can be installed from the releases section. It runs end-to-end tests
+by applying the provided examples and waiting for the expected conditions. Other than that, it enables templating to
+insert dynamic values into the examples and supports running scripts as hooks just before and right after applying
+the examples.
+
 ## Usage
 
-TBD
+```shell
+$ uptest e2e --help
+usage: uptest e2e [<flags>] [<manifest-list>]
+
+Run e2e tests for manifests by applying them to a control plane and waiting until a given condition is met.
+
+Flags:
+  --help                        Show context-sensitive help (also try --help-long and --help-man).
+  --data-source=""              File path of data source that will be used for injection some values.
+  --setup-script=""             Script that will be executed before running tests.
+  --teardown-script=""          Script that will be executed after running tests.
+  --default-timeout=1200        Default timeout in seconds for the test. Timeout could be overridden per resource using
+                                "uptest.upbound.io/timeout" annotation.
+  --default-conditions="Ready"  Comma seperated list of default conditions to wait for a successful test. Conditions could be
+                                overridden per resource using "uptest.upbound.io/conditions" annotation.
+
+Args:
+  [<manifest-list>]  List of manifests. Value of this option will be used to trigger/configure the tests.The possible usage:
+                     'provider-aws/examples/s3/bucket.yaml,provider-gcp/examples/storage/bucket.yaml': The comma separated resources
+                     are used as test inputs. If this option is not set, 'MANIFEST_LIST' env var is used as default.
+```
+
+Uptest expects a running control-plane (a.k.a. k8s + crossplane) where required providers are running and/or required
+configuration were applied.
+
+### Example: 
+
+```shell
+uptest e2e examples/user.yaml,examples/bucket.yaml --setup-script="test/hooks/setup.sh"
+```
+
+### Hooks
+
+There are 4 types of hooks that can be used to customize the test flow:
+
+1. `setup-script`: This hook will be executed before running the tests case. It is useful to set up the control plane
+   before running the tests. For example, you can use it to create a provider config and your cloud credentials. This
+   can be configured via `--setup-script` flag as a relative path to where uptest is executed.
+2. `teardown-script`: This hook will be executed after running the tests case. This can be configured via
+   `--teardown-script` flag as a relative path to where uptest is executed.
+3. `pre-assert-hook`: This hook will be executed before running the assertions and after applying a specific manifest.
+    This can be configured via `uptest.upbound.io/pre-assert-hook` annotation on the manifest as a relative path to the
+    manifest file.
+4. `post-assert-hook`: This hook will be executed after running the assertions. This can be configured via 
+    `uptest.upbound.io/post-assert-hook` annotation on the manifest as a relative path to the manifest file.
+
+> All hooks need to be executables, please make sure to set the executable bit on your scripts, e.g. with `chmod +x`.
+
+### Troubleshooting
+
+Uptest uses [kuttl](https://kuttl.dev/) under the hood and generates a `kuttl` test case based on the provided input.
+You can inspect the generated kuttl test case by checking the temporary test directory which is printed in the beginning
+of uptest e2e output. For example:
+
+```shell
+Running kuttl tests at /var/folders/_5/jc7399qx6cn6t5535npv9z4c0000gn/T/uptest-e2e
+```
 
 ## Report a Bug
 
@@ -13,4 +74,4 @@ open an [issue](https://github.com/upbound/uptest/issues).
 
 ## Licensing
 
-uptest is under the Apache 2.0 license.
+Uptest is under the Apache 2.0 license.
diff --git a/cmd/main.go b/cmd/main.go
@@ -9,6 +9,7 @@ import (
 	"gopkg.in/alecthomas/kingpin.v2"
 
 	"github.com/upbound/uptest/internal"
+	"github.com/upbound/uptest/internal/config"
 )
 
 func main() {
@@ -18,18 +19,27 @@ func main() {
 	}
 
 	var (
-		app         = kingpin.New("uptest", "Automated Test Tool for Upbound Official Providers").DefaultEnvars()
-		exampleList = app.Flag("example-list", "List of example manifests. Value of this option will be used to trigger/configure the tests."+
+		app = kingpin.New("uptest", "Automated Test Tool for Upbound Official Providers").DefaultEnvars()
+
+		e2e          = app.Command("e2e", "Run e2e tests for manifests by applying them to a control plane and waiting until a given condition is met.")
+		manifestList = e2e.Arg("manifest-list", "List of manifests. Value of this option will be used to trigger/configure the tests."+
 			"The possible usage:\n"+
 			"'provider-aws/examples/s3/bucket.yaml,provider-gcp/examples/storage/bucket.yaml': "+
 			"The comma separated resources are used as test inputs.\n"+
-			"If this option is not set, 'EXAMPLE_LIST' env var is used as default.").Envar("EXAMPLE_LIST").String()
-		dataSourcePath = app.Flag("data-source", "File path of data source that will be used for injection some values.").Default("").String()
+			"If this option is not set, 'MANIFEST_LIST' env var is used as default.").Envar("MANIFEST_LIST").String()
+		dataSourcePath = e2e.Flag("data-source", "File path of data source that will be used for injection some values.").Default("").String()
+		setupScript    = e2e.Flag("setup-script", "Script that will be executed before running tests.").Default("").String()
+		teardownScript = e2e.Flag("teardown-script", "Script that will be executed after running tests.").Default("").String()
+
+		defaultTimeout = e2e.Flag("default-timeout", "Default timeout in seconds for the test.\n"+
+			"Timeout could be overridden per resource using \"uptest.upbound.io/timeout\" annotation.").Default("1200").Int()
+		defaultConditions = e2e.Flag("default-conditions", "Comma seperated list of default conditions to wait for a successful test.\n"+
+			"Conditions could be overridden per resource using \"uptest.upbound.io/conditions\" annotation.").Default("Ready").String()
 	)
 	kingpin.MustParse(app.Parse(os.Args[1:]))
 
 	var examplePaths []string
-	for _, e := range strings.Split(*exampleList, ",") {
+	for _, e := range strings.Split(*manifestList, ",") {
 		if e == "" {
 			continue
 		}
@@ -40,9 +50,29 @@ func main() {
 		return
 	}
 
-	o := &internal.AutomatedTestOptions{
-		ExamplePaths:   examplePaths,
-		DataSourcePath: *dataSourcePath,
+	setupPath := ""
+	if *setupScript != "" {
+		setupPath, err = filepath.Abs(*setupScript)
+		if err != nil {
+			kingpin.FatalIfError(err, "cannot get absolute path of setup script")
+		}
+	}
+
+	teardownPath := ""
+	if *teardownScript != "" {
+		teardownPath, err = filepath.Abs(*teardownScript)
+		if err != nil {
+			kingpin.FatalIfError(err, "cannot get absolute path of teardown script")
+		}
 	}
-	kingpin.FatalIfError(internal.RunTest(o), "cannot run automated tests successfully")
+	o := &config.AutomatedTest{
+		ManifestPaths:      examplePaths,
+		DataSourcePath:     *dataSourcePath,
+		SetupScriptPath:    setupPath,
+		TeardownScriptPath: teardownPath,
+		DefaultConditions:  strings.Split(*defaultConditions, ","),
+		DefaultTimeout:     *defaultTimeout,
+	}
+
+	kingpin.FatalIfError(internal.RunTest(o), "cannot run e2e tests successfully")
 }
diff --git a/go.mod b/go.mod
@@ -3,29 +3,46 @@ module github.com/upbound/uptest
 go 1.18
 
 require (
-	github.com/pkg/errors v0.9.1
+	github.com/crossplane/crossplane-runtime v0.18.0
+	github.com/google/go-cmp v0.5.6
 	gopkg.in/alecthomas/kingpin.v2 v2.2.6
-	gopkg.in/yaml.v2 v2.4.0
 	k8s.io/apimachinery v0.23.0
 	sigs.k8s.io/yaml v1.3.0
 )
 
 require (
 	github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect
 	github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/evanphx/json-patch v4.12.0+incompatible // indirect
 	github.com/go-logr/logr v1.2.0 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
-	github.com/google/go-cmp v0.5.6 // indirect
+	github.com/golang/protobuf v1.5.2 // indirect
 	github.com/google/gofuzz v1.1.0 // indirect
+	github.com/googleapis/gnostic v0.5.5 // indirect
+	github.com/imdario/mergo v0.3.12 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
 	golang.org/x/net v0.0.0-20210825183410-e898025ed96a // indirect
+	golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f // indirect
+	golang.org/x/sys v0.0.0-20211029165221-6e7872819dc8 // indirect
+	golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b // indirect
 	golang.org/x/text v0.3.7 // indirect
+	golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac // indirect
+	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
+	google.golang.org/appengine v1.6.7 // indirect
+	google.golang.org/protobuf v1.27.1 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
+	k8s.io/api v0.23.0 // indirect
+	k8s.io/client-go v0.23.0 // indirect
 	k8s.io/klog/v2 v2.30.0 // indirect
+	k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65 // indirect
 	k8s.io/utils v0.0.0-20210930125809-cb0fa318a74b // indirect
+	sigs.k8s.io/controller-runtime v0.11.0 // indirect
 	sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6 // indirect
 	sigs.k8s.io/structured-merge-diff/v4 v4.2.0 // indirect
 )