diff --git a/.github/workflows/pr-comment-trigger.yml b/.github/workflows/pr-comment-trigger.yml new file mode 100644 index 0000000..2a66ded --- /dev/null +++ b/.github/workflows/pr-comment-trigger.yml @@ -0,0 +1,180 @@ +name: Run Uptest + +on: + workflow_call: + inputs: + trigger-keyword: + description: 'Keyword to trigger the workflow, defaults to /test-examples' + default: '/test-examples' + required: false + type: string + go-version: + description: 'Go version to use if building needs to be done' + default: '1.19' + required: false + type: string + secrets: + UPTEST_CLOUD_CREDENTIALS: + description: 'Uptest cloud credentials to be passed to the uptest target as environment variable' + required: true + UPTEST_DATASOURCE: + description: 'A set of key-value pairs to be injected into the uptest' + required: true + +jobs: + debug: + runs-on: ubuntu-latest + steps: + - name: Debug + run: | + echo "Trigger keyword: ${{ inputs.trigger-keyword }}" + echo "Go version: ${{ inputs.go-version }}" + echo "github.event.comment.author_association: ${{ github.event.comment.author_association }}" + echo "github.event.comment.body: ${{ github.event.comment.body }}" + + get-example-list: + if: ${{ (github.event.comment.author_association == 'OWNER' || github.event.comment.author_association == 'MEMBER' || github.event.comment.author_association == 'COLLABORATOR' ) && + github.event.issue.pull_request && + contains(github.event.comment.body, inputs.trigger-keyword ) }} + runs-on: ubuntu-22.04 + outputs: + example_list: ${{ steps.get-example-list-name.outputs.example-list }} + example_hash: ${{ steps.get-example-list-name.outputs.example-hash }} + + steps: + - name: Checkout + uses: actions/checkout@v2 + with: + submodules: true + + - name: Checkout PR + id: checkout-pr + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + gh pr checkout ${{ github.event.issue.number }} + git submodule update --init --recursive + OUTPUT=$(git log -1 --format='%H') + echo "commit-sha=$OUTPUT" >> $GITHUB_OUTPUT + + - name: Prepare The Example List + env: + COMMENT: ${{ github.event.comment.body }} + id: get-example-list-name + run: | + PATHS=$(echo $COMMENT | sed 's/^.*\${{ inputs.trigger-keyword }}="//g' | cut -d '"' -f 1 | sed 's/,/ /g') + EXAMPLE_LIST="" + for P in $PATHS; do EXAMPLE_LIST="${EXAMPLE_LIST},$(find $P -name *.yaml | tr '\n' ',')"; done + + sudo apt-get -y install coreutils + EXAMPLE_HASH=$(echo ${EXAMPLE_LIST} | md5sum | cut -f1 -d" ") + + echo "Examples: ${EXAMPLE_LIST:1}" + echo "Example Hash: ${EXAMPLE_HASH}" + + echo "example-list=${EXAMPLE_LIST:1}" >> $GITHUB_OUTPUT + echo "example-hash=${EXAMPLE_HASH}" >> $GITHUB_OUTPUT + + - name: Create Pending Status Check + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + gh api \ + --method POST \ + -H "Accept: application/vnd.github+json" \ + /repos/${{ github.repository }}/statuses/${{ steps.checkout-pr.outputs.commit-sha }} \ + -f state='pending' \ + -f target_url='https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}' \ + -f description='Running...' \ + -f context="Uptest-${{ steps.get-example-list-name.outputs.example-hash }}" + + uptest: + if: ${{ (github.event.comment.author_association == 'OWNER' || github.event.comment.author_association == 'MEMBER' || github.event.comment.author_association == 'COLLABORATOR' ) && + github.event.issue.pull_request && + contains(github.event.comment.body, inputs.trigger-keyword ) }} + runs-on: ubuntu-22.04 + needs: get-example-list + + steps: + - name: Setup QEMU + uses: docker/setup-qemu-action@v1 + with: + platforms: all + + - name: Checkout + uses: actions/checkout@v2 + with: + submodules: true + + - name: Setup Go + uses: actions/setup-go@v2 + with: + go-version: ${{ inputs.go-version }} + + - name: Checkout PR + id: checkout-pr + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + gh pr checkout ${{ github.event.issue.number }} + git submodule update --init --recursive + OUTPUT=$(git log -1 --format='%H') + echo "commit-sha=$OUTPUT" >> $GITHUB_OUTPUT + + - name: Run Uptest + id: run-uptest + env: + UPTEST_CLOUD_CREDENTIALS: ${{ secrets.UPTEST_CLOUD_CREDENTIALS }} + UPTEST_EXAMPLE_LIST: ${{ needs.get-example-list.outputs.example_list }} + UPTEST_TEST_DIR: ./_output/controlplane-dump + UPTEST_DATASOURCE_PATH: .work/uptest-datasource.yaml + run: | + mkdir -p .work && echo ${{ secrets.UPTEST_DATASOURCE }} > .work/uptest-datasource.yaml + make e2e + + - name: Create Successful Status Check + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + EXAMPLE_HASH: ${{ needs.get-example-list.outputs.example_hash }} + run: | + gh api \ + --method POST \ + -H "Accept: application/vnd.github+json" \ + /repos/${{ github.repository }}/statuses/${{ steps.checkout-pr.outputs.commit-sha }} \ + -f state='success' \ + -f target_url='https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}' \ + -f description='Passed' \ + -f context="Uptest-${EXAMPLE_HASH}" + + - name: Collect Cluster Dump + if: always() + run: | + make controlplane.dump + + - name: Upload Cluster Dump + if: always() + uses: actions/upload-artifact@v3 + with: + name: controlplane-dump + path: ./_output/controlplane-dump + + - name: Cleanup + if: always() + run: | + eval $(make --no-print-directory build.vars) + ${KUBECTL} delete managed --all || true + + - name: Create Unsuccessful Status Check + if: failure() + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + EXAMPLE_HASH: ${{ needs.get-example-list.outputs.example_hash }} + run: | + gh api \ + --method POST \ + -H "Accept: application/vnd.github+json" \ + /repos/${{ github.repository }}/statuses/${{ steps.checkout-pr.outputs.commit-sha }} \ + -f state='failure' \ + -f target_url='https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}' \ + -f description='Failed' \ + -f context="Uptest-${EXAMPLE_HASH}" \ No newline at end of file diff --git a/Makefile b/Makefile index a5d4e17..25cf83e 100644 --- a/Makefile +++ b/Makefile @@ -22,4 +22,11 @@ submodules: @git submodule sync @git submodule update --init --recursive -.PHONY: submodules fallthrough \ No newline at end of file +.PHONY: submodules fallthrough + +-include build/makelib/k8s_tools.mk +-include build/makelib/controlplane.mk + +uptest: + @echo "Running uptest" + @printenv \ No newline at end of file diff --git a/README.md b/README.md index d01d9e7..cfdb2a8 100644 --- a/README.md +++ b/README.md @@ -34,12 +34,71 @@ Args: Uptest expects a running control-plane (a.k.a. k8s + crossplane) where required providers are running and/or required configuration were applied. -### Example: +Example run: ```shell uptest e2e examples/user.yaml,examples/bucket.yaml --setup-script="test/hooks/setup.sh" ``` +### Injecting Dynamic Values (and Datasource) + +Uptest supports injecting dynamic values into the examples by using a data source. The data source is a yaml file +storing key-value pairs. The values can be used in the examples by using the following syntax: + +``` +${data.key} +``` + +Example data source file content: + +```yaml +aws_account_id: 123456789012 +aws_region: us-east-1 +``` + +Example manifest: + +```yaml +apiVersion: athena.aws.upbound.io/v1beta1 +kind: DataCatalog +metadata: + labels: + testing.upbound.io/example-name: example + name: example +spec: + forProvider: + description: Example Athena data catalog + parameters: + function: arn:aws:lambda:${data.aws_region}:${data.aws_account_id}:function:upbound-example-function + region: us-west-1 + tags: + Name: example-athena-data-catalog + type: LAMBDA +``` + +Uptest also supports generating random strings as follows: + +``` +${Rand.RFC1123Subdomain} +``` + +Example Manifest: + +```yaml +apiVersion: s3.aws.upbound.io/v1beta1 +kind: Bucket +metadata: + name: ${Rand.RFC1123Subdomain} + labels: + testing.upbound.io/example-name: s3 +spec: + forProvider: + region: us-west-1 + objectLockEnabled: true + tags: + Name: SampleBucket +``` + ### Hooks There are 6 types of hooks that can be used to customize the test flow: diff --git a/build b/build index b2a9a2d..2d1bdbe 160000 --- a/build +++ b/build @@ -1 +1 @@ -Subproject commit b2a9a2dd051c19c49397fe3e15eb08827a69d103 +Subproject commit 2d1bdbe92eb98b78a5bf78543f26b9a7fe9f7f1b diff --git a/cmd/main.go b/cmd/main.go index cf7ab1b..01318e4 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -1,7 +1,6 @@ package main import ( - "fmt" "os" "path/filepath" "strings" @@ -27,7 +26,7 @@ func main() { "'provider-aws/examples/s3/bucket.yaml,provider-gcp/examples/storage/bucket.yaml': "+ "The comma separated resources are used as test inputs.\n"+ "If this option is not set, 'MANIFEST_LIST' env var is used as default.").Envar("MANIFEST_LIST").String() - dataSourcePath = e2e.Flag("data-source", "File path of data source that will be used for injection some values.").Default("").String() + dataSourcePath = e2e.Flag("data-source", "File path of data source that will be used for injection some values.").Envar("UPTEST_DATASOURCE_PATH").Default("").String() setupScript = e2e.Flag("setup-script", "Script that will be executed before running tests.").Default("").String() teardownScript = e2e.Flag("teardown-script", "Script that will be executed after running tests.").Default("").String() @@ -48,8 +47,7 @@ func main() { examplePaths = append(examplePaths, filepath.Join(cd, filepath.Clean(e))) } if len(examplePaths) == 0 { - fmt.Println("No example files to test.") - return + kingpin.Fatalf("No manifest to test provided.") } setupPath := "" diff --git a/docs/integrating-uptest-for-e2e-testing.md b/docs/integrating-uptest-for-e2e-testing.md new file mode 100644 index 0000000..fc1b60e --- /dev/null +++ b/docs/integrating-uptest-for-e2e-testing.md @@ -0,0 +1,152 @@ +# Integrating Uptest for End to End Testing + +In this tutorial, we will integrate [uptest](https://github.com/upbound/uptest) to a Github repository to automate end to end +testing managed resources. While we will use a `Provider` repository as an example, the process will be identical for a +`Configuration` repository. + +Starting with a provider repository with no end to end testing capability, we will end up having: +- A make target to locally test examples end to end +- A GitHub action triggered for PRs whenever a comment as `/test-examples=` is left + +## Setting up the Make targets + +1. Go to the [demo repository](https://github.com/upbound/demo-uptest-integration) which contains a GitHub provider + generated using upjet and hit the `Use this template` button to initialize your demo repository under your own + GitHub organization. +1. Clone your demo repository on your local and `cd` into the root directory. +2. Initialize build submodule with + + ```bash + make submodules + ``` + +4. First we will add a simple setup script that will deploy a secret and a provider config for our provider. + + ```bash + mkdir -p cluster/test + touch cluster/test/setup.sh + chmod +x cluster/test/setup.sh + + cat < cluster/test/setup.sh + #!/usr/bin/env bash + set -aeuo pipefail + + echo "Running setup.sh" + echo "Creating cloud credential secret..." + \${KUBECTL} -n upbound-system create secret generic provider-secret --from-literal=credentials="{\"token\":\"\${UPTEST_CLOUD_CREDENTIALS}\"}" \ + --dry-run=client -o yaml | \${KUBECTL} apply -f - + + echo "Waiting until provider is healthy..." + \${KUBECTL} wait provider.pkg --all --for condition=Healthy --timeout 5m + + echo "Waiting for all pods to come online..." + \${KUBECTL} -n upbound-system wait --for=condition=Available deployment --all --timeout=5m + + echo "Creating a default provider config..." + cat < + UPTEST_EXAMPLE_LIST=examples/repository/repository.yaml make e2e + ``` + +You should see a `PASS` at the end of logs indicating everything worked fine. + +## Adding the GitHub workflow + +Now we have things working locally, let's add a GitHub workflow to automate end to end testing with CI. + +1. Run the following to add the GitHub workflow definition which will be triggered for `issue_comment` events and will call +uptests reusable workflow: + + ```bash + cat < .github/workflows/e2e.yaml + name: End to End Testing + + on: + issue_comment: + types: [created] + + jobs: + e2e: + uses: upbound/uptest/.github/workflows/pr-comment-trigger.yml@main + secrets: + UPTEST_CLOUD_CREDENTIALS: \${{ secrets.UPTEST_CLOUD_CREDENTIALS }} + UPTEST_DATASOURCE: \${{ secrets.UPTEST_DATASOURCE }} + EOF + ``` + + > See [Injecting Dynamic Values (and Datasource)](../README.md#injecting-dynamic-values-and-datasource) for more + > details on `UPTEST_DATASOURCE` secret. + +1. Commit and push to the `main` branch of the repository. + + ``` + git add .github/workflows/e2e.yaml + git commit -s -m "Add e2e workflow" + git push origin main + ``` + +3. Lastly, we need to add a Repository Secret with our GitHub token. + 1. Go to your repository Settings in GitHub UI. + 2. On the left side, select `Secrets` -> `Actions` under `Security` section. + 3. Hit `New repository secret` + 4. Enter `UPTEST_CLOUD_CREDENTIALS` as `Name` and your GitHub Token as `Secret` and hit `Add secret` + +## Testing via CI + +We are now ready to test our changes end to end via GitHub Actions. We will try that out by opening a test PR. + +1. Go to the `examples/repository/repository.yaml` file and make some wording changes in `description` field, e.g. + Add `CI -` as a prefix. +2. Create a PR with that change. +3. Add the following comment on the PR: + + ``` + /test-examples="examples/repository/repository.yaml" + ``` + +4. Check the Actions and follow how end to end testing goes.