Merge pull request #39 from turkenh/main

Add reusable workflows and Guide for end to end testing with Uptest

.github/workflows/pr-comment-trigger.yml

@@ -0,0 +1,180 @@
+name: Run Uptest
+
+on:
+  workflow_call:
+    inputs:
+      trigger-keyword:
+        description: 'Keyword to trigger the workflow, defaults to /test-examples'
+        default: '/test-examples'
+        required: false
+        type: string
+      go-version:
+        description: 'Go version to use if building needs to be done'
+        default: '1.19'
+        required: false
+        type: string
+    secrets:
+      UPTEST_CLOUD_CREDENTIALS:
+        description: 'Uptest cloud credentials to be passed to the uptest target as environment variable'
+        required: true
+      UPTEST_DATASOURCE:
+        description: 'A set of key-value pairs to be injected into the uptest'
+        required: true
+
+jobs:
+  debug:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Debug
+        run: |
+          echo "Trigger keyword: ${{ inputs.trigger-keyword }}"
+          echo "Go version: ${{ inputs.go-version }}"
+          echo "github.event.comment.author_association: ${{ github.event.comment.author_association }}"
+          echo "github.event.comment.body: ${{ github.event.comment.body }}"
+
+  get-example-list:
+    if: ${{ (github.event.comment.author_association == 'OWNER' || github.event.comment.author_association == 'MEMBER' || github.event.comment.author_association == 'COLLABORATOR' ) &&
+            github.event.issue.pull_request &&
+            contains(github.event.comment.body, inputs.trigger-keyword ) }}
+    runs-on: ubuntu-22.04
+    outputs:
+      example_list: ${{ steps.get-example-list-name.outputs.example-list }}
+      example_hash: ${{ steps.get-example-list-name.outputs.example-hash }}
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          submodules: true
+
+      - name: Checkout PR
+        id: checkout-pr
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          gh pr checkout ${{ github.event.issue.number }}
+          git submodule update --init --recursive
+          OUTPUT=$(git log -1 --format='%H')
+          echo "commit-sha=$OUTPUT" >> $GITHUB_OUTPUT
+
+      - name: Prepare The Example List
+        env:
+          COMMENT: ${{ github.event.comment.body }}
+        id: get-example-list-name
+        run: |
+          PATHS=$(echo $COMMENT | sed 's/^.*\${{ inputs.trigger-keyword }}="//g' | cut -d '"' -f 1 | sed 's/,/ /g')
+          EXAMPLE_LIST=""
+          for P in $PATHS; do EXAMPLE_LIST="${EXAMPLE_LIST},$(find $P -name *.yaml | tr '\n' ',')"; done
+
+          sudo apt-get -y install coreutils
+          EXAMPLE_HASH=$(echo ${EXAMPLE_LIST} | md5sum | cut -f1 -d" ")
+
+          echo "Examples: ${EXAMPLE_LIST:1}"
+          echo "Example Hash: ${EXAMPLE_HASH}"
+
+          echo "example-list=${EXAMPLE_LIST:1}" >> $GITHUB_OUTPUT
+          echo "example-hash=${EXAMPLE_HASH}" >> $GITHUB_OUTPUT
+
+      - name: Create Pending Status Check
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          gh api \
+            --method POST \
+            -H "Accept: application/vnd.github+json" \
+            /repos/${{ github.repository }}/statuses/${{ steps.checkout-pr.outputs.commit-sha }} \
+            -f state='pending' \
+            -f target_url='https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}' \
+            -f description='Running...' \
+            -f context="Uptest-${{ steps.get-example-list-name.outputs.example-hash }}"
+
+  uptest:
+    if: ${{ (github.event.comment.author_association == 'OWNER' || github.event.comment.author_association == 'MEMBER' || github.event.comment.author_association == 'COLLABORATOR' ) &&
+      github.event.issue.pull_request &&
+      contains(github.event.comment.body, inputs.trigger-keyword ) }}
+    runs-on: ubuntu-22.04
+    needs: get-example-list
+
+    steps:
+      - name: Setup QEMU
+        uses: docker/setup-qemu-action@v1
+        with:
+          platforms: all
+
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          submodules: true
+
+      - name: Setup Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: ${{ inputs.go-version }}
+
+      - name: Checkout PR
+        id: checkout-pr
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          gh pr checkout ${{ github.event.issue.number }}
+          git submodule update --init --recursive
+          OUTPUT=$(git log -1 --format='%H')
+          echo "commit-sha=$OUTPUT" >> $GITHUB_OUTPUT
+
+      - name: Run Uptest
+        id: run-uptest
+        env:
+          UPTEST_CLOUD_CREDENTIALS: ${{ secrets.UPTEST_CLOUD_CREDENTIALS }}
+          UPTEST_EXAMPLE_LIST: ${{ needs.get-example-list.outputs.example_list }}
+          UPTEST_TEST_DIR: ./_output/controlplane-dump
+          UPTEST_DATASOURCE_PATH: .work/uptest-datasource.yaml
+        run: |
+          mkdir -p .work && echo ${{ secrets.UPTEST_DATASOURCE }} > .work/uptest-datasource.yaml
+          make e2e
+
+      - name: Create Successful Status Check
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          EXAMPLE_HASH: ${{ needs.get-example-list.outputs.example_hash }}
+        run: |
+          gh api \
+            --method POST \
+            -H "Accept: application/vnd.github+json" \
+            /repos/${{ github.repository }}/statuses/${{ steps.checkout-pr.outputs.commit-sha }} \
+            -f state='success' \
+            -f target_url='https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}' \
+            -f description='Passed' \
+            -f context="Uptest-${EXAMPLE_HASH}"
+
+      - name: Collect Cluster Dump
+        if: always()
+        run: |
+          make controlplane.dump
+
+      - name: Upload Cluster Dump
+        if: always()
+        uses: actions/upload-artifact@v3
+        with:
+          name: controlplane-dump
+          path: ./_output/controlplane-dump
+
+      - name: Cleanup
+        if: always()
+        run: |
+          eval $(make --no-print-directory build.vars)
+          ${KUBECTL} delete managed --all || true
+
+      - name: Create Unsuccessful Status Check
+        if: failure()
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          EXAMPLE_HASH: ${{ needs.get-example-list.outputs.example_hash }}
+        run: |
+          gh api \
+            --method POST \
+            -H "Accept: application/vnd.github+json" \
+            /repos/${{ github.repository }}/statuses/${{ steps.checkout-pr.outputs.commit-sha }} \
+            -f state='failure' \
+            -f target_url='https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}' \
+            -f description='Failed' \
+            -f context="Uptest-${EXAMPLE_HASH}"

Makefile

@@ -22,4 +22,11 @@ submodules:
 	@git submodule sync
 	@git submodule update --init --recursive
 
-.PHONY: submodules fallthrough
+.PHONY: submodules fallthrough
+
+-include build/makelib/k8s_tools.mk
+-include build/makelib/controlplane.mk
+
+uptest:
+	@echo "Running uptest"
+	@printenv

README.md

@@ -34,12 +34,71 @@ Args:
 Uptest expects a running control-plane (a.k.a. k8s + crossplane) where required providers are running and/or required
 configuration were applied.
 
-### Example: 
+Example run: 
 
 ```shell
 uptest e2e examples/user.yaml,examples/bucket.yaml --setup-script="test/hooks/setup.sh"
 ```
 
+### Injecting Dynamic Values (and Datasource)
+
+Uptest supports injecting dynamic values into the examples by using a data source. The data source is a yaml file
+storing key-value pairs. The values can be used in the examples by using the following syntax:
+
+```
+${data.key}
+```
+
+Example data source file content:
+
+```yaml
+aws_account_id: 123456789012
+aws_region: us-east-1
+```
+
+Example manifest:
+
+```yaml
+apiVersion: athena.aws.upbound.io/v1beta1
+kind: DataCatalog
+metadata:
+  labels:
+    testing.upbound.io/example-name: example
+  name: example
+spec:
+  forProvider:
+    description: Example Athena data catalog
+    parameters:
+      function: arn:aws:lambda:${data.aws_region}:${data.aws_account_id}:function:upbound-example-function
+    region: us-west-1
+    tags:
+      Name: example-athena-data-catalog
+    type: LAMBDA
+```
+
+Uptest also supports generating random strings as follows:
+
+```
+${Rand.RFC1123Subdomain}
+```
+
+Example Manifest:
+
+```yaml
+apiVersion: s3.aws.upbound.io/v1beta1
+kind: Bucket
+metadata:
+  name: ${Rand.RFC1123Subdomain}
+  labels:
+    testing.upbound.io/example-name: s3
+spec:
+  forProvider:
+    region: us-west-1
+    objectLockEnabled: true
+    tags:
+      Name: SampleBucket
+```
+
 ### Hooks
 
 There are 6 types of hooks that can be used to customize the test flow:

cmd/main.go

@@ -1,7 +1,6 @@
 package main
 
 import (
-	"fmt"
 	"os"
 	"path/filepath"
 	"strings"
@@ -27,7 +26,7 @@ func main() {
 			"'provider-aws/examples/s3/bucket.yaml,provider-gcp/examples/storage/bucket.yaml': "+
 			"The comma separated resources are used as test inputs.\n"+
 			"If this option is not set, 'MANIFEST_LIST' env var is used as default.").Envar("MANIFEST_LIST").String()
-		dataSourcePath = e2e.Flag("data-source", "File path of data source that will be used for injection some values.").Default("").String()
+		dataSourcePath = e2e.Flag("data-source", "File path of data source that will be used for injection some values.").Envar("UPTEST_DATASOURCE_PATH").Default("").String()
 		setupScript    = e2e.Flag("setup-script", "Script that will be executed before running tests.").Default("").String()
 		teardownScript = e2e.Flag("teardown-script", "Script that will be executed after running tests.").Default("").String()
 
@@ -48,8 +47,7 @@ func main() {
 		examplePaths = append(examplePaths, filepath.Join(cd, filepath.Clean(e)))
 	}
 	if len(examplePaths) == 0 {
-		fmt.Println("No example files to test.")
-		return
+		kingpin.Fatalf("No manifest to test provided.")
 	}
 
 	setupPath := ""