ci: rework e2e tests

This achieves the following goals: 1. Only run the openssl tests for PRs. 2. Only run the servicemesh tests when its files have been touched. 3. Run all other tests every day at night (and send notifications on failure). 4. Keep the manual dispatch workflow. I don't think it's possible to reasonably achieve all of those with a single workflow file, so I've turned e2e.yaml into a reusable workflow that's invoked by other workflow files with different parameters.
edgelesssys · Oct 4, 2024 · 4b00a65 · 4b00a65
1 parent 9458437
commit 4b00a65
Show file tree

Hide file tree

Showing 5 changed files with 217 additions and 35 deletions.
diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml
@@ -1,50 +1,42 @@
 name: e2e test
 
 on:
- workflow_dispatch:
+ workflow_call:
  inputs:
  skip-undeploy:
  description: "Skip undeploy"
- required: false
  type: boolean
- default: false
- pull_request:
- paths-ignore:
- - dev-docs/**
- - docs/**
- - rfc/**
- - tools/asciinema/**
- - tools/vale/**
+ test-name:
+ description: "Test Name"
+ type: string
+ platform:
+ description: "Platform"
+ type: string
+ runner:
+ description: "Runner"
+ type: string
+ self-hosted:
+ description: "Self Hosted"
+ type: boolean
+ send-failure-notifications:
+ description: "Send notifications on failure"
+ type: boolean
 
 env:
  container_registry: ghcr.io/edgelesssys
  azure_resource_group: contrast-ci
  DO_NOT_TRACK: 1
 
 jobs:
- test_matrix:
- strategy:
- matrix:
- platform:
- - name: AKS-CLH-SNP
- runner: ubuntu-22.04
- self-hosted: false
- - name: K3s-QEMU-SNP
- runner: SNP
- self-hosted: true
- - name: K3s-QEMU-TDX
- runner: TDX
- self-hosted: true
- test_name: [servicemesh, openssl, policy, workloadsecret, volumestatefulset]
- fail-fast: false
- name: "${{ matrix.platform.name }} / ${{ matrix.test_name }}"
- runs-on: ${{ matrix.platform.runner }}
+ test:
+ name: "${{ inputs.test-name }}"
+ runs-on: ${{ inputs.runner }}
  permissions:
  contents: read
  packages: write
  steps:
  - uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0
- - if: ${{ !matrix.platform.self-hosted }}
+ - if: ${{ !inputs.self-hosted }}
  uses: ./.github/actions/setup_nix
  with:
  githubToken: ${{ secrets.GITHUB_TOKEN }}
@@ -55,7 +47,7 @@ jobs:
  registry: ghcr.io
  username: ${{ github.actor }}
  password: ${{ secrets.GITHUB_TOKEN }}
- - if: ${{ !matrix.platform.self-hosted }}
+ - if: ${{ !inputs.self-hosted }}
  name: Login to Azure
  uses: azure/login@a65d910e8af852a8061c627c456678983e180302 # v2.2.0
  with:
@@ -67,11 +59,11 @@ jobs:
  container_registry=${{ env.container_registry }}
  azure_resource_group=${{ env.azure_resource_group }}
  EOF
- - if: ${{ !matrix.platform.self-hosted }}
+ - if: ${{ !inputs.self-hosted }}
  name: Get credentials for CI cluster
  run: |
  just get-credentials
- - if: ${{ !matrix.platform.self-hosted }}
+ - if: ${{ !inputs.self-hosted }}
  name: Set sync environment
  run: |
  sync_ip=$(kubectl get svc sync -o jsonpath='{.status.loadBalancer.ingress[0].ip}')
@@ -80,21 +72,27 @@ jobs:
  echo "SYNC_FIFO_UUID=$sync_uuid" | tee -a "$GITHUB_ENV"
  - name: Build and prepare deployments
  run: |
- just coordinator initializer port-forwarder openssl cryptsetup service-mesh-proxy node-installer ${{ matrix.platform.name }}
+ just coordinator initializer port-forwarder openssl cryptsetup service-mesh-proxy node-installer ${{ inputs.platform }}
  - name: E2E Test
  run: |
  nix run .#scripts.get-logs workspace/e2e.namespace &
- nix shell -L .#contrast.e2e --command ${{ matrix.test_name }}.test -test.v \
+ nix shell -L .#contrast.e2e --command ${{ inputs.test-name }}.test -test.v \
  --image-replacements workspace/just.containerlookup \
  --namespace-file workspace/e2e.namespace \
- --platform ${{ matrix.platform.name }} \
+ --platform ${{ inputs.platform }} \
  --skip-undeploy="${{ inputs.skip-undeploy && 'true' || 'false' }}"
  - name: Upload logs
  if: always()
  uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0
  with:
- name: e2e_pod_logs-${{ matrix.platform.name }}-${{ matrix.test_name }}
+ name: e2e_pod_logs-${{ inputs.platform }}-${{ inputs.test-name }}
  path: workspace/namespace-logs
+ - name: Notify teams channel of failure
+ if: ${{ failure() && inputs.send-failure-notifications }}
+ uses: ./.github/actions/post_to_teams
+ with:
+ webhook: ${{ secrets.TEAMS_CI_WEBHOOK }}
+ message: "e2e test ${{ inputs.test-name }} failed"
  - name: Cleanup
  if: cancelled() && !inputs.skip-undeploy
  run: |

diff --git a/.github/workflows/e2e_manual.yaml b/.github/workflows/e2e_manual.yaml
@@ -0,0 +1,74 @@
+name: e2e test
+
+on:
+ workflow_dispatch:
+ inputs:
+ test-name:
+ description: "Test Name"
+ required: true
+ type: choice
+ options:
+ - openssl
+ - servicemesh
+ - policy
+ - workloadsecret
+ - volumestatefulset
+ default: "openssl"
+ platform:
+ description: "Platform"
+ required: true
+ type: choice
+ options:
+ - AKS-CLH-SNP
+ - K3s-QEMU-SNP
+ - K3s-QEMU-TDX
+ skip-undeploy:
+ description: "Skip undeploy"
+ required: false
+ type: boolean
+ default: false
+
+jobs:
+ determine-platform-params:
+ runs-on: ubuntu-22.04
+ outputs:
+ runner: ${{ steps.determine-platform-params.outputs.runner }}
+ self-hosted: ${{ steps.determine-platform-params.outputs.self-hosted }}
+ steps:
+ - name: Determine Platform Parameters
+ id: determine-platform-params
+ run: |
+ case ${{ inputs.platform }} in
+ "AKS-CLH-SNP")
+ echo "runner=ubuntu-22.04" >> "$GITHUB_OUTPUT"
+ echo "self-hosted=false" >> "$GITHUB_OUTPUT"
+ ;;
+ "K3s-QEMU-SNP")
+ echo "runner=SNP" >> "$GITHUB_OUTPUT"
+ echo "self-hosted=true" >> "$GITHUB_OUTPUT"
+ ;;
+ "K3s-QEMU-TDX")
+ echo "runner=TDX" >> "$GITHUB_OUTPUT"
+ echo "self-hosted=true" >> "$GITHUB_OUTPUT"
+ ;;
+ *)
+ echo "Unsupported platform: {{ platform }}"
+ exit 1
+ ;;
+ esac
+
+ test:
+ name: "${{ inputs.platform }}"
+ needs: [determine-platform-params]
+ uses: ./.github/workflows/e2e.yaml
+ with:
+ skip-undeploy: ${{ inputs.skip-undeploy }}
+ test-name: ${{ inputs.test-name }}
+ platform: ${{ inputs.platform }}
+ runner: ${{ needs.determine-platform-params.outputs.runner }}
+ self-hosted: ${{ fromJSON(needs.determine-platform-params.outputs.self-hosted) }}
+ send-failure-notifications: false
+ secrets: inherit
+ permissions:
+ contents: read
+ packages: write
diff --git a/.github/workflows/e2e_nightly.yaml b/.github/workflows/e2e_nightly.yaml
@@ -0,0 +1,35 @@
+name: e2e test
+
+on:
+ schedule:
+ - cron: "30 4 * * *" # 4:30 a.m. every day
+
+jobs:
+ test_matrix:
+ strategy:
+ matrix:
+ platform:
+ - name: AKS-CLH-SNP
+ runner: ubuntu-22.04
+ self-hosted: false
+ - name: K3s-QEMU-SNP
+ runner: SNP
+ self-hosted: true
+ - name: K3s-QEMU-TDX
+ runner: TDX
+ self-hosted: true
+ test-name: [servicemesh, openssl, policy, workloadsecret, volumestatefulset]
+ fail-fast: false
+ name: "${{ matrix.platform.name }}"
+ uses: ./.github/workflows/e2e.yaml
+ with:
+ skip-undeploy: false
+ test-name: ${{ matrix.test-name }}
+ platform: ${{ matrix.platform.name }}
+ runner: ${{ matrix.platform.runner }}
+ self-hosted: ${{ matrix.platform.self-hosted }}
+ send-failure-notifications: true
+ secrets: inherit
+ permissions:
+ contents: read
+ packages: write
diff --git a/.github/workflows/e2e_openssl.yaml b/.github/workflows/e2e_openssl.yaml
@@ -0,0 +1,39 @@
+name: e2e test
+
+on:
+ pull_request:
+ paths-ignore:
+ - dev-docs/**
+ - docs/**
+ - rfc/**
+ - tools/asciinema/**
+ - tools/vale/**
+
+jobs:
+ test_matrix:
+ strategy:
+ matrix:
+ platform:
+ - name: AKS-CLH-SNP
+ runner: ubuntu-22.04
+ self-hosted: false
+ - name: K3s-QEMU-SNP
+ runner: SNP
+ self-hosted: true
+ - name: K3s-QEMU-TDX
+ runner: TDX
+ self-hosted: true
+ fail-fast: false
+ name: "${{ matrix.platform.name }}"
+ uses: ./.github/workflows/e2e.yaml
+ with:
+ skip-undeploy: false
+ test-name: openssl
+ platform: ${{ matrix.platform.name }}
+ runner: ${{ matrix.platform.runner }}
+ self-hosted: ${{ matrix.platform.self-hosted }}
+ send-failure-notifications: false
+ secrets: inherit
+ permissions:
+ contents: read
+ packages: write
diff --git a/.github/workflows/e2e_service_mesh.yaml b/.github/workflows/e2e_service_mesh.yaml
@@ -0,0 +1,36 @@
+name: e2e test
+
+on:
+ pull_request:
+ paths:
+ - e2e/servicemesh/**
+ - service-mesh/**
+
+jobs:
+ test_matrix:
+ strategy:
+ matrix:
+ platform:
+ - name: AKS-CLH-SNP
+ runner: ubuntu-22.04
+ self-hosted: false
+ - name: K3s-QEMU-SNP
+ runner: SNP
+ self-hosted: true
+ - name: K3s-QEMU-TDX
+ runner: TDX
+ self-hosted: true
+ fail-fast: false
+ name: "${{ matrix.platform.name }}"
+ uses: ./.github/workflows/e2e.yaml
+ with:
+ skip-undeploy: false
+ test-name: servicemesh
+ platform: ${{ matrix.platform.name }}
+ runner: ${{ matrix.platform.runner }}
+ self-hosted: ${{ matrix.platform.self-hosted }}
+ send-failure-notifications: false
+ secrets: inherit
+ permissions:
+ contents: read
+ packages: write