e2e-aks-ci.yml

name: E2E on AKS

on:
  schedule:
    - cron:  '5 10 * * *'
  workflow_dispatch:
    inputs:
      ref:
        description: "checkout git branch/tag"
        required: true
        default: "main"
      keep_cluster:
        description: "Keep the cluster afterwards?"
        required: false
        default: "no"

env:
  GOARCH: amd64
  CGO_ENABLED: 0
  GINKGO_NODES: 1
  FLAKE_ATTEMPTS: 1
  AKS_MACHINE_TYPE: 'Standard_D3_v2'

jobs:
  aks-fleet-examples:
    runs-on: ubuntu-latest
    if: >
      github.repository == 'rancher/fleet'

    steps:
      -
        name: Checkout
        uses: actions/checkout@v4
        with:
          submodules: recursive
          fetch-depth: 0
      -
        name: Setup Go
        uses: actions/setup-go@v5
        with:
          go-version-file: 'go.mod'
          check-latest: true
      -
        name: Setup Ginkgo Test Framework
        run: go install github.com/onsi/ginkgo/v2/ginkgo

      # Follow https://github.com/marketplace/actions/azure-login#configure-deployment-credentials
      # az group create --name fleetCI --location eastus2
      # az ad sp create-for-rbac --name "fleetCI" --sdk-auth --role contributor \
      #   --scopes /subscriptions/{id}/resourceGroups/fleetCI
      -
        name: Login to Azure
        uses: azure/login@v2
        with:
          creds: '{"clientId":"${{ secrets.AZURE_CLIENT_ID }}","clientSecret":"${{ secrets.AZURE_CLIENT_SECRET }}","subscriptionId":"${{ secrets.AZURE_SUBSCRIPTION_ID }}","tenantId":"${{ secrets.AZURE_TENANT_ID }}"}'
      -
        name: Create AKS cluster
        id: create-cluster
        # We need to specify bash as a shell when a job is running on windows runner
        shell: bash
        run: |
          id=$RANDOM
          echo "ID=$id" >> $GITHUB_OUTPUT
          az aks create --resource-group fleetCI \
          --node-vm-size ${{ env.AKS_MACHINE_TYPE }} \
          --name fleetCI$id \
          --node-count 2 \
          --generate-ssh-keys

          az aks get-credentials --resource-group fleetCI \
          --name fleetCI$id \
          --file kubeconfig-fleet-ci

          # List existing clusters
          az aks list | jq '.[] | .name + " " + (.powerState|tostring)'
      -
        name: Build fleet binaries
        run: |
          ./.github/scripts/build-fleet-binaries.sh
      -
        name: Set up QEMU
        uses: docker/setup-qemu-action@v3
      -
        name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3
      -
        name: Get UUID
        id: uuid
        run: echo "uuid=$(uuidgen)" >> $GITHUB_OUTPUT
      -
        id: meta-fleet
        uses: docker/metadata-action@v5
        with:
          images: |
            ttl.sh/rancher/fleet-${{ steps.uuid.outputs.uuid }}
          tags: type=raw,value=1h
      -
        uses: docker/build-push-action@v6
        with:
          context: .
          file: package/Dockerfile
          build-args: |
            ARCH=${{ env.GOARCH }}
          push: true
          tags: ${{ steps.meta-fleet.outputs.tags }}
          labels: ${{ steps.meta-fleet.outputs.labels }}
      -
        id: meta-fleet-agent
        uses: docker/metadata-action@v5
        with:
          images: |
            ttl.sh/rancher/fleet-agent-${{ steps.uuid.outputs.uuid }}
          tags: type=raw,value=1h
      -
        uses: docker/build-push-action@v6
        with:
          context: .
          file: package/Dockerfile.agent
          build-args: |
            ARCH=${{ env.GOARCH }}
          push: true
          tags: ${{ steps.meta-fleet-agent.outputs.tags }}
          labels: ${{ steps.meta-fleet-agent.outputs.labels }}
      -
        name: Deploy Fleet
        run: |
          export KUBECONFIG="$GITHUB_WORKSPACE/kubeconfig-fleet-ci"
          kc_node=$(kubectl get nodes --no-headers -o name | head -n 1)
          node=${kc_node#node/}
          echo "${{ steps.meta-fleet.outputs.tags }} ${{ steps.meta-fleet-agent.outputs.tags }}"
          NODE=$node SHARDS='[{"id":"shard0"},{"id":"shard1"},{"id":"shard2"}]' ./.github/scripts/deploy-fleet.sh \
                                       ${{ steps.meta-fleet.outputs.tags }} \
                                       ${{ steps.meta-fleet-agent.outputs.tags }}
      -
        name: Fleet E2E Tests
        env:
          FLEET_E2E_NS: fleet-local
        run: |
          export KUBECONFIG="$GITHUB_WORKSPACE/kubeconfig-fleet-ci"
          ginkgo --github-output --label-filter='!infra-setup' e2e/single-cluster e2e/keep-resources
      -
        name: Acceptance Tests for Examples
        env:
          FLEET_E2E_NS: fleet-local
        run: |
          export KUBECONFIG="$GITHUB_WORKSPACE/kubeconfig-fleet-ci"
          ginkgo --github-output e2e/acceptance/single-cluster-examples
      -
        name: Fleet Tests Requiring Github Secrets
        # These tests can't run for PRs, because PRs don't have access to the secrets
        env:
          FLEET_E2E_NS: fleet-local
          GIT_REPO_URL: "git@github.com:fleetrepoci/testaks.git"
          GIT_REPO_HOST: "github.com"
          GIT_REPO_USER: "git"
          CI_OCI_USERNAME:  ${{ secrets.CI_OCI_USERNAME }}
          CI_OCI_PASSWORD:  ${{ secrets.CI_OCI_PASSWORD }}
        run: |
          export KUBECONFIG="$GITHUB_WORKSPACE/kubeconfig-fleet-ci"
          export GIT_SSH_KEY="$GITHUB_WORKSPACE/id_ecdsa"
          export GIT_SSH_PUBKEY="$GITHUB_WORKSPACE/id_ecdsa.pub"
          echo "${{ secrets.CI_AKS_SSH_KEY }}" > "$GIT_SSH_KEY"
          echo "${{ secrets.CI_AKS_SSH_PUBKEY }}" > "$GIT_SSH_PUBKEY"

          ginkgo --github-output e2e/require-secrets
      -
        name: Delete AKS cluster
        # We always tear down the cluster, to avoid costs. Except when running
        # manually and keep_cluster was set to "yes"
        if: ${{ always() && github.event.inputs.keep_cluster != 'yes' }}
        shell: bash
        run: |
          id="${{ steps.create-cluster.outputs.ID }}"
          az aks delete --resource-group fleetCI --name fleetCI$id --yes