run-api-test.yml

name: API Test

on:
  workflow_call:
    inputs:
      environment:
        type: string
        required: true
        description: "where to test"
  workflow_dispatch:
    inputs:
      environment:
        type: choice
        required: true
        description: "where to test"
        default: "tools"
        options:
          - tools
          - production
  repository_dispatch:
    types:
      - webhook

jobs:
  define-test-matrix:
    runs-on: ubuntu-latest
    environment: ${{ inputs.environment }}
    outputs:
      tests: ${{ steps.tests.outputs.tests }}
    steps:
      - uses: actions/checkout@v4
      - id: tests
        working-directory: bin/si-api-test/tests
        run: |
          # Find .ts files, remove ./ prefix, and format as JSON array
          files=$(find "." -mindepth 1 -maxdepth 1 -type f -name "*.ts" | sed -r "s/\.\/(.*)\.ts/\1/" | sort)
          # Get the number of tests
          test_count=$(echo "$files" | wc -l)

          # Get the list of workspace IDs from the environment variable
          workspace_ids="${{ vars.API_TEST_WORKSPACE_IDS }}"
          echo "workspace_ids found to be $workspace_ids"
          workspace_count=$(echo "$workspace_ids" | tr ',' '\n' | wc -l)

          # Validate that the number of workspace IDs is greater than or equal to the number of tests
          if [ "$workspace_count" -lt "$test_count" ]; then
              echo "Error: The number of workspace IDs ($workspace_count) is less than the number of tests ($test_count)."
              exit 1
          fi

          # Format files as JSON array with correct numbering and sorted order
          indexed_files=$(echo "$files" | awk '{print "{\"name\": \"" $0 "\", \"index\": " NR-1 "}"}' | jq -s .)

          # Ensure indexed_files are formatted correctly
          test_output=$(echo "$indexed_files" | jq -c '.')
          echo "tests=$test_output" >> "$GITHUB_OUTPUT"
          echo "$test_output"

  api-test:
    name: API Test SDF
    environment: ${{ inputs.environment }}
    runs-on: ubuntu-latest
    needs: define-test-matrix
    strategy:
      fail-fast: false
      matrix:
        tests: ${{ fromJSON(needs.define-test-matrix.outputs.tests) }}
    env:
      SDF_API_URL: ${{ vars.SDF_API_URL }}
      AUTH_API_URL: ${{ vars.AUTH_API_URL }}
    outputs:
      last_exit_code: ${{ steps.capture-exit-code.outputs.last_exit_code }}
    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Install Deno
        uses: denoland/setup-deno@v1
        with:
          deno-version: v1.x

      - name: Push failure of exit 53 to test code randomly
        run: |
          [[ "$(( ( RANDOM % 5 )  + 1 ))" == "3" ]] && exit 53
          exit 0

      - name: Run the deno exec with retry
        run: |
          cd bin/si-api-test
          
          echo "Running test ${{ matrix.tests.name }} with index: ${{ matrix.tests.index }}"

          # Split the workspace IDs into an array
          workspace_ids="${{ vars.API_TEST_WORKSPACE_IDS }}"
          IFS=',' read -r -a workspace_array <<< "$workspace_ids"
          
          # Pick the correct workspace ID based on the index
          workspace_id=${workspace_array[${{ matrix.tests.index }}]}

          echo "Using workspace ID: $workspace_id"

          # Retry loop with 5 attempts
          n=0
          max_retries=5
          exit_code=0
          
          until [ $n -ge $max_retries ]
          do
            unset exit_code || echo "exit_code not set"

            # Run the deno task and store exit code in a variable
            deno task run \
            --workspaceId "$workspace_id" \
            --userId ${{ secrets.API_TEST_EMAIL }} \
            --password ${{ secrets.API_TEST_PASSWORD }} \
            --tests ${{ matrix.tests.name }} || exit_code=$?
            
            # Check the exit code
            if [ -z "$exit_code" ]; then
              echo "Deno task succeeded!"
              break
            fi

            n=$((n+1))
            echo "Attempt $n/$max_retries failed with exit code $exit_code! Retrying..."
            sleep 60
          done

          if [ $n -ge $max_retries ]; then
            echo "All $max_retries attempts failed."
            exit_code=0
          fi

          echo "last_exit_code=$exit_code" >> "$GITHUB_OUTPUT"
          exit $last_exit_code

  on-failure:
    runs-on: ubuntu-latest
    needs: api-test
    environment: ${{ inputs.environment }}
    if: ${{ failure() }} #&& github.ref == 'refs/heads/main' }} #TODO(johnrwatson): disable comment before merging
    steps:
      - run: |
          VALID_FAILURE_EXIT_CODES="53"
          echo "consolidated:"
          echo "${{ needs.api-test.outputs.exit_code }}"
          echo "alt:"
          echo "${{ needs.api-test.outputs.exit_code }} | jq"
          exit_codes='[${{ needs.api-test.outputs.exit_code}}]  -r '.[]''
          # Check if any exit code is "bad"
          for code in $exit_codes; do
            if echo "$VALID_FAILURE_EXIT_CODES" | grep -qw "$code"; then
              echo "Bad exit code detected: $code"
              curl --location "${{ secrets.FIREHYDRANT_WEBHOOK_URL }}" \
              --header "Content-Type: application/json" \
              --data "{
                \"summary\": \"API ${{ inputs.environment }} Tests Fail\",
                \"body\": \"API Tests have failed for ${{ inputs.environment }}.\",
                \"links\": [
                  {
                    \"href\": \"https://github.com/systeminit/si/actions/runs/$GITHUB_RUN_ID\",
                    \"text\": \"E2E Test Run ${{ inputs.environment }}\"
                  }
                ],
                \"tags\": [
                  \"service:github\"
                ]
              }" 
            fi
          done
      - run: |
          curl --location "${{ secrets.SLACK_WEBHOOK_URL }}" -X POST \
            --header 'Content-type: application/json' \
            --data "{\"text\": \":si: Failed API Tests for ${{ inputs.environment }}: <https://github.com/systeminit/si/actions/runs/$GITHUB_RUN_ID|:test_tube: Link>\"}"