connectors_tests.yml

name: Connector Ops CI - Connectors Acceptance Tests

concurrency:
  # This is the name of the concurrency group. It is used to prevent concurrent runs of the same workflow.
  #
  # - github.head_ref is only defined on PR runs, it makes sure that the concurrency group is unique for pull requests
  #  ensuring that only one run per pull request is active at a time.
  #
  # - github.run_id is defined on all runs, it makes sure that the concurrency group is unique for workflow dispatches.
  # This allows us to run multiple workflow dispatches in parallel.
  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
  cancel-in-progress: true

on:
  workflow_dispatch:
    inputs:
      test-connectors-options:
        description: "Options to pass to the 'airbyte-ci connectors test' command"
        default: "--modified"
      runner:
        description: "The runner to use for this job"
        default: "ci-runner-connector-test-large-dagger-0-6-4"
  pull_request:
    types:
      - opened
      - synchronize
      - ready_for_review
jobs:
  connectors_ci:
    name: Connectors CI
    timeout-minutes: 1440 # 24 hours
    runs-on: ${{ inputs.runner || 'ci-runner-connector-test-large-dagger-0-6-4'}}
    steps:
      - name: Checkout Airbyte
        uses: actions/checkout@v3
      - name: Check PAT rate limits
        run: |
          ./tools/bin/find_non_rate_limited_PAT \
            ${{ secrets.GH_PAT_BUILD_RUNNER_OSS }} \
            ${{ secrets.GH_PAT_BUILD_RUNNER_BACKUP }}
      - name: Extract branch name [WORKFLOW DISPATCH]
        shell: bash
        if: github.event_name == 'workflow_dispatch'
        run: echo "branch=${GITHUB_REF#refs/heads/}" >> $GITHUB_OUTPUT
        id: extract_branch
      - name: Fetch last commit id from remote branch [PULL REQUESTS]
        if: github.event_name == 'pull_request'
        id: fetch_last_commit_id_pr
        run: echo "commit_id=$(git ls-remote --heads origin ${{ github.head_ref }} | cut -f 1)" >> $GITHUB_OUTPUT
      - name: Fetch last commit id from remote branch [WORKFLOW DISPATCH]
        if: github.event_name == 'workflow_dispatch'
        id: fetch_last_commit_id_wd
        run: echo "commit_id=$(git rev-parse origin/${{ steps.extract_branch.outputs.branch }})" >> $GITHUB_OUTPUT
      - name: Test connectors [WORKFLOW DISPATCH]
        if: github.event_name == 'workflow_dispatch'
        uses: ./.github/actions/run-dagger-pipeline
        with:
          context: "manual"
          docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }}
          docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }}
          gcp_gsm_credentials: ${{ secrets.GCP_GSM_CREDENTIALS }}
          sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }}
          git_branch: ${{ steps.extract_branch.outputs.branch }}
          git_revision: ${{ steps.fetch_last_commit_id_pr.outputs.commit_id }}
          github_token: ${{ env.PAT }}
          s3_build_cache_access_key_id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
          s3_build_cache_secret_key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
          subcommand: "connectors ${{ github.event.inputs.test-connectors-options }} test"
      - name: Test connectors [PULL REQUESTS]
        if: github.event_name == 'pull_request'
        uses: ./.github/actions/run-dagger-pipeline
        with:
          context: "pull_request"
          docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }}
          docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }}
          gcp_gsm_credentials: ${{ secrets.GCP_GSM_CREDENTIALS }}
          sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }}
          git_branch: ${{ github.head_ref }}
          git_revision: ${{ steps.fetch_last_commit_id_pr.outputs.commit_id }}
          github_token: ${{ env.PAT }}
          s3_build_cache_access_key_id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
          s3_build_cache_secret_key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
          subcommand: "connectors --modified test"