airbyte-ci-tests.yml

name: Internal Poetry packages CI

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

on:
  workflow_dispatch:
    inputs:
      airbyte_ci_subcommand:
        description: "Subcommand to pass to the 'airbyte-ci test' command"
        default: "--poetry-package-path=airbyte-ci/connectors/pipelines"
  pull_request:
    types:
      - opened
      - reopened
      - synchronize
jobs:
  changes:
    runs-on: ubuntu-latest
    outputs:
      internal_poetry_packages: ${{ steps.changes.outputs.internal_poetry_packages }}

    steps:
      - name: Checkout Airbyte
        if: github.event_name != 'pull_request'
        uses: actions/checkout@v4
      - id: changes
        uses: dorny/paths-filter@v2
        with:
          # Note: expressions within a filter are OR'ed
          filters: |
            # This list is duplicated in `pipelines/airbyte_ci/test/__init__.py`
            internal_poetry_packages:
            - airbyte-ci/connectors/pipelines/**
            - airbyte-ci/connectors/base_images/**
            - airbyte-ci/connectors/common_utils/**
            - airbyte-ci/connectors/connector_ops/**
            - airbyte-ci/connectors/connectors_qa/**
            - airbyte-ci/connectors/ci_credentials/**
            - airbyte-ci/connectors/metadata_service/lib/**
            - airbyte-ci/connectors/metadata_service/orchestrator/**
            - airbyte-integrations/bases/connector-acceptance-test/**

  run-tests:
    needs: changes
    # We only run the Connectors CI job if there are changes to the connectors on a non-forked PR
    if: needs.changes.outputs.internal_poetry_packages == 'true'
    #name: Internal Poetry packages CI
    # To rename in a follow up PR
    name: Run Airbyte CI tests
    runs-on: tooling-test-large
    permissions:
      pull-requests: read
      statuses: write
    steps:
      # The run-tests job will be triggered if a fork made changes to the internal poetry packages.
      # We don't want forks to make changes to the internal poetry packages.
      # So we fail the job if the PR is from a fork, it will make the required CI check fail.
      - name: Check if PR is from a fork
        id: check-if-pr-is-from-fork
        if: github.event_name == 'pull_request'
        shell: bash
        run: |
          if [ "${{ github.event.pull_request.head.repo.fork }}" == "true" ]; then
            echo "PR is from a fork. Exiting workflow..."
            exit 78
          fi
      - name: Checkout Airbyte
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
          ref: ${{ github.event.pull_request.head.ref }}

      - name: Extract branch name [WORKFLOW DISPATCH]
        shell: bash
        if: github.event_name == 'workflow_dispatch'
        run: echo "branch=${GITHUB_REF#refs/heads/}" >> $GITHUB_OUTPUT
        id: extract_branch
      - name: Fetch last commit id from remote branch [PULL REQUESTS]
        if: github.event_name == 'pull_request'
        id: fetch_last_commit_id_pr
        run: echo "commit_id=$(git ls-remote --heads origin ${{ github.head_ref }} | cut -f 1)" >> $GITHUB_OUTPUT
      - name: Fetch last commit id from remote branch [WORKFLOW DISPATCH]
        if: github.event_name == 'workflow_dispatch'
        id: fetch_last_commit_id_wd
        run: echo "commit_id=$(git rev-parse origin/${{ steps.extract_branch.outputs.branch }})" >> $GITHUB_OUTPUT

      - name: Run poe tasks for modified internal packages [PULL REQUEST]
        if: github.event_name == 'pull_request'
        id: run-airbyte-ci-test-pr
        uses: ./.github/actions/run-airbyte-ci
        with:
          context: "pull_request"
          dagger_cloud_token: ${{ secrets.DAGGER_CLOUD_TOKEN_2 }}
          docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }}
          docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }}
          gcp_gsm_credentials: ${{ secrets.GCP_GSM_CREDENTIALS }}
          gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }}
          git_branch: ${{ github.head_ref }}
          git_revision: ${{ steps.fetch_last_commit_id_pr.outputs.commit_id }}
          github_token: ${{ secrets.GH_PAT_MAINTENANCE_OSS }}
          sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }}
          subcommand: "test --modified"

      - name: Run poe tasks for requested internal packages [WORKFLOW DISPATCH]
        id: run-airbyte-ci-test-workflow-dispatch
        if: github.event_name == 'workflow_dispatch'
        uses: ./.github/actions/run-airbyte-ci
        with:
          context: "manual"
          dagger_cloud_token: ${{ secrets.DAGGER_CLOUD_TOKEN_2 }}
          docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }}
          docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }}
          gcp_gsm_credentials: ${{ secrets.GCP_GSM_CREDENTIALS }}
          gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }}
          git_branch: ${{ steps.extract_branch.outputs.branch }}
          git_revision: ${{ steps.fetch_last_commit_id_pr.outputs.commit_id }}
          github_token: ${{ secrets.GH_PAT_MAINTENANCE_OSS }}
          sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }}
          subcommand: "test ${{ inputs.airbyte_ci_subcommand}}"