e2e_tests.yml

# several test modes:
# - ALL: run all modes below (only useful for manual trigger)
# - CUSTOM: choose git branch and endpoint (only useful for manual trigger)
# - MAIN_STAGING: main branch against STAGING
# - RELEASE_PREPROD: latest release branch against PREPROD
# - MAIN_LTS: main branch against LTS
# - PREV_RELEASE_TESTS_STAGING: SDK from main, previous release tests against STAGING
# - LTS_CYCLE_FIRST_SDK_STAGING: test with first SDK of LTS cycle, against STAGING

name: E2E tests
on:
  workflow_dispatch:
    inputs:
      runE2ETests:
        description: "Run E2E tests?"
        required: true
        type: boolean
        default: true
      runNotebookTests:
        description: "Run notebook tests?"
        required: true
        type: boolean
        default: true
      testMode:
        description: "Test mode: ALL (run all modes), CUSTOM (specify branch and endpoint), MAIN_STAGING (main branch against STAGING), RELEASE_PREPROD (latest release branch against PREPROD), MAIN_LTS (main branch against LTS), PREV_RELEASE_TESTS_STAGING (SDK from main, previous release tests against STAGING), LTS_CYCLE_FIRST_SDK_STAGING (test with first SDK of LTS cycle, against STAGING)"
        required: true
        type: choice
        default: "MAIN_STAGING"
        options:
          - ALL
          - CUSTOM
          - MAIN_STAGING
          - RELEASE_PREPROD
          - MAIN_LTS
          - PREV_RELEASE_TESTS_STAGING
          - LTS_CYCLE_FIRST_SDK_STAGING
      customBranch:
        description: "Custom branch to test a PR (only used if test mode is CUSTOM)"
        required: false
        type: string
        default: ""
      customEndpoint:
        description: "Custom endpoint to test a PR (only used if test mode is CUSTOM)"
        required: false
        type: choice
        default: "STAGING"
        options:
          - "STAGING"
          - "PROD"
          - "PREPROD"
          - "LTS"
  push:
    branches:
      - "release/**"
      - main
  schedule:
    - cron: "0 7,15 * * 1-5" # MAIN_STAGING
    - cron: "0 9 * * 1-5" # MAIN_LTS
    - cron: "0 11 * * 1-5" # PREV_RELEASE_TESTS_STAGING
    - cron: "0 13 * * 1-5" # LTS_CYCLE_FIRST_SDK_STAGING

jobs:
  test_setup:
    name: E2E tests setup
    if: github.repository == 'kili-technology/kili-python-sdk'
    runs-on: ubuntu-latest
    outputs:
      test_mode_array: ${{ steps.setup_step.outputs.TEST_MODE_ARRAY }}
    steps:
      - name: Setup test mode
        id: setup_step
        shell: bash
        run: |
          if [[ "${{ github.event_name }}" == "push" ]]; then

            if [[ "${{ github.ref_name }}" == release/* ]]; then
              # if the push was on a release branch, it can be either
              # the lts branch -> LTS_CYCLE_FIRST_SDK_STAGING mode
              # or the latest release branch -> RELEASE_PREPROD mode
              if [[ "${{ github.ref_name }}" == release/2.129.0 ]]; then
                json_str="{\"array\": [\"LTS_CYCLE_FIRST_SDK_STAGING\"]}"
              else
                json_str="{\"array\": [\"RELEASE_PREPROD\"]}"
              fi

            else
              json_str="{\"array\": [\"MAIN_STAGING\"]}"

            fi

          elif [[ "${{ github.event_name }}" == "schedule" ]]; then

            if [[ "${{ github.event.schedule }}" == "0 7,15 * * 1-5" ]]; then
              json_str="{\"array\": [\"MAIN_STAGING\"]}"

            elif [[ "${{ github.event.schedule }}" == "0 9 * * 1-5" ]]; then
              json_str="{\"array\": [\"MAIN_LTS\"]}"

            elif [[ "${{ github.event.schedule }}" == "0 11 * * 1-5" ]]; then
              json_str="{\"array\": [\"PREV_RELEASE_TESTS_STAGING\"]}"

            elif [[ "${{ github.event.schedule }}" == "0 13 * * 1-5" ]]; then
              json_str="{\"array\": [\"LTS_CYCLE_FIRST_SDK_STAGING\"]}"

            fi

          elif [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then

            if [[ "${{ github.event.inputs.testMode }}" == "ALL" ]]; then
              json_str="{\"array\": [\"MAIN_STAGING\", \"RELEASE_PREPROD\", \"MAIN_LTS\", \"PREV_RELEASE_TESTS_STAGING\", \"LTS_CYCLE_FIRST_SDK_STAGING\"]}"
            else
              json_str="{\"array\": [\"${{ github.event.inputs.testMode }}\"]}"
            fi

          fi

          echo "TEST_MODE_ARRAY=$json_str" >> $GITHUB_OUTPUT

  tests:
    name: ${{ matrix.test_mode }}, ${{ matrix.os }}, ${{ matrix.python-version }}
    if: github.repository == 'kili-technology/kili-python-sdk'
    strategy:
      max-parallel: 1
      matrix:
        test_mode: ${{ fromJson(needs.test_setup.outputs.test_mode_array).array }}
        os:
          - windows-latest
          - ubuntu-latest
        include:
          - os: windows-latest
            python-version: 3.11
          - os: ubuntu-latest
            python-version: 3.8

    runs-on: ${{ matrix.os }}
    needs: [test_setup]
    steps:
      - name: Set test mode
        shell: bash
        run: echo "TEST_MODE=${{ matrix.test_mode }}" >> $GITHUB_ENV

      - name: Check that test mode is valid
        shell: bash
        run: |
          if [[ ${{ env.TEST_MODE }} != "MAIN_STAGING" ]] && \
             [[ ${{ env.TEST_MODE }} != "CUSTOM" ]] && \
             [[ ${{ env.TEST_MODE }} != "RELEASE_PREPROD" ]] && \
             [[ ${{ env.TEST_MODE }} != "MAIN_LTS" ]] && \
             [[ ${{ env.TEST_MODE }} != "PREV_RELEASE_TESTS_STAGING" ]] && \
             [[ ${{ env.TEST_MODE }} != "LTS_CYCLE_FIRST_SDK_STAGING" ]]; then
            echo "TEST_MODE is not valid: ${{ env.TEST_MODE }}"
            exit 1
          fi

      - name: Set first SDK version of LTS cycle
        if: env.TEST_MODE == 'LTS_CYCLE_FIRST_SDK_STAGING'
        shell: bash
        run: echo "LTS_CYCLE_FIRST_SDK_REF=release/2.129.0" >> $GITHUB_ENV # this should be updated when a new LTS cycle starts

      - name: Add latest release branch to github env
        shell: bash
        timeout-minutes: 3
        run: |
          curl https://raw.githubusercontent.com/kili-technology/kili-python-sdk/main/.github/scripts/add_latest_release_branch_to_github_env.py --output add_latest_release_branch_to_github_env.py
          python add_latest_release_branch_to_github_env.py
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

      - name: Set checkout branch/tag and endpoint
        shell: bash
        run: |
          if [[ "${{ env.TEST_MODE }}" == "MAIN_STAGING" ]]; then
            echo "REF=main" >> $GITHUB_ENV
            echo "TEST_AGAINST=STAGING" >> $GITHUB_ENV

          elif [[ "${{ env.TEST_MODE }}" == "RELEASE_PREPROD" ]]; then
            echo "REF=${{ env.LASTEST_RELEASE_BRANCH }}" >> $GITHUB_ENV
            echo "TEST_AGAINST=PREPROD" >> $GITHUB_ENV

          elif [[ "${{ env.TEST_MODE }}" == "MAIN_LTS" ]]; then
            echo "REF=main" >> $GITHUB_ENV
            echo "TEST_AGAINST=LTS" >> $GITHUB_ENV

          elif [[ "${{ env.TEST_MODE }}" == "PREV_RELEASE_TESTS_STAGING" ]]; then
            echo "REF=${{ env.LASTEST_RELEASE_BRANCH }}" >> $GITHUB_ENV
            echo "TEST_AGAINST=STAGING" >> $GITHUB_ENV

          elif [[ "${{ env.TEST_MODE }}" == "LTS_CYCLE_FIRST_SDK_STAGING" ]]; then
            echo "REF=${{ env.LTS_CYCLE_FIRST_SDK_REF }}" >> $GITHUB_ENV
            echo "TEST_AGAINST=STAGING" >> $GITHUB_ENV

          elif [[ "${{ env.TEST_MODE }}" == "CUSTOM" ]]; then
            echo "REF=${{ github.event.inputs.customBranch }}" >> $GITHUB_ENV
            echo "TEST_AGAINST=${{ github.event.inputs.customEndpoint }}" >> $GITHUB_ENV

          fi

      - name: Set KILI_API_ENDPOINT
        shell: bash
        run: |
          if [[ "${{ env.TEST_AGAINST }}" == "STAGING" ]]; then
            echo "KILI_API_ENDPOINT=https://staging.cloud.kili-technology.com/api/label/v2/graphql" >> $GITHUB_ENV
          elif [[ "${{ env.TEST_AGAINST }}" == "PREPROD" ]]; then
            echo "KILI_API_ENDPOINT=https://preproduction.cloud.kili-technology.com/api/label/v2/graphql" >> $GITHUB_ENV
          elif [[ "${{ env.TEST_AGAINST }}" == "LTS" ]]; then
            echo "KILI_API_ENDPOINT=https://lts.cloud.kili-technology.com/api/label/v2/graphql" >> $GITHUB_ENV
          elif [[ "${{ env.TEST_AGAINST }}" == "PROD" ]]; then
            echo "KILI_API_ENDPOINT=https://cloud.kili-technology.com/api/label/v2/graphql" >> $GITHUB_ENV
          else
            echo "TEST_AGAINST is not valid: ${{ env.TEST_AGAINST }}"
            exit 1
          fi

      - name: Check REF is set
        shell: bash
        run: |
          if [[ -z "${{ env.REF }}" ]]; then
            echo "REF is not set"
            exit 1
          fi

      - name: Checkout repo
        uses: actions/checkout@v3
        with:
          ref: ${{ env.REF }}

      - name: Delete src for PREV_RELEASE_TESTS_STAGING mode
        if: env.TEST_MODE == 'PREV_RELEASE_TESTS_STAGING'
        shell: bash
        run: rm -rf src # we want the previous release tests but the kili from main

      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}
          cache: "pip"

      - name: Upgrade pip
        run: python -m pip install --upgrade pip

      - name: Install kili
        shell: bash
        run: |
          if [[ "${{ env.TEST_MODE }}" == "PREV_RELEASE_TESTS_STAGING" ]]; then
            pip install "kili[dev] @ git+https://github.com/kili-technology/kili-python-sdk.git@main"
          else
            pip install -e ".[dev]"
          fi

      - name: Get kili SDK version
        shell: bash
        run: echo "KILI_SDK_VERSION=$(python -c "from kili import __version__; print(__version__)")" >> $GITHUB_ENV

      - name: E2E tests
        if: github.event.inputs.runE2ETests != 'false'
        timeout-minutes: 30
        run: pytest -ra -sv --color yes --code-highlight yes --durations=15 -vv --ignore tests/e2e/test_notebooks.py tests/e2e
        env:
          KILI_API_CLOUD_VISION: ${{ secrets.KILI_API_CLOUD_VISION }}
          KILI_API_ENDPOINT: ${{ env.KILI_API_ENDPOINT }}
          KILI_API_KEY: ${{ secrets[format('KILI_USER_API_KEY_{0}', env.TEST_AGAINST)] }}
          KILI_USER_EMAIL: ${{ secrets[format('KILI_USER_EMAIL_{0}', env.TEST_AGAINST)] }}
          KILI_USER_ID: ${{ secrets[format('KILI_USER_ID_{0}', env.TEST_AGAINST)] }}
          KILI_TEST_DATA_INTEGRATION_ID: ${{ secrets.KILI_TEST_DATA_INTEGRATION_ID }}

      - name: Notebook tests
        # we don't run notebook tests on push to main
        if: github.event.inputs.runNotebookTests != 'false' && (github.event_name != 'push' || github.ref_name != 'main')
        timeout-minutes: 30
        run: pytest -ra -sv --color yes --code-highlight yes --durations=15 -vv tests/e2e/test_notebooks.py
        env:
          KILI_API_CLOUD_VISION: ${{ secrets.KILI_API_CLOUD_VISION }}
          KILI_API_ENDPOINT: ${{ env.KILI_API_ENDPOINT }}
          KILI_API_KEY: ${{ secrets[format('KILI_USER_API_KEY_{0}', env.TEST_AGAINST)] }}
          KILI_USER_EMAIL: ${{ secrets[format('KILI_USER_EMAIL_{0}', env.TEST_AGAINST)] }}
          KILI_USER_ID: ${{ secrets[format('KILI_USER_ID_{0}', env.TEST_AGAINST)] }}
          KILI_TEST_DATA_INTEGRATION_ID: ${{ secrets.KILI_TEST_DATA_INTEGRATION_ID }}
          PYDEVD_DISABLE_FILE_VALIDATION: true

      - name: Check that kili SDK version has not changed
        shell: bash
        run: |
          kili_sdk_version=$(python -c "from kili import __version__; print(__version__)")
          if [[ "${{ env.KILI_SDK_VERSION }}" != "$kili_sdk_version" ]]; then
            echo "Kili has been updated (maybe by notebooks)."
            echo "The version should not be updated during tests."
            echo "version before: ${{ env.KILI_SDK_VERSION }}"
            echo "version after: $kili_sdk_version"
            exit 1
          fi

      - name: Slack notification if failure
        if: failure() && (github.event_name != 'workflow_dispatch') # doesn't notify for manual runs
        id: slack
        uses: slackapi/slack-github-action@v1.24.0
        with:
          payload: |
            {
              "text": "E2E tests '${{ env.TEST_MODE }}' (branch '${{ env.REF }}' against endpoint '${{ env.TEST_AGAINST }}') failed with status '${{ job.status }}'\n${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}",
              "blocks": [
                {
                  "type": "section",
                  "text": {
                    "type": "mrkdwn",
                    "text": "E2E tests '${{ env.TEST_MODE }}' (branch '${{ env.REF }}' against endpoint '${{ env.TEST_AGAINST }}') failed with status '${{ job.status }}'\n${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
                  }
                }
              ]
            }
        env:
          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL_PYTHON_SDK_ALARMING }}
          SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK