Add docs for RC2 wave of providers for 2nd round of Jan 2024 (#37019) #2126
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Licensed to the Apache Software Foundation (ASF) under one | |
| # or more contributor license agreements. See the NOTICE file | |
| # distributed with this work for additional information | |
| # regarding copyright ownership. The ASF licenses this file | |
| # to you under the Apache License, Version 2.0 (the | |
| # "License"); you may not use this file except in compliance | |
| # with the License. You may obtain a copy of the License at | |
| # | |
| # http://www.apache.org/licenses/LICENSE-2.0 | |
| # | |
| # Unless required by applicable law or agreed to in writing, | |
| # software distributed under the License is distributed on an | |
| # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | |
| # KIND, either express or implied. See the License for the | |
| # specific language governing permissions and limitations | |
| # under the License. | |
| # | |
| --- | |
| name: Tests | |
| on: # yamllint disable-line rule:truthy | |
| schedule: | |
| - cron: '28 0 * * *' | |
| push: | |
| branches: ['main', 'v[0-9]+-[0-9]+-test'] | |
| pull_request: | |
| branches: ['main'] | |
| permissions: | |
| # All other permissions are set to none | |
| contents: read | |
| # Technically read access while waiting for images should be more than enough. However, | |
| # there is a bug in GitHub Actions/Packages and in case private repositories are used, you get a permission | |
| # denied error when attempting to just pull private image, changing the token permission to write solves the | |
| # issue. This is not dangerous, because if it is for "apache/airflow", only maintainers can push ci.yml | |
| # changes. If it is for a fork, then the token is read-only anyway. | |
| packages: write | |
| env: | |
| ANSWER: "yes" | |
| DB_RESET: "true" | |
| VERBOSE: "true" | |
| GITHUB_REPOSITORY: ${{ github.repository }} | |
| GITHUB_USERNAME: ${{ github.actor }} | |
| # You can override CONSTRAINTS_GITHUB_REPOSITORY by setting secret in your repo but by default the | |
| # Airflow one is going to be used | |
| CONSTRAINTS_GITHUB_REPOSITORY: >- | |
| ${{ secrets.CONSTRAINTS_GITHUB_REPOSITORY != '' && | |
| secrets.CONSTRAINTS_GITHUB_REPOSITORY || 'apache/airflow' }} | |
| # In builds from forks, this token is read-only. For scheduled/direct push it is WRITE one | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| IMAGE_TAG: "${{ github.event.pull_request.head.sha || github.sha }}" | |
| USE_SUDO: "true" | |
| INCLUDE_SUCCESS_OUTPUTS: "true" | |
| INCLUDE_NOT_READY_PROVIDERS: "true" | |
| AIRFLOW_ENABLE_AIP_44: "true" | |
| MOUNT_SOURCES: "skip" | |
| concurrency: | |
| group: ci-${{ github.event.pull_request.number || github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| build-info: | |
| name: "Build info" | |
| # The runs-on cannot refer to env. or secrets. context, so we have no | |
| # option but to specify a hard-coded list here. This is "safe", the | |
| # runner checks if the user is an owner or collaborator of the repo | |
| # before running the workflow. | |
| runs-on: "ubuntu-22.04" | |
| env: | |
| GITHUB_CONTEXT: ${{ toJson(github) }} | |
| outputs: | |
| cache-directive: ${{ steps.selective-checks.outputs.cache-directive }} | |
| affected-providers-list-as-string: >- | |
| ${{ steps.selective-checks.outputs.affected-providers-list-as-string }} | |
| upgrade-to-newer-dependencies: ${{ steps.selective-checks.outputs.upgrade-to-newer-dependencies }} | |
| python-versions: ${{ steps.selective-checks.outputs.python-versions }} | |
| python-versions-list-as-string: ${{ steps.selective-checks.outputs.python-versions-list-as-string }} | |
| all-python-versions-list-as-string: >- | |
| ${{ steps.selective-checks.outputs.all-python-versions-list-as-string }} | |
| default-python-version: ${{ steps.selective-checks.outputs.default-python-version }} | |
| kubernetes-versions-list-as-string: >- | |
| ${{ steps.selective-checks.outputs.kubernetes-versions-list-as-string }} | |
| kubernetes-combos-list-as-string: >- | |
| ${{ steps.selective-checks.outputs.kubernetes-combos-list-as-string }} | |
| default-kubernetes-version: ${{ steps.selective-checks.outputs.default-kubernetes-version }} | |
| postgres-versions: ${{ steps.selective-checks.outputs.postgres-versions }} | |
| default-postgres-version: ${{ steps.selective-checks.outputs.default-postgres-version }} | |
| mysql-versions: ${{ steps.selective-checks.outputs.mysql-versions }} | |
| default-mysql-version: ${{ steps.selective-checks.outputs.default-mysql-version }} | |
| default-helm-version: ${{ steps.selective-checks.outputs.default-helm-version }} | |
| default-kind-version: ${{ steps.selective-checks.outputs.default-kind-version }} | |
| full-tests-needed: ${{ steps.selective-checks.outputs.full-tests-needed }} | |
| parallel-test-types-list-as-string: >- | |
| ${{ steps.selective-checks.outputs.parallel-test-types-list-as-string }} | |
| postgres-exclude: ${{ steps.selective-checks.outputs.postgres-exclude }} | |
| mysql-exclude: ${{ steps.selective-checks.outputs.mysql-exclude }} | |
| sqlite-exclude: ${{ steps.selective-checks.outputs.sqlite-exclude }} | |
| skip-provider-tests: ${{ steps.selective-checks.outputs.skip-provider-tests }} | |
| run-tests: ${{ steps.selective-checks.outputs.run-tests }} | |
| run-amazon-tests: ${{ steps.selective-checks.outputs.run-amazon-tests }} | |
| run-www-tests: ${{ steps.selective-checks.outputs.run-www-tests }} | |
| run-kubernetes-tests: ${{ steps.selective-checks.outputs.run-kubernetes-tests }} | |
| basic-checks-only: ${{ steps.selective-checks.outputs.basic-checks-only }} | |
| ci-image-build: ${{ steps.selective-checks.outputs.ci-image-build }} | |
| prod-image-build: ${{ steps.selective-checks.outputs.prod-image-build }} | |
| docs-build: ${{ steps.selective-checks.outputs.docs-build }} | |
| mypy-folders: ${{ steps.selective-checks.outputs.mypy-folders }} | |
| needs-mypy: ${{ steps.selective-checks.outputs.needs-mypy }} | |
| needs-helm-tests: ${{ steps.selective-checks.outputs.needs-helm-tests }} | |
| needs-api-tests: ${{ steps.selective-checks.outputs.needs-api-tests }} | |
| needs-api-codegen: ${{ steps.selective-checks.outputs.needs-api-codegen }} | |
| default-branch: ${{ steps.selective-checks.outputs.default-branch }} | |
| default-constraints-branch: ${{ steps.selective-checks.outputs.default-constraints-branch }} | |
| docs-list-as-string: ${{ steps.selective-checks.outputs.docs-list-as-string }} | |
| skip-pre-commits: ${{ steps.selective-checks.outputs.skip-pre-commits }} | |
| providers-compatibility-checks: ${{ steps.selective-checks.outputs.providers-compatibility-checks }} | |
| helm-test-packages: ${{ steps.selective-checks.outputs.helm-test-packages }} | |
| debug-resources: ${{ steps.selective-checks.outputs.debug-resources }} | |
| runs-on: ${{steps.selective-checks.outputs.runs-on}} | |
| is-self-hosted-runner: ${{ steps.selective-checks.outputs.is-self-hosted-runner }} | |
| is-airflow-runner: ${{ steps.selective-checks.outputs.is-airflow-runner }} | |
| is-amd-runner: ${{ steps.selective-checks.outputs.is-amd-runner }} | |
| is-arm-runner: ${{ steps.selective-checks.outputs.is-arm-runner }} | |
| is-vm-runner: ${{ steps.selective-checks.outputs.is-vm-runner }} | |
| is-k8s-runner: ${{ steps.selective-checks.outputs.is-k8s-runner }} | |
| chicken-egg-providers: ${{ steps.selective-checks.outputs.chicken-egg-providers }} | |
| has-migrations: ${{ steps.selective-checks.outputs.has-migrations }} | |
| source-head-repo: ${{ steps.source-run-info.outputs.source-head-repo }} | |
| pull-request-labels: ${{ steps.source-run-info.outputs.pr-labels }} | |
| in-workflow-build: ${{ steps.source-run-info.outputs.in-workflow-build }} | |
| build-job-description: ${{ steps.source-run-info.outputs.build-job-description }} | |
| canary-run: ${{ steps.source-run-info.outputs.canary-run }} | |
| run-coverage: ${{ steps.source-run-info.outputs.run-coverage }} | |
| steps: | |
| - name: Cleanup repo | |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" | |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" | |
| uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false | |
| - name: Fetch incoming commit ${{ github.sha }} with its parent | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ github.sha }} | |
| fetch-depth: 2 | |
| persist-credentials: false | |
| - name: "Install Breeze" | |
| uses: ./.github/actions/breeze | |
| - name: "Retrieve defaults from branch_defaults.py" | |
| id: defaults | |
| # We could retrieve it differently here - by just importing the variables and | |
| # printing them from python code, however we want to have the same code as used in | |
| # the build-images.yml (there we cannot import python code coming from the PR - we need to | |
| # treat the python code as text and extract the variables from there. | |
| run: | | |
| python - <<EOF >> ${GITHUB_ENV} | |
| from pathlib import Path | |
| import re | |
| import sys | |
| DEFAULTS_CONTENT = Path('dev/breeze/src/airflow_breeze/branch_defaults.py').read_text() | |
| BRANCH_PATTERN = r'^AIRFLOW_BRANCH = "(.*)"$' | |
| CONSTRAINTS_BRANCH_PATTERN = r'^DEFAULT_AIRFLOW_CONSTRAINTS_BRANCH = "(.*)"$' | |
| branch = re.search(BRANCH_PATTERN, DEFAULTS_CONTENT, re.MULTILINE).group(1) | |
| constraints_branch = re.search(CONSTRAINTS_BRANCH_PATTERN, DEFAULTS_CONTENT, re.MULTILINE).group(1) | |
| output = f""" | |
| DEFAULT_BRANCH={branch} | |
| DEFAULT_CONSTRAINTS_BRANCH={constraints_branch} | |
| """.strip() | |
| print(output) | |
| # Stdout is redirected to GITHUB_ENV but we also print it to stderr to see it in ci log | |
| print(output, file=sys.stderr) | |
| EOF | |
| - name: "Get information about the Workflow" | |
| id: source-run-info | |
| run: breeze ci get-workflow-info 2>> ${GITHUB_OUTPUT} | |
| - name: Selective checks | |
| id: selective-checks | |
| env: | |
| PR_LABELS: "${{ steps.source-run-info.outputs.pr-labels }}" | |
| COMMIT_REF: "${{ github.sha }}" | |
| VERBOSE: "false" | |
| run: breeze ci selective-check 2>> ${GITHUB_OUTPUT} | |
| - name: env | |
| run: printenv | |
| env: | |
| PR_LABELS: ${{ steps.source-run-info.outputs.pr-labels }} | |
| GITHUB_CONTEXT: ${{ toJson(github) }} | |
| # Push early BuildX cache to GitHub Registry in Apache repository, This cache does not wait for all the | |
| # tests to complete - it is run very early in the build process for "main" merges in order to refresh | |
| # cache using the current constraints. This will speed up cache refresh in cases when pyproject.toml | |
| # changes or in case of Dockerfile changes. Failure in this step is not a problem (at most it will | |
| # delay cache refresh. It does not attempt to upgrade to newer dependencies. | |
| # We only push CI cache as PROD cache usually does not gain as much from fresh cache because | |
| # it uses prepared airflow and provider packages that invalidate the cache anyway most of the time | |
| push-early-buildx-cache-to-github-registry: | |
| timeout-minutes: 50 | |
| name: "Push Early Image Cache" | |
| runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}} | |
| needs: | |
| - build-info | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| platform: ["linux/amd64", "linux/arm64"] | |
| env: | |
| RUNS_ON: "${{needs.build-info.outputs.runs-on}}" | |
| UPGRADE_TO_NEWER_DEPENDENCIES: false | |
| VERSION_SUFFIX_FOR_PYPI: "dev0" | |
| continue-on-error: true | |
| steps: | |
| - name: Cleanup repo | |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" | |
| if: needs.build-info.outputs.canary-run == 'true' && needs.build-info.outputs.default-branch == 'main' | |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" | |
| uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false | |
| if: needs.build-info.outputs.canary-run == 'true' && needs.build-info.outputs.default-branch == 'main' | |
| - name: "Install Breeze" | |
| uses: ./.github/actions/breeze | |
| if: needs.build-info.outputs.canary-run == 'true' && needs.build-info.outputs.default-branch == 'main' | |
| - name: "Start ARM instance" | |
| run: ./scripts/ci/images/ci_start_arm_instance_and_connect_to_docker.sh | |
| if: > | |
| matrix.platform == 'linux/arm64' && needs.build-info.outputs.canary-run == 'true' | |
| && needs.build-info.outputs.default-branch == 'main' | |
| - name: "Push CI cache ${{ matrix.platform }}" | |
| run: > | |
| breeze ci-image build | |
| --builder airflow_cache | |
| --prepare-buildx-cache | |
| --run-in-parallel | |
| --platform ${{ matrix.platform }} | |
| env: | |
| DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}} | |
| COMMIT_SHA: ${{ github.sha }} | |
| if: needs.build-info.outputs.canary-run == 'true' && needs.build-info.outputs.default-branch == 'main' | |
| - name: "Push CI latest image ${{ matrix.platform }}" | |
| run: > | |
| breeze ci-image build | |
| --tag-as-latest --push --run-in-parallel --platform ${{ matrix.platform }} | |
| env: | |
| DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}} | |
| COMMIT_SHA: ${{ github.sha }} | |
| # We only push "amd" image as it is really only needed for any kind of automated builds in CI | |
| # and currently there is not an easy way to make multi-platform image from two separate builds | |
| if: > | |
| matrix.platform == 'linux/amd64' && needs.build-info.outputs.canary-run == 'true' | |
| && needs.build-info.outputs.default-branch == 'main' | |
| - name: "Stop ARM instance" | |
| run: ./scripts/ci/images/ci_stop_arm_instance.sh | |
| if: > | |
| always() && matrix.platform == 'linux/arm64' && needs.build-info.outputs.canary-run == 'true' | |
| && needs.build-info.outputs.default-branch == 'main' | |
| - name: "Clean docker cache for ${{ matrix.platform }}" | |
| run: docker system prune --all --force | |
| if: > | |
| matrix.platform == 'linux/amd64' && needs.build-info.outputs.canary-run == 'true' | |
| && needs.build-info.outputs.default-branch == 'main' | |
| # Check that after earlier cache push, breeze command will build quickly | |
| check-that-image-builds-quickly: | |
| timeout-minutes: 5 | |
| name: "Check that image builds quickly" | |
| runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}} | |
| needs: | |
| - build-info | |
| - push-early-buildx-cache-to-github-registry | |
| env: | |
| RUNS_ON: "${{needs.build-info.outputs.runs-on}}" | |
| UPGRADE_TO_NEWER_DEPENDENCIES: false | |
| PLATFORM: "linux/amd64" | |
| if: > | |
| needs.build-info.outputs.canary-run == 'true' | |
| && needs.build-info.outputs.default-branch == 'main' | |
| steps: | |
| - name: Cleanup repo | |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" | |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" | |
| uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false | |
| - name: "Install Breeze" | |
| uses: ./.github/actions/breeze | |
| - name: "Check that image builds quickly" | |
| run: breeze shell --max-time 120 | |
| build-ci-images: | |
| timeout-minutes: 80 | |
| name: >- | |
| ${{needs.build-info.outputs.build-job-description}} CI images | |
| ${{needs.build-info.outputs.all-python-versions-list-as-string}} | |
| runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}} | |
| needs: [build-info] | |
| env: | |
| DEFAULT_BRANCH: ${{ needs.build-info.outputs.default-branch }} | |
| DEFAULT_CONSTRAINTS_BRANCH: ${{ needs.build-info.outputs.default-constraints-branch }} | |
| RUNS_ON: "${{needs.build-info.outputs.runs-on}}" | |
| # Force more parallelism for build even on public images | |
| PARALLELISM: 6 | |
| VERSION_SUFFIX_FOR_PYPI: "dev0" | |
| steps: | |
| - name: Cleanup repo | |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" | |
| if: needs.build-info.outputs.in-workflow-build == 'true' | |
| - uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ needs.build-info.outputs.targetCommitSha }} | |
| persist-credentials: false | |
| if: needs.build-info.outputs.in-workflow-build == 'true' | |
| - name: > | |
| Build CI Images | |
| ${{needs.build-info.outputs.all-python-versions-list-as-string}}:${{env.IMAGE_TAG}} | |
| uses: ./.github/actions/build-ci-images | |
| if: needs.build-info.outputs.in-workflow-build == 'true' | |
| env: | |
| UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} | |
| DOCKER_CACHE: ${{ needs.build-info.outputs.cache-directive }} | |
| PYTHON_VERSIONS: ${{needs.build-info.outputs.all-python-versions-list-as-string}} | |
| DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}} | |
| BUILD_TIMEOUT_MINUTES: 70 | |
| - name: Verify CI images ${{ env.PYTHON_VERSIONS }}:${{ env.IMAGE_TAG }} | |
| run: breeze ci-image verify --run-in-parallel | |
| env: | |
| PYTHON_VERSIONS: ${{needs.build-info.outputs.all-python-versions-list-as-string}} | |
| DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}} | |
| if: needs.build-info.outputs.in-workflow-build == 'true' | |
| run-breeze-tests: | |
| timeout-minutes: 10 | |
| name: Breeze unit tests | |
| runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}} | |
| needs: [build-info] | |
| steps: | |
| - name: Cleanup repo | |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" | |
| - uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false | |
| - uses: actions/setup-python@v4 | |
| with: | |
| python-version: "${{needs.build-info.outputs.default-python-version}}" | |
| cache: 'pip' | |
| cache-dependency-path: ./dev/breeze/setup* | |
| - run: pip install --editable ./dev/breeze/ | |
| - run: python -m pytest -n auto --color=yes | |
| working-directory: ./dev/breeze/ | |
| tests-www: | |
| timeout-minutes: 10 | |
| name: React WWW tests | |
| runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}} | |
| needs: [build-info] | |
| if: needs.build-info.outputs.run-www-tests == 'true' | |
| steps: | |
| - name: Cleanup repo | |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" | |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" | |
| uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false | |
| - name: "Setup node" | |
| uses: actions/setup-node@v3 | |
| with: | |
| node-version: 14 | |
| - name: "Cache eslint" | |
| uses: actions/cache@v3 | |
| with: | |
| path: 'airflow/www/node_modules' | |
| key: ${{ runner.os }}-www-node-modules-${{ hashFiles('airflow/www/**/yarn.lock') }} | |
| - run: yarn --cwd airflow/www/ install --frozen-lockfile --non-interactive | |
| - run: yarn --cwd airflow/www/ run test | |
| env: | |
| FORCE_COLOR: 2 | |
| test-openapi-client: | |
| timeout-minutes: 10 | |
| name: "Test OpenAPI client" | |
| runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}} | |
| needs: [build-info] | |
| if: needs.build-info.outputs.needs-api-codegen == 'true' | |
| steps: | |
| - name: Cleanup repo | |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" | |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 2 | |
| persist-credentials: false | |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: "apache/airflow-client-python" | |
| fetch-depth: 1 | |
| persist-credentials: false | |
| path: ./airflow-client-python | |
| - name: "Install Breeze" | |
| uses: ./.github/actions/breeze | |
| - name: "Generate client with breeze" | |
| run: > | |
| breeze release-management prepare-python-client --package-format both | |
| --version-suffix-for-pypi dev0 --python-client-repo ./airflow-client-python | |
| - name: "Show diff" | |
| run: git diff --color HEAD | |
| working-directory: ./airflow-client-python | |
| - name: Install hatch | |
| run: | | |
| python -m pip install --upgrade pipx | |
| pipx ensurepath | |
| pipx install hatch --force | |
| - name: Run tests | |
| run: hatch run run-coverage | |
| env: | |
| HATCH_ENV: "test" | |
| working-directory: ./clients/python | |
| - name: "Install Airflow in editable mode with fab for webserver tests" | |
| run: pip install -e ".[fab]" | |
| - name: "Install Python client" | |
| run: pip install ./dist/apache_airflow_client-*.whl | |
| - name: "Initialize Airflow DB and start webserver" | |
| run: | | |
| airflow db init | |
| # Let scheduler runs a few loops and get all DAG files from example DAGs serialized to DB | |
| airflow scheduler --num-runs 100 | |
| airflow users create --username admin --password admin --firstname Admin --lastname Admin \ | |
| --role Admin --email [email protected] | |
| killall python || true # just in case there is a webserver running in the background | |
| nohup airflow webserver --port 8080 & | |
| echo "Started webserver" | |
| env: | |
| AIRFLOW__API__AUTH_BACKENDS: airflow.api.auth.backend.session,airflow.api.auth.backend.basic_auth | |
| AIRFLOW__WEBSERVER__EXPOSE_CONFIG: "True" | |
| AIRFLOW__CORE__LOAD_EXAMPLES: "True" | |
| - name: "Waiting for the webserver to be available" | |
| run: | | |
| timeout 30 bash -c 'until nc -z $0 $1; do echo "sleeping"; sleep 1; done' localhost 8080 | |
| sleep 5 | |
| - name: "Run test python client" | |
| run: python ./clients/python/test_python_client.py | |
| env: | |
| FORCE_COLOR: "standard" | |
| - name: "Stop running webserver" | |
| run: killall python || true # just in case there is a webserver running in the background | |
| if: always() | |
| - name: "Upload python client packages" | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: python-client-packages | |
| path: ./dist/apache_airflow_client-* | |
| retention-days: 7 | |
| if-no-files-found: error | |
| test-git-clone-on-windows: | |
| timeout-minutes: 5 | |
| name: "Test git clone on Windows" | |
| runs-on: "windows-latest" | |
| needs: [build-info] | |
| steps: | |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 2 | |
| persist-credentials: false | |
| wait-for-ci-images: | |
| timeout-minutes: 120 | |
| name: "Wait for CI images" | |
| runs-on: "ubuntu-22.04" | |
| needs: [build-info, build-ci-images] | |
| if: needs.build-info.outputs.ci-image-build == 'true' | |
| env: | |
| RUNS_ON: "${{needs.build-info.outputs.runs-on}}" | |
| BACKEND: sqlite | |
| # Force more parallelism for pull even on public images | |
| PARALLELISM: 6 | |
| steps: | |
| - name: Cleanup repo | |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" | |
| if: needs.build-info.outputs.in-workflow-build == 'false' | |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" | |
| uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false | |
| if: needs.build-info.outputs.in-workflow-build == 'false' | |
| - name: "Install Breeze" | |
| uses: ./.github/actions/breeze | |
| if: needs.build-info.outputs.in-workflow-build == 'false' | |
| - name: Wait for CI images ${{ env.PYTHON_VERSIONS }}:${{ env.IMAGE_TAG }} | |
| id: wait-for-images | |
| run: breeze ci-image pull --run-in-parallel --wait-for-image --tag-as-latest | |
| env: | |
| PYTHON_VERSIONS: ${{ needs.build-info.outputs.python-versions-list-as-string }} | |
| DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}} | |
| if: needs.build-info.outputs.in-workflow-build == 'false' | |
| generate-constraints: | |
| permissions: | |
| contents: read | |
| timeout-minutes: 70 | |
| name: > | |
| Generate constraints with CI image verifying | |
| ${{needs.build-info.outputs.all-python-versions-list-as-string}} | |
| runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}} | |
| needs: [build-info, wait-for-ci-images] | |
| env: | |
| RUNS_ON: "${{ needs.build-info.outputs.runs-on }}" | |
| PYTHON_VERSIONS: ${{needs.build-info.outputs.all-python-versions-list-as-string}} | |
| DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}} | |
| VERSION_SUFFIX_FOR_PYPI: "dev0" | |
| if: needs.build-info.outputs.ci-image-build == 'true' | |
| steps: | |
| - name: Cleanup repo | |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" | |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" | |
| uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false | |
| - name: "Install Breeze" | |
| uses: ./.github/actions/breeze | |
| - name: Pull CI images ${{ env.PYTHON_VERSIONS }}:${{ env.IMAGE_TAG }} | |
| run: breeze ci-image pull --run-in-parallel --tag-as-latest | |
| - name: Verify CI images ${{ env.PYTHON_VERSIONS }}:${{ env.IMAGE_TAG }} | |
| run: breeze ci-image verify --run-in-parallel | |
| env: | |
| PYTHON_VERSIONS: ${{needs.build-info.outputs.all-python-versions-list-as-string}} | |
| DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}} | |
| - name: "Source constraints" | |
| shell: bash | |
| run: > | |
| breeze release-management generate-constraints --run-in-parallel | |
| --airflow-constraints-mode constraints-source-providers | |
| - name: "No providers constraints" | |
| shell: bash | |
| timeout-minutes: 25 | |
| run: > | |
| breeze release-management generate-constraints --run-in-parallel | |
| --airflow-constraints-mode constraints-no-providers | |
| - name: "Prepare chicken-eggs provider packages" | |
| # In case of provider packages which use latest dev0 version of providers, we should prepare them | |
| # from the source code, not from the PyPI because they have apache-airflow>=X.Y.Z dependency | |
| # And when we prepare them from sources they will have apache-airflow>=X.Y.Z.dev0 | |
| shell: bash | |
| run: > | |
| breeze release-management prepare-provider-packages --include-not-ready-providers | |
| --package-format wheel --version-suffix-for-pypi dev0 | |
| ${{ needs.build-info.outputs.chicken-egg-providers }} | |
| if: needs.build-info.outputs.chicken-egg-providers != '' | |
| - name: "PyPI constraints" | |
| shell: bash | |
| timeout-minutes: 25 | |
| run: > | |
| breeze release-management generate-constraints --run-in-parallel | |
| --airflow-constraints-mode constraints | |
| --chicken-egg-providers "${{ needs.build-info.outputs.chicken-egg-providers }}" | |
| - name: "Dependency upgrade summary" | |
| shell: bash | |
| run: | | |
| for PYTHON_VERSION in ${{ env.PYTHON_VERSIONS }}; do | |
| echo "Summarizing Python $PYTHON_VERSION" | |
| cat "files/constraints-${PYTHON_VERSION}"/*.md >> $GITHUB_STEP_SUMMARY || true | |
| done | |
| - name: "Upload constraint artifacts" | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: constraints | |
| path: ./files/constraints-*/constraints-*.txt | |
| retention-days: 7 | |
| if-no-files-found: error | |
| static-checks: | |
| timeout-minutes: 45 | |
| name: "Static checks" | |
| runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}} | |
| needs: [build-info, wait-for-ci-images] | |
| env: | |
| RUNS_ON: "${{needs.build-info.outputs.runs-on}}" | |
| PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}" | |
| UPGRADE_TO_NEWER_DEPENDENCIES: "${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}" | |
| if: needs.build-info.outputs.basic-checks-only == 'false' | |
| steps: | |
| - name: Cleanup repo | |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" | |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" | |
| uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false | |
| - name: > | |
| Prepare breeze & CI image: ${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}} | |
| uses: ./.github/actions/prepare_breeze_and_image | |
| id: breeze | |
| - name: Cache pre-commit envs | |
| uses: actions/cache@v3 | |
| with: | |
| path: ~/.cache/pre-commit | |
| # yamllint disable-line rule:line-length | |
| key: "pre-commit-${{steps.breeze.outputs.host-python-version}}-${{ hashFiles('.pre-commit-config.yaml') }}" | |
| restore-keys: | | |
| pre-commit-${{steps.breeze.outputs.host-python-version}}- | |
| - name: "Static checks" | |
| run: breeze static-checks --all-files --show-diff-on-failure --color always --initialize-environment | |
| env: | |
| VERBOSE: "false" | |
| SKIP: ${{ needs.build-info.outputs.skip-pre-commits }} | |
| COLUMNS: "250" | |
| SKIP_GROUP_OUTPUT: "true" | |
| DEFAULT_BRANCH: ${{ needs.build-info.outputs.default-branch }} | |
| RUFF_FORMAT: "github" | |
| # Runs static checks for groups of files in the repository in a single process without passing . | |
| # List of files | |
| mypy: | |
| timeout-minutes: 45 | |
| name: "MyPy checks" | |
| runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}} | |
| needs: [build-info, wait-for-ci-images] | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| mypy-folder: ${{fromJson(needs.build-info.outputs.mypy-folders)}} | |
| env: | |
| RUNS_ON: "${{needs.build-info.outputs.runs-on}}" | |
| PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}" | |
| UPGRADE_TO_NEWER_DEPENDENCIES: "${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}" | |
| steps: | |
| - name: Cleanup repo | |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" | |
| if: needs.build-info.outputs.needs-mypy == 'true' | |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" | |
| uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false | |
| if: needs.build-info.outputs.needs-mypy == 'true' | |
| - name: > | |
| Prepare breeze & CI image: ${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}} | |
| uses: ./.github/actions/prepare_breeze_and_image | |
| id: breeze | |
| if: needs.build-info.outputs.needs-mypy == 'true' | |
| - name: Cache pre-commit envs | |
| uses: actions/cache@v3 | |
| with: | |
| path: ~/.cache/pre-commit | |
| # yamllint disable-line rule:line-length | |
| key: "pre-commit-${{steps.breeze.outputs.host-python-version}}-${{ hashFiles('.pre-commit-config.yaml') }}" | |
| restore-keys: | | |
| pre-commit-${{steps.breeze.outputs.host-python-version}}- | |
| if: needs.build-info.outputs.needs-mypy == 'true' | |
| - name: "MyPy checks for ${{ matrix.mypy-folder }}" | |
| run: | | |
| pip install pre-commit | |
| pre-commit run --color always --verbose --hook-stage manual mypy-${{matrix.mypy-folder}} --all-files | |
| env: | |
| VERBOSE: "false" | |
| COLUMNS: "250" | |
| SKIP_GROUP_OUTPUT: "true" | |
| DEFAULT_BRANCH: ${{ needs.build-info.outputs.default-branch }} | |
| RUFF_FORMAT: "github" | |
| if: needs.build-info.outputs.needs-mypy == 'true' | |
| # Those checks are run if no image needs to be built for checks. This is for simple changes that | |
| # Do not touch any of the python code or any of the important files that might require building | |
| # The CI Docker image and they can be run entirely using the pre-commit virtual environments on host | |
| static-checks-basic-checks-only: | |
| timeout-minutes: 30 | |
| name: "Static checks: basic checks only" | |
| runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}} | |
| needs: [build-info] | |
| env: | |
| RUNS_ON: "${{needs.build-info.outputs.runs-on}}" | |
| if: needs.build-info.outputs.basic-checks-only == 'true' | |
| steps: | |
| - name: Cleanup repo | |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" | |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" | |
| uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false | |
| - name: "Setup python" | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: "${{needs.build-info.outputs.default-python-version}}" | |
| cache: 'pip' | |
| cache-dependency-path: ./dev/breeze/setup* | |
| - name: "Install Breeze" | |
| uses: ./.github/actions/breeze | |
| id: breeze | |
| - name: Cache pre-commit envs | |
| uses: actions/cache@v3 | |
| with: | |
| path: ~/.cache/pre-commit | |
| # yamllint disable-line rule:line-length | |
| key: "pre-commit-${{steps.breeze.outputs.host-python-version}}-${{ hashFiles('.pre-commit-config.yaml') }}" | |
| restore-keys: "\ | |
| pre-commit-${{steps.breeze.outputs.host-python-version}}-\ | |
| ${{ hashFiles('.pre-commit-config.yaml') }}\n | |
| pre-commit-${{steps.breeze.outputs.host-python-version}}-" | |
| - name: Fetch incoming commit ${{ github.sha }} with its parent | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ github.sha }} | |
| fetch-depth: 2 | |
| persist-credentials: false | |
| - name: "Static checks: basic checks only" | |
| run: > | |
| breeze static-checks --show-diff-on-failure --color always --initialize-environment | |
| --skip-image-upgrade-check --commit-ref "${{ github.sha }}" | |
| env: | |
| VERBOSE: "false" | |
| SKIP_BREEZE_PRE_COMMITS: "true" | |
| SKIP: ${{ needs.build-info.outputs.skip-pre-commits }} | |
| COLUMNS: "250" | |
| build-docs: | |
| timeout-minutes: 60 | |
| name: "Build docs" | |
| runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}} | |
| needs: [build-info, wait-for-ci-images] | |
| if: needs.build-info.outputs.docs-build == 'true' | |
| env: | |
| RUNS_ON: "${{needs.build-info.outputs.runs-on}}" | |
| PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}" | |
| steps: | |
| - name: Cleanup repo | |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" | |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" | |
| uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false | |
| - name: > | |
| Prepare breeze & CI image: ${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}} | |
| uses: ./.github/actions/prepare_breeze_and_image | |
| - uses: actions/cache@v3 | |
| id: cache-doc-inventories | |
| with: | |
| path: ./docs/_inventory_cache/ | |
| key: docs-inventory-${{ hashFiles('pyproject.toml;') }} | |
| restore-keys: | | |
| docs-inventory-${{ hashFiles('pyproject.toml;') }} | |
| docs-inventory- | |
| - name: "Build docs" | |
| run: > | |
| breeze build-docs ${{ needs.build-info.outputs.docs-list-as-string }} --docs-only | |
| - name: "Clone airflow-site" | |
| run: > | |
| git clone https://github.com/apache/airflow-site.git ${GITHUB_WORKSPACE}/airflow-site && | |
| echo "AIRFLOW_SITE_DIRECTORY=${GITHUB_WORKSPACE}/airflow-site" >> "$GITHUB_ENV" | |
| if: needs.build-info.outputs.full-tests-needed == 'true' | |
| - name: "Publish docs" | |
| run: > | |
| breeze release-management publish-docs | |
| --override-versioned --run-in-parallel | |
| ${{ needs.build-info.outputs.docs-list-as-string }} | |
| if: needs.build-info.outputs.full-tests-needed == 'true' | |
| - name: "Generate back references for providers" | |
| run: breeze release-management add-back-references all-providers | |
| if: needs.build-info.outputs.full-tests-needed == 'true' | |
| - name: "Generate back references for apache-airflow" | |
| run: breeze release-management add-back-references apache-airflow | |
| if: needs.build-info.outputs.full-tests-needed == 'true' | |
| - name: "Generate back references for docker-stack" | |
| run: breeze release-management add-back-references docker-stack | |
| if: needs.build-info.outputs.full-tests-needed == 'true' | |
| - name: "Generate back references for helm-chart" | |
| run: breeze release-management add-back-references helm-chart | |
| if: needs.build-info.outputs.full-tests-needed == 'true' | |
| - name: Configure AWS credentials | |
| uses: aws-actions/configure-aws-credentials@010d0da01d0b5a38af31e9c3470dbfdabdecca3a # v4.0.1 | |
| if: needs.build-info.outputs.canary-run == 'true' && needs.build-info.outputs.default-branch == 'main' | |
| with: | |
| aws-access-key-id: ${{ secrets.DOCS_AWS_ACCESS_KEY_ID }} | |
| aws-secret-access-key: ${{ secrets.DOCS_AWS_SECRET_ACCESS_KEY }} | |
| aws-region: eu-central-1 | |
| - name: "Upload documentation to AWS S3" | |
| if: needs.build-info.outputs.canary-run == 'true' && needs.build-info.outputs.default-branch == 'main' | |
| run: aws s3 sync --delete ./files/documentation s3://apache-airflow-docs | |
| spellcheck-docs: | |
| timeout-minutes: 120 | |
| name: "Spellcheck docs" | |
| runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}} | |
| needs: [build-info, wait-for-ci-images] | |
| if: needs.build-info.outputs.docs-build == 'true' | |
| env: | |
| RUNS_ON: "${{needs.build-info.outputs.runs-on}}" | |
| PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}" | |
| steps: | |
| - name: Cleanup repo | |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" | |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" | |
| uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false | |
| - name: > | |
| Prepare breeze & CI image: ${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}} | |
| uses: ./.github/actions/prepare_breeze_and_image | |
| - uses: actions/cache@v3 | |
| id: cache-doc-inventories | |
| with: | |
| path: ./docs/_inventory_cache/ | |
| key: docs-inventory-${{ hashFiles('pyproject.toml;') }} | |
| restore-keys: | | |
| docs-inventory-${{ hashFiles('pyproject.toml;') }} | |
| docs-inventory- | |
| - name: "Spellcheck docs" | |
| run: > | |
| breeze build-docs ${{ needs.build-info.outputs.docs-list-as-string }} --spellcheck-only | |
| prepare-test-provider-packages-wheel: | |
| timeout-minutes: 80 | |
| name: "Provider packages wheel (verify)" | |
| runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}} | |
| needs: [build-info, wait-for-ci-images] | |
| env: | |
| RUNS_ON: "${{needs.build-info.outputs.runs-on}}" | |
| PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}" | |
| if: needs.build-info.outputs.skip-provider-tests != 'true' | |
| steps: | |
| - name: Cleanup repo | |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" | |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" | |
| uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false | |
| - name: > | |
| Prepare breeze & CI image: ${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}} | |
| uses: ./.github/actions/prepare_breeze_and_image | |
| - name: "Cleanup dist files" | |
| run: rm -fv ./dist/* | |
| - name: "Prepare provider documentation" | |
| run: > | |
| breeze release-management prepare-provider-documentation --include-not-ready-providers | |
| --non-interactive | |
| - name: "Prepare provider packages: wheel" | |
| run: > | |
| breeze release-management prepare-provider-packages --include-not-ready-providers | |
| --version-suffix-for-pypi dev0 --package-format wheel | |
| - name: "Prepare airflow package: wheel" | |
| run: breeze release-management prepare-airflow-package --version-suffix-for-pypi dev0 | |
| - name: "Verify wheel packages with twine" | |
| run: pipx install twine --force && twine check dist/*.whl | |
| - name: "Test providers issue generation automatically" | |
| run: > | |
| breeze release-management generate-issue-content-providers | |
| --only-available-in-dist --disable-progress | |
| - name: "Generate source constraints from CI image" | |
| shell: bash | |
| run: > | |
| breeze release-management generate-constraints | |
| --airflow-constraints-mode constraints-source-providers | |
| - name: "Install and verify all provider packages and airflow via wheel files" | |
| run: > | |
| breeze release-management verify-provider-packages | |
| --use-packages-from-dist | |
| --package-format wheel | |
| --use-airflow-version wheel | |
| --airflow-constraints-reference default | |
| --providers-constraints-location | |
| /files/constraints-${{env.PYTHON_MAJOR_MINOR_VERSION}}/constraints-source-providers-${{env.PYTHON_MAJOR_MINOR_VERSION}}.txt | |
| env: | |
| AIRFLOW_SKIP_CONSTRAINTS: "${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}" | |
| - name: "Prepare airflow package: wheel without suffix and skipping the tag check" | |
| run: > | |
| breeze release-management prepare-provider-packages --skip-tag-check --package-format wheel | |
| prepare-install-provider-packages-sdist: | |
| timeout-minutes: 80 | |
| name: "Provider packages sdist (install)" | |
| runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}} | |
| needs: [build-info, wait-for-ci-images] | |
| env: | |
| RUNS_ON: "${{needs.build-info.outputs.runs-on}}" | |
| PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}" | |
| if: > | |
| needs.build-info.outputs.canary-run == 'true' && | |
| needs.build-info.outputs.skip-provider-tests != 'true' | |
| steps: | |
| - name: Cleanup repo | |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" | |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" | |
| uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false | |
| - name: > | |
| Prepare breeze & CI image: ${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}} | |
| uses: ./.github/actions/prepare_breeze_and_image | |
| - name: "Cleanup dist files" | |
| run: rm -fv ./dist/* | |
| - name: "Prepare provider packages: sdist" | |
| run: > | |
| breeze release-management prepare-provider-packages --include-not-ready-providers | |
| --version-suffix-for-pypi dev0 --package-format sdist | |
| ${{ needs.build-info.outputs.affected-providers-list-as-string }} | |
| - name: "Prepare airflow package: sdist" | |
| run: > | |
| breeze release-management prepare-airflow-package | |
| --version-suffix-for-pypi dev0 --package-format sdist | |
| - name: "Verify sdist packages with twine" | |
| run: pipx install twine --force && twine check dist/*.tar.gz | |
| - name: "Generate source constraints from CI image" | |
| shell: bash | |
| run: > | |
| breeze release-management generate-constraints | |
| --airflow-constraints-mode constraints-source-providers | |
| - name: "Install all provider packages and airflow via sdist files" | |
| run: > | |
| breeze release-management install-provider-packages | |
| --use-packages-from-dist | |
| --package-format sdist | |
| --use-airflow-version sdist | |
| --airflow-constraints-reference default | |
| --providers-constraints-location | |
| /files/constraints-${{env.PYTHON_MAJOR_MINOR_VERSION}}/constraints-source-providers-${{env.PYTHON_MAJOR_MINOR_VERSION}}.txt | |
| --run-in-parallel | |
| if: needs.build-info.outputs.affected-providers-list-as-string == '' | |
| - name: "Install affected provider packages and airflow via sdist files" | |
| run: > | |
| breeze release-management install-provider-packages | |
| --use-packages-from-dist | |
| --package-format sdist | |
| --use-airflow-version sdist | |
| --airflow-constraints-reference default | |
| --providers-constraints-location | |
| /files/constraints-${{env.PYTHON_MAJOR_MINOR_VERSION}}/constraints-source-providers-${{env.PYTHON_MAJOR_MINOR_VERSION}}.txt | |
| --run-in-parallel | |
| if: needs.build-info.outputs.affected-providers-list-as-string != '' | |
| providers-compatibility-checks: | |
| timeout-minutes: 80 | |
| name: > | |
| Compat ${{matrix.airflow-version}}:P${{matrix.python-version}} provider check | |
| runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}} | |
| needs: [build-info, wait-for-ci-images] | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: ${{fromJson(needs.build-info.outputs.providers-compatibility-checks)}} | |
| env: | |
| RUNS_ON: "${{needs.build-info.outputs.runs-on}}" | |
| PYTHON_MAJOR_MINOR_VERSION: "${{matrix.python-version}}" | |
| VERSION_SUFFIX_FOR_PYPI: "dev0" | |
| if: needs.build-info.outputs.skip-provider-tests != 'true' | |
| steps: | |
| - name: Cleanup repo | |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" | |
| if: contains(fromJson(needs.build-info.outputs.python-versions),matrix.python-version) | |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" | |
| uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false | |
| - name: > | |
| Prepare breeze & CI image: ${{matrix.python-version}}:${{env.IMAGE_TAG}} | |
| uses: ./.github/actions/prepare_breeze_and_image | |
| - name: "Cleanup dist files" | |
| run: rm -fv ./dist/* | |
| - name: "Prepare provider packages: wheel" | |
| run: > | |
| breeze release-management prepare-provider-packages --include-not-ready-providers | |
| --package-format wheel | |
| - name: > | |
| Remove incompatible Airflow | |
| ${{matrix.airflow-version}}:Python ${{matrix.python-version}} provider packages | |
| run: | | |
| rm -vf ${{ matrix.remove-providers }} | |
| working-directory: ./dist | |
| if: matrix.remove-providers != '' | |
| - name: "Download airflow package: wheel" | |
| run: | | |
| pip download "apache-airflow==${{matrix.airflow-version}}" -d dist --no-deps | |
| - name: > | |
| Install and verify all provider packages and airflow on | |
| Airflow ${{matrix.airflow-version}}:Python ${{matrix.python-version}} | |
| run: > | |
| breeze release-management verify-provider-packages | |
| --use-packages-from-dist | |
| --package-format wheel | |
| --use-airflow-version wheel | |
| --airflow-constraints-reference constraints-${{matrix.airflow-version}} | |
| --providers-skip-constraints | |
| test-airflow-release-commands: | |
| timeout-minutes: 80 | |
| name: "Test Airflow release commands" | |
| runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}} | |
| needs: [build-info, wait-for-ci-images] | |
| if: > | |
| needs.build-info.outputs.is-self-hosted-runner == 'true' | |
| && needs.build-info.outputs.is-airflow-runner == 'false' | |
| env: | |
| RUNS_ON: "${{needs.build-info.outputs.runs-on}}" | |
| PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}" | |
| steps: | |
| - name: Cleanup repo | |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" | |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" | |
| uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false | |
| - name: > | |
| Prepare breeze & CI image: ${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}} | |
| uses: ./.github/actions/prepare_breeze_and_image | |
| - name: "Cleanup dist files" | |
| run: rm -fv ./dist/* | |
| - name: "Install required packages" | |
| run: python -m pip install --editable ./dev/breeze/ | |
| - name: "Check Airflow create minor branch command" | |
| run: breeze release-management create-minor-branch --version-branch 2-4 -a y | |
| - name: "Check Airflow RC process command" | |
| run: | | |
| breeze release-management start-rc-process --version 2.4.3rc1 --previous-version 2.4.2 -a y | |
| - name: "Check Airflow release process command" | |
| run: | | |
| breeze release-management start-release --release-candidate 2.4.3rc1 --previous-release 2.4.2 -a y | |
| tests-helm: | |
| timeout-minutes: 80 | |
| name: "Unit tests Helm: ${{matrix.helm-test-package}}" | |
| runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}} | |
| needs: [build-info, wait-for-ci-images] | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| helm-test-package: ${{fromJson(needs.build-info.outputs.helm-test-packages)}} | |
| env: | |
| RUNS_ON: "${{needs.build-info.outputs.runs-on}}" | |
| PARALLEL_TEST_TYPES: "Helm" | |
| BACKEND: "none" | |
| DB_RESET: "false" | |
| PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}" | |
| JOB_ID: "helm-tests" | |
| USE_XDIST: "true" | |
| if: > | |
| needs.build-info.outputs.needs-helm-tests == 'true' && needs.build-info.outputs.default-branch == 'main' | |
| steps: | |
| - name: Cleanup repo | |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" | |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" | |
| uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false | |
| - name: > | |
| Prepare breeze & CI image: ${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}} | |
| uses: ./.github/actions/prepare_breeze_and_image | |
| - name: "Helm Unit Tests: ${{ matrix.helm-test-package }}" | |
| run: breeze testing helm-tests --helm-test-package "${{ matrix.helm-test-package }}" | |
| tests-helm-release: | |
| timeout-minutes: 80 | |
| name: "Release Helm" | |
| runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}} | |
| needs: [build-info] | |
| env: | |
| RUNS_ON: "${{needs.build-info.outputs.runs-on}}" | |
| PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}" | |
| if: > | |
| needs.build-info.outputs.needs-helm-tests == 'true' && needs.build-info.outputs.default-branch == 'main' | |
| steps: | |
| - name: Cleanup repo | |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" | |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" | |
| uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false | |
| - name: "Install Breeze" | |
| uses: ./.github/actions/breeze | |
| - name: Setup git for tagging | |
| run: | | |
| git config --global user.email "[email protected]" | |
| git config --global user.name "Your Name" | |
| - name: "Remove old artifacts" | |
| run: rm -rf dist/* | |
| - name: "Setup k8s/helm environment" | |
| run: breeze k8s setup-env | |
| - name: "Install helm gpg plugin" | |
| run: > | |
| breeze k8s shell -c "helm plugin install https://github.com/technosophos/helm-gpg | |
| --version 6303407eb63deaeb1b2f24de611e3468a27ec05b" | |
| - name: "Helm release tarball" | |
| run: > | |
| breeze release-management prepare-helm-chart-tarball --ignore-version-check --override-tag | |
| --skip-tag-signing --version 0.0.0 --version-suffix dev0 | |
| - name: Generate GPG key for signing | |
| run: gpg --batch --passphrase '' --quick-gen-key [email protected] default default | |
| - name: "Helm release package" | |
| run: > | |
| breeze release-management prepare-helm-chart-package --sign-email [email protected] | |
| - name: "Sign artifacts for ASF distribution" | |
| run: ./dev/sign.sh dist/airflow-*.tgz dist/airflow-*-source.tar.gz | |
| env: | |
| SIGN_WITH: [email protected] | |
| - name: "Upload Helm artifacts" | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: Helm artifacts | |
| path: ./dist/airflow-* | |
| retention-days: 7 | |
| if-no-files-found: error | |
| tests-postgres: | |
| timeout-minutes: 130 | |
| name: > | |
| DB:Postgres${{matrix.postgres-version}},Py${{matrix.python-version}}: | |
| ${{needs.build-info.outputs.parallel-test-types-list-as-string}} | |
| runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}} | |
| needs: [build-info, wait-for-ci-images] | |
| strategy: | |
| matrix: | |
| python-version: "${{fromJson(needs.build-info.outputs.python-versions)}}" | |
| postgres-version: "${{fromJson(needs.build-info.outputs.postgres-versions)}}" | |
| exclude: "${{fromJson(needs.build-info.outputs.postgres-exclude)}}" | |
| fail-fast: false | |
| env: | |
| RUNS_ON: "${{needs.build-info.outputs.runs-on}}" | |
| PARALLEL_TEST_TYPES: "${{needs.build-info.outputs.parallel-test-types-list-as-string}}" | |
| PR_LABELS: "${{needs.build-info.outputs.pull-request-labels}}" | |
| FULL_TESTS_NEEDED: "${{needs.build-info.outputs.full-tests-needed}}" | |
| DEBUG_RESOURCES: "${{needs.build-info.outputs.debug-resources}}" | |
| BACKEND: "postgres" | |
| PYTHON_MAJOR_MINOR_VERSION: "${{matrix.python-version}}" | |
| POSTGRES_VERSION: "${{matrix.postgres-version}}" | |
| BACKEND_VERSION: "${{matrix.postgres-version}}" | |
| JOB_ID: "postgres-${{matrix.postgres-version}}-${{matrix.python-version}}" | |
| ENABLE_COVERAGE: "${{needs.build-info.outputs.run-coverage}}" | |
| if: needs.build-info.outputs.run-tests == 'true' | |
| steps: | |
| - name: Cleanup repo | |
| shell: bash | |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" | |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" | |
| uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false | |
| - name: "Prepare breeze & CI image: ${{matrix.python-version}}:${{env.IMAGE_TAG}}" | |
| uses: ./.github/actions/prepare_breeze_and_image | |
| - name: > | |
| Migration Tests: | |
| ${{matrix.python-version}}:${{needs.build-info.outputs.parallel-test-types-list-as-string}} | |
| uses: ./.github/actions/migration_tests | |
| - name: > | |
| Tests: ${{matrix.python-version}}:${{needs.build-info.outputs.parallel-test-types-list-as-string}} | |
| run: > | |
| breeze testing db-tests | |
| --parallel-test-types "${{needs.build-info.outputs.parallel-test-types-list-as-string}}" | |
| - name: "Tests ARM Pytest collection: ${{matrix.python-version}}" | |
| run: breeze testing db-tests --collect-only --remove-arm-packages | |
| if: matrix.postgres-version == needs.build-info.outputs.default-postgres-version | |
| - name: > | |
| Post Tests success: Postgres" | |
| uses: ./.github/actions/post_tests_success | |
| if: success() | |
| - name: > | |
| Post Tests failure: Postgres" | |
| uses: ./.github/actions/post_tests_failure | |
| if: failure() | |
| tests-postgres-boto: | |
| timeout-minutes: 130 | |
| name: > | |
| DB:LatestBoto${{needs.build-info.outputs.default-postgres-version}}, | |
| Py${{needs.build-info.outputs.default-python-version}}: | |
| ${{needs.build-info.outputs.parallel-test-types-list-as-string}} | |
| runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}} | |
| needs: [build-info, wait-for-ci-images] | |
| env: | |
| RUNS_ON: "${{needs.build-info.outputs.runs-on}}" | |
| PARALLEL_TEST_TYPES: "${{needs.build-info.outputs.parallel-test-types-list-as-string}}" | |
| PR_LABELS: "${{needs.build-info.outputs.pull-request-labels}}" | |
| FULL_TESTS_NEEDED: "${{needs.build-info.outputs.full-tests-needed}}" | |
| DEBUG_RESOURCES: "${{needs.build-info.outputs.debug-resources}}" | |
| BACKEND: "postgres" | |
| ENABLE_COVERAGE: "${{needs.build-info.outputs.run-coverage}}" | |
| PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}" | |
| PYTHON_VERSION: "${needs.build-info.outputs.default-python-version}}" | |
| POSTGRES_VERSION: "${{needs.build-info.outputs.default-postgres-version}}" | |
| BACKEND_VERSION: "${{needs.build-info.outputs.default-postgres-version}}" | |
| UPGRADE_BOTO: "true" | |
| JOB_ID: > | |
| postgres-boto-${{needs.build-info.outputs.default-python-version}}- | |
| ${{needs.build-info.outputs.default-postgres-version}} | |
| if: needs.build-info.outputs.run-tests == 'true' && needs.build-info.outputs.run-amazon-tests == 'true' | |
| steps: | |
| - name: Cleanup repo | |
| shell: bash | |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" | |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" | |
| uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false | |
| - name: > | |
| Prepare breeze & CI image: ${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}} | |
| uses: ./.github/actions/prepare_breeze_and_image | |
| - name: > | |
| Tests: ${{matrix.python-version}}:${{needs.build-info.outputs.parallel-test-types-list-as-string}} | |
| run: > | |
| breeze testing db-tests | |
| --parallel-test-types "${{needs.build-info.outputs.parallel-test-types-list-as-string}}" | |
| - name: > | |
| Post Tests success: ${{needs.build-info.outputs.default-python-version}}:Boto" | |
| uses: ./.github/actions/post_tests_success | |
| if: success() | |
| - name: > | |
| Post Tests failure: ${{needs.build-info.outputs.default-python-version}}:Boto" | |
| uses: ./.github/actions/post_tests_failure | |
| if: failure() | |
| tests-postgres-min-sqlalchemy: | |
| timeout-minutes: 130 | |
| name: > | |
| DB:MinSQLAlchemy${{needs.build-info.outputs.default-postgres-version}}, | |
| Py${{needs.build-info.outputs.default-python-version}}: | |
| ${{needs.build-info.outputs.parallel-test-types-list-as-string}} | |
| runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}} | |
| needs: [build-info, wait-for-ci-images] | |
| env: | |
| RUNS_ON: "${{needs.build-info.outputs.runs-on}}" | |
| PARALLEL_TEST_TYPES: "${{needs.build-info.outputs.parallel-test-types-list-as-string}}" | |
| PR_LABELS: "${{needs.build-info.outputs.pull-request-labels}}" | |
| FULL_TESTS_NEEDED: "${{needs.build-info.outputs.full-tests-needed}}" | |
| DEBUG_RESOURCES: "${{needs.build-info.outputs.debug-resources}}" | |
| BACKEND: "postgres" | |
| ENABLE_COVERAGE: "${{needs.build-info.outputs.run-coverage}}" | |
| PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}" | |
| PYTHON_VERSION: "${needs.build-info.outputs.default-python-version}}" | |
| POSTGRES_VERSION: "${{needs.build-info.outputs.default-postgres-version}}" | |
| BACKEND_VERSION: "${{needs.build-info.outputs.default-postgres-version}}" | |
| DOWNGRADE_SQLALCHEMY: "true" | |
| JOB_ID: > | |
| postgres-min-sqlalchemy-${{needs.build-info.outputs.default-python-version}}- | |
| ${{needs.build-info.outputs.default-postgres-version}} | |
| if: needs.build-info.outputs.run-tests == 'true' | |
| steps: | |
| - name: Cleanup repo | |
| shell: bash | |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" | |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" | |
| uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false | |
| - name: > | |
| Prepare breeze & CI image: ${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}} | |
| uses: ./.github/actions/prepare_breeze_and_image | |
| - name: > | |
| Tests: ${{matrix.python-version}}:${{needs.build-info.outputs.parallel-test-types-list-as-string}} | |
| run: > | |
| breeze testing db-tests | |
| --parallel-test-types "${{needs.build-info.outputs.parallel-test-types-list-as-string}}" | |
| - name: > | |
| Post Tests success: ${{needs.build-info.outputs.default-python-version}}:MinSQLAlchemy" | |
| uses: ./.github/actions/post_tests_success | |
| if: success() | |
| - name: > | |
| Post Tests failure: ${{needs.build-info.outputs.default-python-version}}:MinSQLAlchemy" | |
| uses: ./.github/actions/post_tests_failure | |
| if: failure() | |
| tests-postgres-pendulum-2: | |
| timeout-minutes: 130 | |
| name: > | |
| DB:Postgres${{needs.build-info.outputs.default-postgres-version}}, | |
| Pendulum2,Py${{needs.build-info.outputs.default-python-version}}: | |
| ${{needs.build-info.outputs.parallel-test-types-list-as-string}} | |
| runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}} | |
| needs: [build-info, wait-for-ci-images] | |
| env: | |
| RUNS_ON: "${{needs.build-info.outputs.runs-on}}" | |
| PARALLEL_TEST_TYPES: "${{needs.build-info.outputs.parallel-test-types-list-as-string}}" | |
| PR_LABELS: "${{needs.build-info.outputs.pull-request-labels}}" | |
| FULL_TESTS_NEEDED: "${{needs.build-info.outputs.full-tests-needed}}" | |
| DEBUG_RESOURCES: "${{needs.build-info.outputs.debug-resources}}" | |
| BACKEND: "postgres" | |
| ENABLE_COVERAGE: "${{needs.build-info.outputs.run-coverage}}" | |
| PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}" | |
| PYTHON_VERSION: "${needs.build-info.outputs.default-python-version}}" | |
| POSTGRES_VERSION: "${{needs.build-info.outputs.default-postgres-version}}" | |
| BACKEND_VERSION: "${{needs.build-info.outputs.default-postgres-version}}" | |
| DOWNGRADE_PENDULUM: "true" | |
| JOB_ID: > | |
| postgres-pendulum-2-${{needs.build-info.outputs.default-python-version}}- | |
| ${{needs.build-info.outputs.default-postgres-version}} | |
| if: needs.build-info.outputs.run-tests == 'true' | |
| steps: | |
| - name: Cleanup repo | |
| shell: bash | |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" | |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" | |
| uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false | |
| - name: > | |
| Prepare breeze & CI image: ${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}} | |
| uses: ./.github/actions/prepare_breeze_and_image | |
| - name: > | |
| Tests: ${{matrix.python-version}}:${{needs.build-info.outputs.parallel-test-types-list-as-string}} | |
| run: > | |
| breeze testing db-tests | |
| --parallel-test-types "${{needs.build-info.outputs.parallel-test-types-list-as-string}}" | |
| - name: > | |
| Post Tests success: ${{needs.build-info.outputs.default-python-version}}:Pendulum2" | |
| uses: ./.github/actions/post_tests_success | |
| if: success() | |
| - name: > | |
| Post Tests failure: ${{needs.build-info.outputs.default-python-version}}:Pendulum2" | |
| uses: ./.github/actions/post_tests_failure | |
| if: failure() | |
| tests-postgres-in-progress-features-disabled: | |
| timeout-minutes: 130 | |
| name: > | |
| DB:InProgressDisabledPostgres${{needs.build-info.outputs.default-postgres-version}}, | |
| Py${{needs.build-info.outputs.default-python-version}}: | |
| ${{needs.build-info.outputs.parallel-test-types-list-as-string}} | |
| runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}} | |
| needs: [build-info, wait-for-ci-images] | |
| env: | |
| RUNS_ON: "${{needs.build-info.outputs.runs-on}}" | |
| PARALLEL_TEST_TYPES: "${{needs.build-info.outputs.parallel-test-types-list-as-string}}" | |
| PR_LABELS: "${{needs.build-info.outputs.pull-request-labels}}" | |
| FULL_TESTS_NEEDED: "${{needs.build-info.outputs.full-tests-needed}}" | |
| DEBUG_RESOURCES: "${{needs.build-info.outputs.debug-resources}}" | |
| BACKEND: "postgres" | |
| ENABLE_COVERAGE: "${{needs.build-info.outputs.run-coverage}}" | |
| PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}" | |
| PYTHON_VERSION: "${needs.build-info.outputs.default-python-version}}" | |
| POSTGRES_VERSION: "${{needs.build-info.outputs.default-postgres-version}}" | |
| BACKEND_VERSION: "${{needs.build-info.outputs.default-postgres-version}}" | |
| AIRFLOW_ENABLE_AIP_44: "false" | |
| JOB_ID: > | |
| postgres-in-progress-disabled-${{needs.build-info.outputs.default-python-version}}- | |
| ${{needs.build-info.outputs.default-postgres-version}} | |
| if: needs.build-info.outputs.run-tests == 'true' && needs.build-info.outputs.run-amazon-tests == 'true' | |
| steps: | |
| - name: Cleanup repo | |
| shell: bash | |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" | |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" | |
| uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false | |
| - name: > | |
| Prepare breeze & CI image: ${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}} | |
| uses: ./.github/actions/prepare_breeze_and_image | |
| - name: > | |
| Tests: ${{matrix.python-version}}:${{needs.build-info.outputs.parallel-test-types-list-as-string}} | |
| run: > | |
| breeze testing db-tests | |
| --parallel-test-types "${{needs.build-info.outputs.parallel-test-types-list-as-string}}" | |
| - name: > | |
| Post Tests success: ${{needs.build-info.outputs.default-python-version}}:FeaturesDisabled" | |
| uses: ./.github/actions/post_tests_success | |
| if: success() | |
| - name: > | |
| Post Tests failure: ${{needs.build-info.outputs.default-python-version}}:FeaturesDisabled" | |
| uses: ./.github/actions/post_tests_failure | |
| if: failure() | |
| tests-mysql: | |
| timeout-minutes: 130 | |
| name: > | |
| DB:MySQL${{matrix.mysql-version}}, Py${{matrix.python-version}}: | |
| ${{needs.build-info.outputs.parallel-test-types-list-as-string}} | |
| runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}} | |
| needs: [build-info, wait-for-ci-images] | |
| strategy: | |
| matrix: | |
| python-version: "${{fromJson(needs.build-info.outputs.python-versions)}}" | |
| mysql-version: "${{fromJson(needs.build-info.outputs.mysql-versions)}}" | |
| exclude: "${{fromJson(needs.build-info.outputs.mysql-exclude)}}" | |
| fail-fast: false | |
| env: | |
| RUNS_ON: "${{needs.build-info.outputs.runs-on}}" | |
| PR_LABELS: "${{needs.build-info.outputs.pull-request-labels}}" | |
| FULL_TESTS_NEEDED: "${{needs.build-info.outputs.full-tests-needed}}" | |
| PARALLEL_TEST_TYPES: "${{needs.build-info.outputs.parallel-test-types-list-as-string}}" | |
| DEBUG_RESOURCES: "${{needs.build-info.outputs.debug-resources}}" | |
| BACKEND: "mysql" | |
| ENABLE_COVERAGE: "${{needs.build-info.outputs.run-coverage}}" | |
| PYTHON_MAJOR_MINOR_VERSION: "${{matrix.python-version}}" | |
| MYSQL_VERSION: "${{matrix.mysql-version}}" | |
| BACKEND_VERSION: "${{matrix.mysql-version}}" | |
| JOB_ID: "mysql-${{matrix.mysql-version}}-${{matrix.python-version}}" | |
| if: needs.build-info.outputs.run-tests == 'true' | |
| steps: | |
| - name: Cleanup repo | |
| shell: bash | |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" | |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" | |
| uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false | |
| - name: "Prepare breeze & CI image: ${{matrix.python-version}}:${{env.IMAGE_TAG}}" | |
| uses: ./.github/actions/prepare_breeze_and_image | |
| - name: > | |
| Migration Tests: | |
| ${{matrix.python-version}}:${{needs.build-info.outputs.parallel-test-types-list-as-string}} | |
| uses: ./.github/actions/migration_tests | |
| - name: > | |
| Tests: ${{matrix.python-version}}:${{needs.build-info.outputs.parallel-test-types-list-as-string}} | |
| run: > | |
| breeze testing db-tests | |
| --parallel-test-types "${{needs.build-info.outputs.parallel-test-types-list-as-string}}" | |
| - name: > | |
| Post Tests success: MySQL" | |
| uses: ./.github/actions/post_tests_success | |
| if: success() | |
| - name: > | |
| Post Tests failure: MySQL" | |
| uses: ./.github/actions/post_tests_failure | |
| if: failure() | |
| tests-sqlite: | |
| timeout-minutes: 130 | |
| name: > | |
| DB:Sqlite Py${{matrix.python-version}}: ${{needs.build-info.outputs.parallel-test-types-list-as-string}} | |
| runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}} | |
| needs: [build-info, wait-for-ci-images] | |
| strategy: | |
| matrix: | |
| python-version: ${{ fromJson(needs.build-info.outputs.python-versions) }} | |
| exclude: ${{ fromJson(needs.build-info.outputs.sqlite-exclude) }} | |
| fail-fast: false | |
| if: needs.build-info.outputs.run-tests == 'true' | |
| env: | |
| RUNS_ON: "${{needs.build-info.outputs.runs-on}}" | |
| PARALLEL_TEST_TYPES: "${{needs.build-info.outputs.parallel-test-types-list-as-string}}" | |
| PR_LABELS: "${{needs.build-info.outputs.pull-request-labels}}" | |
| PYTHON_MAJOR_MINOR_VERSION: "${{matrix.python-version}}" | |
| FULL_TESTS_NEEDED: "${{needs.build-info.outputs.full-tests-needed}}" | |
| DEBUG_RESOURCES: "${{needs.build-info.outputs.debug-resources}}" | |
| BACKEND: "sqlite" | |
| ENABLE_COVERAGE: "${{needs.build-info.outputs.run-coverage}}" | |
| BACKEND_VERSION: "" | |
| JOB_ID: "sqlite-${{matrix.python-version}}" | |
| steps: | |
| - name: Cleanup repo | |
| shell: bash | |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" | |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" | |
| uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false | |
| - name: "Prepare breeze & CI image: ${{matrix.python-version}}:${{env.IMAGE_TAG}}" | |
| uses: ./.github/actions/prepare_breeze_and_image | |
| - name: > | |
| Migration Tests: | |
| ${{matrix.python-version}}:${{needs.build-info.outputs.parallel-test-types-list-as-string}} | |
| uses: ./.github/actions/migration_tests | |
| - name: > | |
| Tests: ${{matrix.python-version}}:${{needs.build-info.outputs.parallel-test-types-list-as-string}} | |
| run: > | |
| breeze testing db-tests | |
| --parallel-test-types "${{needs.build-info.outputs.parallel-test-types-list-as-string}}" | |
| - name: > | |
| Post Tests success: Sqlite" | |
| uses: ./.github/actions/post_tests_success | |
| if: success() | |
| - name: > | |
| Post Tests failure: Sqlite" | |
| uses: ./.github/actions/post_tests_failure | |
| if: failure() | |
| tests-integration-postgres: | |
| timeout-minutes: 130 | |
| name: Integration Tests Postgres | |
| runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}} | |
| needs: [build-info, wait-for-ci-images] | |
| env: | |
| RUNS_ON: "${{needs.build-info.outputs.runs-on}}" | |
| PARALLEL_TEST_TYPES: "${{needs.build-info.outputs.parallel-test-types-list-as-string}}" | |
| PR_LABELS: "${{needs.build-info.outputs.pull-request-labels}}" | |
| FULL_TESTS_NEEDED: "${{needs.build-info.outputs.full-tests-needed}}" | |
| DEBUG_RESOURCES: "${{needs.build-info.outputs.debug-resources}}" | |
| BACKEND: "postgres" | |
| ENABLE_COVERAGE: "${{needs.build-info.outputs.run-coverage}}" | |
| PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}" | |
| POSTGRES_VERSION: "${{needs.build-info.outputs.default-postgres-version}}" | |
| BACKEND_VERSION: "${{needs.build-info.outputs.default-postgres-version}}" | |
| JOB_ID: "integration" | |
| SKIP_PROVIDER_TESTS: "${{needs.build-info.outputs.skip-provider-tests}}" | |
| if: needs.build-info.outputs.run-tests == 'true' | |
| steps: | |
| - name: Cleanup repo | |
| shell: bash | |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" | |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" | |
| uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false | |
| - name: "Prepare breeze & CI image: ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{env.IMAGE_TAG}}" | |
| uses: ./.github/actions/prepare_breeze_and_image | |
| - name: "Integration Tests Postgres: cassandra" | |
| run: | | |
| breeze testing integration-tests --integration cassandra | |
| breeze down | |
| if: needs.build-info.outputs.is-airflow-runner != 'true' | |
| - name: "Integration Tests Postgres: mongo" | |
| run: | | |
| breeze testing integration-tests --integration mongo | |
| breeze down | |
| if: needs.build-info.outputs.is-airflow-runner != 'true' | |
| - name: "Integration Tests Postgres: pinot" | |
| run: | | |
| breeze testing integration-tests --integration pinot | |
| breeze down | |
| if: needs.build-info.outputs.is-airflow-runner != 'true' | |
| - name: "Integration Tests Postgres: celery" | |
| run: | | |
| breeze testing integration-tests --integration celery | |
| breeze down | |
| if: needs.build-info.outputs.is-airflow-runner != 'true' | |
| - name: "Integration Tests Postgres: trino, kerberos" | |
| run: | | |
| breeze testing integration-tests --integration trino --integration kerberos | |
| breeze down | |
| if: needs.build-info.outputs.is-airflow-runner != 'true' | |
| - name: "Integration Tests Postgres: Kafka" | |
| run: | | |
| breeze testing integration-tests --integration kafka | |
| breeze down | |
| if: needs.build-info.outputs.is-airflow-runner != 'true' | |
| - name: "Integration Tests Postgres: all-testable" | |
| run: breeze testing integration-tests --integration all-testable | |
| if: needs.build-info.outputs.is-airflow-runner == 'true' | |
| - name: > | |
| Post Tests success: Integration Postgres" | |
| uses: ./.github/actions/post_tests_success | |
| if: success() | |
| - name: > | |
| Post Tests failure: Integration Postgres" | |
| uses: ./.github/actions/post_tests_failure | |
| if: failure() | |
| tests-integration-mysql: | |
| timeout-minutes: 130 | |
| name: Integration Tests MySQL | |
| runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}} | |
| needs: [build-info, wait-for-ci-images] | |
| env: | |
| RUNS_ON: "${{needs.build-info.outputs.runs-on}}" | |
| PARALLEL_TEST_TYPES: "${{needs.build-info.outputs.parallel-test-types-list-as-string}}" | |
| PR_LABELS: "${{needs.build-info.outputs.pull-request-labels}}" | |
| FULL_TESTS_NEEDED: "${{needs.build-info.outputs.full-tests-needed}}" | |
| DEBUG_RESOURCES: "${{needs.build-info.outputs.debug-resources}}" | |
| BACKEND: "mysql" | |
| ENABLE_COVERAGE: "${{needs.build-info.outputs.run-coverage}}" | |
| PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}" | |
| MYSQL_VERSION: "${{needs.build-info.outputs.default-mysql-version}}" | |
| BACKEND_VERSION: "${{needs.build-info.outputs.default-mysql-version}}" | |
| JOB_ID: "integration" | |
| SKIP_PROVIDER_TESTS: "${{needs.build-info.outputs.skip-provider-tests}}" | |
| if: needs.build-info.outputs.run-tests == 'true' | |
| steps: | |
| - name: Cleanup repo | |
| shell: bash | |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" | |
| if: needs.build-info.outputs.is-airflow-runner == 'true' | |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" | |
| uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false | |
| if: needs.build-info.outputs.is-airflow-runner == 'true' | |
| - name: "Prepare breeze & CI image: ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{env.IMAGE_TAG}}" | |
| uses: ./.github/actions/prepare_breeze_and_image | |
| if: needs.build-info.outputs.is-airflow-runner == 'true' | |
| - name: "Integration Tests MySQL: all-testable" | |
| run: breeze testing integration-tests --integration all-testable | |
| if: needs.build-info.outputs.is-airflow-runner == 'true' | |
| - name: > | |
| Post Tests success: Integration MySQL" | |
| uses: ./.github/actions/post_tests_success | |
| if: needs.build-info.outputs.is-airflow-runner == 'true' | |
| - name: > | |
| Post Tests failure: Integration MySQL" | |
| uses: ./.github/actions/post_tests_failure | |
| if: failure() | |
| tests-quarantined: | |
| timeout-minutes: 60 | |
| name: "Quarantined tests" | |
| runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}} | |
| continue-on-error: true | |
| needs: [build-info, wait-for-ci-images] | |
| env: | |
| RUNS_ON: "${{needs.build-info.outputs.runs-on}}" | |
| TEST_TYPE: "All-Quarantined" | |
| PR_LABELS: "${{needs.build-info.outputs.pull-request-labels}}" | |
| PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}" | |
| DEBUG_RESOURCES: "${{needs.build-info.outputs.debug-resources}}" | |
| JOB_ID: "quarantined-${{needs.build-info.outputs.default-python-version}}" | |
| ENABLE_COVERAGE: "${{needs.build-info.outputs.run-coverage}}" | |
| if: needs.build-info.outputs.run-tests == 'true' && needs.build-info.outputs.is-airflow-runner == 'true' | |
| steps: | |
| - name: Cleanup repo | |
| shell: bash | |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" | |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" | |
| uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false | |
| - name: > | |
| Prepare breeze & CI image: ${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}} | |
| uses: ./.github/actions/prepare_breeze_and_image | |
| - name: > | |
| Tests: postgres:${{needs.build-info.outputs.default-python-version}}:Quarantined | |
| run: breeze testing tests || true | |
| env: | |
| BACKEND: "postgres" | |
| BACKEND_VERSION: ${{needs.build-info.outputs.default-postgres-version}} | |
| POSTGRES_VERSION: ${{needs.build-info.outputs.default-postgres-version}} | |
| - name: > | |
| Tests: mysql:${{needs.build-info.outputs.default-python-version}}:Quarantined | |
| run: breeze testing tests || true | |
| env: | |
| BACKEND: "mysql" | |
| BACKEND_VERSION: ${{needs.build-info.outputs.default-mysql-version}} | |
| MYSQL_VERSION: ${{needs.build-info.outputs.default-mysql-version}} | |
| - name: > | |
| Tests: sqlite:${{needs.build-info.outputs.default-python-version}}:Quarantined | |
| run: breeze testing tests || true | |
| env: | |
| BACKEND: "sqlite" | |
| BACKEND_VERSION: "" | |
| - name: > | |
| Post Tests success: Quarantined" | |
| uses: ./.github/actions/post_tests_success | |
| if: success() | |
| - name: > | |
| Post Tests failure: Quarantined" | |
| uses: ./.github/actions/post_tests_failure | |
| if: failure() | |
| tests-no-db: | |
| timeout-minutes: 60 | |
| name: > | |
| Non-DB: Py${{matrix.python-version}}:${{needs.build-info.outputs.parallel-test-types-list-as-string}} | |
| runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}} | |
| needs: [build-info, wait-for-ci-images] | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python-version: ${{ fromJson(needs.build-info.outputs.python-versions) }} | |
| env: | |
| RUNS_ON: "${{needs.build-info.outputs.runs-on}}" | |
| PR_LABELS: "${{needs.build-info.outputs.pull-request-labels}}" | |
| PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}" | |
| DEBUG_RESOURCES: "${{needs.build-info.outputs.debug-resources}}" | |
| JOB_ID: "quarantined-${{needs.build-info.outputs.default-python-version}}" | |
| ENABLE_COVERAGE: "${{needs.build-info.outputs.run-coverage}}" | |
| if: needs.build-info.outputs.run-tests == 'true' | |
| steps: | |
| - name: Cleanup repo | |
| shell: bash | |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" | |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" | |
| uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false | |
| - name: > | |
| Prepare breeze & CI image: ${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}} | |
| uses: ./.github/actions/prepare_breeze_and_image | |
| - name: "Tests: ${{matrix.python-version}}:Non-DB" | |
| run: > | |
| breeze testing non-db-tests | |
| --parallel-test-types "${{needs.build-info.outputs.parallel-test-types-list-as-string}}" | |
| - name: "Post Tests success: Non-DB" | |
| uses: ./.github/actions/post_tests_success | |
| if: success() | |
| - name: "Post Tests failure: Non-DB" | |
| uses: ./.github/actions/post_tests_failure | |
| if: failure() | |
| tests-no-db-pendulum-2: | |
| timeout-minutes: 60 | |
| name: > | |
| Non-DB: Pendulum2, Py${{needs.build-info.outputs.default-python-version}}: | |
| ${{needs.build-info.outputs.parallel-test-types-list-as-string}} | |
| runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}} | |
| needs: [build-info, wait-for-ci-images] | |
| env: | |
| RUNS_ON: "${{needs.build-info.outputs.runs-on}}" | |
| PR_LABELS: "${{needs.build-info.outputs.pull-request-labels}}" | |
| PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}" | |
| DEBUG_RESOURCES: "${{needs.build-info.outputs.debug-resources}}" | |
| JOB_ID: "quarantined-${{needs.build-info.outputs.default-python-version}}" | |
| ENABLE_COVERAGE: "${{needs.build-info.outputs.run-coverage}}" | |
| DOWNGRADE_PENDULUM: "true" | |
| if: needs.build-info.outputs.run-tests == 'true' | |
| steps: | |
| - name: Cleanup repo | |
| shell: bash | |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" | |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" | |
| uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false | |
| - name: > | |
| Prepare breeze & CI image: ${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}} | |
| uses: ./.github/actions/prepare_breeze_and_image | |
| - name: "Tests: ${{matrix.python-version}}:Non-DB-Pendulum2" | |
| run: > | |
| breeze testing non-db-tests | |
| --parallel-test-types "${{needs.build-info.outputs.parallel-test-types-list-as-string}}" | |
| - name: "Post Tests success: Non-DB-Pendulum2" | |
| uses: ./.github/actions/post_tests_success | |
| if: success() | |
| - name: "Post Tests failure: Non-DB-Pendulum2" | |
| uses: ./.github/actions/post_tests_failure | |
| if: failure() | |
| summarize-warnings: | |
| timeout-minutes: 15 | |
| name: "Summarize warnings" | |
| runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}} | |
| needs: | |
| - build-info | |
| - tests-postgres | |
| - tests-sqlite | |
| - tests-mysql | |
| - tests-quarantined | |
| - tests-integration-postgres | |
| - tests-integration-mysql | |
| - tests-no-db | |
| env: | |
| RUNS_ON: "${{needs.build-info.outputs.runs-on}}" | |
| steps: | |
| - name: Cleanup repo | |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" | |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" | |
| uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false | |
| - name: "Download all artifacts from the current build" | |
| uses: actions/download-artifact@v3 | |
| with: | |
| path: ./artifacts | |
| - name: "Summarize all warnings" | |
| run: | | |
| ls -R ./artifacts/ | |
| cat ./artifacts/test-warnings*/* | sort | uniq | |
| echo | |
| echo Total number of unique warnings $(cat ./artifacts/test-warnings*/* | sort | uniq | wc -l) | |
| build-prod-images: | |
| timeout-minutes: 80 | |
| name: > | |
| ${{needs.build-info.outputs.build-job-description}} PROD images (main) | |
| ${{needs.build-info.outputs.all-python-versions-list-as-string}} | |
| runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}} | |
| needs: [build-info, build-ci-images] | |
| env: | |
| DEFAULT_BRANCH: ${{ needs.build-info.outputs.default-branch }} | |
| DEFAULT_CONSTRAINTS_BRANCH: ${{ needs.build-info.outputs.default-constraints-branch }} | |
| RUNS_ON: "${{needs.build-info.outputs.runs-on}}" | |
| BACKEND: sqlite | |
| VERSION_SUFFIX_FOR_PYPI: "dev0" | |
| DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}} | |
| # Force more parallelism for build even on public images | |
| PARALLELISM: 6 | |
| steps: | |
| - name: Cleanup repo | |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" | |
| if: > | |
| needs.build-info.outputs.in-workflow-build == 'true' && | |
| needs.build-info.outputs.default-branch == 'main' | |
| - uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ needs.build-info.outputs.targetCommitSha }} | |
| persist-credentials: false | |
| if: > | |
| needs.build-info.outputs.in-workflow-build == 'true' && | |
| needs.build-info.outputs.default-branch == 'main' | |
| - name: "Install Breeze" | |
| uses: ./.github/actions/breeze | |
| if: > | |
| needs.build-info.outputs.in-workflow-build == 'true' && | |
| needs.build-info.outputs.default-branch == 'main' | |
| - name: > | |
| Build PROD Images | |
| ${{needs.build-info.outputs.all-python-versions-list-as-string}}:${{env.IMAGE_TAG}} | |
| uses: ./.github/actions/build-prod-images | |
| if: > | |
| needs.build-info.outputs.in-workflow-build == 'true' && | |
| needs.build-info.outputs.default-branch == 'main' | |
| with: | |
| build-provider-packages: ${{ needs.build-info.outputs.default-branch == 'main' }} | |
| chicken-egg-providers: ${{ needs.build-info.outputs.chicken-egg-providers }} | |
| env: | |
| UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} | |
| DOCKER_CACHE: ${{ needs.build-info.outputs.cache-directive }} | |
| PYTHON_VERSIONS: ${{needs.build-info.outputs.all-python-versions-list-as-string}} | |
| DEBUG_RESOURCES: ${{ needs.build-info.outputs.debug-resources }} | |
| build-prod-images-bullseye: | |
| timeout-minutes: 80 | |
| name: > | |
| Build Bullseye PROD images (main) | |
| ${{needs.build-info.outputs.all-python-versions-list-as-string}} | |
| runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}} | |
| needs: [build-info, build-ci-images] | |
| if: needs.build-info.outputs.canary-run == 'true' | |
| env: | |
| DEFAULT_BRANCH: ${{ needs.build-info.outputs.default-branch }} | |
| DEFAULT_CONSTRAINTS_BRANCH: ${{ needs.build-info.outputs.default-constraints-branch }} | |
| RUNS_ON: "${{needs.build-info.outputs.runs-on}}" | |
| BACKEND: sqlite | |
| VERSION_SUFFIX_FOR_PYPI: "dev0" | |
| DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}} | |
| # Force more parallelism for build even on public images | |
| PARALLELISM: 6 | |
| steps: | |
| - name: Cleanup repo | |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" | |
| if: > | |
| needs.build-info.outputs.in-workflow-build == 'true' && | |
| needs.build-info.outputs.default-branch == 'main' | |
| - uses: actions/checkout@v3 | |
| with: | |
| ref: ${{ needs.build-info.outputs.targetCommitSha }} | |
| persist-credentials: false | |
| submodules: recursive | |
| if: > | |
| needs.build-info.outputs.in-workflow-build == 'true' && | |
| needs.build-info.outputs.default-branch == 'main' | |
| - name: "Install Breeze" | |
| uses: ./.github/actions/breeze | |
| if: > | |
| needs.build-info.outputs.in-workflow-build == 'true' && | |
| needs.build-info.outputs.default-branch == 'main' | |
| - name: > | |
| Build Bullseye PROD Images | |
| ${{needs.build-info.outputs.all-python-versions-list-as-string}}:${{env.IMAGE_TAG}} | |
| uses: ./.github/actions/build-prod-images | |
| if: > | |
| needs.build-info.outputs.in-workflow-build == 'true' && | |
| needs.build-info.outputs.default-branch == 'main' | |
| with: | |
| build-provider-packages: ${{ needs.build-info.outputs.default-branch == 'main' }} | |
| chicken-egg-providers: ${{ needs.build-info.outputs.chicken-egg-providers }} | |
| env: | |
| UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} | |
| DOCKER_CACHE: ${{ needs.build-info.outputs.cache-directive }} | |
| PYTHON_VERSIONS: ${{needs.build-info.outputs.all-python-versions-list-as-string}} | |
| DEBUG_RESOURCES: ${{ needs.build-info.outputs.debug-resources }} | |
| DEBIAN_VERSION: "bullseye" | |
| # Do not override the "bookworm" image - just push a new bullseye image | |
| # TODO: improve caching for that build | |
| IMAGE_TAG: "bullseye-${{ github.event.pull_request.head.sha || github.sha }}" | |
| build-prod-images-mysql-client: | |
| timeout-minutes: 80 | |
| name: > | |
| Build MySQL Client PROD images (main) | |
| ${{needs.build-info.outputs.all-python-versions-list-as-string}} | |
| runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}} | |
| needs: [build-info, build-ci-images] | |
| if: needs.build-info.outputs.canary-run == 'true' | |
| env: | |
| DEFAULT_BRANCH: ${{ needs.build-info.outputs.default-branch }} | |
| DEFAULT_CONSTRAINTS_BRANCH: ${{ needs.build-info.outputs.default-constraints-branch }} | |
| RUNS_ON: "${{needs.build-info.outputs.runs-on}}" | |
| BACKEND: sqlite | |
| VERSION_SUFFIX_FOR_PYPI: "dev0" | |
| DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}} | |
| # Force more parallelism for build even on public images | |
| PARALLELISM: 6 | |
| steps: | |
| - name: Cleanup repo | |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" | |
| if: > | |
| needs.build-info.outputs.in-workflow-build == 'true' && | |
| needs.build-info.outputs.default-branch == 'main' | |
| - uses: actions/checkout@v3 | |
| with: | |
| ref: ${{ needs.build-info.outputs.targetCommitSha }} | |
| persist-credentials: false | |
| submodules: recursive | |
| if: > | |
| needs.build-info.outputs.in-workflow-build == 'true' && | |
| needs.build-info.outputs.default-branch == 'main' | |
| - name: "Install Breeze" | |
| uses: ./.github/actions/breeze | |
| if: > | |
| needs.build-info.outputs.in-workflow-build == 'true' && | |
| needs.build-info.outputs.default-branch == 'main' | |
| - name: > | |
| Build MySQL Client PROD Images | |
| ${{needs.build-info.outputs.all-python-versions-list-as-string}}:${{env.IMAGE_TAG}} | |
| uses: ./.github/actions/build-prod-images | |
| if: > | |
| needs.build-info.outputs.in-workflow-build == 'true' && | |
| needs.build-info.outputs.default-branch == 'main' | |
| with: | |
| build-provider-packages: ${{ needs.build-info.outputs.default-branch == 'main' }} | |
| chicken-egg-providers: ${{ needs.build-info.outputs.chicken-egg-providers }} | |
| env: | |
| UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} | |
| DOCKER_CACHE: ${{ needs.build-info.outputs.cache-directive }} | |
| PYTHON_VERSIONS: ${{needs.build-info.outputs.all-python-versions-list-as-string}} | |
| DEBUG_RESOURCES: ${{ needs.build-info.outputs.debug-resources }} | |
| INSTALL_MYSQL_CLIENT_TYPE: "mysql" | |
| # Do not override the "mariadb" (original) image - just push a new mysql image | |
| # TODO: improve caching for that build | |
| IMAGE_TAG: "bullseye-${{ github.event.pull_request.head.sha || github.sha }}" | |
| build-prod-images-release-branch: | |
| timeout-minutes: 80 | |
| name: > | |
| ${{needs.build-info.outputs.build-job-description}} PROD images (v2_*_test) | |
| ${{needs.build-info.outputs.all-python-versions-list-as-string}} | |
| runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}} | |
| needs: [build-info, generate-constraints] | |
| env: | |
| DEFAULT_BRANCH: ${{ needs.build-info.outputs.default-branch }} | |
| DEFAULT_CONSTRAINTS_BRANCH: ${{ needs.build-info.outputs.default-constraints-branch }} | |
| RUNS_ON: "${{needs.build-info.outputs.runs-on}}" | |
| BACKEND: sqlite | |
| VERSION_SUFFIX_FOR_PYPI: "dev0" | |
| DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}} | |
| # Force more parallelism for build even on public images | |
| PARALLELISM: 6 | |
| steps: | |
| - name: Cleanup repo | |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" | |
| if: > | |
| needs.build-info.outputs.in-workflow-build == 'true' && | |
| needs.build-info.outputs.default-branch != 'main' | |
| - uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ needs.build-info.outputs.targetCommitSha }} | |
| persist-credentials: false | |
| if: > | |
| needs.build-info.outputs.in-workflow-build == 'true' && | |
| needs.build-info.outputs.default-branch != 'main' | |
| - name: "Install Breeze" | |
| uses: ./.github/actions/breeze | |
| if: > | |
| needs.build-info.outputs.in-workflow-build == 'true' && | |
| needs.build-info.outputs.default-branch != 'main' | |
| - name: > | |
| Build PROD Images | |
| ${{needs.build-info.outputs.all-python-versions-list-as-string}}:${{env.IMAGE_TAG}} | |
| uses: ./.github/actions/build-prod-images | |
| if: > | |
| needs.build-info.outputs.in-workflow-build == 'true' && | |
| needs.build-info.outputs.default-branch != 'main' | |
| with: | |
| build-provider-packages: ${{ needs.build-info.outputs.default-branch == 'main' }} | |
| chicken-egg-providers: ${{ needs.build-info.outputs.chicken-egg-providers }} | |
| env: | |
| UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} | |
| DOCKER_CACHE: ${{ needs.build-info.outputs.cache-directive }} | |
| PYTHON_VERSIONS: ${{needs.build-info.outputs.all-python-versions-list-as-string}} | |
| DEBUG_RESOURCES: ${{ needs.build-info.outputs.debug-resources }} | |
| build-prod-images-bullseye-release-branch: | |
| timeout-minutes: 80 | |
| name: > | |
| Build Bullseye PROD images (v2_*_test) | |
| ${{needs.build-info.outputs.all-python-versions-list-as-string}} | |
| runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}} | |
| needs: [build-info, generate-constraints] | |
| if: needs.build-info.outputs.canary-run == 'true' | |
| env: | |
| DEFAULT_BRANCH: ${{ needs.build-info.outputs.default-branch }} | |
| DEFAULT_CONSTRAINTS_BRANCH: ${{ needs.build-info.outputs.default-constraints-branch }} | |
| RUNS_ON: "${{needs.build-info.outputs.runs-on}}" | |
| BACKEND: sqlite | |
| VERSION_SUFFIX_FOR_PYPI: "dev0" | |
| DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}} | |
| # Force more parallelism for build even on public images | |
| PARALLELISM: 6 | |
| steps: | |
| - name: Cleanup repo | |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" | |
| if: > | |
| needs.build-info.outputs.in-workflow-build == 'true' && | |
| needs.build-info.outputs.default-branch != 'main' | |
| - uses: actions/checkout@v3 | |
| with: | |
| ref: ${{ needs.build-info.outputs.targetCommitSha }} | |
| persist-credentials: false | |
| submodules: recursive | |
| if: > | |
| needs.build-info.outputs.in-workflow-build == 'true' && | |
| needs.build-info.outputs.default-branch != 'main' | |
| - name: "Install Breeze" | |
| uses: ./.github/actions/breeze | |
| if: > | |
| needs.build-info.outputs.in-workflow-build == 'true' && | |
| needs.build-info.outputs.default-branch != 'main' | |
| - name: > | |
| Build Bullseye PROD Images | |
| ${{needs.build-info.outputs.all-python-versions-list-as-string}}:${{env.IMAGE_TAG}} | |
| uses: ./.github/actions/build-prod-images | |
| if: > | |
| needs.build-info.outputs.in-workflow-build == 'true' && | |
| needs.build-info.outputs.default-branch != 'main' | |
| with: | |
| build-provider-packages: ${{ needs.build-info.outputs.default-branch == 'main' }} | |
| chicken-egg-providers: ${{ needs.build-info.outputs.chicken-egg-providers }} | |
| env: | |
| UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} | |
| DOCKER_CACHE: ${{ needs.build-info.outputs.cache-directive }} | |
| PYTHON_VERSIONS: ${{needs.build-info.outputs.all-python-versions-list-as-string}} | |
| DEBUG_RESOURCES: ${{ needs.build-info.outputs.debug-resources }} | |
| DEBIAN_VERSION: "bullseye" | |
| # Do not override the "bookworm" image - just push a new bullseye image | |
| # TODO: improve caching for that build | |
| IMAGE_TAG: "bullseye-${{ github.event.pull_request.head.sha || github.sha }}" | |
| build-prod-images-mysql-release-branch: | |
| timeout-minutes: 80 | |
| name: > | |
| Build MySQL PROD images (v2_*_test) | |
| ${{needs.build-info.outputs.all-python-versions-list-as-string}} | |
| runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}} | |
| needs: [build-info, generate-constraints] | |
| if: needs.build-info.outputs.canary-run == 'true' | |
| env: | |
| DEFAULT_BRANCH: ${{ needs.build-info.outputs.default-branch }} | |
| DEFAULT_CONSTRAINTS_BRANCH: ${{ needs.build-info.outputs.default-constraints-branch }} | |
| RUNS_ON: "${{needs.build-info.outputs.runs-on}}" | |
| BACKEND: sqlite | |
| VERSION_SUFFIX_FOR_PYPI: "dev0" | |
| DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}} | |
| # Force more parallelism for build even on public images | |
| PARALLELISM: 6 | |
| steps: | |
| - name: Cleanup repo | |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" | |
| if: > | |
| needs.build-info.outputs.in-workflow-build == 'true' && | |
| needs.build-info.outputs.default-branch != 'main' | |
| - uses: actions/checkout@v3 | |
| with: | |
| ref: ${{ needs.build-info.outputs.targetCommitSha }} | |
| persist-credentials: false | |
| submodules: recursive | |
| if: > | |
| needs.build-info.outputs.in-workflow-build == 'true' && | |
| needs.build-info.outputs.default-branch != 'main' | |
| - name: "Install Breeze" | |
| uses: ./.github/actions/breeze | |
| if: > | |
| needs.build-info.outputs.in-workflow-build == 'true' && | |
| needs.build-info.outputs.default-branch != 'main' | |
| - name: > | |
| Build Bullseye PROD Images | |
| ${{needs.build-info.outputs.all-python-versions-list-as-string}}:${{env.IMAGE_TAG}} | |
| uses: ./.github/actions/build-prod-images | |
| if: > | |
| needs.build-info.outputs.in-workflow-build == 'true' && | |
| needs.build-info.outputs.default-branch != 'main' | |
| with: | |
| build-provider-packages: ${{ needs.build-info.outputs.default-branch == 'main' }} | |
| chicken-egg-providers: ${{ needs.build-info.outputs.chicken-egg-providers }} | |
| env: | |
| UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} | |
| DOCKER_CACHE: ${{ needs.build-info.outputs.cache-directive }} | |
| PYTHON_VERSIONS: ${{needs.build-info.outputs.all-python-versions-list-as-string}} | |
| DEBUG_RESOURCES: ${{ needs.build-info.outputs.debug-resources }} | |
| INSTALL_MYSQL_CLIENT_TYPE: "mysql" | |
| # Do not override the "mariadb" image - just push a new mysql image | |
| # TODO: improve caching for that build | |
| IMAGE_TAG: "mysql-${{ github.event.pull_request.head.sha || github.sha }}" | |
| wait-for-prod-images: | |
| timeout-minutes: 80 | |
| name: "Wait for PROD images" | |
| runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}} | |
| needs: [build-info, wait-for-ci-images, build-prod-images, build-prod-images-release-branch] | |
| if: needs.build-info.outputs.prod-image-build == 'true' | |
| env: | |
| RUNS_ON: "${{needs.build-info.outputs.runs-on}}" | |
| BACKEND: sqlite | |
| PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}" | |
| # Force more parallelism for pull even on public images | |
| PARALLELISM: 6 | |
| steps: | |
| - name: Cleanup repo | |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" | |
| if: needs.build-info.outputs.in-workflow-build == 'false' | |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" | |
| uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false | |
| if: needs.build-info.outputs.in-workflow-build == 'false' | |
| - name: "Install Breeze" | |
| uses: ./.github/actions/breeze | |
| if: needs.build-info.outputs.in-workflow-build == 'false' | |
| - name: Wait for PROD images ${{ env.PYTHON_VERSIONS }}:${{ env.IMAGE_TAG }} | |
| # We wait for the images to be available either from "build-images.yml' run as pull_request_target | |
| # or from build-prod-images (or build-prod-images-release-branch) above. | |
| # We are utilising single job to wait for all images because this job merely waits | |
| # For the images to be available. | |
| run: breeze prod-image pull --wait-for-image --run-in-parallel | |
| env: | |
| PYTHON_VERSIONS: ${{ needs.build-info.outputs.python-versions-list-as-string }} | |
| DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}} | |
| if: needs.build-info.outputs.in-workflow-build == 'false' | |
| test-examples-of-prod-image-building: | |
| timeout-minutes: 60 | |
| name: "Test examples of production image building" | |
| runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}} | |
| needs: [build-info, wait-for-prod-images] | |
| if: needs.build-info.outputs.prod-image-build == 'true' | |
| steps: | |
| - name: Cleanup repo | |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" | |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 2 | |
| persist-credentials: false | |
| - name: "Install Breeze" | |
| uses: ./.github/actions/breeze | |
| - name: Pull PROD image "${{needs.build-info.outputs.default-python-version}}":${{ env.IMAGE_TAG }} | |
| run: breeze prod-image pull --tag-as-latest | |
| env: | |
| PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}" | |
| - name: "Setup python" | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{needs.build-info.outputs.default-python-version}} | |
| cache: 'pip' | |
| cache-dependency-path: ./dev/requirements.txt | |
| - name: "Test examples of PROD image building" | |
| # yamllint disable-line rule:line-length | |
| run: > | |
| cd ./docker_tests && | |
| python -m pip install -r requirements.txt && | |
| TEST_IMAGE="ghcr.io/${{env.GITHUB_REPOSITORY}}/${{needs.build-info.outputs.default-branch}}/prod/python${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{env.IMAGE_TAG}}" | |
| python -m pytest test_examples_of_prod_image_building.py -n auto --color=yes | |
| env: | |
| PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}" | |
| test-docker-compose-quick-start: | |
| timeout-minutes: 60 | |
| name: "Docker-compose quick start with PROD image verifying" | |
| runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}} | |
| needs: [build-info, wait-for-prod-images] | |
| if: needs.build-info.outputs.prod-image-build == 'true' | |
| env: | |
| PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}" | |
| steps: | |
| - name: Cleanup repo | |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" | |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 2 | |
| persist-credentials: false | |
| - name: "Install Breeze" | |
| uses: ./.github/actions/breeze | |
| - name: Pull PROD images ${{ env.PYTHON_VERSIONS }}:${{ env.IMAGE_TAG }} | |
| run: breeze prod-image pull --run-in-parallel --tag-as-latest | |
| env: | |
| PYTHON_VERSIONS: ${{ needs.build-info.outputs.python-versions-list-as-string }} | |
| # Force more parallelism for pull even on public images | |
| PARALLELISM: 6 | |
| - name: Verify PROD images ${{ env.PYTHON_VERSIONS }}:${{ env.IMAGE_TAG }} | |
| run: breeze prod-image verify --run-in-parallel | |
| env: | |
| PYTHON_VERSIONS: ${{needs.build-info.outputs.all-python-versions-list-as-string}} | |
| DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}} | |
| - name: "Test docker-compose quick start" | |
| run: breeze testing docker-compose-tests | |
| tests-kubernetes: | |
| timeout-minutes: 240 | |
| name: "\ | |
| K8S System:${{matrix.executor}} - ${{matrix.use-standard-naming}} - \ | |
| ${{needs.build-info.outputs.kubernetes-versions-list-as-string}}" | |
| runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}} | |
| needs: [build-info, wait-for-prod-images] | |
| strategy: | |
| matrix: | |
| executor: [KubernetesExecutor, CeleryExecutor, LocalExecutor] | |
| use-standard-naming: [true, false] | |
| fail-fast: false | |
| env: | |
| RUNS_ON: "${{needs.build-info.outputs.runs-on}}" | |
| DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}} | |
| if: > | |
| ( needs.build-info.outputs.run-kubernetes-tests == 'true' || | |
| needs.build-info.outputs.needs-helm-tests == 'true') | |
| steps: | |
| - name: Cleanup repo | |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" | |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" | |
| uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false | |
| - name: "Install Breeze" | |
| uses: ./.github/actions/breeze | |
| id: breeze | |
| - name: Pull PROD images ${{ env.PYTHON_VERSIONS }}:${{ env.IMAGE_TAG }} | |
| run: breeze prod-image pull --run-in-parallel --tag-as-latest | |
| env: | |
| PYTHON_VERSIONS: ${{ needs.build-info.outputs.python-versions-list-as-string }} | |
| # Force more parallelism for pull even on public images | |
| PARALLELISM: 6 | |
| - name: "Cache bin folder with tools for kubernetes testing" | |
| uses: actions/cache@v3 | |
| with: | |
| path: ".build/.k8s-env" | |
| key: "\ | |
| k8s-env-${{steps.breeze.outputs.host-python-version}}-\ | |
| ${{ hashFiles('scripts/ci/kubernetes/k8s_requirements.txt','pyproject.toml') }}" | |
| - name: Run complete K8S tests ${{needs.build-info.outputs.kubernetes-combos-list-as-string}} | |
| run: breeze k8s run-complete-tests --run-in-parallel --upgrade | |
| env: | |
| PYTHON_VERSIONS: ${{ needs.build-info.outputs.python-versions-list-as-string }} | |
| KUBERNETES_VERSIONS: ${{needs.build-info.outputs.kubernetes-versions-list-as-string}} | |
| EXECUTOR: ${{matrix.executor}} | |
| USE_STANDARD_NAMING: ${{matrix.use-standard-naming}} | |
| VERBOSE: "false" | |
| - name: Upload KinD logs on failure ${{needs.build-info.outputs.kubernetes-combos-list-as-string}} | |
| uses: actions/upload-artifact@v3 | |
| if: failure() || cancelled() | |
| with: | |
| name: kind-logs-${{matrix.executor}}-${{matrix.use-standard-naming}} | |
| path: /tmp/kind_logs_* | |
| retention-days: 7 | |
| - name: > | |
| Upload test resource logs on failure ${{needs.build-info.outputs.kubernetes-combos-list-as-string}} | |
| uses: actions/upload-artifact@v3 | |
| if: failure() || cancelled() | |
| with: | |
| name: k8s-test-resources-${{matrix.executor}}-${{matrix.use-standard-naming}} | |
| path: /tmp/k8s_test_resources_* | |
| retention-days: 7 | |
| - name: "Delete clusters just in case they are left" | |
| run: breeze k8s delete-cluster --all | |
| if: always() | |
| update-constraints: | |
| permissions: | |
| contents: write | |
| packages: write | |
| timeout-minutes: 80 | |
| name: "Update constraints" | |
| runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}} | |
| needs: | |
| - build-info | |
| - build-docs | |
| - spellcheck-docs | |
| - wait-for-ci-images | |
| - wait-for-prod-images | |
| - static-checks | |
| - mypy | |
| - tests-sqlite | |
| - tests-mysql | |
| - tests-postgres | |
| - tests-no-db | |
| - tests-integration-postgres | |
| - tests-integration-mysql | |
| - generate-constraints | |
| env: | |
| RUNS_ON: "${{needs.build-info.outputs.runs-on}}" | |
| DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}} | |
| PYTHON_VERSIONS: ${{ needs.build-info.outputs.python-versions-list-as-string }} | |
| if: needs.build-info.outputs.upgrade-to-newer-dependencies != 'false' | |
| steps: | |
| - name: Cleanup repo | |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" | |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" | |
| uses: actions/checkout@v4 | |
| with: | |
| # Needed to perform push action | |
| persist-credentials: false | |
| - name: "Set constraints branch name" | |
| id: constraints-branch | |
| run: ./scripts/ci/constraints/ci_branch_constraints.sh >> ${GITHUB_OUTPUT} | |
| - name: Checkout ${{ steps.constraints-branch.outputs.branch }} | |
| uses: actions/checkout@v4 | |
| with: | |
| path: "constraints" | |
| ref: ${{ steps.constraints-branch.outputs.branch }} | |
| persist-credentials: true | |
| fetch-depth: 0 | |
| - name: "Download constraints from the constraints generated by build CI image" | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: constraints | |
| path: ./files | |
| - name: "Diff in constraints for ${{needs.build-info.outputs.python-versions}}" | |
| run: ./scripts/ci/constraints/ci_diff_constraints.sh | |
| # only commit and push constraints in canary runs (main) | |
| - name: "Commit changed constraint files for ${{needs.build-info.outputs.python-versions}}" | |
| run: ./scripts/ci/constraints/ci_commit_constraints.sh | |
| if: needs.build-info.outputs.canary-run == 'true' | |
| - name: "Push changes" | |
| if: needs.build-info.outputs.canary-run == 'true' | |
| working-directory: "constraints" | |
| run: | |
| git push | |
| # Push BuildX cache to GitHub Registry in Apache repository, if all tests are successful and build | |
| # is executed as result of direct push to "main" or one of the "vX-Y-test" branches | |
| # It rebuilds all images using just-pushed constraints using buildx and pushes them to registry | |
| # It will automatically check if a new python image was released and will pull the latest one if needed | |
| push-buildx-cache-to-github-registry: | |
| timeout-minutes: 120 | |
| name: "Push Image Cache" | |
| runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}} | |
| needs: | |
| - build-info | |
| - update-constraints | |
| if: needs.build-info.outputs.canary-run == 'true' | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| platform: ["linux/amd64", "linux/arm64"] | |
| env: | |
| RUNS_ON: "${{needs.build-info.outputs.runs-on}}" | |
| VERSION_SUFFIX_FOR_PYPI: "dev0" | |
| steps: | |
| - name: Cleanup repo | |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" | |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" | |
| uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false | |
| - name: > | |
| Prepare breeze & CI image: ${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}} | |
| uses: ./.github/actions/prepare_breeze_and_image | |
| env: | |
| # Always use default Python version of CI image for preparing packages | |
| PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}" | |
| - name: "Cleanup dist and context file" | |
| run: rm -fv ./dist/* ./docker-context-files/* | |
| - name: "Prepare airflow package for PROD build" | |
| run: breeze release-management prepare-airflow-package --package-format wheel | |
| env: | |
| VERSION_SUFFIX_FOR_PYPI: "dev0" | |
| - name: "Prepare providers packages for PROD build" | |
| run: > | |
| breeze release-management prepare-provider-packages | |
| --package-list-file ./dev/prod_image_installed_providers.txt | |
| --package-format wheel | |
| env: | |
| VERSION_SUFFIX_FOR_PYPI: "dev0" | |
| if: needs.build-info.outputs.default-branch == 'main' | |
| - name: "Start ARM instance" | |
| run: ./scripts/ci/images/ci_start_arm_instance_and_connect_to_docker.sh | |
| if: matrix.platform == 'linux/arm64' | |
| - name: "Push CI cache ${{ matrix.platform }}" | |
| run: > | |
| breeze ci-image build | |
| --builder airflow_cache | |
| --prepare-buildx-cache | |
| --run-in-parallel | |
| --platform ${{ matrix.platform }} | |
| env: | |
| DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}} | |
| COMMIT_SHA: ${{ github.sha }} | |
| - name: "Push CI latest image ${{ matrix.platform }}" | |
| run: > | |
| breeze ci-image build | |
| --tag-as-latest --push --run-in-parallel --platform ${{ matrix.platform }} | |
| env: | |
| DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}} | |
| COMMIT_SHA: ${{ github.sha }} | |
| if: matrix.platform == 'linux/amd64' | |
| - name: "Copy dist packages to docker-context files" | |
| run: cp -v --no-preserve=mode,ownership ./dist/*.whl ./docker-context-files | |
| - name: "Push PROD cache ${{ matrix.python-version }} ${{ matrix.platform }}" | |
| run: > | |
| breeze prod-image build | |
| --builder airflow_cache | |
| --install-packages-from-context | |
| --run-in-parallel | |
| --airflow-constraints-mode constraints-source-providers | |
| --prepare-buildx-cache | |
| --platform ${{ matrix.platform }} | |
| env: | |
| COMMIT_SHA: ${{ github.sha }} | |
| - name: "Push PROD latest image ${{ matrix.platform }}" | |
| run: > | |
| breeze prod-image build --tag-as-latest --install-packages-from-context | |
| --push --run-in-parallel --platform ${{ matrix.platform }} | |
| env: | |
| DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}} | |
| COMMIT_SHA: ${{ github.sha }} | |
| if: matrix.platform == 'linux/amd64' | |
| - name: "Stop ARM instance" | |
| run: ./scripts/ci/images/ci_stop_arm_instance.sh | |
| if: always() && matrix.platform == 'linux/arm64' | |
| # This is only a check if ARM images are successfully building when committer runs PR from | |
| # Apache repository. This is needed in case you want to fix failing cache job in "canary" run | |
| # There is no point in running this one in "canary" run, because the above step is doing the | |
| # same build anyway. | |
| build-ci-arm-images: | |
| timeout-minutes: 50 | |
| name: > | |
| Build CI ARM images | |
| ${{needs.build-info.outputs.all-python-versions-list-as-string}} | |
| runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}} | |
| needs: | |
| - build-info | |
| - build-docs | |
| - spellcheck-docs | |
| - static-checks | |
| - mypy | |
| - tests-sqlite | |
| - tests-mysql | |
| - tests-postgres | |
| - tests-no-db | |
| - tests-integration-postgres | |
| - tests-integration-mysql | |
| env: | |
| DEFAULT_BRANCH: ${{ needs.build-info.outputs.default-branch }} | |
| DEFAULT_CONSTRAINTS_BRANCH: ${{ needs.build-info.outputs.default-constraints-branch }} | |
| RUNS_ON: "${{needs.build-info.outputs.runs-on}}" | |
| # Force more parallelism for build even on small instances | |
| PARALLELISM: 6 | |
| if: > | |
| needs.build-info.outputs.in-workflow-build == 'true' && | |
| needs.build-info.outputs.canary-run != 'true' | |
| steps: | |
| - name: Cleanup repo | |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" | |
| - uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ needs.build-info.outputs.targetCommitSha }} | |
| persist-credentials: false | |
| - name: "Install Breeze" | |
| uses: ./.github/actions/breeze | |
| - name: "Start ARM instance" | |
| run: ./scripts/ci/images/ci_start_arm_instance_and_connect_to_docker.sh | |
| - name: > | |
| Build CI ARM images ${{ env.IMAGE_TAG }} | |
| ${{needs.build-info.outputs.all-python-versions-list-as-string}}:${{env.IMAGE_TAG}} | |
| run: > | |
| breeze ci-image build --run-in-parallel --builder airflow_cache --platform "linux/arm64" | |
| env: | |
| UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} | |
| DOCKER_CACHE: ${{ needs.build-info.outputs.cache-directive }} | |
| PYTHON_VERSIONS: ${{needs.build-info.outputs.all-python-versions-list-as-string}} | |
| DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}} | |
| COMMIT_SHA: ${{ github.sha }} | |
| - name: "Stop ARM instance" | |
| run: ./scripts/ci/images/ci_stop_arm_instance.sh | |
| if: always() |