CI Main #1672
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI Main | |
| on: | |
| merge_group: | |
| # Allows you to run this workflow manually from the Actions tab | |
| workflow_dispatch: | |
| schedule: | |
| - cron: "0 * * * *" | |
| pull_request: | |
| # runs for the same workflow are cancelled on PRs but not on master | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }} | |
| cancel-in-progress: true | |
| permissions: read-all | |
| env: | |
| CI_COMMIT_SHA: ${{ github.sha }} | |
| CI_COMMIT_REF_PROTECTED: ${{ github.ref_protected }} | |
| CI_JOB_NAME: ${{ github.job }} | |
| CI_JOB_ID: ${{ github.job }} # github does not expose this variable https://github.com/orgs/community/discussions/8945 | |
| CI_JOB_URL: "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" | |
| CI_PIPELINE_SOURCE: ${{ github.event_name }} | |
| CI_PROJECT_DIR: ${{ github.workspace }} | |
| CI_MERGE_REQUEST_TARGET_BRANCH_NAME: ${{ github.event.pull_request.base.ref }} | |
| ROOT_PIPELINE_ID: ${{ github.run_id }} | |
| BAZEL_STARTUP_ARGS: "--output_base=/var/tmp/bazel-output/" | |
| RUSTFLAGS: "--remap-path-prefix=${CI_PROJECT_DIR}=/ic" | |
| AWS_SHARED_CREDENTIALS_CONTENT: ${{ secrets.AWS_SHARED_CREDENTIALS_FILE }} | |
| DOCKER_HUB_USER: ${{ secrets.DOCKER_HUB_USER }} | |
| DOCKER_HUB_PASSWORD_RO: ${{ secrets.DOCKER_HUB_PASSWORD_RO }} | |
| CI_MERGE_REQUEST_TITLE: ${{ github.event.pull_request.title }} | |
| BUILDEVENT_APIKEY: ${{ secrets.HONEYCOMB_API_TOKEN }} | |
| BUILDEVENT_DATASET: "github-ci-dfinity" | |
| jobs: | |
| bazel-test-all: | |
| name: Bazel Test All | |
| runs-on: | |
| labels: bazel-runner-large | |
| container: | |
| image: ghcr.io/dfinity/ic-build@sha256:eb85228ebf7511e2589f86788345eb3d1c8144914a8a2fa771d4347ddacac413 | |
| timeout-minutes: 90 | |
| if: ${{ vars.RUN_CI == 'true' }} # needed to avoid running on public dfinity org until published | |
| env: | |
| TITLE: ${{ github.event.pull_request.title }} | |
| steps: | |
| - name: Before script | |
| if: always() | |
| id: before-script | |
| shell: bash | |
| run: | | |
| git config --global http.postBuffer 524288000 | |
| git config --global safe.directory '*' | |
| if [ -e /__w/cache ]; then sudo ln -s /__w/cache /; fi | |
| if [ -n "${NODE_NAME:-}" ]; then echo "Node: $NODE_NAME"; fi | |
| - name: Checkout | |
| uses: actions/checkout@v3 # v4 does not work with bazel-runner-large | |
| if: ${{ github.event_name == 'pull_request' }} | |
| with: | |
| fetch-depth: 256 | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| if: ${{ github.event_name != 'pull_request' }} | |
| - name: Docker RO SA login | |
| id: docker-ro-login | |
| shell: bash | |
| run: | | |
| docker login -u "$DOCKER_HUB_USER" -p "$DOCKER_HUB_PASSWORD_RO" | |
| if which docker-bin 2>/dev/null; then | |
| docker-bin login -u "$DOCKER_HUB_USER" -p "$DOCKER_HUB_PASSWORD_RO" | |
| fi | |
| - name: Run Bazel Test All | |
| id: bazel-test-all | |
| uses: ./.github/actions/bazel-test-all/ | |
| with: | |
| BAZEL_COMMAND: "test" | |
| BAZEL_TARGETS: "//... --deleted_packages=gitlab-ci/src/gitlab_config" | |
| BAZEL_CI_CONFIG: "--config=ci --repository_cache=/cache/bazel" | |
| # check if PR title contains release and set timeout filters accordingly | |
| BAZEL_EXTRA_ARGS_RULES: ${{ contains(env.TITLE, 'release') && '--test_timeout_filters=short,moderate' || '' }} | |
| BAZEL_EXTRA_ARGS: "--keep_going --verbose_failures" | |
| # run on diff only if it is a pull request, otherwise run all targets | |
| RUN_ON_DIFF_ONLY: ${{ contains(github.event_name, 'pull_request') && 'true' || 'false'}} | |
| HONEYCOMB_API_TOKEN: ${{ secrets.HONEYCOMB_API_TOKEN }} | |
| - name: Upload bazel-targets | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: bazel-targets | |
| retention-days: 1 | |
| if-no-files-found: error | |
| path: | | |
| bazel-targets | |
| - name: Bazel Clean | |
| if: always() | |
| run: bazel clean | |
| bazel-build-all-config-check: | |
| runs-on: | |
| labels: bazel-runner-large | |
| container: | |
| image: ghcr.io/dfinity/ic-build@sha256:eb85228ebf7511e2589f86788345eb3d1c8144914a8a2fa771d4347ddacac413 | |
| timeout-minutes: 90 | |
| if: ${{ vars.RUN_CI == 'true' }} # needed to avoid running on public dfinity org until published | |
| name: Bazel Build All Config Check | |
| steps: | |
| - name: Before script | |
| if: always() | |
| id: before-script | |
| shell: bash | |
| run: | | |
| git config --global http.postBuffer 524288000 | |
| git config --global safe.directory '*' | |
| if [ -e /__w/cache ]; then sudo ln -s /__w/cache /; fi | |
| if [ -n "${NODE_NAME:-}" ]; then echo "Node: $NODE_NAME"; fi | |
| - name: Checkout | |
| uses: actions/checkout@v3 # v4 does not work with bazel-runner-large | |
| if: ${{ github.event_name == 'pull_request' }} | |
| with: | |
| fetch-depth: 256 | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| if: ${{ github.event_name != 'pull_request' }} | |
| - name: Docker RO SA login | |
| id: docker-ro-login | |
| shell: bash | |
| run: | | |
| docker login -u "$DOCKER_HUB_USER" -p "$DOCKER_HUB_PASSWORD_RO" | |
| if which docker-bin 2>/dev/null; then | |
| docker-bin login -u "$DOCKER_HUB_USER" -p "$DOCKER_HUB_PASSWORD_RO" | |
| fi | |
| - name: Run bazel build --config=check //rs/... | |
| id: bazel-build-config-check | |
| uses: ./.github/actions/bazel-test-all/ | |
| with: | |
| BAZEL_COMMAND: "build" | |
| BAZEL_TARGETS: "//rs/..." | |
| BAZEL_CI_CONFIG: "--config=check" | |
| # run on diff only if it is a pull request, otherwise run all targets | |
| RUN_ON_DIFF_ONLY: ${{ contains(github.event_name, 'pull_request') && 'true' || 'false'}} | |
| - name: Bazel Clean | |
| if: always() | |
| run: bazel clean | |
| bazel-test-darwin-x86-64: | |
| name: Bazel Test Darwin x86-64 | |
| timeout-minutes: 120 | |
| if: ${{ vars.RUN_CI == 'true' && github.event_name != 'merge_group' && false }} # disable until we have more macos runners | |
| runs-on: | |
| labels: macos | |
| steps: | |
| - name: Before script | |
| if: always() | |
| id: before-script | |
| shell: bash | |
| run: | | |
| git config --global http.postBuffer 524288000 | |
| git config --global safe.directory '*' | |
| if [ -e /__w/cache ]; then sudo ln -s /__w/cache /; fi | |
| if [ -n "${NODE_NAME:-}" ]; then echo "Node: $NODE_NAME"; fi | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3 | |
| id: filter | |
| if: ${{ github.event_name == 'pull_request' }} | |
| with: | |
| filters: | | |
| bazel-test-darwin-x86-64: | |
| - '.github/workflows/ci-main.yml' | |
| - '.bazelrc' | |
| - '.bazelversion' | |
| - '**/*.bazel' | |
| - '**/*.bzl' | |
| - '**/*.lock' | |
| - '**/*.rs' | |
| - '**/*.toml' | |
| - name: Docker RO SA login | |
| id: docker-ro-login | |
| shell: bash | |
| run: | | |
| docker login -u "$DOCKER_HUB_USER" -p "$DOCKER_HUB_PASSWORD_RO" | |
| if which docker-bin 2>/dev/null; then | |
| docker-bin login -u "$DOCKER_HUB_USER" -p "$DOCKER_HUB_PASSWORD_RO" | |
| fi | |
| - name: Run Bazel Test Darwin x86-64 | |
| id: bazel-test-darwin-x86-64 | |
| if: steps.filter.outputs.bazel-test-darwin-x86-64 != 'false' | |
| uses: ./.github/actions/bazel-test-all/ | |
| with: | |
| BAZEL_CI_CONFIG: "--config=ci --config macos_ci" | |
| BAZEL_COMMAND: test | |
| BAZEL_EXTRA_ARGS: "--test_tag_filters=test_macos" | |
| BAZEL_STARTUP_ARGS: "--output_base /var/tmp/bazel-output//${ROOT_PIPELINE_ID}" | |
| BAZEL_TARGETS: "//rs/... //publish/binaries/..." | |
| HONEYCOMB_API_TOKEN: ${{ secrets.HONEYCOMB_API_TOKEN }} | |
| - name: No run | |
| if: steps.filter.outputs.bazel-test-darwin-x86-64 == 'false' | |
| run: echo "No changes, skipping run" | |
| - name: Purge Bazel Output | |
| shell: bash | |
| run: | | |
| sudo rm -rf /var/tmp/bazel-output//${ROOT_PIPELINE_ID} | |
| bazel-build-fuzzers: | |
| name: Bazel Build Fuzzers | |
| runs-on: | |
| labels: dind-runner-large | |
| container: | |
| image: ghcr.io/dfinity/ic-build@sha256:eb85228ebf7511e2589f86788345eb3d1c8144914a8a2fa771d4347ddacac413 | |
| timeout-minutes: 60 | |
| if: ${{ vars.RUN_CI == 'true' }} | |
| steps: | |
| - name: Before script | |
| if: always() | |
| id: before-script | |
| shell: bash | |
| run: | | |
| git config --global http.postBuffer 524288000 | |
| git config --global safe.directory '*' | |
| if [ -e /__w/cache ]; then sudo ln -s /__w/cache /; fi | |
| if [ -n "${NODE_NAME:-}" ]; then echo "Node: $NODE_NAME"; fi | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Run Bazel Build Fuzzers | |
| id: bazel-build-fuzzers | |
| uses: ./.github/actions/bazel-test-all/ | |
| with: | |
| BAZEL_COMMAND: "build" | |
| BAZEL_TARGETS: "//rs/..." | |
| BAZEL_CI_CONFIG: "--config=ci" | |
| BAZEL_EXTRA_ARGS: "--keep_going --config=fuzzing --build_tag_filters=libfuzzer" | |
| - name: Bazel Clean | |
| if: always() | |
| run: bazel clean | |
| bazel-build-fuzzers-afl: | |
| name: Bazel Build Fuzzers AFL | |
| runs-on: | |
| labels: dind-runner-large | |
| container: | |
| image: ghcr.io/dfinity/ic-build@sha256:eb85228ebf7511e2589f86788345eb3d1c8144914a8a2fa771d4347ddacac413 | |
| timeout-minutes: 60 | |
| if: ${{ vars.RUN_CI == 'true' }} | |
| steps: | |
| - name: Before script | |
| if: always() | |
| id: before-script | |
| shell: bash | |
| run: | | |
| git config --global http.postBuffer 524288000 | |
| git config --global safe.directory '*' | |
| if [ -e /__w/cache ]; then sudo ln -s /__w/cache /; fi | |
| if [ -n "${NODE_NAME:-}" ]; then echo "Node: $NODE_NAME"; fi | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Run Bazel Build Fuzzers AFL | |
| id: bazel-build-fuzzers-afl | |
| uses: ./.github/actions/bazel-test-all/ | |
| with: | |
| BAZEL_COMMAND: "build" | |
| BAZEL_TARGETS: "//rs/..." | |
| BAZEL_CI_CONFIG: "--config=ci" | |
| BAZEL_EXTRA_ARGS: "--keep_going --config=afl" | |
| - name: Bazel Clean | |
| if: always() | |
| run: bazel clean | |
| python-ci-tests: | |
| name: Python CI Tests | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 30 | |
| if: ${{ vars.RUN_CI == 'true' }} | |
| steps: | |
| - name: Before script | |
| if: always() | |
| id: before-script | |
| shell: bash | |
| run: | | |
| git config --global http.postBuffer 524288000 | |
| git config --global safe.directory '*' | |
| if [ -e /__w/cache ]; then sudo ln -s /__w/cache /; fi | |
| if [ -n "${NODE_NAME:-}" ]; then echo "Node: $NODE_NAME"; fi | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Set up Python | |
| uses: actions/setup-python@v2 | |
| with: | |
| python-version: '3.10' | |
| - name: Run Python CI Tests | |
| id: python-ci-tests | |
| shell: bash | |
| run: | | |
| set -xeuo pipefail | |
| export PYTHONPATH=$PWD/gitlab-ci/src:$PWD/gitlab-ci/src/dependencies | |
| pip3 install --ignore-installed -r requirements.txt | |
| cd gitlab-ci/src | |
| pytest --ignore=gitlab_config/ --ignore=git_changes/ -v -o junit_family=xunit1 \ | |
| --junitxml=../../test_report.xml --cov=. --cov-report=term \ | |
| --cov-report=term-missing --cov-report=html --cov-branch | |
| build-ic: | |
| name: Build IC | |
| runs-on: | |
| labels: bazel-runner-large | |
| container: | |
| image: ghcr.io/dfinity/ic-build@sha256:eb85228ebf7511e2589f86788345eb3d1c8144914a8a2fa771d4347ddacac413 | |
| timeout-minutes: 90 | |
| if: ${{ vars.RUN_CI == 'true' && github.event_name != 'merge_group' }} | |
| steps: | |
| - name: Before script | |
| if: always() | |
| id: before-script | |
| shell: bash | |
| run: | | |
| git config --global http.postBuffer 524288000 | |
| git config --global safe.directory '*' | |
| if [ -e /__w/cache ]; then sudo ln -s /__w/cache /; fi | |
| if [ -n "${NODE_NAME:-}" ]; then echo "Node: $NODE_NAME"; fi | |
| - name: Checkout | |
| uses: actions/checkout@v3 # v4 does not work with bazel-runner-large | |
| if: ${{ github.event_name == 'pull_request' }} | |
| with: | |
| fetch-depth: 256 | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| if: ${{ github.event_name != 'pull_request' }} | |
| - name: Docker RO SA login | |
| id: docker-ro-login | |
| shell: bash | |
| run: | | |
| docker login -u "$DOCKER_HUB_USER" -p "$DOCKER_HUB_PASSWORD_RO" | |
| if which docker-bin 2>/dev/null; then | |
| docker-bin login -u "$DOCKER_HUB_USER" -p "$DOCKER_HUB_PASSWORD_RO" | |
| fi | |
| - name: Run Build IC | |
| id: build-ic | |
| shell: bash | |
| run: | | |
| set -eExuo pipefail | |
| rm -rf "/cache/job/${CI_JOB_ID}/${ROOT_PIPELINE_ID}" | |
| mkdir -p "/cache/job/${CI_JOB_ID}/${ROOT_PIPELINE_ID}/artifacts" | |
| ln -s "/cache/job/${CI_JOB_ID}/${ROOT_PIPELINE_ID}/artifacts" /__w/ic/ic/artifacts | |
| buildevents cmd "$ROOT_PIPELINE_ID" "$CI_JOB_ID" build-command -- \ | |
| "$CI_PROJECT_DIR"/gitlab-ci/src/ci-scripts/build-ic.sh | |
| rm -rf "/cache/job/${CI_JOB_ID}/${ROOT_PIPELINE_ID}" | |
| env: | |
| RUN_ON_DIFF_ONLY: "true" | |
| BAZEL_COMMAND: "build" | |
| - name: Upload build-ic.tar | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: build-ic | |
| retention-days: 1 | |
| if-no-files-found: error | |
| path: | | |
| build-ic.tar | |
| build-determinism: | |
| name: Build Determinism | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 30 | |
| if: ${{ vars.RUN_CI == 'true' }} | |
| needs: [build-ic, bazel-test-all] | |
| strategy: | |
| matrix: | |
| include: | |
| - TARGET: "//publish/binaries:upload" | |
| PATH0: "release" | |
| PATH1: "build-ic/release" | |
| SETUPOS_FLAG: "false" | |
| - TARGET: "//publish/canisters:upload" | |
| PATH0: "canisters" | |
| PATH1: "build-ic/canisters" | |
| SETUPOS_FLAG: "false" | |
| - TARGET: "//ic-os/guestos/envs/prod:upload_disk-img" | |
| PATH0: "guest-os/update-img" | |
| PATH1: "build-ic/icos/guestos" | |
| SETUPOS_FLAG: "false" | |
| - TARGET: "//ic-os/hostos/envs/prod:upload_update-img" | |
| PATH0: "host-os/update-img" | |
| PATH1: "build-ic/icos/hostos" | |
| SETUPOS_FLAG: "false" | |
| - TARGET: "//ic-os/setupos/envs/prod:upload_disk-img" | |
| PATH0: "setup-os/disk-img" | |
| PATH1: "build-ic/icos/setupos" | |
| SETUPOS_FLAG: "true" | |
| steps: | |
| - name: Before script | |
| if: always() | |
| id: before-script | |
| shell: bash | |
| run: | | |
| git config --global http.postBuffer 524288000 | |
| git config --global safe.directory '*' | |
| if [ -e /__w/cache ]; then sudo ln -s /__w/cache /; fi | |
| if [ -n "${NODE_NAME:-}" ]; then echo "Node: $NODE_NAME"; fi | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Download bazel-targets [bazel-test-all] | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: bazel-targets | |
| - name: Download build-ic.tar [build-ic] | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: build-ic | |
| - name: Build Determinism Test | |
| id: build-determinism | |
| shell: bash | |
| run: | | |
| set -eExuo pipefail | |
| sudo apt update && sudo apt install -y curl | |
| "$CI_PROJECT_DIR"/gitlab-ci/src/ci-scripts/build-determinism.sh | |
| env: | |
| TARGET: ${{ matrix.TARGET }} | |
| PATH0: ${{ matrix.PATH0 }} | |
| PATH1: ${{ matrix.PATH1 }} | |
| SETUPOS_FLAG: ${{ matrix.SETUPOS_FLAG }} | |
| cargo-clippy-linux: | |
| name: Cargo Clippy Linux | |
| runs-on: | |
| labels: dind-runner-large | |
| container: | |
| image: ghcr.io/dfinity/ic-build@sha256:eb85228ebf7511e2589f86788345eb3d1c8144914a8a2fa771d4347ddacac413 | |
| timeout-minutes: 60 | |
| if: ${{ vars.RUN_CI == 'true' }} | |
| steps: | |
| - name: Before script | |
| if: always() | |
| id: before-script | |
| shell: bash | |
| run: | | |
| git config --global http.postBuffer 524288000 | |
| git config --global safe.directory '*' | |
| if [ -e /__w/cache ]; then sudo ln -s /__w/cache /; fi | |
| if [ -n "${NODE_NAME:-}" ]; then echo "Node: $NODE_NAME"; fi | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Filter Rust Files [*.{rs,toml,lock}] | |
| uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3 | |
| if: ${{ github.event_name == 'pull_request' || github.event_name == 'merge_group' }} | |
| id: filter | |
| with: | |
| filters: | | |
| cargo: | |
| - "**/*.rs" | |
| - "**/*.toml" | |
| - "**/*.lock" | |
| - name: Run Cargo Clippy Linux | |
| id: cargo-clippy-linux | |
| if: steps.filter.outputs.cargo == 'true' | |
| shell: bash | |
| run: | | |
| set -eExuo pipefail | |
| buildevents cmd "$ROOT_PIPELINE_ID" "$CI_JOB_ID" build-command -- \ | |
| "$CI_PROJECT_DIR"/gitlab-ci/src/ci-scripts/rust-lint.sh | |
| - name: No run | |
| if: steps.filter.outputs.cargo == 'false' | |
| run: echo "No cargo changes, skipping run" | |
| cargo-build-release-linux: | |
| name: Cargo Build Release Linux | |
| runs-on: | |
| labels: dind-runner-large | |
| container: | |
| image: ghcr.io/dfinity/ic-build@sha256:eb85228ebf7511e2589f86788345eb3d1c8144914a8a2fa771d4347ddacac413 | |
| if: ${{ vars.RUN_CI == 'true' }} | |
| timeout-minutes: 60 | |
| steps: | |
| - name: Before script | |
| if: always() | |
| id: before-script | |
| shell: bash | |
| run: | | |
| git config --global http.postBuffer 524288000 | |
| git config --global safe.directory '*' | |
| if [ -e /__w/cache ]; then sudo ln -s /__w/cache /; fi | |
| if [ -n "${NODE_NAME:-}" ]; then echo "Node: $NODE_NAME"; fi | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Filter Rust Files [*.{rs,toml,lock}] | |
| uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3 | |
| if: ${{ github.event_name == 'pull_request' || github.event_name == 'merge_group' }} | |
| id: filter | |
| with: | |
| filters: | | |
| cargo: | |
| - "**/*.rs" | |
| - "**/*.toml" | |
| - "**/*.lock" | |
| - name: Run Cargo Build Release Linux | |
| id: cargo-build-release-linux | |
| if: steps.filter.outputs.cargo == 'true' | |
| shell: bash | |
| run: | | |
| set -eExuo pipefail | |
| buildevents cmd "$ROOT_PIPELINE_ID" "$CI_JOB_ID" build-command -- \ | |
| cargo build --release | |
| - name: No run | |
| if: steps.filter.outputs.cargo == 'false' | |
| run: echo "No cargo changes, skipping run" |