Allow custom images in tests #2109
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # ============= | |
| # This file is automatically generated from the templates in stackabletech/operator-templating | |
| # DON'T MANUALLY EDIT THIS FILE | |
| # ============= | |
| --- | |
| name: Stackable Build Pipeline | |
| on: | |
| push: | |
| branches: | |
| - main | |
| - staging | |
| - trying | |
| - "renovate/**" | |
| tags: | |
| - '[0-9][0-9].[0-9]+.[0-9]+' | |
| pull_request: | |
| merge_group: | |
| schedule: | |
| # Run every Saturday morning: https://crontab.guru/#15_3_*_*_6 | |
| - cron: '15 3 * * 6' | |
| workflow_dispatch: | |
| env: | |
| CARGO_TERM_COLOR: always | |
| CARGO_INCREMENTAL: '0' | |
| CARGO_PROFILE_DEV_DEBUG: '0' | |
| RUST_TOOLCHAIN_VERSION: "1.80.1" | |
| RUSTFLAGS: "-D warnings" | |
| RUSTDOCFLAGS: "-D warnings" | |
| RUST_LOG: "info" | |
| DEV_REPO_HELM_URL: https://repo.stackable.tech/repository/helm-dev | |
| TEST_REPO_HELM_URL: https://repo.stackable.tech/repository/helm-test | |
| STABLE_REPO_HELM_URL: https://repo.stackable.tech/repository/helm-stable | |
| jobs: | |
| # Identify unused dependencies | |
| run_udeps: | |
| name: Run Cargo Udeps | |
| runs-on: ubuntu-latest | |
| env: | |
| RUSTC_BOOTSTRAP: 1 | |
| steps: | |
| - name: Install host dependencies | |
| uses: awalsh128/cache-apt-pkgs-action@a6c3917cc929dd0345bfb2d3feaf9101823370ad # v1.4.2 | |
| with: | |
| packages: protobuf-compiler krb5-user libkrb5-dev libclang-dev liblzma-dev libssl-dev pkg-config apt-transport-https | |
| version: ubuntu-latest | |
| - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 | |
| with: | |
| submodules: recursive | |
| - uses: dtolnay/rust-toolchain@7b1c307e0dcbda6122208f10795a713336a9b35a | |
| with: | |
| toolchain: ${{ env.RUST_TOOLCHAIN_VERSION }} | |
| - uses: Swatinem/rust-cache@23bce251a8cd2ffc3c1075eaa2367cf899916d84 # v2.7.3 | |
| with: | |
| key: udeps | |
| cache-all-crates: "true" | |
| - uses: stackabletech/cargo-install-action@cargo-udeps | |
| - run: cargo udeps --workspace --all-targets | |
| # This job evaluates the github environment to determine why this action is running and selects the appropriate | |
| # target repository for published Helm charts based on this. | |
| # | |
| # The following scenarios are identified: | |
| # - all pull requests land in the test repository: | |
| # condition: github.event_name == "pull_request" | |
| # repository: test | |
| # | |
| # - all tagged releases land in stable: | |
| # condition: github.event_name == 'push' & github.ref.startswith('refs/tags/') | |
| # repository: stable | |
| # | |
| # - all pushes to main (i.e. PR-merges) and all scheduled/manual workflow runs on main land in dev: | |
| # condition: ( github.event_name == 'push' | github.event_name == 'schedule' | github.event_name == 'workflow_dispatch' ) & github.ref == 'refs/heads/main' | |
| # repository: dev | |
| # | |
| # Any other scenarios (e.g. when a branch is created/pushed) will cause the publish step to be skipped, most commonly this is expected to happen for the | |
| # branches that the GitHub merge queue feature uses internally for which the checks need to run, but we do not want artifacts to be published. | |
| select_helm_repo: | |
| name: Select target helm repository based on action trigger | |
| runs-on: ubuntu-latest | |
| outputs: | |
| helm_repository: ${{ steps.selecthelmrepo.outputs.helm_repo }} | |
| steps: | |
| - id: selecthelmrepo | |
| env: | |
| TRIGGER: ${{ github.event_name }} | |
| GITHUB_REF: ${{ github.ref }} | |
| run: | | |
| if [[ "$TRIGGER" == "pull_request" ]]; then | |
| echo "exporting test as target helm repo: ${{ env.TEST_REPO_HELM_URL }}" | |
| echo "helm_repo=${{ env.TEST_REPO_HELM_URL }}" >> "$GITHUB_OUTPUT" | |
| elif [[ ( "$TRIGGER" == "push" || "$TRIGGER" == "schedule" || "$TRIGGER" == "workflow_dispatch" ) && "$GITHUB_REF" == "refs/heads/main" ]]; then | |
| echo "exporting dev as target helm repo: ${{ env.DEV_REPO_HELM_URL }}" | |
| echo "helm_repo=${{ env.DEV_REPO_HELM_URL }}" >> "$GITHUB_OUTPUT" | |
| elif [[ "$TRIGGER" == "push" && $GITHUB_REF == refs/tags/* ]]; then | |
| echo "exporting stable as target helm repo: ${{ env.STABLE_REPO_HELM_URL }}" | |
| echo "helm_repo=${{ env.STABLE_REPO_HELM_URL }}" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "Unknown trigger and ref combination encountered, skipping publish step: $TRIGGER $GITHUB_REF" | |
| echo "helm_repo=skip" >> "$GITHUB_OUTPUT" | |
| fi | |
| run_cargodeny: | |
| name: Run Cargo Deny | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| checks: | |
| - advisories | |
| - bans licenses sources | |
| # Prevent sudden announcement of a new advisory from failing ci: | |
| continue-on-error: ${{ matrix.checks == 'advisories' }} | |
| steps: | |
| - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 | |
| with: | |
| submodules: recursive | |
| - uses: EmbarkStudios/cargo-deny-action@8371184bd11e21dcf8ac82ebf8c9c9f74ebf7268 # v2.0.1 | |
| with: | |
| command: check ${{ matrix.checks }} | |
| run_rustfmt: | |
| name: Run Rustfmt | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 | |
| with: | |
| submodules: recursive | |
| - uses: dtolnay/rust-toolchain@7b1c307e0dcbda6122208f10795a713336a9b35a | |
| with: | |
| toolchain: ${{ env.RUST_TOOLCHAIN_VERSION }} | |
| components: rustfmt | |
| - run: cargo fmt --all -- --check | |
| run_clippy: | |
| name: Run Clippy | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Install host dependencies | |
| uses: awalsh128/cache-apt-pkgs-action@a6c3917cc929dd0345bfb2d3feaf9101823370ad # v1.4.2 | |
| with: | |
| packages: protobuf-compiler krb5-user libkrb5-dev libclang-dev liblzma-dev libssl-dev pkg-config apt-transport-https | |
| version: ubuntu-latest | |
| - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 | |
| with: | |
| submodules: recursive | |
| - uses: dtolnay/rust-toolchain@7b1c307e0dcbda6122208f10795a713336a9b35a | |
| with: | |
| toolchain: ${{ env.RUST_TOOLCHAIN_VERSION }} | |
| components: clippy | |
| - uses: Swatinem/rust-cache@23bce251a8cd2ffc3c1075eaa2367cf899916d84 # v2.7.3 | |
| with: | |
| key: clippy | |
| cache-all-crates: "true" | |
| - name: Run clippy action to produce annotations | |
| uses: giraffate/clippy-action@13b9d32482f25d29ead141b79e7e04e7900281e0 # v1.0.1 | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| if: env.GITHUB_TOKEN != null | |
| with: | |
| clippy_flags: --all-targets -- -D warnings | |
| reporter: 'github-pr-review' | |
| github_token: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Run clippy manually without annotations | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| if: env.GITHUB_TOKEN == null | |
| run: cargo clippy --color never -q --all-targets -- -D warnings | |
| run_rustdoc: | |
| name: Run RustDoc | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Install host dependencies | |
| uses: awalsh128/cache-apt-pkgs-action@a6c3917cc929dd0345bfb2d3feaf9101823370ad # v1.4.2 | |
| with: | |
| packages: protobuf-compiler krb5-user libkrb5-dev libclang-dev liblzma-dev libssl-dev pkg-config apt-transport-https | |
| version: ubuntu-latest | |
| - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 | |
| with: | |
| submodules: recursive | |
| - uses: dtolnay/rust-toolchain@7b1c307e0dcbda6122208f10795a713336a9b35a | |
| with: | |
| toolchain: ${{ env.RUST_TOOLCHAIN_VERSION }} | |
| components: rustfmt | |
| - uses: Swatinem/rust-cache@23bce251a8cd2ffc3c1075eaa2367cf899916d84 # v2.7.3 | |
| with: | |
| key: doc | |
| cache-all-crates: "true" | |
| - run: cargo doc --document-private-items | |
| run_tests: | |
| name: Run Cargo Tests | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Install host dependencies | |
| uses: awalsh128/cache-apt-pkgs-action@a6c3917cc929dd0345bfb2d3feaf9101823370ad # v1.4.2 | |
| with: | |
| packages: protobuf-compiler krb5-user libkrb5-dev libclang-dev liblzma-dev libssl-dev pkg-config apt-transport-https | |
| version: ubuntu-latest | |
| - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 | |
| with: | |
| submodules: recursive | |
| - uses: dtolnay/rust-toolchain@7b1c307e0dcbda6122208f10795a713336a9b35a | |
| with: | |
| toolchain: ${{ env.RUST_TOOLCHAIN_VERSION }} | |
| - uses: Swatinem/rust-cache@23bce251a8cd2ffc3c1075eaa2367cf899916d84 # v2.7.3 | |
| with: | |
| key: test | |
| cache-all-crates: "true" | |
| - run: cargo test | |
| # Similar to check_charts, this tries to render the README, and see if there are unintended changes. | |
| # This will save us from merging changes to the wrong file (instead of the templated source), and from | |
| # forgetting to render out modifications to the README. | |
| check_readme: | |
| name: Check if committed README is the one we would render from the available parts | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 | |
| with: | |
| submodules: recursive | |
| - uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5.2.0 | |
| with: | |
| python-version: '3.12' | |
| - name: Install jinja2-cli | |
| run: pip install jinja2-cli==0.8.2 | |
| - name: Regenerate charts | |
| run: make render-readme | |
| - name: Check if committed README were up to date | |
| run: git diff --exit-code | |
| - name: Git Diff showed uncommitted changes | |
| if: ${{ failure() }} | |
| uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 | |
| with: | |
| script: | | |
| core.setFailed('Committed README are not up to date, please make sure to apply them to the templated partials, and re-commit!') | |
| # This job cleans up the CRDs and Helm charts, followed by rebuilding them | |
| # It then runs a `git diff` and fails the entire workflow, if any difference is encountered. | |
| # | |
| # Since CRD files are generated during the 'cargo build' process we need to run this once after | |
| # removing the CRD files to ensure that the checked in versions match what the code expects. | |
| # | |
| # The reason for this step is, that developers are expected to check in up-to-date versions of charts | |
| # as we'd otherwise have to build these in CI and commit them back to the PR, which | |
| # creates all kinds of problems. | |
| # This failsafe simply aborts anything that has not had charts rebuilt before pushing. | |
| check_charts: | |
| name: Check if committed Helm charts are up to date | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Install host dependencies | |
| uses: awalsh128/cache-apt-pkgs-action@a6c3917cc929dd0345bfb2d3feaf9101823370ad # v1.4.2 | |
| with: | |
| packages: protobuf-compiler krb5-user libkrb5-dev libclang-dev liblzma-dev libssl-dev pkg-config apt-transport-https | |
| version: ubuntu-latest | |
| - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 | |
| with: | |
| submodules: recursive | |
| - name: Set up Helm | |
| uses: azure/setup-helm@fe7b79cd5ee1e45176fcad797de68ecaf3ca4814 # v4.2.0 | |
| with: | |
| version: v3.16.1 | |
| - name: Set up cargo | |
| uses: dtolnay/rust-toolchain@7b1c307e0dcbda6122208f10795a713336a9b35a | |
| with: | |
| toolchain: ${{ env.RUST_TOOLCHAIN_VERSION }} | |
| - uses: Swatinem/rust-cache@23bce251a8cd2ffc3c1075eaa2367cf899916d84 # v2.7.3 | |
| with: | |
| key: charts | |
| cache-all-crates: "true" | |
| - name: Regenerate charts | |
| run: make regenerate-charts | |
| - name: Check if committed charts were up to date | |
| run: git diff --exit-code | |
| - name: Git Diff showed uncommitted changes | |
| if: ${{ failure() }} | |
| uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 | |
| with: | |
| script: | | |
| core.setFailed('Committed charts were not up to date, please regenerate and re-commit!') | |
| tests_passed: | |
| name: All tests passed | |
| needs: | |
| - run_udeps | |
| - run_cargodeny | |
| - run_clippy | |
| - run_rustfmt | |
| - run_rustdoc | |
| - run_tests | |
| - check_charts | |
| - check_readme | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: log | |
| run: echo All tests have passed! | |
| package_and_publish: | |
| name: Package Charts, Build Docker Image and publish them - ${{ matrix.runner }} | |
| needs: | |
| - tests_passed | |
| - select_helm_repo | |
| strategy: | |
| matrix: | |
| runner: ["ubuntu-latest", "ubicloud-standard-8-arm"] | |
| runs-on: ${{ matrix.runner }} | |
| timeout-minutes: 120 | |
| permissions: | |
| id-token: write | |
| env: | |
| NEXUS_PASSWORD: ${{ secrets.NEXUS_PASSWORD }} | |
| HELM_REPO: ${{ needs.select_helm_repo.outputs.helm_repository }} | |
| OCI_REGISTRY_SDP_PASSWORD: ${{ secrets.HARBOR_ROBOT_SDP_GITHUB_ACTION_BUILD_SECRET }} | |
| OCI_REGISTRY_SDP_USERNAME: "robot$sdp+github-action-build" | |
| OCI_REGISTRY_SDP_CHARTS_PASSWORD: ${{ secrets.HARBOR_ROBOT_SDP_CHARTS_GITHUB_ACTION_BUILD_SECRET }} | |
| OCI_REGISTRY_SDP_CHARTS_USERNAME: "robot$sdp-charts+github-action-build" | |
| if: needs.select_helm_repo.outputs.helm_repository != 'skip' | |
| outputs: | |
| IMAGE_TAG: ${{ steps.printtag.outputs.IMAGE_TAG }} | |
| steps: | |
| - name: Install host dependencies | |
| uses: awalsh128/cache-apt-pkgs-action@a6c3917cc929dd0345bfb2d3feaf9101823370ad # v1.4.2 | |
| with: | |
| packages: protobuf-compiler krb5-user libkrb5-dev libclang-dev liblzma-dev libssl-dev pkg-config apt-transport-https | |
| version: ${{ matrix.runner }} | |
| - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 | |
| with: | |
| submodules: recursive | |
| - uses: cachix/install-nix-action@ba0dd844c9180cbf77aa72a116d6fbc515d0e87b # v27 | |
| - uses: dtolnay/rust-toolchain@7b1c307e0dcbda6122208f10795a713336a9b35a | |
| with: | |
| toolchain: ${{ env.RUST_TOOLCHAIN_VERSION }} | |
| components: rustfmt | |
| # This step checks if the current run was triggered by a push to a pr (or a pr being created). | |
| # If this is the case it changes the version of this project in all Cargo.toml files to include the suffix | |
| # "-pr<prnumber>" so that the published artifacts can be linked to this PR. | |
| - uses: stackabletech/cargo-install-action@main | |
| with: | |
| crate: cargo-edit | |
| bin: cargo-set-version | |
| - name: Update version if PR | |
| if: ${{ github.event_name == 'pull_request' }} | |
| run: cargo set-version --offline --workspace 0.0.0-pr${{ github.event.pull_request.number }} | |
| # Recreate charts and publish charts and docker image. The "-e" is needed as we want to override the | |
| # default value in the makefile if called from this action, but not otherwise (i.e. when called locally). | |
| # This is needed for the HELM_REPO variable. | |
| - name: Install cosign | |
| uses: sigstore/cosign-installer@4959ce089c160fddf62f7b42464195ba1a56d382 # v3.6.0 | |
| - name: Install syft | |
| uses: anchore/sbom-action/download-syft@61119d458adab75f756bc0b9e4bde25725f86a7a # v0.17.2 | |
| - name: Build Docker image and Helm chart | |
| run: | | |
| # Installing helm and yq on ubicloud-standard-8-arm only | |
| if [ "$(arch)" = "aarch64" ]; then | |
| curl https://baltocdn.com/helm/signing.asc | gpg --dearmor | sudo tee /usr/share/keyrings/helm.gpg > /dev/null | |
| echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/helm.gpg] https://baltocdn.com/helm/stable/debian/ all main" | sudo tee /etc/apt/sources.list.d/helm-stable-debian.list | |
| sudo apt-get -y update | |
| sudo apt-get -y install helm | |
| sudo wget https://github.com/mikefarah/yq/releases/latest/download/yq_linux_arm64 -O /usr/bin/yq && sudo chmod +x /usr/bin/yq | |
| fi | |
| make -e build | |
| - name: Publish Docker image and Helm chart | |
| if: ${{ !github.event.pull_request.head.repo.fork }} | |
| run: | | |
| # We want to publish helmcharts only once as they have a common name, while still publishing both images with architecture specific tags | |
| if [ "$(uname -m)" = "x86_64" ]; then | |
| make -e publish | |
| else | |
| make -e docker-publish | |
| fi | |
| # Output the name of the published image to the Job output for later use | |
| - id: printtag | |
| name: Output image name and tag | |
| if: ${{ !github.event.pull_request.head.repo.fork }} | |
| run: echo "IMAGE_TAG=$(make -e print-docker-tag)" >> "$GITHUB_OUTPUT" | |
| create_manifest_list: | |
| name: Build and publish manifest list | |
| needs: | |
| - package_and_publish | |
| runs-on: ubuntu-latest | |
| permissions: | |
| id-token: write | |
| env: | |
| NEXUS_PASSWORD: ${{ secrets.NEXUS_PASSWORD }} | |
| OCI_REGISTRY_SDP_PASSWORD: ${{ secrets.HARBOR_ROBOT_SDP_GITHUB_ACTION_BUILD_SECRET }} | |
| OCI_REGISTRY_SDP_USERNAME: "robot$sdp+github-action-build" | |
| OCI_REGISTRY_SDP_CHARTS_PASSWORD: ${{ secrets.HARBOR_ROBOT_SDP_CHARTS_GITHUB_ACTION_BUILD_SECRET }} | |
| OCI_REGISTRY_SDP_CHARTS_USERNAME: "robot$sdp-charts+github-action-build" | |
| steps: | |
| - name: Install cosign | |
| uses: sigstore/cosign-installer@4959ce089c160fddf62f7b42464195ba1a56d382 # v3.6.0 | |
| - name: Checkout | |
| uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 | |
| with: | |
| submodules: recursive | |
| # This step checks if the current run was triggered by a push to a pr (or a pr being created). | |
| # If this is the case it changes the version of this project in all Cargo.toml files to include the suffix | |
| # "-pr<prnumber>" so that the published artifacts can be linked to this PR. | |
| - uses: stackabletech/cargo-install-action@main | |
| with: | |
| crate: cargo-edit | |
| bin: cargo-set-version | |
| - name: Update version if PR | |
| if: ${{ github.event_name == 'pull_request' }} | |
| run: cargo set-version --offline --workspace 0.0.0-pr${{ github.event.pull_request.number }} | |
| - name: Build manifest list | |
| run: | | |
| # Creating manifest list | |
| make -e docker-manifest-list-build | |
| # Pushing and signing manifest list | |
| make -e docker-manifest-list-publish | |
| openshift_preflight: | |
| name: Run the OpenShift Preflight check on the published images | |
| if: ${{ !github.event.pull_request.head.repo.fork }} | |
| needs: | |
| - create_manifest_list | |
| - package_and_publish | |
| runs-on: ubuntu-latest | |
| env: | |
| IMAGE_TAG: ${{ needs.package_and_publish.outputs.IMAGE_TAG }} | |
| steps: | |
| - name: Install preflight | |
| run: | | |
| wget https://github.com/redhat-openshift-ecosystem/openshift-preflight/releases/download/1.10.0/preflight-linux-amd64 | |
| chmod +x preflight-linux-amd64 | |
| - name: Check container | |
| run: | | |
| ARCH_FOR_PREFLIGHT="$(arch | sed -e 's#x86_64#amd64#' | sed -e 's#aarch64#arm64#')" | |
| ./preflight-linux-amd64 check container "$IMAGE_TAG" --platform "${ARCH_FOR_PREFLIGHT}" > preflight.out | |
| - name: "Passed?" | |
| run: '[ "$(jq -r .passed < preflight.out)" == true ]' |