From 826db24f9d39dd9d3049e9a4a9d35769617c1ab8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eduardo=20Leegwater=20Sim=C3=B5es?= Date: Thu, 17 Oct 2024 13:12:07 +0200 Subject: [PATCH 1/3] Delete Rust workflows --- .github/workflows/ci.yml | 253 ----------------------------- .github/workflows/dependencies.yml | 155 ------------------ 2 files changed, 408 deletions(-) delete mode 100644 .github/workflows/ci.yml delete mode 100644 .github/workflows/dependencies.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml deleted file mode 100644 index 003c1e5d7ebbe..0000000000000 --- a/.github/workflows/ci.yml +++ /dev/null @@ -1,253 +0,0 @@ -# This file defines our primary CI workflow that runs on pull requests -# and also on pushes to special branches (auto, try). -# -# The actual definition of the executed jobs is calculated by a Python -# script located at src/ci/github-actions/calculate-job-matrix.py, which -# uses job definition data from src/ci/github-actions/jobs.yml. -# You should primarily modify the `jobs.yml` file if you want to modify -# what jobs are executed in CI. - -name: CI -on: - push: - branches: - - auto - - try - - try-perf - - automation/bors/try - pull_request: - branches: - - "**" - -permissions: - contents: read - packages: write - -defaults: - run: - # On Linux, macOS, and Windows, use the system-provided bash as the default - # shell. (This should only make a difference on Windows, where the default - # shell is PowerShell.) - shell: bash - -concurrency: - # For a given workflow, if we push to the same branch, cancel all previous builds on that branch. - # We add an exception for try builds (try branch) and unrolled rollup builds (try-perf), which - # are all triggered on the same branch, but which should be able to run concurrently. - group: ${{ github.workflow }}-${{ ((github.ref == 'refs/heads/try' || github.ref == 'refs/heads/try-perf') && github.sha) || github.ref }} - cancel-in-progress: true -env: - TOOLSTATE_REPO: "https://github.com/rust-lang-nursery/rust-toolstate" - # This will be empty in PR jobs. - TOOLSTATE_REPO_ACCESS_TOKEN: ${{ secrets.TOOLSTATE_REPO_ACCESS_TOKEN }} -jobs: - # The job matrix for `calculate_matrix` is defined in src/ci/github-actions/jobs.yml. - # It calculates which jobs should be executed, based on the data of the ${{ github }} context. - # If you want to modify CI jobs, take a look at src/ci/github-actions/jobs.yml. - calculate_matrix: - name: Calculate job matrix - runs-on: ubuntu-latest - outputs: - jobs: ${{ steps.jobs.outputs.jobs }} - run_type: ${{ steps.jobs.outputs.run_type }} - steps: - - name: Checkout the source code - uses: actions/checkout@v4 - - name: Calculate the CI job matrix - env: - COMMIT_MESSAGE: ${{ github.event.head_commit.message }} - run: python3 src/ci/github-actions/calculate-job-matrix.py >> $GITHUB_OUTPUT - id: jobs - job: - name: ${{ matrix.name }} - needs: [ calculate_matrix ] - runs-on: "${{ matrix.os }}" - defaults: - run: - shell: ${{ contains(matrix.os, 'windows') && 'msys2 {0}' || 'bash' }} - timeout-minutes: 240 - env: - CI_JOB_NAME: ${{ matrix.image }} - CARGO_REGISTRIES_CRATES_IO_PROTOCOL: sparse - # commit of PR sha or commit sha. `GITHUB_SHA` is not accurate for PRs. - HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }} - DOCKER_TOKEN: ${{ secrets.GITHUB_TOKEN }} - SCCACHE_BUCKET: rust-lang-ci-sccache2 - CACHE_DOMAIN: ci-caches.rust-lang.org - continue-on-error: ${{ matrix.continue_on_error || false }} - strategy: - matrix: - # Check the `calculate_matrix` job to see how is the matrix defined. - include: ${{ fromJSON(needs.calculate_matrix.outputs.jobs) }} - steps: - - if: contains(matrix.os, 'windows') - uses: msys2/setup-msys2@v2.22.0 - with: - # i686 jobs use mingw32. x86_64 and cross-compile jobs use mingw64. - msystem: ${{ contains(matrix.name, 'i686') && 'mingw32' || 'mingw64' }} - # don't try to download updates for already installed packages - update: false - # don't try to use the msys that comes built-in to the github runner, - # so we can control what is installed (i.e. not python) - release: true - # Inherit the full path from the Windows environment, with MSYS2's */bin/ - # dirs placed in front. This lets us run Windows-native Python etc. - path-type: inherit - install: > - make - - - name: disable git crlf conversion - run: git config --global core.autocrlf false - - - name: checkout the source code - uses: actions/checkout@v4 - with: - fetch-depth: 2 - - # Rust Log Analyzer can't currently detect the PR number of a GitHub - # Actions build on its own, so a hint in the log message is needed to - # point it in the right direction. - - name: configure the PR in which the error message will be posted - run: echo "[CI_PR_NUMBER=$num]" - env: - num: ${{ github.event.number }} - if: needs.calculate_matrix.outputs.run_type == 'pr' - - - name: add extra environment variables - run: src/ci/scripts/setup-environment.sh - env: - # Since it's not possible to merge `${{ matrix.env }}` with the other - # variables in `job..env`, the variables defined in the matrix - # are passed to the `setup-environment.sh` script encoded in JSON, - # which then uses log commands to actually set them. - EXTRA_VARIABLES: ${{ toJson(matrix.env) }} - - - name: setup upstream remote - run: src/ci/scripts/setup-upstream-remote.sh - - - name: ensure the channel matches the target branch - run: src/ci/scripts/verify-channel.sh - - - name: collect CPU statistics - run: src/ci/scripts/collect-cpu-stats.sh - - - name: show the current environment - run: src/ci/scripts/dump-environment.sh - - - name: install awscli - run: src/ci/scripts/install-awscli.sh - - - name: install sccache - run: src/ci/scripts/install-sccache.sh - - - name: select Xcode - run: src/ci/scripts/select-xcode.sh - - - name: install clang - run: src/ci/scripts/install-clang.sh - - - name: install tidy - run: src/ci/scripts/install-tidy.sh - - - name: install WIX - run: src/ci/scripts/install-wix.sh - - - name: disable git crlf conversion - run: src/ci/scripts/disable-git-crlf-conversion.sh - - - name: checkout submodules - run: src/ci/scripts/checkout-submodules.sh - - - name: install MinGW - run: src/ci/scripts/install-mingw.sh - - - name: install ninja - run: src/ci/scripts/install-ninja.sh - - - name: enable ipv6 on Docker - run: src/ci/scripts/enable-docker-ipv6.sh - - # Disable automatic line ending conversion (again). On Windows, when we're - # installing dependencies, something switches the git configuration directory or - # re-enables autocrlf. We've not tracked down the exact cause -- and there may - # be multiple -- but this should ensure submodules are checked out with the - # appropriate line endings. - - name: disable git crlf conversion - run: src/ci/scripts/disable-git-crlf-conversion.sh - - - name: ensure line endings are correct - run: src/ci/scripts/verify-line-endings.sh - - - name: ensure backported commits are in upstream branches - run: src/ci/scripts/verify-backported-commits.sh - - - name: ensure the stable version number is correct - run: src/ci/scripts/verify-stable-version-number.sh - - - name: run the build - # Redirect stderr to stdout to avoid reordering the two streams in the GHA logs. - run: src/ci/scripts/run-build-from-ci.sh 2>&1 - env: - AWS_ACCESS_KEY_ID: ${{ env.CACHES_AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.CACHES_AWS_ACCESS_KEY_ID)] }} - - - name: create github artifacts - run: src/ci/scripts/create-doc-artifacts.sh - - - name: upload artifacts to github - uses: actions/upload-artifact@v4 - with: - # name is set in previous step - name: ${{ env.DOC_ARTIFACT_NAME }} - path: obj/artifacts/doc - if-no-files-found: ignore - retention-days: 5 - - - name: upload artifacts to S3 - run: src/ci/scripts/upload-artifacts.sh - env: - AWS_ACCESS_KEY_ID: ${{ env.ARTIFACTS_AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.ARTIFACTS_AWS_ACCESS_KEY_ID)] }} - # Adding a condition on DEPLOY=1 or DEPLOY_ALT=1 is not needed as all deploy - # builders *should* have the AWS credentials available. Still, explicitly - # adding the condition is helpful as this way CI will not silently skip - # deploying artifacts from a dist builder if the variables are misconfigured, - # erroring about invalid credentials instead. - if: github.event_name == 'push' || env.DEPLOY == '1' || env.DEPLOY_ALT == '1' - - - name: upload job metrics to DataDog - if: needs.calculate_matrix.outputs.run_type != 'pr' - env: - DATADOG_SITE: datadoghq.com - DATADOG_API_KEY: ${{ secrets.DATADOG_API_KEY }} - DD_GITHUB_JOB_NAME: ${{ matrix.name }} - run: | - npm install -g @datadog/datadog-ci@^2.x.x - python3 src/ci/scripts/upload-build-metrics.py build/cpu-usage.csv - - # This job isused to tell bors the final status of the build, as there is no practical way to detect - # when a workflow is successful listening to webhooks only in our current bors implementation (homu). - outcome: - name: bors build finished - runs-on: ubuntu-latest - needs: [ calculate_matrix, job ] - # !cancelled() executes the job regardless of whether the previous jobs passed or failed - if: ${{ !cancelled() && contains(fromJSON('["auto", "try"]'), needs.calculate_matrix.outputs.run_type) }} - steps: - - name: checkout the source code - uses: actions/checkout@v4 - with: - fetch-depth: 2 - # Calculate the exit status of the whole CI workflow. - # If all dependent jobs were successful, this exits with 0 (and the outcome job continues successfully). - # If a some dependent job has failed, this exits with 1. - - name: calculate the correct exit status - run: jq --exit-status 'all(.result == "success" or .result == "skipped")' <<< '${{ toJson(needs) }}' - # Publish the toolstate if an auto build succeeds (just before push to master) - - name: publish toolstate - run: src/ci/publish_toolstate.sh - shell: bash - if: needs.calculate_matrix.outputs.run_type == 'auto' - env: - TOOLSTATE_ISSUES_API_URL: https://api.github.com/repos/rust-lang/rust/issues - TOOLSTATE_PUBLISH: 1 diff --git a/.github/workflows/dependencies.yml b/.github/workflows/dependencies.yml deleted file mode 100644 index b7b5a03bd41f3..0000000000000 --- a/.github/workflows/dependencies.yml +++ /dev/null @@ -1,155 +0,0 @@ -# Automatically run `cargo update` periodically - ---- -name: Bump dependencies in Cargo.lock -on: - schedule: - # Run weekly - - cron: '0 0 * * Sun' - workflow_dispatch: - # Needed so we can run it manually -permissions: - contents: read -defaults: - run: - shell: bash -env: - # So cargo doesn't complain about unstable features - RUSTC_BOOTSTRAP: 1 - PR_TITLE: Weekly `cargo update` - PR_MESSAGE: | - Automation to keep dependencies in `Cargo.lock` current. - - The following is the output from `cargo update`: - COMMIT_MESSAGE: "cargo update \n\n" - -jobs: - not-waiting-on-bors: - if: github.repository_owner == 'rust-lang' - name: skip if S-waiting-on-bors - runs-on: ubuntu-latest - steps: - - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - # Fetch state and labels of PR - # Or exit successfully if PR does not exist - JSON=$(gh pr view cargo_update --repo $GITHUB_REPOSITORY --json labels,state || exit 0) - STATE=$(echo "$JSON" | jq -r '.state') - WAITING_ON_BORS=$(echo "$JSON" | jq '.labels[] | any(.name == "S-waiting-on-bors"; .)') - - # Exit with error if open and S-waiting-on-bors - if [[ "$STATE" == "OPEN" && "$WAITING_ON_BORS" == "true" ]]; then - exit 1 - fi - - update: - if: github.repository_owner == 'rust-lang' - name: update dependencies - needs: not-waiting-on-bors - runs-on: ubuntu-latest - steps: - - name: checkout the source code - uses: actions/checkout@v4 - with: - submodules: recursive - - name: install the bootstrap toolchain - run: | - # Extract the stage0 version - TOOLCHAIN=$(awk -F= '{a[$1]=$2} END {print(a["compiler_version"] "-" a["compiler_date"])}' src/stage0) - # Install and set as default - rustup toolchain install --no-self-update --profile minimal $TOOLCHAIN - rustup default $TOOLCHAIN - - - name: cargo update compiler & tools - # Remove first line that always just says "Updating crates.io index" - run: | - echo -e "\ncompiler & tools dependencies:" >> cargo_update.log - cargo update 2>&1 | sed '/crates.io index/d' | tee -a cargo_update.log - - name: cargo update library - run: | - echo -e "\nlibrary dependencies:" >> cargo_update.log - cargo update --manifest-path library/Cargo.toml 2>&1 | sed '/crates.io index/d' | tee -a cargo_update.log - - name: cargo update rustbook - run: | - echo -e "\nrustbook dependencies:" >> cargo_update.log - cargo update --manifest-path src/tools/rustbook/Cargo.toml 2>&1 | sed '/crates.io index/d' | tee -a cargo_update.log - - name: upload Cargo.lock artifact for use in PR - uses: actions/upload-artifact@v4 - with: - name: Cargo-lock - path: | - Cargo.lock - library/Cargo.lock - src/tools/rustbook/Cargo.lock - retention-days: 1 - - name: upload cargo-update log artifact for use in PR - uses: actions/upload-artifact@v4 - with: - name: cargo-updates - path: cargo_update.log - retention-days: 1 - - pr: - if: github.repository_owner == 'rust-lang' - name: amend PR - needs: update - runs-on: ubuntu-latest - permissions: - contents: write - pull-requests: write - steps: - - name: checkout the source code - uses: actions/checkout@v4 - - - name: download Cargo.lock from update job - uses: actions/download-artifact@v4 - with: - name: Cargo-lock - - name: download cargo-update log from update job - uses: actions/download-artifact@v4 - with: - name: cargo-updates - - - name: craft PR body and commit message - run: | - echo "${COMMIT_MESSAGE}" > commit.txt - cat cargo_update.log >> commit.txt - - echo "${PR_MESSAGE}" > body.md - echo '```txt' >> body.md - cat cargo_update.log >> body.md - echo '```' >> body.md - - - name: commit - run: | - git config user.name github-actions - git config user.email github-actions@github.com - git switch --force-create cargo_update - git add ./Cargo.lock ./library/Cargo.lock ./src/tools/rustbook/Cargo.lock - git commit --no-verify --file=commit.txt - - - name: push - run: git push --no-verify --force --set-upstream origin cargo_update - - - name: edit existing open pull request - id: edit - # Don't fail job if we need to open new PR - continue-on-error: true - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - # Exit with error if PR is closed - STATE=$(gh pr view cargo_update --repo $GITHUB_REPOSITORY --json state --jq '.state') - if [[ "$STATE" != "OPEN" ]]; then - exit 1 - fi - - gh pr edit cargo_update --title "${PR_TITLE}" --body-file body.md --repo $GITHUB_REPOSITORY - - - name: open new pull request - # Only run if there wasn't an existing PR - if: steps.edit.outcome != 'success' - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: gh pr create --title "${PR_TITLE}" --body-file body.md --repo $GITHUB_REPOSITORY From 2e1df30b8962c3fccebf5ca588d7eee5a10731f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eduardo=20Leegwater=20Sim=C3=B5es?= Date: Thu, 17 Oct 2024 13:14:19 +0200 Subject: [PATCH 2/3] Add compiler building workflow The compiler is built for the following hosts: - x86_64-unknown-linux-gnu - x86_64-apple-darwin - aarch64-apple-darwin And produces code for the following targets: - The host triple - wasm32-unknown-unknown - wasm64-unknown-unknown --- .github/workflows/dusk.yml | 64 ++++++++++++++++++++++++++++++++++++++ config.template.toml | 25 +++++++++++++++ 2 files changed, 89 insertions(+) create mode 100644 .github/workflows/dusk.yml create mode 100644 config.template.toml diff --git a/.github/workflows/dusk.yml b/.github/workflows/dusk.yml new file mode 100644 index 0000000000000..a178a592467ce --- /dev/null +++ b/.github/workflows/dusk.yml @@ -0,0 +1,64 @@ +# Instructions for GitHub to build Dusk's smart contract compilers. + +on: + pull_request: + push: + tags: + - "v*.*.*" + branches: + - master + +name: Dusk Contract Compilers + +jobs: + target: + strategy: + matrix: + target: [ + x86_64-unknown-linux-gnu, + x86_64-apple-darwin, + aarch64-apple-darwin, + ] + include: + - os: ubuntu-latest + target: x86_64-unknown-linux-gnu + - os: macos-latest-large + target: x86_64-apple-darwin + - os: macos-latest-xlarge + target: aarch64-apple-darwin + runs-on: ${{ matrix.os }} + steps: + - name: Check out repository + uses: actions/checkout@v4 + with: + fetch-depth: 16 + + - name: Setup dependencies + if: matrix.os == 'ubuntu-latest' + run: sudo apt update && sudo apt install -y ninja-build + + - name: Setup dependencies + if: startsWith(matrix.os, 'macos') + run: | + brew install ninja gettext && + brew link --force gettext + - name: Generate configuration + run: | + HOST=${{ matrix.target }} envsubst < config.template.toml > config.toml && + cat config.toml + - name: Run build + run: ./x.py dist + + - name: Set artifact name + id: artifact-name + run: | + if [ "${{ github.ref_type }}" == "tag" ]; then + echo "name=duskc-${{ matrix.target }}-${{ github.ref_name }}" >> "$GITHUB_OUTPUT" + else + echo "name=duskc-${{ matrix.target }}-${{ github.sha }}" >> "$GITHUB_OUTPUT" + fi + - name: Upload build + uses: actions/upload-artifact@v3 + with: + name: ${{ steps.artifact-name.outputs.name }} + path: build/dist/ diff --git a/config.template.toml b/config.template.toml new file mode 100644 index 0000000000000..8fd0d693a60a3 --- /dev/null +++ b/config.template.toml @@ -0,0 +1,25 @@ +change-id = 131075 + +[llvm] +download-ci-llvm = false +clang = true + +[build] +host = [ "${HOST}" ] +target = [ + "${HOST}", + "wasm32-unknown-unknown", + "wasm64-unknown-unknown" +] +docs = false +extended = true + +[rust] +channel = "nightly" +description = "Dusk Network's Smart Contract Compiler" + +lld = true + +[dist] +compression-formats = ["gz"] +compression-profile = "best" From 8f8a58c5fece64cfd156fd5915ba371d706de867 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eduardo=20Leegwater=20Sim=C3=B5es?= Date: Thu, 17 Oct 2024 18:20:47 +0200 Subject: [PATCH 3/3] workflow FIXUP --- .github/workflows/dusk.yml | 30 +++++++++++++++++++++++------- config.template.toml | 17 +++++++++++------ 2 files changed, 34 insertions(+), 13 deletions(-) diff --git a/.github/workflows/dusk.yml b/.github/workflows/dusk.yml index a178a592467ce..0c375349b2bb0 100644 --- a/.github/workflows/dusk.yml +++ b/.github/workflows/dusk.yml @@ -26,6 +26,7 @@ jobs: target: x86_64-apple-darwin - os: macos-latest-xlarge target: aarch64-apple-darwin + runs-on: ${{ matrix.os }} steps: - name: Check out repository @@ -33,21 +34,36 @@ jobs: with: fetch-depth: 16 - - name: Setup dependencies + - name: Setup dependencies (x86_64-linux) if: matrix.os == 'ubuntu-latest' - run: sudo apt update && sudo apt install -y ninja-build + run: | + sudo apt update && \ + sudo apt install -y ninja-build && \ + touch .env + + - name: Setup dependencies (x86_64-apple) + if: startsWith(matrix.target, 'x86_64-apple') + run: | + brew install ninja llvm lld && + echo 'export PATH="/usr/local/opt/llvm/bin:$PATH"' >> .env && + echo 'export LDFLAGS="-L/usr/local/opt/llvm/lib"' >> .env && + echo 'export CPPFLAGS="-I/usr/local/opt/llvm/include"' >> .env - - name: Setup dependencies - if: startsWith(matrix.os, 'macos') + - name: Setup dependencies (aarch64-apple) + if: startsWith(matrix.target, 'aarch64-apple') run: | - brew install ninja gettext && - brew link --force gettext + brew install ninja llvm lld && + echo 'export PATH="/opt/homebrew/opt/llvm/bin:$PATH"' >> .env && + echo 'export LDFLAGS="-L/opt/homebrew/opt/llvm/lib"' >> .env && + echo 'export CPPFLAGS="-I/opt/homebrew/opt/llvm/include"' >> .env + - name: Generate configuration run: | HOST=${{ matrix.target }} envsubst < config.template.toml > config.toml && cat config.toml + - name: Run build - run: ./x.py dist + run: source .env && ./x.py dist - name: Set artifact name id: artifact-name diff --git a/config.template.toml b/config.template.toml index 8fd0d693a60a3..b9b4be6a4c932 100644 --- a/config.template.toml +++ b/config.template.toml @@ -1,25 +1,30 @@ change-id = 131075 -[llvm] -download-ci-llvm = false -clang = true - [build] -host = [ "${HOST}" ] +build-stage = 2 +test-stage = 2 +doc-stage = 2 + target = [ "${HOST}", "wasm32-unknown-unknown", "wasm64-unknown-unknown" ] -docs = false + extended = true +[llvm] +download-ci-llvm = false + [rust] channel = "nightly" description = "Dusk Network's Smart Contract Compiler" +download-rustc = false lld = true +llvm-bitcode-linker = true + [dist] compression-formats = ["gz"] compression-profile = "best"