From 01384f0ef124464862cba40f75a8dd12f6e2f10e Mon Sep 17 00:00:00 2001 From: gera Date: Thu, 27 Jun 2024 15:42:27 +0200 Subject: [PATCH] ci: try-runtime gh action --- .github/workflows/clippy-check.yml | 7 +-- .github/workflows/try-runtime.yml | 50 +++++++++-------- .gitlab-ci.yml | 16 ++---- .gitlab/build.yml | 47 +++++++++++++++- .gitlab/check-skip.yml | 15 ------ .gitlab/check.yml | 34 ++++++++++++ .gitlab/release.yml | 60 --------------------- .gitlab/test.yml | 87 +++++++++++------------------- .maintain/build-image.sh | 57 ++++++++++---------- .maintain/push-image.sh | 36 ++++++------- 10 files changed, 193 insertions(+), 216 deletions(-) delete mode 100644 .gitlab/check-skip.yml create mode 100644 .gitlab/check.yml delete mode 100644 .gitlab/release.yml diff --git a/.github/workflows/clippy-check.yml b/.github/workflows/clippy-check.yml index 42fb04ac3..90189a8ef 100644 --- a/.github/workflows/clippy-check.yml +++ b/.github/workflows/clippy-check.yml @@ -1,17 +1,18 @@ on: push name: Clippy check +concurrency: + group: ${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/master' }} -# Make sure CI fails on all warnings, including Clippy lints env: RUSTFLAGS: "-Dwarnings" jobs: clippy_check: - runs-on: [self-hosted] + runs-on: ubuntu-latest services: docker: image: docker:20.10.8-dind - privileged: true container: image: paritytech/ci-unified:bullseye-1.77.0 steps: diff --git a/.github/workflows/try-runtime.yml b/.github/workflows/try-runtime.yml index e1dd96120..d39f55cd7 100644 --- a/.github/workflows/try-runtime.yml +++ b/.github/workflows/try-runtime.yml @@ -1,33 +1,41 @@ name: try-runtime Workflow +concurrency: + group: ${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/master' }} -on: [pull_request] - +on: push jobs: - build: + test-try-runtime: strategy: matrix: - runtime: [ peregrine, spiritnet ] - uri: [ 'wss://peregrine.kilt.io:443', 'wss://peregrine.kilt.io:443' ] - runs-on: [ self-hosted ] - services: - docker: - image: docker:20.10.8-dind - privileged: true + runtime: [peregrine, spiritnet] + uri: ['wss://peregrine.kilt.io', 'wss://spiritnet.kilt.io'] + exclude: + - runtime: peregrine + uri: 'wss://spiritnet.kilt.io' + - runtime: spiritnet + uri: 'wss://peregrine.kilt.io' + runs-on: ubuntu-latest container: image: paritytech/ci-unified:bullseye-1.77.0 steps: - name: Checkout code - uses: actions/checkout@v2 + uses: actions/checkout@v4 + + - name: script + run: | + echo "Running ${{ matrix.runtime }} runtime migration check" + + echo "---------- Downloading try-runtime CLI ----------" + curl -sL https://github.com/paritytech/try-runtime-cli/releases/download/v0.7.0/try-runtime-x86_64-unknown-linux-musl -o try-runtime + chmod +x ./try-runtime + echo "Using try-runtime-cli version:" + ./try-runtime --version - - name: Build - run: cargo build --release -p ${{ matrix.runtime }}-runtime --features try-runtime --locked + echo "---------- Building ${{ matrix.runtime }} runtime ----------" + cargo build --release --locked -p ${{ matrix.runtime }}-runtime --features try-runtime - - name: Run tests - uses: NodleCode/action-try-runtime@v0.5.1 - with: - url: ${matrix.uri} - snap: snapshots/eden-snapshot-full - runtime: target/release/wbuild/${metrix.runtime}-runtime/peregrine_runtime.compact.compressed.wasm - checks: "all" - options: "----disable-spec-version-check" + echo "---------- Executing on-runtime-upgrade for ${{ matrix.uri }} ----------" + ./try-runtime --runtime ./target/release/wbuild/${{ matrix.runtime }}-runtime/${{ matrix.runtime }}_runtime.compact.compressed.wasm \ + on-runtime-upgrade --disable-spec-version-check --checks=all ${{ matrix.subcommand_extra_args }} live --uri ${{ matrix.uri }} diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 659e64a59..b4174482d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,27 +1,19 @@ +default: + interruptible: true include: - - local: '.gitlab/check-skip.yml' + - local: '.gitlab/check.yml' - local: '.gitlab/test.yml' - local: '.gitlab/build.yml' - - local: '.gitlab/release.yml' variables: CARGO_HOME: $CI_PROJECT_DIR/.cargo CARGO_TARGET_DIR: $CI_PROJECT_DIR/target -default: - cache: - key: ${CI_COMMIT_REF_SLUG}-rust - paths: - - target - - .cargo/bin/ - - .cargo/registry/index/ - - .cargo/registry/cache/ - - .cargo/git/db/ workflow: auto_cancel: on_new_commit: interruptible stages: + - check - test - build - - release diff --git a/.gitlab/build.yml b/.gitlab/build.yml index 4a4e1bc78..7e5fdf2a8 100644 --- a/.gitlab/build.yml +++ b/.gitlab/build.yml @@ -1,5 +1,4 @@ build: - interruptible: true timeout: 2 hours image: name: kiltprotocol/kilt-ci:2.7.31 @@ -28,4 +27,50 @@ build: - "if [[ ! -z ${CI_COMMIT_BRANCH} ]]; then ./.maintain/push-image.sh build latest-${CI_COMMIT_BRANCH}; fi" - "if [[ ! -z ${CI_COMMIT_TAG} ]]; then ./.maintain/push-image.sh build ${CI_COMMIT_TAG}; fi" - "if [[ ! -z ${CI_COMMIT_TAG} && ! -z ${CI_COMMIT_TAG##*-rc*} && ! -z ${CI_COMMIT_TAG##*dev-*} ]]; then ./.maintain/push-image.sh build latest; fi" + cache: + key: cache-build + paths: + - target/ + - .cargo/bin/ + - .cargo/registry/index/ + - .cargo/registry/cache/ + - .cargo/git/db/ +build-wasm: + image: + name: paritytech/srtool:1.77.0 + entrypoint: [""] + stage: build + only: + - develop + - master + - /^[0-9]+(?:\.[0-9]+){2}(?:-(rc)*([0-9])+)?$/ + parallel: + matrix: + - RUNTIME: "peregrine" + ENDPOINT: "wss://peregrine.kilt.io" + - RUNTIME: "spiritnet" + ENDPOINT: "wss://spiritnet.kilt.io" + script: + - export PACKAGE=${RUNTIME}-runtime + - export RUNTIME_DIR=runtimes/${RUNTIME} + - export PARACHAIN_PALLET_ID=0x50 + - export AUTHORIZE_UPGRADE_PREFIX=0x02 + - cp -r * /build + - /srtool/build build + - subwasm meta --format=json+scale /out/${RUNTIME}_runtime.compact.wasm > /out/${RUNTIME}-metadata.json + - subwasm get -o ${RUNTIME}-live.wasm ${ENDPOINT} + - subwasm diff --no-color ${RUNTIME}-live.wasm /out/${RUNTIME}_runtime.compact.wasm | tee /out/${RUNTIME}-diff.txt + - mkdir ./out + - mv /out/* ./out/ + cache: + key: cache-wasm + paths: + - /build + - /out + artifacts: + paths: + - out/*.wasm + - out/*.json + - out/*.txt + expire_in: 12 week diff --git a/.gitlab/check-skip.yml b/.gitlab/check-skip.yml deleted file mode 100644 index 4d3b91b86..000000000 --- a/.gitlab/check-skip.yml +++ /dev/null @@ -1,15 +0,0 @@ -.check_skip_rust: - before_script: - - |- - if echo "$CI_COMMIT_TITLE" | grep -q "ci-skip-rust"; then - echo "Skipping rust tests due to commit title" - exit 0 - fi - -.check_skip_integration_tests: - before_script: - - |- - if echo "$CI_COMMIT_TITLE" | grep -q "ci-skip-integration-tests"; then - echo "Skipping integration tests due to commit title" - exit 0 - fi diff --git a/.gitlab/check.yml b/.gitlab/check.yml new file mode 100644 index 000000000..449e1c1f9 --- /dev/null +++ b/.gitlab/check.yml @@ -0,0 +1,34 @@ +.check_skip_rust: + before_script: + - |- + if echo "$CI_COMMIT_TITLE" | grep -q "ci-skip-rust"; then + echo "Skipping rust tests due to commit title" + exit 0 + fi + +.check_skip_integration_tests: + before_script: + - |- + if echo "$CI_COMMIT_TITLE" | grep -q "ci-skip-integration-tests"; then + echo "Skipping integration tests due to commit title" + exit 0 + fi + +cargo-clippy: + extends: .check_skip_rust + image: paritytech/ci-unified:bullseye-1.77.0 + stage: check + variables: + RUSTFLAGS: "-D warnings" + script: + - cargo clippy --all-targets --locked --workspace --quiet + - cargo clippy --all-targets --all-features --locked --workspace --quiet + +check-toml-format: + extends: .check_skip_rust + image: paritytech/ci-unified:bullseye-1.77.0 + stage: check + cache: {} + script: + - cargo fmt -- --check + - taplo fmt --check diff --git a/.gitlab/release.yml b/.gitlab/release.yml deleted file mode 100644 index bf32bb84a..000000000 --- a/.gitlab/release.yml +++ /dev/null @@ -1,60 +0,0 @@ -build-wasm-peregrine: - interruptible: true - image: - name: paritytech/srtool:1.70.0 - entrypoint: [""] - stage: release - only: - - develop - - master - - /^[0-9]+(?:\.[0-9]+){2}(?:-(rc)*([0-9])+)?$/ - script: - - export PACKAGE=peregrine-runtime - - export RUNTIME_DIR=runtimes/peregrine - - export PARACHAIN_PALLET_ID=0x50 - - export AUTHORIZE_UPGRADE_PREFIX=0x02 - - cp -r * /build - - /srtool/build build - - subwasm meta --format=json+scale /out/peregrine_runtime.compact.wasm > /out/peregrine-metadata.json - - subwasm get -o peregrine-live.wasm wss://peregrine.kilt.io - - subwasm diff --no-color peregrine-live.wasm /out/peregrine_runtime.compact.wasm | tee /out/peregrine-diff.txt - - mkdir ./out - - mv /out/* ./out/ - artifacts: - paths: - - out/*.wasm - - out/*.json - - out/*.txt - expire_in: 12 week - -build-wasm-spiritnet: - interruptible: true - image: - name: paritytech/srtool:1.70.0 - entrypoint: [""] - stage: release - only: - - develop - - master - - /^[0-9]+(?:\.[0-9]+){2}(?:-(rc)*([0-9])+)?$/ - needs: - - build-wasm-peregrine - script: - - export PACKAGE=spiritnet-runtime - - export RUNTIME_DIR=runtimes/spiritnet - - export PARACHAIN_PALLET_ID=0x50 - - export AUTHORIZE_UPGRADE_PREFIX=0x02 - - cp -r * /build - - /srtool/build build - - subwasm meta --format=json+scale /out/spiritnet_runtime.compact.wasm > /out/spiritnet-metadata.json - - subwasm get -o spiritnet-live.wasm wss://spiritnet.kilt.io - - subwasm diff --no-color spiritnet-live.wasm /out/spiritnet_runtime.compact.wasm | tee /out/spiritnet-diff.txt - - mkdir ./out - - mv /out/* ./out/ - artifacts: - paths: - - out/*.wasm - - out/*.json - - out/*.txt - expire_in: 12 week - diff --git a/.gitlab/test.yml b/.gitlab/test.yml index afd358266..6c7bb97ad 100644 --- a/.gitlab/test.yml +++ b/.gitlab/test.yml @@ -1,49 +1,46 @@ -clippy-and-docs: - extends: .check_skip_rust - interruptible: true - timeout: 2 hours - image: paritytech/ci-unified:bullseye-1.77.0 - stage: test - script: - - cargo clippy --all-features --all-targets --locked -- -D warnings - - RUSTDOCFLAGS='-D warnings' cargo doc --all-features --no-deps --locked - -fmt: - interruptible: true - extends: .check_skip_rust - image: paritytech/ci-unified:bullseye-1.77.0 - stage: test - script: - - cargo fmt -- --check - - taplo fmt --check - test: - interruptible: true extends: .check_skip_rust timeout: 2 hours image: paritytech/ci-unified:bullseye-1.77.0 stage: test + cache: + key: cache-test + paths: + - target/ + - .cargo/bin/ + - .cargo/registry/index/ + - .cargo/registry/cache/ + - .cargo/git/db/ script: - cargo test --all --all-targets --locked test-features: extends: .check_skip_rust - interruptible: true timeout: 2 hours image: paritytech/ci-unified:bullseye-1.77.0 stage: test needs: - test + cache: + key: cache-test + paths: + - target/ + - .cargo/bin/ + - .cargo/registry/index/ + - .cargo/registry/cache/ + - .cargo/git/db/ script: - cargo test --all --all-features --all-targets --locked integration-tests: extends: .check_skip_integration_tests - interruptible: true timeout: 30 minutes image: paritytech/ci-unified:bullseye-1.77.0 stage: test - cache: {} + cache: + key: cache-integration-tests + paths: + - ./integration-tests/chopsticks/node_modules variables: CI: "true" script: @@ -56,51 +53,27 @@ integration-tests: - yarn lint - yarn test:CI -build-wasm-try-runtime: - interruptible: true +test-try-runtime: parallel: matrix: - RUNTIME: "peregrine" + ENDPOINT: "wss://peregrine.kilt.io:443" - RUNTIME: "spiritnet" - image: - name: paritytech/ci-unified:bullseye-1.77.0 - entrypoint: [""] - stage: release + ENDPOINT: "wss://spiritnet.kilt.io:443" + image: paritytech/ci-unified:bullseye-1.77.0 + stage: test only: - develop - master - /^[0-9]+(?:\.[0-9]+){2}(?:-(rc)*([0-9])+)?$/ + variables: + TRY_RUNTIME_CLI_VERSION_TAG: v0.7.0 script: + # Build - cargo build --release -p ${RUNTIME}-runtime --features try-runtime - mkdir ./out - mv target/release/wbuild/${RUNTIME}-runtime/${RUNTIME}_runtime.compact.compressed.wasm ./dangerous_${RUNTIME}.try-runtime.wasm - artifacts: - name: ${RUNTIME}_try-runtime - paths: - - ./dangerous_${RUNTIME}.try-runtime.wasm - expire_in: 12 week - -test-try-runtime: - extends: .check_skip_rust - interruptible: true - parallel: - matrix: - - RUNTIME: "peregrine" - ENDPOINT: "wss://peregrine.kilt.io:443" - - RUNTIME: "spiritnet" - ENDPOINT: "wss://spiritnet.kilt.io:443" - timeout: 2 hours - image: paritytech/ci-unified:bullseye-1.77.0 - stage: release - variables: - TRY_RUNTIME_CLI_VERSION_TAG: v0.7.0 - needs: - - build-wasm-try-runtime - artifacts: - paths: - - out/* - when: on_success - script: + # Test - cargo install --git https://github.com/paritytech/try-runtime-cli --tag ${TRY_RUNTIME_CLI_VERSION_TAG} --locked - ./.cargo/bin/try-runtime --version - - ./try-runtime --runtime ./out/dangerous_${RUNTIME}.try-runtime.wasm on-runtime-upgrade --disable-spec-version-check --checks=all live --uri=${ENDPOINT} + - ./try-runtime --runtime ./dangerous_${RUNTIME}.try-runtime.wasm on-runtime-upgrade --disable-spec-version-check --checks=all live --uri=${ENDPOINT} diff --git a/.maintain/build-image.sh b/.maintain/build-image.sh index 0a1948485..ad88929b4 100755 --- a/.maintain/build-image.sh +++ b/.maintain/build-image.sh @@ -1,40 +1,39 @@ #!/bin/bash +set -e + target_tag=$1 +# Build the builder image and push it in the background docker build \ --target builder \ --cache-from $AWS_REGISTRY/kilt-parachain/collator:builder \ -t $AWS_REGISTRY/kilt-parachain/collator:builder \ - . + . & docker push $AWS_REGISTRY/kilt-parachain/collator:builder & -# build parachain image and standalone image -docker build \ - --cache-from $AWS_REGISTRY/kilt-parachain/collator:builder \ - --cache-from $AWS_REGISTRY/kilt-parachain/collator:$target_tag \ - --build-arg NODE_TYPE=kilt-parachain \ - -t local/kilt-node:$target_tag \ - . -docker build \ - --cache-from $AWS_REGISTRY/kilt-parachain/collator:builder \ - --cache-from $AWS_REGISTRY/kilt/prototype-chain:$target_tag \ - --build-arg NODE_TYPE=standalone-node \ - -t local/standalone-node:$target_tag \ - . +wait -# build DIP provider and consumer templates -PROVIDER_BIN_NAME="dip-provider-node-template" -docker build \ - --cache-from $AWS_REGISTRY/kilt-parachain/collator:builder \ - --cache-from $AWS_REGISTRY/$PROVIDER_BIN_NAME:$target_tag \ - --build-arg NODE_TYPE=$PROVIDER_BIN_NAME \ - -t local/$PROVIDER_BIN_NAME:$target_tag \ - . -CONSUMER_BIN_NAME="dip-consumer-node-template" -docker build \ - --cache-from $AWS_REGISTRY/kilt-parachain/collator:builder \ - --cache-from $AWS_REGISTRY/$CONSUMER_BIN_NAME:$target_tag \ - --build-arg NODE_TYPE=$CONSUMER_BIN_NAME \ - -t local/$CONSUMER_BIN_NAME:$target_tag \ - . +# Build and tag images in parallel +build_and_tag() { + local node_type=$1 + local image_name=$2 + local cache_image=$3 + + docker build \ + --cache-from $AWS_REGISTRY/kilt-parachain/collator:builder \ + --cache-from $AWS_REGISTRY/$cache_image:$target_tag \ + --build-arg NODE_TYPE=$node_type \ + -t local/$image_name:$target_tag \ + . +} + +build_and_tag "kilt-parachain" "kilt-node" "kilt-parachain/collator" & + +build_and_tag "standalone-node" "standalone-node" "kilt/prototype-chain" & + +build_and_tag "dip-provider-node-template" "dip-provider-node-template" "kilt-parachain/collator" & + +build_and_tag "dip-consumer-node-template" "dip-consumer-node-template" "kilt-parachain/collator" & + +wait diff --git a/.maintain/push-image.sh b/.maintain/push-image.sh index b83c0808d..b007fee9d 100755 --- a/.maintain/push-image.sh +++ b/.maintain/push-image.sh @@ -6,26 +6,26 @@ target_tag=$2 PROVIDER_BIN_NAME="dip-provider-node-template" CONSUMER_BIN_NAME="dip-consumer-node-template" -# publish to docker hub -docker tag local/standalone-node:$source_tag ${DOCKER_HUB_STANDALONE}:$target_tag -docker tag local/kilt-node:$source_tag ${DOCKER_HUB_PARACHAIN}:$target_tag -docker tag local/$PROVIDER_BIN_NAME:$source_tag ${DOCKER_HUB_DIP_PROVIDER_TEMPLATE}:$target_tag -docker tag local/$CONSUMER_BIN_NAME:$source_tag ${DOCKER_HUB_DIP_CONSUMER_TEMPLATE}:$target_tag +# Function to tag and push images +tag_and_push() { + local source=$1 + local target=$2 + local registry=$3 -docker push ${DOCKER_HUB_STANDALONE}:$target_tag & -docker push ${DOCKER_HUB_PARACHAIN}:$target_tag & -docker push ${DOCKER_HUB_DIP_PROVIDER_TEMPLATE}:$target_tag & -docker push ${DOCKER_HUB_DIP_CONSUMER_TEMPLATE}:$target_tag & + docker tag $source $target + docker push $target & +} -# publish to AWS -docker tag local/standalone-node:$source_tag $AWS_REGISTRY/kilt/prototype-chain:$target_tag -docker tag local/kilt-node:$source_tag $AWS_REGISTRY/kilt-parachain/collator:$target_tag -docker tag local/$PROVIDER_BIN_NAME:$source_tag $AWS_REGISTRY/$PROVIDER_BIN_NAME:$target_tag -docker tag local/$CONSUMER_BIN_NAME:$source_tag $AWS_REGISTRY/$CONSUMER_BIN_NAME:$target_tag +# Tag and push to Docker Hub +tag_and_push local/standalone-node:$source_tag ${DOCKER_HUB_STANDALONE}:$target_tag +tag_and_push local/kilt-node:$source_tag ${DOCKER_HUB_PARACHAIN}:$target_tag +tag_and_push local/$PROVIDER_BIN_NAME:$source_tag ${DOCKER_HUB_DIP_PROVIDER_TEMPLATE}:$target_tag +tag_and_push local/$CONSUMER_BIN_NAME:$source_tag ${DOCKER_HUB_DIP_CONSUMER_TEMPLATE}:$target_tag -docker push $AWS_REGISTRY/kilt/prototype-chain:$target_tag & -docker push $AWS_REGISTRY/kilt-parachain/collator:$target_tag & -docker push $AWS_REGISTRY/$PROVIDER_BIN_NAME:$target_tag & -docker push $AWS_REGISTRY/$CONSUMER_BIN_NAME:$target_tag & +# Tag and push to AWS +tag_and_push local/standalone-node:$source_tag $AWS_REGISTRY/kilt/prototype-chain:$target_tag +tag_and_push local/kilt-node:$source_tag $AWS_REGISTRY/kilt-parachain/collator:$target_tag +tag_and_push local/$PROVIDER_BIN_NAME:$source_tag $AWS_REGISTRY/$PROVIDER_BIN_NAME:$target_tag +tag_and_push local/$CONSUMER_BIN_NAME:$source_tag $AWS_REGISTRY/$CONSUMER_BIN_NAME:$target_tag wait