Merge branch 'oty-umbrella-test' into oty-test-umbrella

.github/scripts/common/lib.sh

@@ -369,7 +369,7 @@ function relative_parent() {
 # used as Github Workflow Matrix. This call is exposed by the `scan` command and can be used as:
 # podman run --rm -it -v /.../fellowship-runtimes:/build docker.io/chevdor/srtool:1.70.0-0.11.1 scan
 function find_runtimes() {
-    libs=($(git grep -I -r --cached --max-depth 20 --files-with-matches 'construct_runtime!' -- '*lib.rs'))
+    libs=($(git grep -I -r --cached --max-depth 20 --files-with-matches '[frame_support::runtime]!' -- '*lib.rs'))
     re=".*-runtime$"
     JSON=$(jq --null-input '{ "include": [] }')
 
@@ -434,3 +434,13 @@ check_release_id() {
   fi
 
 }
+
+# Get latest release tag
+#
+# input: none
+# output: latest_release_tag
+get_latest_release_tag() {
+    TOKEN="Authorization: Bearer $GITHUB_TOKEN"
+    latest_release_tag=$(curl -s -H "$TOKEN" $api_base/paritytech/polkadot-sdk/releases/latest | jq -r '.tag_name')
+    printf $latest_release_tag
+}

.github/workflows/check-links.yml

@@ -3,8 +3,8 @@ name: Check links
 on:
   pull_request:
     paths:
-      - "*.rs"
-      - "*.prdoc"
+      - "**.rs"
+      - "**.prdoc"
       - ".github/workflows/check-links.yml"
       - ".config/lychee.toml"
     types: [opened, synchronize, reopened, ready_for_review]

.github/workflows/release-30_publish_release_draft.yml

@@ -0,0 +1,155 @@
+name: Release - Publish draft
+
+on:
+  push:
+    tags:
+      # Catches v1.2.3 and v1.2.3-rc1
+      - v[0-9]+.[0-9]+.[0-9]+*
+
+  workflow_dispatch:
+    inputs:
+      version:
+        description: Current release/rc version
+
+jobs:
+  get-rust-versions:
+    runs-on: ubuntu-latest
+    outputs:
+      rustc-stable: ${{ steps.get-rust-versions.outputs.stable }}
+    steps:
+      - id: get-rust-versions
+        run: |
+          RUST_STABLE_VERSION=$(curl -sS https://raw.githubusercontent.com/paritytech/scripts/master/dockerfiles/ci-unified/Dockerfile | grep -oP 'ARG RUST_STABLE_VERSION=\K[^ ]+')
+          echo "stable=$RUST_STABLE_VERSION" >> $GITHUB_OUTPUT
+
+  build-runtimes:
+    uses: "./.github/workflows/srtool.yml"
+    with:
+       excluded_runtimes: "substrate-test bp cumulus-test kitchensink minimal-template parachain-template penpal polkadot-test seedling shell frame-try sp solochain-template"
+
+  publish-release-draft:
+    runs-on: ubuntu-latest
+    needs: [get-rust-versions, build-runtimes]
+    outputs:
+      release_url: ${{ steps.create-release.outputs.html_url }}
+      asset_upload_url: ${{ steps.create-release.outputs.upload_url }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0
+
+      - name: Prepare tooling
+        run: |
+          URL=https://github.com/chevdor/tera-cli/releases/download/v0.2.4/tera-cli_linux_amd64.deb
+          wget $URL -O tera.deb
+          sudo dpkg -i tera.deb
+          tera --version
+
+      - name: Download artifacts
+        uses: actions/download-artifact@c850b930e6ba138125429b7e5c93fc707a7f8427 # v4.1.4
+
+      - name: Prepare draft
+        id: draft
+        env:
+          RUSTC_STABLE: ${{ needs.get-rust-versions.outputs.rustc-stable }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          ASSET_HUB_ROCOCO_DIGEST: ${{ github.workspace}}/asset-hub-rococo-runtime/asset-hub-rococo-srtool-digest.json
+          ASSET_HUB_WESTEND_DIGEST: ${{ github.workspace}}/asset-hub-westend-runtime/asset-hub-westend-srtool-digest.json
+          BRIDGE_HUB_ROCOCO_DIGEST: ${{ github.workspace}}/bridge-hub-rococo-runtime/bridge-hub-rococo-srtool-digest.json
+          BRIDGE_HUB_WESTEND_DIGEST: ${{ github.workspace}}/bridge-hub-westend-runtime/bridge-hub-westend-srtool-digest.json
+          COLLECTIVES_WESTEND_DIGEST: ${{ github.workspace}}/collectives-westend-runtime/collectives-westend-srtool-digest.json
+          CONTRACTS_ROCOCO_DIGEST: ${{ github.workspace}}/contracts-rococo-runtime/contracts-rococo-srtool-digest.json
+          CORETIME_ROCOCO_DIGEST: ${{ github.workspace}}/coretime-rococo-runtime/coretime-rococo-srtool-digest.json
+          CORETIME_WESTEND_DIGEST: ${{ github.workspace}}/coretime-westend-runtime/coretime-westend-srtool-digest.json
+          GLUTTON_WESTEND_DIGEST: ${{ github.workspace}}/glutton-westend-runtime/glutton-westend-srtool-digest.json
+          PEOPLE_ROCOCO_DIGEST: ${{ github.workspace}}/people-rococo-runtime/people-rococo-srtool-digest.json
+          PEOPLE_WESTEND_DIGEST: ${{ github.workspace}}/people-westend-runtime/people-westend-srtool-digest.json
+          ROCOCO_DIGEST: ${{ github.workspace}}/rococo-runtime/rococo-srtool-digest.json
+          WESTEND_DIGEST: ${{ github.workspace}}/westend-runtime/westend-srtool-digest.json
+        run: |
+          . ./.github/scripts/common/lib.sh
+
+          export REF1=$(get_latest_release_tag)
+          if [[ -z "${{ inputs.version }}" ]]; then
+            export REF2="${{ github.ref }}"
+          else
+            export REF2="${{ inputs.version }}"
+          fi
+          echo "REL_TAG=$REF2" >> $GITHUB_ENV
+          export VERSION=$(echo "$REF2" | sed -E 's/^v([0-9]+\.[0-9]+\.[0-9]+).*$/\1/')
+
+          ./scripts/release/build-changelogs.sh
+
+          echo "Checking the folder state"
+          pwd
+          ls -la scripts/release
+
+      - name: Archive artifact context.json
+        uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
+        with:
+          name: release-notes-context
+          path: |
+            scripts/release/context.json
+            **/*-srtool-digest.json
+
+      - name: Create draft release
+        id: create-release
+        uses: actions/create-release@0cb9c9b65d5d1901c1f53e5e66eaf4afd303e70e # v1.1.4
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          tag_name: ${{ env.REL_TAG }}
+          release_name: Polkadot ${{ env.REL_TAG }}
+          body_path: ${{ github.workspace}}/scripts/release/RELEASE_DRAFT.md
+          draft: true
+
+  publish-runtimes:
+    needs: [ build-runtimes, publish-release-draft ]
+    continue-on-error: true
+    runs-on: ubuntu-latest
+    strategy:
+      matrix: ${{ fromJSON(needs.build-runtimes.outputs.published_runtimes) }}
+
+    steps:
+      - name: Checkout sources
+        uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0
+
+      - name: Download artifacts
+        uses: actions/download-artifact@c850b930e6ba138125429b7e5c93fc707a7f8427 # v4.1.4
+
+      - name: Get runtime info
+        env:
+          JSON: release-notes-context/${{ matrix.chain }}-runtime/${{ matrix.chain }}-srtool-digest.json
+        run: |
+          >>$GITHUB_ENV echo ASSET=$(find ${{ matrix.chain }}-runtime -name '*.compact.compressed.wasm')
+          >>$GITHUB_ENV echo SPEC=$(<${JSON} jq -r .runtimes.compact.subwasm.core_version.specVersion)
+
+      - name: Upload compressed ${{ matrix.chain }} v${{ env.SPEC }} wasm
+        if: ${{ matrix.chain  != 'rococo-parachain' }}
+        uses: actions/upload-release-asset@e8f9f06c4b078e705bd2ea027f0926603fc9b4d5 #v1.0.2
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          upload_url: ${{ needs.publish-release-draft.outputs.asset_upload_url }}
+          asset_path: ${{ env.ASSET }}
+          asset_name: ${{ matrix.chain }}_runtime-v${{ env.SPEC }}.compact.compressed.wasm
+          asset_content_type: application/wasm
+
+  post_to_matrix:
+    runs-on: ubuntu-latest
+    needs: publish-release-draft
+    strategy:
+      matrix:
+        channel:
+          - name: "Team: RelEng Internal"
+            room: '!GvAyzgCDgaVrvibaAF:parity.io'
+
+    steps:
+      - name: Send Matrix message to ${{ matrix.channel.name }}
+        uses: s3krit/matrix-message-action@70ad3fb812ee0e45ff8999d6af11cafad11a6ecf # v0.0.3
+        with:
+          room_id: ${{ matrix.channel.room }}
+          access_token: ${{ secrets.RELEASENOTES_MATRIX_V2_ACCESS_TOKEN }}
+          server: m.parity.io
+          message: |
+            **New version of polkadot tagged**: ${{ github.ref }}<br/>
+            Draft release created: ${{ needs.publish-release-draft.outputs.release_url }}

.github/workflows/srtool.yml

@@ -12,6 +12,13 @@ on:
       - release-v[0-9]+.[0-9]+.[0-9]+*
       - release-cumulus-v[0-9]+*
       - release-polkadot-v[0-9]+*
+  workflow_call:
+    inputs:
+      excluded_runtimes:
+        type: string
+    outputs:
+      published_runtimes:
+        value: ${{ jobs.find-runtimes.outputs.runtime }}
 
   schedule:
     - cron: "00 02 * * 1" # 2AM weekly on monday
@@ -39,7 +46,7 @@ jobs:
 
         - name: Scan runtimes
           env:
-            EXCLUDED_RUNTIMES: "substrate-test"
+            EXCLUDED_RUNTIMES: ${{ inputs.excluded_runtimes }}:"substrate-test"
           run: |
             . ./.github/scripts/common/lib.sh
 
@@ -85,16 +92,6 @@ jobs:
           echo "Compact Runtime: ${{ steps.srtool_build.outputs.wasm }}"
           echo "Compressed Runtime: ${{ steps.srtool_build.outputs.wasm_compressed }}"
 
-      # it takes a while to build the runtime, so let's save the artifact as soon as we have it
-      - name: Archive Artifacts for ${{ matrix.chain }}
-        uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2
-        with:
-          name: ${{ matrix.chain }}-runtime
-          path: |
-            ${{ steps.srtool_build.outputs.wasm }}
-            ${{ steps.srtool_build.outputs.wasm_compressed }}
-            ${{ matrix.chain }}-srtool-digest.json
-
       # We now get extra information thanks to subwasm
       - name: Install subwasm
         run: |
@@ -125,11 +122,14 @@ jobs:
             tee ${{ matrix.chain }}-diff.txt
 
       - name: Archive Subwasm results
-        uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2
+        uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
         with:
           name: ${{ matrix.chain }}-runtime
           path: |
             ${{ matrix.chain }}-info.json
             ${{ matrix.chain }}-compressed-info.json
             ${{ matrix.chain }}-metadata.json
             ${{ matrix.chain }}-diff.txt
+            ${{ steps.srtool_build.outputs.wasm }}
+            ${{ steps.srtool_build.outputs.wasm_compressed }}
+            ${{ matrix.chain }}-srtool-digest.json

.gitlab-ci.yml

@@ -125,10 +125,10 @@ default:
     - cp $FL_FORKLIFT_CONFIG ~/.forklift/config.toml
     - shopt -s expand_aliases
     - export PATH=$PATH:$(pwd)
-    - |
+    - >
       if [ "$FORKLIFT_BYPASS" != "true" ]; then
-      echo "FORKLIFT_BYPASS not set, creating alias cargo='forklift cargo'"
-      alias cargo="forklift cargo"
+      echo "FORKLIFT_BYPASS not set, creating alias cargo='forklift cargo'"; 
+      alias cargo="forklift cargo";
       fi
     #
     - echo "FL_FORKLIFT_VERSION ${FL_FORKLIFT_VERSION}"

.gitlab/pipeline/build.yml

@@ -91,40 +91,39 @@ build-rustdoc:
     - .run-immediately
   variables:
     SKIP_WASM_BUILD: 1
-    RUSTDOCFLAGS: "--default-theme=ayu --html-in-header ./docs/sdk/headers/header.html --extend-css ./docs/sdk/headers/theme.css"
+    RUSTDOCFLAGS: "-Dwarnings --default-theme=ayu --html-in-header ./docs/sdk/assets/header.html --extend-css ./docs/sdk/assets/theme.css --html-after-content ./docs/sdk/assets/after-content.html"
   artifacts:
     name: "${CI_JOB_NAME}_${CI_COMMIT_REF_NAME}-doc"
     when: on_success
     expire_in: 1 days
     paths:
       - ./crate-docs/
   script:
-    # FIXME: it fails with `RUSTDOCFLAGS="-Dwarnings"` and `--all-features`
-    - time cargo doc --features try-runtime,experimental --workspace --no-deps
+    - time cargo doc --all-features --workspace --no-deps
     - rm -f ./target/doc/.lock
     - mv ./target/doc ./crate-docs
     # Inject Simple Analytics (https://www.simpleanalytics.com/) privacy preserving tracker into
     # all .html files
-    - |
+    - >
       inject_simple_analytics() {
-        local path="$1"
-        local script_content="<script async defer src=\"https://apisa.parity.io/latest.js\"></script><noscript><img src=\"https://apisa.parity.io/latest.js\" alt=\"\" referrerpolicy=\"no-referrer-when-downgrade\" /></noscript>"
+        local path="$1";
+        local script_content="<script async defer src=\"https://apisa.parity.io/latest.js\"></script><noscript><img src=\"https://apisa.parity.io/latest.js\" alt=\"\" referrerpolicy=\"no-referrer-when-downgrade\" /></noscript>";
 
         # Function that inject script into the head of an html file using sed.
         process_file() {
-            local file="$1"
-            echo "Adding Simple Analytics script to $file"
-            sed -i "s|</head>|$script_content</head>|" "$file"
-        }
-        export -f process_file
-        # xargs runs process_file in seperate shells without access to outer variables.
-        # to make script_content available inside process_file, export it as an env var here.
-        export script_content
+            local file="$1";
+            echo "Adding Simple Analytics script to $file";
+            sed -i "s|</head>|$script_content</head>|" "$file";
+        };
+        export -f process_file;
+        # xargs runs process_file in separate shells without access to outer variables.
+        # make script_content available inside process_file, export it as an env var here.
+        export script_content;
 
         # Modify .html files in parallel using xargs, otherwise it can take a long time.
-        find "$path" -name '*.html' | xargs -I {} -P "$(nproc)" bash -c 'process_file "$@"' _ {}
-      }
-      inject_simple_analytics "./crate-docs"
+        find "$path" -name '*.html' | xargs -I {} -P "$(nproc)" bash -c 'process_file "$@"' _ {};
+      };
+      inject_simple_analytics "./crate-docs";
     - echo "<meta http-equiv=refresh content=0;url=polkadot_sdk_docs/index.html>" > ./crate-docs/index.html
 
 build-implementers-guide:

.gitlab/pipeline/check.yml

@@ -7,8 +7,8 @@ cargo-clippy:
   variables:
     RUSTFLAGS: "-D warnings"
   script:
-    - SKIP_WASM_BUILD=1 cargo clippy --all-targets --locked --workspace
-    - SKIP_WASM_BUILD=1 cargo clippy --all-targets --all-features --locked --workspace
+    - SKIP_WASM_BUILD=1 cargo clippy --all-targets --locked --workspace --quiet
+    - SKIP_WASM_BUILD=1 cargo clippy --all-targets --all-features --locked --workspace --quiet
 
 check-try-runtime:
   stage: check
@@ -104,23 +104,20 @@ check-toml-format:
     - .docker-env
     - .test-pr-refs
   script:
-    - |
-      export RUST_LOG=remote-ext=debug,runtime=debug
-
-      echo "---------- Downloading try-runtime CLI ----------"
-      curl -sL https://github.com/paritytech/try-runtime-cli/releases/download/v0.5.4/try-runtime-x86_64-unknown-linux-musl -o try-runtime
-      chmod +x ./try-runtime
-      echo "Using try-runtime-cli version:"
-      ./try-runtime --version
-
-      echo "---------- Building ${PACKAGE} runtime ----------"
-      time cargo build --release --locked -p "$PACKAGE" --features try-runtime
-
-      echo "---------- Executing on-runtime-upgrade for ${NETWORK} ----------"
+    - export RUST_LOG=remote-ext=debug,runtime=debug
+    - echo "---------- Downloading try-runtime CLI ----------"
+    - curl -sL https://github.com/paritytech/try-runtime-cli/releases/download/v0.5.4/try-runtime-x86_64-unknown-linux-musl -o try-runtime
+    - chmod +x ./try-runtime
+    - echo "Using try-runtime-cli version:"
+    - ./try-runtime --version
+    - echo "---------- Building ${PACKAGE} runtime ----------"
+    - time cargo build --release --locked -p "$PACKAGE" --features try-runtime
+    - echo "---------- Executing on-runtime-upgrade for ${NETWORK} ----------"
+    - >
       time ./try-runtime ${COMMAND_EXTRA_ARGS} \
-          --runtime ./target/release/wbuild/"$PACKAGE"/"$WASM" \
-          on-runtime-upgrade --disable-spec-version-check --checks=all ${SUBCOMMAND_EXTRA_ARGS} live --uri ${URI}
-      sleep 5
+        --runtime ./target/release/wbuild/"$PACKAGE"/"$WASM" \
+        on-runtime-upgrade --disable-spec-version-check --checks=all ${SUBCOMMAND_EXTRA_ARGS} live --uri ${URI}
+    - sleep 5
 
 # Check runtime migrations for Parity managed relay chains
 check-runtime-migration-westend:

.gitlab/pipeline/publish.yml

@@ -113,17 +113,17 @@ trigger_workflow:
       artifacts: true
   script:
     - echo "Triggering workflow"
-    - |
+    - >
       for benchmark in $(ls charts/*.json); do
-        export bencmark_name=$(basename $benchmark)
-        echo "Benchmark: $bencmark_name"
-        export benchmark_dir=$(echo $bencmark_name | sed 's/\.json//')
+        export benchmark_name=$(basename $benchmark);
+        echo "Benchmark: $benchmark_name";
+        export benchmark_dir=$(echo $benchmark_name | sed 's/\.json//');
         curl -q -X POST \
               -H "Accept: application/vnd.github.v3+json" \
               -H "Authorization: token $GITHUB_TOKEN" \
-               https://api.github.com/repos/paritytech/${CI_PROJECT_NAME}/actions/workflows/subsystem-benchmarks.yml/dispatches \
-                -d '{"ref":"refs/heads/master","inputs":{"benchmark-data-dir-path":"'$benchmark_dir'","output-file-path":"'$bencmark_name'"}}'
-        sleep 300
+              https://api.github.com/repos/paritytech/${CI_PROJECT_NAME}/actions/workflows/subsystem-benchmarks.yml/dispatches \
+              -d "{\"ref\":\"refs/heads/master\",\"inputs\":{\"benchmark-data-dir-path\":\"$benchmark_dir\",\"output-file-path\":\"$benchmark_name\"}}";
+        sleep 300;
       done
   allow_failure: true