github/workflows: add ARM macos build binaries job (#6427)

# Description This PR adds the required changes to release `polkadot`, `polkadot-parachain` and `polkadot-omni-node` binaries built on Apple Sillicon macos. ## Integration This addresses requests from the community for such binaries: #802, and they should be part of the Github release page. ## Review Notes Test on paritytech-stg solely focused on macos binaries: https://github.com/paritytech-stg/polkadot-sdk/actions/runs/11824692766/job/32946793308, except the steps related to `pgpkms` (which need AWS credentials, missing from paritytech-stg). The binary names don't have a `darwin-arm` identifier, and conflict with the existing x86_64-linux binaries. I haven't tested building everything on `paritytech-stg` because the x86_64-linux builds run on `unbutu-latest-m` which isn't enabled on `pairtytech-stg` (and I haven't asked CI team to enable one), so testing how to go around naming conflicts should be covered next. ### TODO - [x] Test the workflow start to end (especially the last bits related to uploading the binaries on S3 and ensuring the previous binaries and the new ones coexist harmoniously on S3/action artifacts storage without naming conflicts) @EgorPopelyaev - [x] Publish the arm binaries on the Github release page - to clarify what's needed @iulianbarbu . Current practice is to manually publish the binaries built via `release-build-rc.yml` workflow, taken from S3. Would be great to have the binaries there in the first place before working on automating this, but I would also do it in a follow up PR. ### Follow ups - [ ] unify the binaries building under `release-30_publish_release_draft.yml` maybe? - [ ] automate binary artifacts upload to S3 in `release-30_publish_release_draft.yml` --------- Signed-off-by: Iulian Barbu <[email protected]> Co-authored-by: EgorPopelyaev <[email protected]>
paritytech · Nov 21, 2024 · 1f7765b · 1f7765b
1 parent 56d97c3
commit 1f7765b
Show file tree

Hide file tree

Showing 7 changed files with 376 additions and 27 deletions.
diff --git a/.github/scripts/release/build-linux-release.sh b/.github/scripts/release/build-linux-release.sh
@@ -3,6 +3,8 @@
 # This is used to build our binaries:
 # - polkadot
 # - polkadot-parachain
+# - polkadot-omni-node 
+#
 # set -e
 
 BIN=$1
@@ -21,7 +23,7 @@ time cargo build --profile $PROFILE --locked --verbose --bin $BIN --package $PAC
 echo "Artifact target: $ARTIFACTS"
 
 cp ./target/$PROFILE/$BIN "$ARTIFACTS"
-pushd "$ARTIFACTS" > /dev/nul
+pushd "$ARTIFACTS" > /dev/null
 sha256sum "$BIN" | tee "$BIN.sha256"
 
 EXTRATAG="$($ARTIFACTS/$BIN --version |

diff --git a/.github/scripts/release/build-macos-release.sh b/.github/scripts/release/build-macos-release.sh
@@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+
+# This is used to build our binaries:
+# - polkadot
+# - polkadot-parachain
+# - polkadot-omni-node
+# set -e
+
+BIN=$1
+PACKAGE=${2:-$BIN}
+
+PROFILE=${PROFILE:-production}
+# parity-macos runner needs a path where it can
+# write, so make it relative to github workspace.
+ARTIFACTS=$GITHUB_WORKSPACE/artifacts/$BIN
+VERSION=$(git tag -l --contains HEAD | grep -E "^v.*")
+
+echo "Artifacts will be copied into $ARTIFACTS"
+mkdir -p "$ARTIFACTS"
+
+git log --pretty=oneline -n 1
+time cargo build --profile $PROFILE --locked --verbose --bin $BIN --package $PACKAGE
+
+echo "Artifact target: $ARTIFACTS"
+
+cp ./target/$PROFILE/$BIN "$ARTIFACTS"
+pushd "$ARTIFACTS" > /dev/null
+sha256sum "$BIN" | tee "$BIN.sha256"
+
+EXTRATAG="$($ARTIFACTS/$BIN --version |
+    sed -n -r 's/^'$BIN' ([0-9.]+.*-[0-9a-f]{7,13})-.*$/\1/p')"
+
+EXTRATAG="${VERSION}-${EXTRATAG}-$(cut -c 1-8 $ARTIFACTS/$BIN.sha256)"
+
+echo "$BIN version = ${VERSION} (EXTRATAG = ${EXTRATAG})"
+echo -n ${VERSION} > "$ARTIFACTS/VERSION"
+echo -n ${EXTRATAG} > "$ARTIFACTS/EXTRATAG"
diff --git a/.github/scripts/release/release_lib.sh b/.github/scripts/release/release_lib.sh
@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-# Set the new version by replacing the value of the constant given as patetrn
+# Set the new version by replacing the value of the constant given as pattern
 # in the file.
 #
 # input: pattern, version, file
@@ -119,21 +119,23 @@ set_polkadot_parachain_binary_version() {
 
 
 upload_s3_release() {
-  alias aws='podman run --rm -it docker.io/paritytech/awscli -e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e AWS_BUCKET aws'
-
-  product=$1
-  version=$2
-
-  echo "Working on product: $product "
-  echo "Working on version: $version "
-
-  echo "Current content, should be empty on new uploads:"
-  aws s3 ls "s3://releases.parity.io/polkadot/${version}/" --recursive --human-readable --summarize || true
-  echo "Content to be uploaded:"
-  artifacts="artifacts/$product/"
-  ls "$artifacts"
-  aws s3 sync --acl public-read "$artifacts" "s3://releases.parity.io/polkadot/${version}/"
-  echo "Uploaded files:"
-  aws s3 ls "s3://releases.parity.io/polkadot/${version}/" --recursive --human-readable --summarize
-  echo "✅ The release should be at https://releases.parity.io/polkadot/${version}"
+    alias aws='podman run --rm -it docker.io/paritytech/awscli -e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e AWS_BUCKET aws'
+
+    product=$1
+    version=$2
+    target=$3
+
+    echo "Working on product:  $product "
+    echo "Working on version:  $version "
+    echo "Working on platform: $target "
+
+    echo "Current content, should be empty on new uploads:"
+    aws s3 ls "s3://releases.parity.io/${product}/${version}/${target}" --recursive --human-readable --summarize || true
+    echo "Content to be uploaded:"
+    artifacts="artifacts/$product/"
+    ls "$artifacts"
+    aws s3 sync --acl public-read "$artifacts" "s3://releases.parity.io/${product}/${version}/${target}"
+    echo "Uploaded files:"
+    aws s3 ls "s3://releases.parity.io/${product}/${version}/${target}" --recursive --human-readable --summarize
+    echo "✅ The release should be at https://releases.parity.io/${product}/${version}/${target}"
 }
diff --git a/.github/workflows/release-30_publish_release_draft.yml b/.github/workflows/release-30_publish_release_draft.yml
@@ -34,7 +34,7 @@ jobs:
     strategy:
       matrix:
         # Tuples of [package, binary-name]
-        binary: [ [frame-omni-bencher, frame-omni-bencher], [staging-chain-spec-builder, chain-spec-builder], [polkadot-omni-node, polkadot-omni-node] ]
+        binary: [ [frame-omni-bencher, frame-omni-bencher], [staging-chain-spec-builder, chain-spec-builder] ]
     steps:
       - name: Checkout sources
         uses: actions/checkout@6d193bf28034eafb982f37bd894289fe649468fc # v4.0.0
@@ -161,7 +161,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        binary: [frame-omni-bencher, chain-spec-builder, polkadot-omni-node]
+        binary: [frame-omni-bencher, chain-spec-builder]
 
     steps:
       - name: Download artifacts

diff --git a/.github/workflows/release-build-rc.yml b/.github/workflows/release-build-rc.yml
@@ -10,6 +10,7 @@ on:
         options:
           - polkadot
           - polkadot-parachain
+          - polkadot-omni-node
           - all
 
       release_tag:
@@ -47,6 +48,7 @@ jobs:
       binary: '["polkadot", "polkadot-prepare-worker", "polkadot-execute-worker"]'
       package: polkadot
       release_tag: ${{ needs.validate-inputs.outputs.release_tag }}
+      target: x86_64-unknown-linux-gnu
     secrets:
       PGP_KMS_KEY:  ${{ secrets.PGP_KMS_KEY }}
       PGP_KMS_HASH:  ${{ secrets.PGP_KMS_HASH }}
@@ -68,6 +70,95 @@ jobs:
       binary: '["polkadot-parachain"]'
       package: "polkadot-parachain-bin"
       release_tag: ${{ needs.validate-inputs.outputs.release_tag }}
+      target: x86_64-unknown-linux-gnu
+    secrets:
+      PGP_KMS_KEY:  ${{ secrets.PGP_KMS_KEY }}
+      PGP_KMS_HASH:  ${{ secrets.PGP_KMS_HASH }}
+      AWS_ACCESS_KEY_ID:  ${{ secrets.AWS_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+      AWS_DEFAULT_REGION:  ${{ secrets.AWS_DEFAULT_REGION }}
+      AWS_RELEASE_ACCESS_KEY_ID: ${{ secrets.AWS_RELEASE_ACCESS_KEY_ID }}
+      AWS_RELEASE_SECRET_ACCESS_KEY: ${{ secrets.AWS_RELEASE_SECRET_ACCESS_KEY }}
+    permissions:
+      id-token: write
+      attestations: write
+      contents: read
+
+  build-polkadot-omni-node-binary:
+    needs: [validate-inputs]
+    if: ${{ inputs.binary == 'polkadot-omni-node' || inputs.binary == 'all' }}
+    uses: "./.github/workflows/release-reusable-rc-buid.yml"
+    with:
+      binary: '["polkadot-omni-node"]'
+      package: "polkadot-omni-node"
+      release_tag: ${{ needs.validate-inputs.outputs.release_tag }}
+      target: x86_64-unknown-linux-gnu
+    secrets:
+      PGP_KMS_KEY:  ${{ secrets.PGP_KMS_KEY }}
+      PGP_KMS_HASH:  ${{ secrets.PGP_KMS_HASH }}
+      AWS_ACCESS_KEY_ID:  ${{ secrets.AWS_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+      AWS_DEFAULT_REGION:  ${{ secrets.AWS_DEFAULT_REGION }}
+      AWS_RELEASE_ACCESS_KEY_ID: ${{ secrets.AWS_RELEASE_ACCESS_KEY_ID }}
+      AWS_RELEASE_SECRET_ACCESS_KEY: ${{ secrets.AWS_RELEASE_SECRET_ACCESS_KEY }}
+    permissions:
+      id-token: write
+      attestations: write
+      contents: read
+
+  build-polkadot-macos-binary:
+    needs: [validate-inputs]
+    if: ${{ inputs.binary == 'polkadot' || inputs.binary == 'all' }}
+    uses: "./.github/workflows/release-reusable-rc-buid.yml"
+    with:
+      binary: '["polkadot", "polkadot-prepare-worker", "polkadot-execute-worker"]'
+      package: polkadot
+      release_tag: ${{ needs.validate-inputs.outputs.release_tag }}
+      target: aarch64-apple-darwin
+    secrets:
+      PGP_KMS_KEY:  ${{ secrets.PGP_KMS_KEY }}
+      PGP_KMS_HASH:  ${{ secrets.PGP_KMS_HASH }}
+      AWS_ACCESS_KEY_ID:  ${{ secrets.AWS_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+      AWS_DEFAULT_REGION:  ${{ secrets.AWS_DEFAULT_REGION }}
+      AWS_RELEASE_ACCESS_KEY_ID: ${{ secrets.AWS_RELEASE_ACCESS_KEY_ID }}
+      AWS_RELEASE_SECRET_ACCESS_KEY: ${{ secrets.AWS_RELEASE_SECRET_ACCESS_KEY }}
+    permissions:
+      id-token: write
+      attestations: write
+      contents: read
+
+  build-polkadot-parachain-macos-binary:
+    needs: [validate-inputs]
+    if: ${{ inputs.binary == 'polkadot-parachain' || inputs.binary == 'all' }}
+    uses: "./.github/workflows/release-reusable-rc-buid.yml"
+    with:
+      binary: '["polkadot-parachain"]'
+      package: "polkadot-parachain-bin"
+      release_tag: ${{ needs.validate-inputs.outputs.release_tag }}
+      target: aarch64-apple-darwin
+    secrets:
+      PGP_KMS_KEY:  ${{ secrets.PGP_KMS_KEY }}
+      PGP_KMS_HASH:  ${{ secrets.PGP_KMS_HASH }}
+      AWS_ACCESS_KEY_ID:  ${{ secrets.AWS_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+      AWS_DEFAULT_REGION:  ${{ secrets.AWS_DEFAULT_REGION }}
+      AWS_RELEASE_ACCESS_KEY_ID: ${{ secrets.AWS_RELEASE_ACCESS_KEY_ID }}
+      AWS_RELEASE_SECRET_ACCESS_KEY: ${{ secrets.AWS_RELEASE_SECRET_ACCESS_KEY }}
+    permissions:
+      id-token: write
+      attestations: write
+      contents: read
+
+  build-polkadot-omni-node-macos-binary:
+    needs: [validate-inputs]
+    if: ${{ inputs.binary == 'polkadot-omni-node' || inputs.binary == 'all' }}
+    uses: "./.github/workflows/release-reusable-rc-buid.yml"
+    with:
+      binary: '["polkadot-omni-node"]'
+      package: "polkadot-omni-node"
+      release_tag: ${{ needs.validate-inputs.outputs.release_tag }}
+      target: aarch64-apple-darwin
     secrets:
       PGP_KMS_KEY:  ${{ secrets.PGP_KMS_KEY }}
       PGP_KMS_HASH:  ${{ secrets.PGP_KMS_HASH }}