Publish RC container images

.github/workflows/check-licenses.yml

@@ -8,7 +8,7 @@ jobs:
     runs-on: ubuntu-22.04
     steps:
       - name: Checkout sources
-        uses: actions/checkout@v3.3.0
+        uses: actions/checkout@v3
       - uses: actions/[email protected]
         with:
           node-version: '18.x'

.github/workflows/release-40_publish-rc-image.yml

@@ -0,0 +1,89 @@
+name: Release - Publish RC Container image
+# see https://github.com/paritytech/release-engineering/issues/97#issuecomment-1651372277
+
+on:
+  workflow_dispatch:
+    inputs:
+      release_id:
+        description: 'Release ID'
+        required: true
+        type: string
+        # TODO: to be removed, this eases testing
+        default: "62185729"
+      registry:
+        description: "Container registry"
+        required: true
+        type: string
+        default: docker.io/parity
+
+env:
+  RELEASE_ID: ${{ inputs.release_id }}
+  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+  # paritytech
+  OWNER: ${{ github.repository_owner }}
+  # polkadot
+  REPO: ${{ github.event.repository.name }}
+
+jobs:
+  fetch-artifacts:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout sources
+        uses: actions/checkout@v3
+
+      - name: Fetch all artifacts
+        run: |
+          . ./scripts/ci/common/lib.sh
+          fetch_release_artifacts
+
+      - name: Cache the artifacts
+        uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 # v3.3.1
+        with:
+          key: artifacts-${{ github.sha }}
+          path: |
+            artifacts/**/*
+
+  build-container:
+    runs-on: ubuntu-latest
+    needs: fetch-artifacts
+
+    strategy:
+      matrix:
+        binary: ["polkadot", "staking-miner"]
+
+    steps:
+      - name: Checkout sources
+        uses: actions/checkout@v3
+
+      - name: Get artifacts from cache
+        uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 # v3.3.1
+        with:
+          key: artifacts-${{ github.sha }}
+          path: |
+            artifacts/**/*
+
+      - name: Check sha256 ${{ matrix.binary }}
+        working-directory: ./artifacts
+        run: |
+          . ../scripts/ci/common/lib.sh
+
+          echo "Checking binary ${{ matrix.binary }}"
+          check_sha256 ${{ matrix.binary }} && echo "OK" || echo "ERR"
+
+      # - name: Check GPG ${{ matrix.binary }}
+      #   working-directory: ./artifacts
+      #   run: |
+      #     . ../scripts/ci/common/lib.sh
+      #     # TODO import relevant GPG keys
+      #     check_gpg ${{ matrix.binary }}
+
+      - name: Build Injected Container image for ${{ matrix.binary }}
+        run: |
+          echo "Building container for ${{ matrix.binary }}"
+
+      - name: Push Container image
+        env:
+          IMAGE: ${{ inputs.registry }}/${{ matrix.binary }}
+        run: |
+          echo "TODO Pushing image to $IMAGE"

.github/workflows/release-50_publish-docker-release.yml

@@ -30,7 +30,7 @@ jobs:
         uses: docker/build-push-action@v4
         with:
           push: true
-          file: scripts/ci/dockerfiles/polkadot_injected_release.Dockerfile
+          file: scripts/ci/dockerfiles/polkadot/polkadot_injected_debian.Dockerfile
           tags: |
             parity/polkadot:latest
             parity/polkadot:${{ github.event.release.tag_name }}

.github/workflows/release-51_publish-docker-manual.yml

@@ -37,7 +37,7 @@ jobs:
         uses: docker/build-push-action@v4
         with:
           push: true
-          file: scripts/ci/dockerfiles/polkadot_injected_release.Dockerfile
+          file: scripts/ci/dockerfiles/polkadot/polkadot_injected_debian.Dockerfile
           tags: |
             parity/polkadot:latest
             parity/polkadot:${{ github.event.inputs.version }}

.gitignore

@@ -10,3 +10,6 @@ polkadot.*
 !polkadot.service
 .DS_Store
 .env
+
+artifacts
+release.json

scripts/ci/common/lib.sh

@@ -193,3 +193,53 @@ check_bootnode(){
     echo "    Bootnode appears unreachable"
     return 1
 }
+
+# Assumes the ENV are set:
+# - RELEASE_ID
+# - GITHUB_TOKEN
+# - OWNER
+# - REPO
+fetch_release_artifacts() {
+  echo "Release ID : $RELEASE_ID"
+  echo "Owner      : $OWNER"
+  echo "Repo       : $REPO"
+
+  curl -L -s \
+    -H "Accept: application/vnd.github+json" \
+    -H "Authorization: Bearer ${GITHUB_TOKEN}" \
+    -H "X-GitHub-Api-Version: 2022-11-28" \
+    https://api.github.com/repos/$OWNER/$REPO/releases/$RELEASE_ID > release.json
+
+  # Get Asset ids
+  ids=($(cat release.json | jq -r '.assets[].id'))
+  count=$(cat release.json | jq '.assets|length')
+
+  # Fetch artifacts
+  mkdir -p artifacts
+  pushd artifacts > /dev/null
+
+  iter=1
+  for id in "${ids[@]}"
+  do
+      echo " - $iter/$count: downloading asset id: $id..."
+      curl -s -OJ -L -H "Accept: application/octet-stream" \
+          -H "Authorization: Token ${GITHUB_TOKEN}" \
+          "https://api.github.com/repos/${OWNER}/${REPO}/releases/assets/$id"
+      iter=$((iter + 1))
+  done
+
+  ls -al --color
+  popd > /dev/null
+}
+
+# Check the checksum for a given binary
+function check_sha256() {
+    echo "Checking SHA256 for $1"
+    shasum -qc $1.sha256
+}
+
+# Check the GPG signature for a given binary
+function check_gpg() {
+    echo "Checking GPG Signature for $1"
+    gpg --verify -q $1.asc $1
+}

scripts/ci/dockerfiles/adder-collator/build-injected.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+
+# Sample call:
+# $0 /path/to/folder_with_binary
+# This script replace the former dedicated Dockerfile
+# and shows how to use the generic binary_injected.dockerfile
+
+PROJECT_ROOT=`git rev-parse --show-toplevel`
+
+export BINARY=adder-collator,undying-collator
+export BIN_FOLDER=$1
+
+$PROJECT_ROOT/scripts/ci/dockerfiles/build-injected.sh

scripts/ci/dockerfiles/adder-collator/test-build.sh

@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+
+TMP=$(mktemp -d)
+ENGINE=${ENGINE:-podman}
+
+# TODO: Switch to /bin/bash when the image is built from parity/base-bin
+
+# Fetch some binaries
+$ENGINE run --user root --rm -it \
+  --pull always \
+  -v "$TMP:/export" \
+  --entrypoint /usr/bin/bash \
+  paritypr/colander:master -c \
+  'cp "$(which adder-collator)" /export'
+
+$ENGINE run --user root --rm -it \
+  --pull always \
+  -v "$TMP:/export" \
+  --entrypoint /usr/bin/bash \
+  paritypr/colander:master -c \
+  'cp "$(which undying-collator)" /export'
+
+./build-injected.sh $TMP

scripts/ci/dockerfiles/binary_injected.Dockerfile

@@ -0,0 +1,71 @@
+FROM docker.io/parity/base-bin
+
+# This file allows building a Generic container image
+# based on one or multiple pre-built Linux binaries.
+# Some defaults are set to polkadot but all can be overriden.
+
+SHELL ["/bin/bash", "-c"]
+
+# metadata
+ARG VCS_REF
+ARG BUILD_DATE
+ARG IMAGE_NAME
+
+# That can be a single one or a comma separated list
+ARG BINARY=polkadot
+
+ARG TAGS
+ARG BIN_FOLDER=.
+ARG DOC_URL=https://github.com/paritytech/polkadot
+ARG DESCRIPTION="Polkadot: a platform for web3"
+ARG AUTHORS="[email protected]"
+ARG VENDOR="Parity Technologies"
+ARG VOLUMES
+ARG PORTS
+
+LABEL io.parity.image.authors=${AUTHORS} \
+	io.parity.image.vendor="${VENDOR}" \
+	io.parity.image.revision="${VCS_REF}" \
+	io.parity.image.title="${IMAGE_NAME}" \
+	io.parity.image.created="${BUILD_DATE}" \
+	io.parity.image.documentation="${DOC_URL}" \
+	io.parity.image.description="${DESCRIPTION}" \
+	io.parity.image.source="https://github.com/paritytech/polkadot/blob/${VCS_REF}/scripts/ci/dockerfiles/binary_injected.Dockerfile"
+
+USER root
+WORKDIR /app
+
+# add polkadot binary to docker image
+# sample for polkadot: COPY ./polkadot ./polkadot-*-worker /usr/local/bin/
+COPY entrypoint.sh .
+COPY "bin/*" "/usr/local/bin/"
+RUN chmod -R a+rx "/usr/local/bin"
+
+USER parity
+ENV BINARY=${BINARY}
+
+# check that all the executables works in this container
+# TODO: There may be several
+# RUN bash -c IFS=',' read -r -a BINARIES <<< "$BINARY" \
+# 	for bin in "${BINARIES[@]}"; do \
+# 		/usr/local/bin/$bin --version \
+# 	done
+
+ENV VOLUMES=$VOLUMES
+ENV TAGS=$TAGS
+ENV PORTS=$PORTS
+
+# TODO: change that, we may have multiple BINARIES
+# TODO: we need a VAR for VOLUMES
+# If defined, VOLUME cannot be empty
+#VOLUME $VOLUMES
+
+# TODO: we need a VAR for PORTS
+# If defined, EXPOSE cannot be empty
+# EXPOSE $PORTS
+
+# ENTRYPOINT
+ENTRYPOINT ["/app/entrypoint.sh"]
+
+# We call the help by default
+CMD ["--help"]

scripts/ci/dockerfiles/build-injected.sh

@@ -0,0 +1,70 @@
+#!/usr/bin/env bash
+set -e
+
+# This script allows building a Container Image from a Linux
+# binary that is injected into a base-image.
+
+ENGINE=${ENGINE:-podman}
+CONTEXT=$(mktemp -d)
+
+# The following line ensure we know the project root
+PROJECT_ROOT=`git rev-parse --show-toplevel`
+DOCKERFILE=${DOCKERFILE:-$PROJECT_ROOT/scripts/ci/dockerfiles/binary_injected.Dockerfile}
+VERSION_TOML=$(grep "^version " $PROJECT_ROOT/Cargo.toml | grep -oE "([0-9\.]+-?[0-9]+)")
+
+#n The following VAR have default that can be overriden
+OWNER=${OWNER:-parity}
+
+# We may get 1..n binaries, comma separated
+BINARY=${BINARY:-polkadot}
+IFS=',' read -r -a BINARIES <<< "$BINARY"
+
+VERSION=${VERSION:-$VERSION_TOML}
+BIN_FOLDER=${BIN_FOLDER:-.}
+
+IMAGE=${IMAGE:-${OWNER}/${BINARIES[0]}}
+DESCRIPTION_DEFAULT="Injected Container image built for ${BINARY[*]}"
+DESCRIPTION=${DESCRIPTION:-$DESCRIPTION_DEFAULT}
+
+# Build the image
+echo "Using engine: $ENGINE"
+echo "Using Dockerfile: $DOCKERFILE"
+echo "Using context: $CONTEXT"
+echo "Building ${IMAGE}:latest container image for ${BINARY[*]} v${VERSION} from ${BIN_FOLDER} hang on!"
+echo "BIN_FOLDER=$BIN_FOLDER"
+echo "CONTEXT=$CONTEXT"
+
+# We need all binaries and resources available in the Container build "CONTEXT"
+mkdir -p $CONTEXT/bin
+for bin in "${BINARIES[@]}"
+do
+  echo "Copying $BIN_FOLDER/$bin to context: $CONTEXT/bin"
+  cp "$BIN_FOLDER/$bin" "$CONTEXT/bin"
+done
+
+cp "$PROJECT_ROOT/scripts/ci/dockerfiles/entrypoint.sh" "$CONTEXT"
+
+echo "Building image: $IMAGE"
+
+# time \
+$ENGINE build \
+    --format docker \
+    --build-arg BUILD_DATE=$(date -u '+%Y-%m-%dT%H:%M:%SZ') \
+    --build-arg IMAGE_NAME="${IMAGE}" \
+    --build-arg BINARY="${BINARY}" \
+    --build-arg BIN_FOLDER="${BIN_FOLDER}" \
+    --build-arg DESCRIPTION="${DESCRIPTION}" \
+    -f $DOCKERFILE \
+    -t ${IMAGE}:latest \
+    -t ${IMAGE}:v${VERSION} \
+    $CONTEXT
+
+# Show the list of available images for this repo
+echo "Your Container image for ${IMAGE} is ready"
+$ENGINE images | grep ${IMAGE}
+
+# Check the final image
+$ENGINE run --rm -it "${IMAGE}:latest" --version
+
+# Check binary-ies
+$ENGINE run --rm -it --entrypoint /bin/bash "${IMAGE}:latest" -c 'echo BINARY: $BINARY'