From e813323b74b57bf64b49b9fe96a231d900578263 Mon Sep 17 00:00:00 2001 From: Chevdor Date: Fri, 11 Aug 2023 15:28:39 +0200 Subject: [PATCH] Publish RC container images (#7556) * WIP * Add missing checkout * Add debuggin * Fix VAR name * Bug fix * Rework jobs * Revert "Rework jobs" This reverts commit 2bfa79fd3ae633c17403b838f9a5025f0f7fc3f3. * Add cache * Add temp default for testing * Add missing checkout * Fix patch * Comment out the GPG check for now * Rename polkadot_injected_release into a more appropriate polkadot_injected_debian * Refactoring / renaming * Introduce a generic image for binary injection * Flag files to be deleted and changes to be done * WIP * Fix multi binaries images * Add test build scripts * Remove old file, add polkadot build-injected script * Fix doc * Fix tagging * Add build of the injected container * Fix for docker * Remove the need for TTY * Handling container publishing * Fix owner and registry * Fix vars * Fix repo * Fix var naming * Fix case when there is no tag * Fix case with no tag * Handle error * Fix spacings * Fix tags * Remove unnecessary grep that may fail * Add final check * Clean up and introduce GPG check * Add doc * Add doc * Update doc/docker.md Co-authored-by: Mira Ressel * type Co-authored-by: Mira Ressel * Fix used VAR * Improve doc * ci: Update .build-push-image jobs to use the new build-injected.sh * ci: fix path to build-injected.sh script * Rename the release artifacts folder to prevent confusion due to a similar folder in the gitlab CI * ci: check out polkadot repo in .build-push-image This seems far cleaner than copying the entire scripts/ folder into our job artifacts. * feat(build-injected.sh): make PROJECT_ROOT configurable This lets us avoid a dependency on git in our CI image. * ci: build injected images with buildah * ci: pass full image names to zombienet * Add missing ignore --------- Co-authored-by: Mira Ressel --- .github/workflows/check-licenses.yml | 2 +- .../workflows/release-40_publish-rc-image.yml | 132 ++++++++++++++++++ .../release-50_publish-docker-release.yml | 2 +- .../release-51_publish-docker-manual.yml | 2 +- .gitignore | 4 + .gitlab-ci.yml | 36 +++-- doc/docker.md | 86 +++++++----- scripts/ci/common/lib.sh | 70 ++++++++++ .../adder-collator/build-injected.sh | 13 ++ .../dockerfiles/adder-collator/test-build.sh | 23 +++ .../ci/dockerfiles/binary_injected.Dockerfile | 48 +++++++ scripts/ci/dockerfiles/build-injected.sh | 92 ++++++++++++ .../dockerfiles/collator_injected.Dockerfile | 49 ------- scripts/ci/dockerfiles/entrypoint.sh | 18 +++ .../ci/dockerfiles/malus/build-injected.sh | 14 ++ scripts/ci/dockerfiles/malus/test-build.sh | 19 +++ .../ci/dockerfiles/malus_injected.Dockerfile | 50 ------- scripts/ci/dockerfiles/polkadot/README.md | 2 + .../ci/dockerfiles/polkadot/build-injected.sh | 13 ++ scripts/ci/dockerfiles/polkadot/build.sh | 27 ---- .../dockerfiles/polkadot/docker-compose.yml | 13 +- .../polkadot_Dockerfile.README.md | 0 .../polkadot/polkadot_builder.Dockerfile | 2 +- .../polkadot_injected_debian.Dockerfile} | 2 +- scripts/ci/dockerfiles/polkadot/test-build.sh | 18 +++ .../polkadot_injected_debug.Dockerfile | 48 ------- .../ci/dockerfiles/staking-miner/README.md | 37 +++++ .../staking-miner/build-injected.sh | 13 ++ scripts/ci/dockerfiles/staking-miner/build.sh | 13 ++ .../staking-miner_builder.Dockerfile | 11 +- .../staking-miner_injected.Dockerfile | 43 ------ .../dockerfiles/staking-miner/test-build.sh | 18 +++ scripts/ci/gitlab/pipeline/build.yml | 4 - scripts/ci/gitlab/pipeline/publish.yml | 54 +++---- utils/staking-miner/README.md | 6 +- 35 files changed, 661 insertions(+), 323 deletions(-) create mode 100644 .github/workflows/release-40_publish-rc-image.yml create mode 100755 scripts/ci/dockerfiles/adder-collator/build-injected.sh create mode 100755 scripts/ci/dockerfiles/adder-collator/test-build.sh create mode 100644 scripts/ci/dockerfiles/binary_injected.Dockerfile create mode 100755 scripts/ci/dockerfiles/build-injected.sh delete mode 100644 scripts/ci/dockerfiles/collator_injected.Dockerfile create mode 100755 scripts/ci/dockerfiles/entrypoint.sh create mode 100755 scripts/ci/dockerfiles/malus/build-injected.sh create mode 100755 scripts/ci/dockerfiles/malus/test-build.sh delete mode 100644 scripts/ci/dockerfiles/malus_injected.Dockerfile create mode 100755 scripts/ci/dockerfiles/polkadot/build-injected.sh delete mode 100755 scripts/ci/dockerfiles/polkadot/build.sh rename scripts/ci/dockerfiles/{ => polkadot}/polkadot_Dockerfile.README.md (100%) rename scripts/ci/dockerfiles/{polkadot_injected_release.Dockerfile => polkadot/polkadot_injected_debian.Dockerfile} (95%) create mode 100755 scripts/ci/dockerfiles/polkadot/test-build.sh delete mode 100644 scripts/ci/dockerfiles/polkadot_injected_debug.Dockerfile create mode 100644 scripts/ci/dockerfiles/staking-miner/README.md create mode 100755 scripts/ci/dockerfiles/staking-miner/build-injected.sh create mode 100755 scripts/ci/dockerfiles/staking-miner/build.sh delete mode 100644 scripts/ci/dockerfiles/staking-miner/staking-miner_injected.Dockerfile create mode 100755 scripts/ci/dockerfiles/staking-miner/test-build.sh diff --git a/.github/workflows/check-licenses.yml b/.github/workflows/check-licenses.yml index a4c8d5d97424..522037b6827c 100644 --- a/.github/workflows/check-licenses.yml +++ b/.github/workflows/check-licenses.yml @@ -8,7 +8,7 @@ jobs: runs-on: ubuntu-22.04 steps: - name: Checkout sources - uses: actions/checkout@v3.3.0 + uses: actions/checkout@v3 - uses: actions/setup-node@v3.7.0 with: node-version: '18.x' diff --git a/.github/workflows/release-40_publish-rc-image.yml b/.github/workflows/release-40_publish-rc-image.yml new file mode 100644 index 000000000000..a821eaa033fd --- /dev/null +++ b/.github/workflows/release-40_publish-rc-image.yml @@ -0,0 +1,132 @@ +name: Release - Publish RC Container image +# see https://github.com/paritytech/release-engineering/issues/97#issuecomment-1651372277 + +on: + workflow_dispatch: + inputs: + release_id: + description: | + Release ID. + You can find it using the command: + curl -s \ + -H "Authorization: Bearer ${GITHUB_TOKEN}" https://api.github.com/repos/$OWNER/$REPO/releases | \ + jq '.[] | { name: .name, id: .id }' + required: true + type: string + registry: + description: "Container registry" + required: true + type: string + default: docker.io + owner: + description: Owner of the container image repo + required: true + type: string + default: parity + +env: + RELEASE_ID: ${{ inputs.release_id }} + ENGINE: docker + REGISTRY: ${{ inputs.registry }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + DOCKER_OWNER: ${{ inputs.owner || github.repository_owner }} + REPO: ${{ github.repository }} + ARTIFACT_FOLDER: release-artifacts + +jobs: + fetch-artifacts: + runs-on: ubuntu-latest + + steps: + - name: Checkout sources + uses: actions/checkout@v3 + + - name: Fetch all artifacts + run: | + . ./scripts/ci/common/lib.sh + fetch_release_artifacts + + - name: Cache the artifacts + uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 # v3.3.1 + with: + key: artifacts-${{ github.sha }} + path: | + ${ARTIFACT_FOLDER}/**/* + + build-container: + runs-on: ubuntu-latest + needs: fetch-artifacts + + strategy: + matrix: + binary: ["polkadot", "staking-miner"] + + steps: + - name: Checkout sources + uses: actions/checkout@v3 + + - name: Get artifacts from cache + uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 # v3.3.1 + with: + key: artifacts-${{ github.sha }} + path: | + ${ARTIFACT_FOLDER}/**/* + + - name: Check sha256 ${{ matrix.binary }} + working-directory: ${ARTIFACT_FOLDER} + run: | + . ../scripts/ci/common/lib.sh + + echo "Checking binary ${{ matrix.binary }}" + check_sha256 ${{ matrix.binary }} && echo "OK" || echo "ERR" + + - name: Check GPG ${{ matrix.binary }} + working-directory: ${ARTIFACT_FOLDER} + run: | + . ../scripts/ci/common/lib.sh + import_gpg_keys + check_gpg ${{ matrix.binary }} + + - name: Fetch commit and tag + id: fetch_refs + run: | + release=release-${{ inputs.release_id }} && \ + echo "release=${release}" >> $GITHUB_OUTPUT + + commit=$(git rev-parse --short HEAD) && \ + echo "commit=${commit}" >> $GITHUB_OUTPUT + + tag=$(git name-rev --tags --name-only $(git rev-parse HEAD)) && \ + [ "${tag}" != "undefined" ] && echo "tag=${tag}" >> $GITHUB_OUTPUT || \ + echo "No tag, doing without" + + - name: Build Injected Container image for ${{ matrix.binary }} + env: + BIN_FOLDER: ${ARTIFACT_FOLDER} + BINARY: ${{ matrix.binary }} + TAGS: ${{join(steps.fetch_refs.outputs.*, ',')}} + run: | + echo "Building container for ${{ matrix.binary }}" + ./scripts/ci/dockerfiles/build-injected.sh + + - name: Login to Dockerhub + uses: docker/login-action@v2 + with: + username: ${{ inputs.owner }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Push Container image for ${{ matrix.binary }} + id: docker_push + env: + BINARY: ${{ matrix.binary }} + run: | + $ENGINE images | grep ${BINARY} + $ENGINE push --all-tags ${REGISTRY}/${DOCKER_OWNER}/${BINARY} + + - name: Check version for the published image for ${{ matrix.binary }} + env: + BINARY: ${{ matrix.binary }} + RELEASE_TAG: ${{ steps.fetch_refs.outputs.release }} + run: | + echo "Checking tag ${RELEASE_TAG} for image ${REGISTRY}/${DOCKER_OWNER}/${BINARY}" + $ENGINE run -i ${REGISTRY}/${DOCKER_OWNER}/${BINARY}:${RELEASE_TAG} --version diff --git a/.github/workflows/release-50_publish-docker-release.yml b/.github/workflows/release-50_publish-docker-release.yml index a6bf19162a46..81e5caa718f3 100644 --- a/.github/workflows/release-50_publish-docker-release.yml +++ b/.github/workflows/release-50_publish-docker-release.yml @@ -30,7 +30,7 @@ jobs: uses: docker/build-push-action@v4 with: push: true - file: scripts/ci/dockerfiles/polkadot_injected_release.Dockerfile + file: scripts/ci/dockerfiles/polkadot/polkadot_injected_debian.Dockerfile tags: | parity/polkadot:latest parity/polkadot:${{ github.event.release.tag_name }} diff --git a/.github/workflows/release-51_publish-docker-manual.yml b/.github/workflows/release-51_publish-docker-manual.yml index 0c973d33b71c..919769f8700d 100644 --- a/.github/workflows/release-51_publish-docker-manual.yml +++ b/.github/workflows/release-51_publish-docker-manual.yml @@ -37,7 +37,7 @@ jobs: uses: docker/build-push-action@v4 with: push: true - file: scripts/ci/dockerfiles/polkadot_injected_release.Dockerfile + file: scripts/ci/dockerfiles/polkadot/polkadot_injected_debian.Dockerfile tags: | parity/polkadot:latest parity/polkadot:${{ github.event.inputs.version }} diff --git a/.gitignore b/.gitignore index 0c6913dac340..61ef9e91a55e 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,7 @@ polkadot.* !polkadot.service .DS_Store .env + +artifacts +release-artifacts +release.json diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5056012e588e..5a84bbfeba85 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -159,31 +159,39 @@ default: - if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1 .build-push-image: + variables: + CI_IMAGE: "${BUILDAH_IMAGE}" + + REGISTRY: "docker.io" + DOCKER_OWNER: "paritypr" + DOCKER_USER: "${PARITYPR_USER}" + DOCKER_PASS: "${PARITYPR_PASS}" + IMAGE: "${REGISTRY}/${DOCKER_OWNER}/${IMAGE_NAME}" + + ENGINE: "${BUILDAH_COMMAND}" + BUILDAH_FORMAT: "docker" + SKIP_IMAGE_VALIDATION: 1 + + PROJECT_ROOT: "." + BIN_FOLDER: "./artifacts" + VCS_REF: "${CI_COMMIT_SHA}" + before_script: - !reference [.common-before-script, before_script] - test -s ./artifacts/VERSION || exit 1 - test -s ./artifacts/EXTRATAG || exit 1 - - VERSION="$(cat ./artifacts/VERSION)" + - export VERSION="$(cat ./artifacts/VERSION)" - EXTRATAG="$(cat ./artifacts/EXTRATAG)" - echo "Polkadot version = ${VERSION} (EXTRATAG = ${EXTRATAG})" script: - test "$DOCKER_USER" -a "$DOCKER_PASS" || ( echo "no docker credentials provided"; exit 1 ) - - cd ./artifacts - - $BUILDAH_COMMAND build - --format=docker - --build-arg VCS_REF="${CI_COMMIT_SHA}" - --build-arg BUILD_DATE="$(date -u '+%Y-%m-%dT%H:%M:%SZ')" - --build-arg IMAGE_NAME="${IMAGE_NAME}" - --tag "$IMAGE_NAME:$VERSION" - --tag "$IMAGE_NAME:$EXTRATAG" - --file ${DOCKERFILE} . - # The job will success only on the protected branch + - TAGS="${VERSION},${EXTRATAG}" scripts/ci/dockerfiles/build-injected.sh - echo "$DOCKER_PASS" | - buildah login --username "$DOCKER_USER" --password-stdin docker.io + buildah login --username "$DOCKER_USER" --password-stdin "${REGISTRY}" - $BUILDAH_COMMAND info - - $BUILDAH_COMMAND push --format=v2s2 "$IMAGE_NAME:$VERSION" - - $BUILDAH_COMMAND push --format=v2s2 "$IMAGE_NAME:$EXTRATAG" + - $BUILDAH_COMMAND push --format=v2s2 "$IMAGE:$VERSION" + - $BUILDAH_COMMAND push --format=v2s2 "$IMAGE:$EXTRATAG" after_script: - buildah logout --all diff --git a/doc/docker.md b/doc/docker.md index e8b7fa74732e..f20c2d001edd 100644 --- a/doc/docker.md +++ b/doc/docker.md @@ -1,43 +1,58 @@ -# Using Docker +# Using Containers + +The following commands should work no matter if you use Docker or Podman. In general, Podman is recommended. All commands are "engine neutral" so you can use the container engine of your choice while still being able to copy/paste the commands below. + +Let's start defining Podman as our engine: +``` +ENGINE=podman +``` + +If you prefer to stick with Docker, use: +``` +ENGINE=docker +``` ## The easiest way -The easiest/faster option to run Polkadot in Docker is to use the latest release images. These are small images that use the latest official release of the Polkadot binary, pulled from our package repository. +The easiest/faster option to run Polkadot in Docker is to use the latest release images. These are small images that use the latest official release of the Polkadot binary, pulled from our Debian package. -**_Following examples are running on westend chain and without SSL. They can be used to quick start and learn how Polkadot needs to be configured. Please find out how to secure your node, if you want to operate it on the internet. Do not expose RPC and WS ports, if they are not correctly configured._** +**_The following examples are running on westend chain and without SSL. They can be used to quick start and learn how Polkadot needs to be configured. Please find out how to secure your node, if you want to operate it on the internet. Do not expose RPC and WS ports, if they are not correctly configured._** Let's first check the version we have. The first time you run this command, the Polkadot docker image will be downloaded. This takes a bit of time and bandwidth, be patient: ```bash -docker run --rm -it parity/polkadot:latest --version +$ENGINE run --rm -it parity/polkadot:latest --version ``` You can also pass any argument/flag that Polkadot supports: ```bash -docker run --rm -it parity/polkadot:latest --chain westend --name "PolkaDocker" +$ENGINE run --rm -it parity/polkadot:latest --chain westend --name "PolkaDocker" ``` ## Examples -Once you are done experimenting and picking the best node name :) you can start Polkadot as daemon, exposes the Polkadot ports and mount a volume that will keep your blockchain data locally. Make sure that you set the ownership of your local directory to the Polkadot user that is used by the container. Set user id 1000 and group id 1000, by running `chown 1000.1000 /my/local/folder -R` if you use a bind mount. - -To start a Polkadot node on default rpc port 9933 and default p2p port 30333 use the following command. If you want to connect to rpc port 9933, then must add Polkadot startup parameter: `--rpc-external`. +Once you are done experimenting and picking the best node name :) you can start Polkadot as daemon, exposes the Polkadot ports and mount a volume that will keep your blockchain data locally. Make sure that you set the ownership of your local directory to the Polkadot user that is used by the container. -```bash -docker run -d -p 30333:30333 -p 9933:9933 -v /my/local/folder:/polkadot parity/polkadot:latest --chain westend --rpc-external --rpc-cors all -``` +Set user id 1000 and group id 1000, by running `chown 1000.1000 /my/local/folder -R` if you use a bind mount. -Additionally if you want to have custom node name you can add the `--name "YourName"` at the end +To start a Polkadot node on default rpc port 9933 and default p2p port 30333 use the following command. If you want to connect to rpc port 9933, then must add Polkadot startup parameter: `--rpc-external`. ```bash -docker run -d -p 30333:30333 -p 9933:9933 -v /my/local/folder:/polkadot parity/polkadot:latest --chain westend --rpc-external --rpc-cors all --name "PolkaDocker" +$ENGINE run -d -p 30333:30333 -p 9933:9933 \ + -v /my/local/folder:/polkadot \ + parity/polkadot:latest \ + --chain westend --rpc-external --rpc-cors all \ + --name "PolkaDocker ``` If you also want to expose the webservice port 9944 use the following command: ```bash -docker run -d -p 30333:30333 -p 9933:9933 -p 9944:9944 -v /my/local/folder:/polkadot parity/polkadot:latest --chain westend --ws-external --rpc-external --rpc-cors all --name "PolkaDocker" +$ENGINE run -d -p 30333:30333 -p 9933:9933 -p 9944:9944 \ + -v /my/local/folder:/polkadot \ + parity/polkadot:latest \ + --chain westend --ws-external --rpc-external --rpc-cors all --name "PolkaDocker" ``` ## Using Docker compose @@ -55,17 +70,19 @@ services: - 30333:30333 # p2p port - 9933:9933 # rpc port - 9944:9944 # ws port + - 9615:9615 # Prometheus port volumes: - /my/local/folder:/polkadot command: [ "--name", "PolkaDocker", "--ws-external", "--rpc-external", + "--prometheus-external", "--rpc-cors", "all" ] ``` -With following docker-compose.yml you can set up a node and use polkadot-js-apps as the front end on port 80. After starting the node use a browser and enter your Docker host IP in the URL field: __ +With following `docker-compose.yml` you can set up a node and use polkadot-js-apps as the front end on port 80. After starting the node use a browser and enter your Docker host IP in the URL field: __ ```bash version: '2' @@ -78,10 +95,12 @@ services: - 30333:30333 # p2p port - 9933:9933 # rpc port - 9944:9944 # ws port + - 9615:9615 # Prometheus port command: [ "--name", "PolkaDocker", "--ws-external", "--rpc-external", + "--prometheus-external", "--rpc-cors", "all" ] @@ -100,27 +119,30 @@ Chain syncing will utilize all available memory and CPU power your server has to If running on a low resource VPS, use `--memory` and `--cpus` to limit the resources used. E.g. To allow a maximum of 512MB memory and 50% of 1 CPU, use `--cpus=".5" --memory="512m"`. Read more about limiting a container's resources [here](https://docs.docker.com/config/containers/resource_constraints). -Start a shell session with the daemon: -```bash -docker exec -it $(docker ps -q) bash; -``` +## Build your own image -Check the current version: +There are 3 options to build a polkadot container image: +- using the builder image +- using the injected "Debian" image +- using the generic injected image -```bash -polkadot --version -``` +### Builder image -## Build your own image +To get up and running with the smallest footprint on your system, you may use an existing Polkadot Container image. -To get up and running with the smallest footprint on your system, you may use the Polkadot Docker image. -You can build it yourself (it takes a while...) in the shell session of the daemon: +You may also build a polkadot container image yourself (it takes a while...) using the container specs `scripts/ci/dockerfiles/polkadot/polkadot_builder.Dockerfile`. -```bash -cd scripts/ci/dockerfiles/polkadot -./build.sh -``` +### Debian injected + +The Debian injected image is how the official polkadot container image is produced. It relies on the Debian package that is published upon each release. The Debian injected image is usually available a few minutes after a new release is published. +It has the benefit of relying on the GPG signatures embedded in the Debian package. + +### Generic injected + +For simple testing purposes, the easiest option for polkadot and also random binaries, is to use the `binary_injected.Dockerfile` container spec. This option is less secure since the injected binary is not checked at all but it has the benefit to be simple. This option requires to already have a valid `polkadot` binary, compiled for Linux. + +This binary is then simply copied inside the `parity/base-bin` image. ## Reporting issues @@ -128,8 +150,8 @@ If you run into issues with Polkadot when using docker, please run the following (replace the tag with the appropriate one if you do not use latest): ```bash -docker run --rm -it parity/polkadot:latest --version +$ENGINE run --rm -it parity/polkadot:latest --version ``` This will show you the Polkadot version as well as the git commit ref that was used to build your container. -Just paste that in the issue you create. +You can now paste the version information in a [new issue](https://github.com/paritytech/polkadot/issues/new/choose). diff --git a/scripts/ci/common/lib.sh b/scripts/ci/common/lib.sh index 2e94feb150ce..00abe9a1d8d4 100755 --- a/scripts/ci/common/lib.sh +++ b/scripts/ci/common/lib.sh @@ -193,3 +193,73 @@ check_bootnode(){ echo " Bootnode appears unreachable" return 1 } + +# Assumes the ENV are set: +# - RELEASE_ID +# - GITHUB_TOKEN +# - REPO in the form paritytech/polkadot +fetch_release_artifacts() { + echo "Release ID : $RELEASE_ID" + echo "Repo : $REPO" + echo "ARTIFACT_FOLDER: $ARTIFACT_FOLDER" + + curl -L -s \ + -H "Accept: application/vnd.github+json" \ + -H "Authorization: Bearer ${GITHUB_TOKEN}" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + https://api.github.com/repos/${REPO}/releases/$RELEASE_ID > release.json + + # Get Asset ids + ids=($(jq -r '.assets[].id' < release.json )) + count=$(jq '.assets|length' < release.json ) + + # Fetch artifacts + mkdir -p ${ARTIFACT_FOLDER} + pushd ${ARTIFACT_FOLDER} > /dev/null + + iter=1 + for id in "${ids[@]}" + do + echo " - $iter/$count: downloading asset id: $id..." + curl -s -OJ -L -H "Accept: application/octet-stream" \ + -H "Authorization: Token ${GITHUB_TOKEN}" \ + "https://api.github.com/repos/${REPO}/releases/assets/$id" + iter=$((iter + 1)) + done + + ls -al --color + popd > /dev/null +} + +# Check the checksum for a given binary +function check_sha256() { + echo "Checking SHA256 for $1" + shasum -qc $1.sha256 +} + +# Import GPG keys of the release team members +# This is done in parallel as it can take a while sometimes +function import_gpg_keys() { + GPG_KEYSERVER=${GPG_KEYSERVER:-"keyserver.ubuntu.com"} + SEC="9D4B2B6EB8F97156D19669A9FF0812D491B96798" + WILL="2835EAF92072BC01D188AF2C4A092B93E97CE1E2" + EGOR="E6FC4D4782EB0FA64A4903CCDB7D3555DD3932D3" + MARA="533C920F40E73A21EEB7E9EBF27AEA7E7594C9CF" + MORGAN="2E92A9D8B15D7891363D1AE8AF9E6C43F7F8C4CF" + + echo "Importing GPG keys from $GPG_KEYSERVER in parallel" + for key in $SEC $WILL $EGOR $MARA $MORGAN; do + ( + echo "Importing GPG key $key" + gpg --no-tty --quiet --keyserver $GPG_KEYSERVER --recv-keys $key + echo -e "5\ny\n" | gpg --no-tty --command-fd 0 --expert --edit-key $key trust; + ) & + done + wait +} + +# Check the GPG signature for a given binary +function check_gpg() { + echo "Checking GPG Signature for $1" + gpg --no-tty --verify -q $1.asc $1 +} diff --git a/scripts/ci/dockerfiles/adder-collator/build-injected.sh b/scripts/ci/dockerfiles/adder-collator/build-injected.sh new file mode 100755 index 000000000000..9a1857bc7ab4 --- /dev/null +++ b/scripts/ci/dockerfiles/adder-collator/build-injected.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +# Sample call: +# $0 /path/to/folder_with_binary +# This script replace the former dedicated Dockerfile +# and shows how to use the generic binary_injected.dockerfile + +PROJECT_ROOT=`git rev-parse --show-toplevel` + +export BINARY=adder-collator,undying-collator +export BIN_FOLDER=$1 + +$PROJECT_ROOT/scripts/ci/dockerfiles/build-injected.sh diff --git a/scripts/ci/dockerfiles/adder-collator/test-build.sh b/scripts/ci/dockerfiles/adder-collator/test-build.sh new file mode 100755 index 000000000000..171e0309f807 --- /dev/null +++ b/scripts/ci/dockerfiles/adder-collator/test-build.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash + +TMP=$(mktemp -d) +ENGINE=${ENGINE:-podman} + +# TODO: Switch to /bin/bash when the image is built from parity/base-bin + +# Fetch some binaries +$ENGINE run --user root --rm -i \ + --pull always \ + -v "$TMP:/export" \ + --entrypoint /usr/bin/bash \ + paritypr/colander:master -c \ + 'cp "$(which adder-collator)" /export' + +$ENGINE run --user root --rm -i \ + --pull always \ + -v "$TMP:/export" \ + --entrypoint /usr/bin/bash \ + paritypr/colander:master -c \ + 'cp "$(which undying-collator)" /export' + +./build-injected.sh $TMP diff --git a/scripts/ci/dockerfiles/binary_injected.Dockerfile b/scripts/ci/dockerfiles/binary_injected.Dockerfile new file mode 100644 index 000000000000..cee81a2eb8ae --- /dev/null +++ b/scripts/ci/dockerfiles/binary_injected.Dockerfile @@ -0,0 +1,48 @@ +FROM docker.io/parity/base-bin + +# This file allows building a Generic container image +# based on one or multiple pre-built Linux binaries. +# Some defaults are set to polkadot but all can be overriden. + +SHELL ["/bin/bash", "-c"] + +# metadata +ARG VCS_REF +ARG BUILD_DATE +ARG IMAGE_NAME + +# That can be a single one or a comma separated list +ARG BINARY=polkadot + +ARG BIN_FOLDER=. +ARG DOC_URL=https://github.com/paritytech/polkadot +ARG DESCRIPTION="Polkadot: a platform for web3" +ARG AUTHORS="devops-team@parity.io" +ARG VENDOR="Parity Technologies" + +LABEL io.parity.image.authors=${AUTHORS} \ + io.parity.image.vendor="${VENDOR}" \ + io.parity.image.revision="${VCS_REF}" \ + io.parity.image.title="${IMAGE_NAME}" \ + io.parity.image.created="${BUILD_DATE}" \ + io.parity.image.documentation="${DOC_URL}" \ + io.parity.image.description="${DESCRIPTION}" \ + io.parity.image.source="https://github.com/paritytech/polkadot/blob/${VCS_REF}/scripts/ci/dockerfiles/binary_injected.Dockerfile" + +USER root +WORKDIR /app + +# add polkadot binary to docker image +# sample for polkadot: COPY ./polkadot ./polkadot-*-worker /usr/local/bin/ +COPY entrypoint.sh . +COPY "bin/*" "/usr/local/bin/" +RUN chmod -R a+rx "/usr/local/bin" + +USER parity +ENV BINARY=${BINARY} + +# ENTRYPOINT +ENTRYPOINT ["/app/entrypoint.sh"] + +# We call the help by default +CMD ["--help"] diff --git a/scripts/ci/dockerfiles/build-injected.sh b/scripts/ci/dockerfiles/build-injected.sh new file mode 100755 index 000000000000..d0e7fee3646e --- /dev/null +++ b/scripts/ci/dockerfiles/build-injected.sh @@ -0,0 +1,92 @@ +#!/usr/bin/env bash +set -e + +# This script allows building a Container Image from a Linux +# binary that is injected into a base-image. + +ENGINE=${ENGINE:-podman} + +if [ "$ENGINE" == "podman" ]; then + PODMAN_FLAGS="--format docker" +else + PODMAN_FLAGS="" +fi + +CONTEXT=$(mktemp -d) +REGISTRY=${REGISTRY:-docker.io} + +# The following line ensure we know the project root +PROJECT_ROOT=${PROJECT_ROOT:-$(git rev-parse --show-toplevel)} +DOCKERFILE=${DOCKERFILE:-$PROJECT_ROOT/scripts/ci/dockerfiles/binary_injected.Dockerfile} +VERSION_TOML=$(grep "^version " $PROJECT_ROOT/Cargo.toml | grep -oE "([0-9\.]+-?[0-9]+)") + +#n The following VAR have default that can be overriden +DOCKER_OWNER=${DOCKER_OWNER:-parity} + +# We may get 1..n binaries, comma separated +BINARY=${BINARY:-polkadot} +IFS=',' read -r -a BINARIES <<< "$BINARY" + +VERSION=${VERSION:-$VERSION_TOML} +BIN_FOLDER=${BIN_FOLDER:-.} + +IMAGE=${IMAGE:-${REGISTRY}/${DOCKER_OWNER}/${BINARIES[0]}} +DESCRIPTION_DEFAULT="Injected Container image built for ${BINARY}" +DESCRIPTION=${DESCRIPTION:-$DESCRIPTION_DEFAULT} + +VCS_REF=${VCS_REF:-01234567} + +# Build the image +echo "Using engine: $ENGINE" +echo "Using Dockerfile: $DOCKERFILE" +echo "Using context: $CONTEXT" +echo "Building ${IMAGE}:latest container image for ${BINARY} v${VERSION} from ${BIN_FOLDER} hang on!" +echo "BIN_FOLDER=$BIN_FOLDER" +echo "CONTEXT=$CONTEXT" + +# We need all binaries and resources available in the Container build "CONTEXT" +mkdir -p $CONTEXT/bin +for bin in "${BINARIES[@]}" +do + echo "Copying $BIN_FOLDER/$bin to context: $CONTEXT/bin" + cp "$BIN_FOLDER/$bin" "$CONTEXT/bin" +done + +cp "$PROJECT_ROOT/scripts/ci/dockerfiles/entrypoint.sh" "$CONTEXT" + +echo "Building image: ${IMAGE}" + +TAGS=${TAGS[@]:-latest} +IFS=',' read -r -a TAG_ARRAY <<< "$TAGS" +TAG_ARGS=" " + +echo "The image ${IMAGE} will be tagged with ${TAG_ARRAY[*]}" +for tag in "${TAG_ARRAY[@]}"; do + TAG_ARGS+="--tag ${IMAGE}:${tag} " +done + +echo "$TAG_ARGS" + +# time \ +$ENGINE build \ + ${PODMAN_FLAGS} \ + --build-arg VCS_REF="${VCS_REF}" \ + --build-arg BUILD_DATE=$(date -u '+%Y-%m-%dT%H:%M:%SZ') \ + --build-arg IMAGE_NAME="${IMAGE}" \ + --build-arg BINARY="${BINARY}" \ + --build-arg BIN_FOLDER="${BIN_FOLDER}" \ + --build-arg DESCRIPTION="${DESCRIPTION}" \ + ${TAG_ARGS} \ + -f "${DOCKERFILE}" \ + ${CONTEXT} + +echo "Your Container image for ${IMAGE} is ready" +$ENGINE images + +if [[ -z "${SKIP_IMAGE_VALIDATION}" ]]; then + echo "Check the image ${IMAGE}:${TAG_ARRAY[0]}" + $ENGINE run --rm -i "${IMAGE}:${TAG_ARRAY[0]}" --version + + echo "Query binaries" + $ENGINE run --rm -i --entrypoint /bin/bash "${IMAGE}:${TAG_ARRAY[0]}" -c 'echo BINARY: $BINARY' +fi diff --git a/scripts/ci/dockerfiles/collator_injected.Dockerfile b/scripts/ci/dockerfiles/collator_injected.Dockerfile deleted file mode 100644 index 91b8cb0057bf..000000000000 --- a/scripts/ci/dockerfiles/collator_injected.Dockerfile +++ /dev/null @@ -1,49 +0,0 @@ -# this file copies from scripts/ci/dockerfiles/Dockerfile and changes only the binary name -FROM docker.io/library/ubuntu:20.04 - -# metadata -ARG VCS_REF -ARG BUILD_DATE -ARG IMAGE_NAME - -LABEL io.parity.image.authors="devops-team@parity.io" \ - io.parity.image.vendor="Parity Technologies" \ - io.parity.image.title="${IMAGE_NAME}" \ - io.parity.image.description="Injected adder-collator Docker image" \ - io.parity.image.source="https://github.com/paritytech/polkadot/blob/${VCS_REF}/scripts/ci/dockerfiles/collator_injected.Dockerfile" \ - io.parity.image.revision="${VCS_REF}" \ - io.parity.image.created="${BUILD_DATE}" \ - io.parity.image.documentation="https://github.com/paritytech/polkadot/" - -# show backtraces -ENV RUST_BACKTRACE 1 - -# install tools and dependencies -RUN apt-get update && \ - DEBIAN_FRONTEND=noninteractive apt-get install -y \ - libssl1.1 \ - ca-certificates && \ -# apt cleanup - apt-get autoremove -y && \ - apt-get clean && \ - find /var/lib/apt/lists/ -type f -not -name lock -delete; \ -# add user and link ~/.local/share/adder-collator to /data - useradd -m -u 1000 -U -s /bin/sh -d /adder-collator adder-collator && \ - mkdir -p /data /adder-collator/.local/share && \ - chown -R adder-collator:adder-collator /data && \ - ln -s /data /adder-collator/.local/share/polkadot - -# add adder-collator binary to docker image -COPY ./adder-collator /usr/local/bin -COPY ./undying-collator /usr/local/bin - -USER adder-collator - -# check if executable works in this container -RUN /usr/local/bin/adder-collator --version -RUN /usr/local/bin/undying-collator --version - -EXPOSE 30333 9933 9944 -VOLUME ["/adder-collator"] - -ENTRYPOINT ["/usr/local/bin/adder-collator"] diff --git a/scripts/ci/dockerfiles/entrypoint.sh b/scripts/ci/dockerfiles/entrypoint.sh new file mode 100755 index 000000000000..eaa815faf6a4 --- /dev/null +++ b/scripts/ci/dockerfiles/entrypoint.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash + +# Sanity check +if [ -z "$BINARY" ] +then + echo "BINARY ENV not defined, this should never be the case. Aborting..." + exit 1 +fi + +# If the user built the image with multiple binaries, +# we consider the first one to be the canonical one +# To start with another binary, the user can either: +# - use the --entrypoint option +# - pass the ENV BINARY with a single binary +IFS=',' read -r -a BINARIES <<< "$BINARY" +BIN0=${BINARIES[0]} +echo "Starting binary $BIN0" +$BIN0 $@ diff --git a/scripts/ci/dockerfiles/malus/build-injected.sh b/scripts/ci/dockerfiles/malus/build-injected.sh new file mode 100755 index 000000000000..99bd5fde1d5a --- /dev/null +++ b/scripts/ci/dockerfiles/malus/build-injected.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +# Sample call: +# $0 /path/to/folder_with_binary +# This script replace the former dedicated Dockerfile +# and shows how to use the generic binary_injected.dockerfile + +PROJECT_ROOT=`git rev-parse --show-toplevel` + +export BINARY=malus,polkadot-execute-worker,polkadot-prepare-worker +export BIN_FOLDER=$1 +# export TAGS=... + +$PROJECT_ROOT/scripts/ci/dockerfiles/build-injected.sh diff --git a/scripts/ci/dockerfiles/malus/test-build.sh b/scripts/ci/dockerfiles/malus/test-build.sh new file mode 100755 index 000000000000..3114e9e2adf1 --- /dev/null +++ b/scripts/ci/dockerfiles/malus/test-build.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash + +TMP=$(mktemp -d) +ENGINE=${ENGINE:-podman} + +export TAGS=latest,beta,7777,1.0.2-rc23 + +# Fetch some binaries +$ENGINE run --user root --rm -i \ + --pull always \ + -v "$TMP:/export" \ + --entrypoint /bin/bash \ + paritypr/malus:7217 -c \ + 'cp "$(which malus)" /export' + +echo "Checking binaries we got:" +ls -al $TMP + +./build-injected.sh $TMP diff --git a/scripts/ci/dockerfiles/malus_injected.Dockerfile b/scripts/ci/dockerfiles/malus_injected.Dockerfile deleted file mode 100644 index fa429b5f142a..000000000000 --- a/scripts/ci/dockerfiles/malus_injected.Dockerfile +++ /dev/null @@ -1,50 +0,0 @@ -FROM debian:bullseye-slim - -# metadata -ARG VCS_REF -ARG BUILD_DATE -ARG IMAGE_NAME - -LABEL io.parity.image.authors="devops-team@parity.io" \ - io.parity.image.vendor="Parity Technologies" \ - io.parity.image.title="${IMAGE_NAME}" \ - io.parity.image.description="Malus - the nemesis of polkadot" \ - io.parity.image.source="https://github.com/paritytech/polkadot/blob/${VCS_REF}/scripts/ci/dockerfiles/malus.Dockerfile" \ - io.parity.image.revision="${VCS_REF}" \ - io.parity.image.created="${BUILD_DATE}" \ - io.parity.image.documentation="https://github.com/paritytech/polkadot/" - -# show backtraces -ENV RUST_BACKTRACE 1 - -# install tools and dependencies -RUN apt-get update && \ - DEBIAN_FRONTEND=noninteractive apt-get install -y \ - ca-certificates \ - curl \ - libssl1.1 \ - tini && \ -# apt cleanup - apt-get autoremove -y && \ - apt-get clean && \ - find /var/lib/apt/lists/ -type f -not -name lock -delete; \ -# add user - groupadd --gid 10000 nonroot && \ - useradd --home-dir /home/nonroot \ - --create-home \ - --shell /bin/bash \ - --gid nonroot \ - --groups nonroot \ - --uid 10000 nonroot - - -# add malus binary to docker image -COPY ./malus ./polkadot-execute-worker ./polkadot-prepare-worker /usr/local/bin - -USER nonroot - -# check if executable works in this container -RUN /usr/local/bin/malus --version - -# Tini allows us to avoid several Docker edge cases, see https://github.com/krallin/tini. -ENTRYPOINT ["tini", "--", "/bin/bash"] diff --git a/scripts/ci/dockerfiles/polkadot/README.md b/scripts/ci/dockerfiles/polkadot/README.md index 9ddf324bb29c..e331d8984c2c 100644 --- a/scripts/ci/dockerfiles/polkadot/README.md +++ b/scripts/ci/dockerfiles/polkadot/README.md @@ -1,7 +1,9 @@ # Self built Docker image The Polkadot repo contains several options to build Docker images for Polkadot. + This folder contains a self-contained image that does not require a Linux pre-built binary. + Instead, building the image is possible on any host having docker installed and will build Polkadot inside Docker. That also means that no Rust toolchain is required on the host machine for the build to succeed. diff --git a/scripts/ci/dockerfiles/polkadot/build-injected.sh b/scripts/ci/dockerfiles/polkadot/build-injected.sh new file mode 100755 index 000000000000..22774c7b7122 --- /dev/null +++ b/scripts/ci/dockerfiles/polkadot/build-injected.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +# Sample call: +# $0 /path/to/folder_with_binary +# This script replace the former dedicated Dockerfile +# and shows how to use the generic binary_injected.dockerfile + +PROJECT_ROOT=`git rev-parse --show-toplevel` + +export BINARY=polkadot,polkadot-execute-worker,polkadot-prepare-worker +export BIN_FOLDER=$1 + +$PROJECT_ROOT/scripts/ci/dockerfiles/build-injected.sh diff --git a/scripts/ci/dockerfiles/polkadot/build.sh b/scripts/ci/dockerfiles/polkadot/build.sh deleted file mode 100755 index d00c9108bd8c..000000000000 --- a/scripts/ci/dockerfiles/polkadot/build.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/usr/bin/env bash -set -e - -pushd . - -# The following line ensure we run from the project root -PROJECT_ROOT=`git rev-parse --show-toplevel` -cd $PROJECT_ROOT - -# Find the current version from Cargo.toml -VERSION=`grep "^version" ./cli/Cargo.toml | egrep -o "([0-9\.]+-?[0-9]+)"` -GITUSER=parity -GITREPO=polkadot - -# Build the image -echo "Building ${GITUSER}/${GITREPO}:latest docker image, hang on!" -time docker build \ - -f ./scripts/ci/dockerfiles/polkadot/polkadot_builder.Dockerfile \ - -t ${GITUSER}/${GITREPO}:latest \ - -t ${GITUSER}/${GITREPO}:v${VERSION} \ - . - -# Show the list of available images for this repo -echo "Your Docker image for $GITUSER/$GITREPO is ready" -docker images | grep ${GITREPO} - -popd diff --git a/scripts/ci/dockerfiles/polkadot/docker-compose.yml b/scripts/ci/dockerfiles/polkadot/docker-compose.yml index 978191af88c1..524b1164796a 100644 --- a/scripts/ci/dockerfiles/polkadot/docker-compose.yml +++ b/scripts/ci/dockerfiles/polkadot/docker-compose.yml @@ -1,23 +1,22 @@ version: '3' services: polkadot: + image: parity/polkadot:latest + ports: - "127.0.0.1:30333:30333/tcp" - "127.0.0.1:9933:9933/tcp" - image: parity/polkadot:latest + - "127.0.0.1:9944:9944/tcp" + - "127.0.0.1:9615:9615/tcp" + volumes: - "polkadot-data:/data" + command: | --unsafe-rpc-external --unsafe-ws-external --rpc-cors all --prometheus-external - ports: - - "30333:30333" - - "9933:9933" - - "9944:9944" - - "9615:9615" - volumes: polkadot-data: diff --git a/scripts/ci/dockerfiles/polkadot_Dockerfile.README.md b/scripts/ci/dockerfiles/polkadot/polkadot_Dockerfile.README.md similarity index 100% rename from scripts/ci/dockerfiles/polkadot_Dockerfile.README.md rename to scripts/ci/dockerfiles/polkadot/polkadot_Dockerfile.README.md diff --git a/scripts/ci/dockerfiles/polkadot/polkadot_builder.Dockerfile b/scripts/ci/dockerfiles/polkadot/polkadot_builder.Dockerfile index 6e31298432f7..f263c836bbfe 100644 --- a/scripts/ci/dockerfiles/polkadot/polkadot_builder.Dockerfile +++ b/scripts/ci/dockerfiles/polkadot/polkadot_builder.Dockerfile @@ -7,7 +7,7 @@ COPY . /polkadot RUN cargo build --locked --release # This is the 2nd stage: a very small image where we copy the Polkadot binary." -FROM docker.io/library/ubuntu:20.04 +FROM docker.io/parity/base-bin:latest LABEL description="Multistage Docker image for Polkadot: a platform for web3" \ io.parity.image.type="builder" \ diff --git a/scripts/ci/dockerfiles/polkadot_injected_release.Dockerfile b/scripts/ci/dockerfiles/polkadot/polkadot_injected_debian.Dockerfile similarity index 95% rename from scripts/ci/dockerfiles/polkadot_injected_release.Dockerfile rename to scripts/ci/dockerfiles/polkadot/polkadot_injected_debian.Dockerfile index 74b5c7f48f88..e2c72dcfe2e9 100644 --- a/scripts/ci/dockerfiles/polkadot_injected_release.Dockerfile +++ b/scripts/ci/dockerfiles/polkadot/polkadot_injected_debian.Dockerfile @@ -11,7 +11,7 @@ LABEL io.parity.image.authors="devops-team@parity.io" \ io.parity.image.vendor="Parity Technologies" \ io.parity.image.title="parity/polkadot" \ io.parity.image.description="Polkadot: a platform for web3. This is the official Parity image with an injected binary." \ - io.parity.image.source="https://github.com/paritytech/polkadot/blob/${VCS_REF}/scripts/ci/dockerfiles/polkadot_injected_release.Dockerfile" \ + io.parity.image.source="https://github.com/paritytech/polkadot/blob/${VCS_REF}/scripts/ci/dockerfiles/polkadot/polkadot_injected_debian.Dockerfile" \ io.parity.image.revision="${VCS_REF}" \ io.parity.image.created="${BUILD_DATE}" \ io.parity.image.documentation="https://github.com/paritytech/polkadot/" diff --git a/scripts/ci/dockerfiles/polkadot/test-build.sh b/scripts/ci/dockerfiles/polkadot/test-build.sh new file mode 100755 index 000000000000..d2d904561cb5 --- /dev/null +++ b/scripts/ci/dockerfiles/polkadot/test-build.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash + +TMP=$(mktemp -d) +ENGINE=${ENGINE:-podman} + +# You need to build an injected image first + +# Fetch some binaries +$ENGINE run --user root --rm -i \ + -v "$TMP:/export" \ + --entrypoint /bin/bash \ + parity/polkadot -c \ + 'cp "$(which polkadot)" /export' + +echo "Checking binaries we got:" +tree $TMP + +./build-injected.sh $TMP diff --git a/scripts/ci/dockerfiles/polkadot_injected_debug.Dockerfile b/scripts/ci/dockerfiles/polkadot_injected_debug.Dockerfile deleted file mode 100644 index aebbbdcf1b7f..000000000000 --- a/scripts/ci/dockerfiles/polkadot_injected_debug.Dockerfile +++ /dev/null @@ -1,48 +0,0 @@ -FROM docker.io/library/ubuntu:20.04 - -# metadata -ARG VCS_REF -ARG BUILD_DATE -ARG IMAGE_NAME - -LABEL io.parity.image.authors="devops-team@parity.io" \ - io.parity.image.vendor="Parity Technologies" \ - io.parity.image.title="${IMAGE_NAME}" \ - io.parity.image.description="Polkadot: a platform for web3" \ - io.parity.image.source="https://github.com/paritytech/polkadot/blob/${VCS_REF}/scripts/ci/dockerfiles/polkadot_injected_debug.Dockerfile" \ - io.parity.image.revision="${VCS_REF}" \ - io.parity.image.created="${BUILD_DATE}" \ - io.parity.image.documentation="https://github.com/paritytech/polkadot/" - -# show backtraces -ENV RUST_BACKTRACE 1 - -# install tools and dependencies -RUN apt-get update && \ - DEBIAN_FRONTEND=noninteractive apt-get install -y \ - libssl1.1 \ - ca-certificates && \ -# apt cleanup - apt-get autoremove -y && \ - apt-get clean && \ - find /var/lib/apt/lists/ -type f -not -name lock -delete; \ -# add user and link ~/.local/share/polkadot to /data - useradd -m -u 1000 -U -s /bin/sh -d /polkadot polkadot && \ - mkdir -p /data /polkadot/.local/share && \ - chown -R polkadot:polkadot /data && \ - ln -s /data /polkadot/.local/share/polkadot - -# add polkadot binary to docker image -COPY ./polkadot ./polkadot-execute-worker ./polkadot-prepare-worker /usr/local/bin - -USER polkadot - -# check if executable works in this container -RUN /usr/local/bin/polkadot --version -RUN /usr/local/bin/polkadot-execute-worker --version -RUN /usr/local/bin/polkadot-prepare-worker --version - -EXPOSE 30333 9933 9944 -VOLUME ["/polkadot"] - -ENTRYPOINT ["/usr/local/bin/polkadot"] diff --git a/scripts/ci/dockerfiles/staking-miner/README.md b/scripts/ci/dockerfiles/staking-miner/README.md new file mode 100644 index 000000000000..3610e1130316 --- /dev/null +++ b/scripts/ci/dockerfiles/staking-miner/README.md @@ -0,0 +1,37 @@ +# staking-miner container image + +## Build using the Builder + +``` +./build.sh +``` + +## Build the injected Image + +You first need a valid Linux binary to inject. Let's assume this binary is located in `BIN_FOLDER`. + +``` +./build-injected.sh "$BIN_FOLDER" +``` + +## Test + +Here is how to test the image. We can generate a valid seed but the staking-miner will quickly notice that our +account is not funded and "does not exist". + +You may pass any ENV supported by the binary and must provide at least a few such as `SEED` and `URI`: +``` +ENV SEED="" +ENV URI="wss://rpc.polkadot.io:443" +ENV RUST_LOG="info" +``` + +``` +export SEED=$(subkey generate -n polkadot --output-type json | jq -r .secretSeed) +podman run --rm -it \ + -e URI="wss://rpc.polkadot.io:443" \ + -e RUST_LOG="info" \ + -e SEED \ + localhost/parity/staking-miner \ + dry-run seq-phragmen +``` diff --git a/scripts/ci/dockerfiles/staking-miner/build-injected.sh b/scripts/ci/dockerfiles/staking-miner/build-injected.sh new file mode 100755 index 000000000000..536636df6a91 --- /dev/null +++ b/scripts/ci/dockerfiles/staking-miner/build-injected.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +# Sample call: +# $0 /path/to/folder_with_staking-miner_binary +# This script replace the former dedicated staking-miner "injected" Dockerfile +# and shows how to use the generic binary_injected.dockerfile + +PROJECT_ROOT=`git rev-parse --show-toplevel` + +export BINARY=staking-miner +export BIN_FOLDER=$1 + +$PROJECT_ROOT/scripts/ci/dockerfiles/build-injected.sh diff --git a/scripts/ci/dockerfiles/staking-miner/build.sh b/scripts/ci/dockerfiles/staking-miner/build.sh new file mode 100755 index 000000000000..67c82afcd2ce --- /dev/null +++ b/scripts/ci/dockerfiles/staking-miner/build.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +# Sample call: +# $0 /path/to/folder_with_staking-miner_binary +# This script replace the former dedicated staking-miner "injected" Dockerfile +# and shows how to use the generic binary_injected.dockerfile + +PROJECT_ROOT=`git rev-parse --show-toplevel` +ENGINE=podman + +echo "Building the staking-miner using the Builder image" +echo "PROJECT_ROOT=$PROJECT_ROOT" +$ENGINE build -t staking-miner -f staking-miner_builder.Dockerfile "$PROJECT_ROOT" diff --git a/scripts/ci/dockerfiles/staking-miner/staking-miner_builder.Dockerfile b/scripts/ci/dockerfiles/staking-miner/staking-miner_builder.Dockerfile index a1932095fd4c..0ae77f36c79d 100644 --- a/scripts/ci/dockerfiles/staking-miner/staking-miner_builder.Dockerfile +++ b/scripts/ci/dockerfiles/staking-miner/staking-miner_builder.Dockerfile @@ -4,17 +4,17 @@ FROM paritytech/ci-linux:production as builder ARG VCS_REF ARG BUILD_DATE ARG IMAGE_NAME="staking-miner" -ARG PROFILE=release +ARG PROFILE=production LABEL description="This is the build stage. Here we create the binary." WORKDIR /app COPY . /app -RUN cargo build --locked --$PROFILE --package staking-miner +RUN cargo build --locked --profile $PROFILE --package staking-miner # ===== SECOND STAGE ====== -FROM docker.io/library/ubuntu:20.04 +FROM docker.io/parity/base-bin:latest LABEL description="This is the 2nd stage: a very small image where we copy the binary." LABEL io.parity.image.authors="devops-team@parity.io" \ io.parity.image.vendor="Parity Technologies" \ @@ -28,13 +28,10 @@ LABEL io.parity.image.authors="devops-team@parity.io" \ ARG PROFILE=release COPY --from=builder /app/target/$PROFILE/staking-miner /usr/local/bin -RUN useradd -u 1000 -U -s /bin/sh miner && \ - rm -rf /usr/bin /usr/sbin - # show backtraces ENV RUST_BACKTRACE 1 -USER miner +USER parity ENV SEED="" ENV URI="wss://rpc.polkadot.io" diff --git a/scripts/ci/dockerfiles/staking-miner/staking-miner_injected.Dockerfile b/scripts/ci/dockerfiles/staking-miner/staking-miner_injected.Dockerfile deleted file mode 100644 index 4901ab4a3736..000000000000 --- a/scripts/ci/dockerfiles/staking-miner/staking-miner_injected.Dockerfile +++ /dev/null @@ -1,43 +0,0 @@ -FROM docker.io/library/ubuntu:20.04 - -# metadata -ARG VCS_REF -ARG BUILD_DATE -ARG IMAGE_NAME="staking-miner" - -LABEL io.parity.image.authors="devops-team@parity.io" \ - io.parity.image.vendor="Parity Technologies" \ - io.parity.image.title="${IMAGE_NAME}" \ - io.parity.image.description="${IMAGE_NAME} for substrate based chains" \ - io.parity.image.source="https://github.com/paritytech/polkadot/blob/${VCS_REF}/scripts/ci/dockerfiles/${IMAGE_NAME}/${IMAGE_NAME}_injected.Dockerfile" \ - io.parity.image.revision="${VCS_REF}" \ - io.parity.image.created="${BUILD_DATE}" \ - io.parity.image.documentation="https://github.com/paritytech/polkadot/" - -# show backtraces -ENV RUST_BACKTRACE 1 - -# install tools and dependencies -RUN apt-get update && \ - DEBIAN_FRONTEND=noninteractive apt-get install -y \ - libssl1.1 \ - ca-certificates && \ -# apt cleanup - apt-get autoremove -y && \ - apt-get clean && \ - find /var/lib/apt/lists/ -type f -not -name lock -delete; \ - useradd -u 1000 -U -s /bin/sh miner - -# add binary to docker image -COPY ./staking-miner /usr/local/bin - -USER miner - -ENV SEED="" -ENV URI="wss://rpc.polkadot.io" -ENV RUST_LOG="info" - -# check if the binary works in this container -RUN /usr/local/bin/staking-miner --version - -ENTRYPOINT [ "/usr/local/bin/staking-miner" ] diff --git a/scripts/ci/dockerfiles/staking-miner/test-build.sh b/scripts/ci/dockerfiles/staking-miner/test-build.sh new file mode 100755 index 000000000000..0ce74e2df296 --- /dev/null +++ b/scripts/ci/dockerfiles/staking-miner/test-build.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash + +TMP=$(mktemp -d) +ENGINE=${ENGINE:-podman} + +# You need to build an injected image first + +# Fetch some binaries +$ENGINE run --user root --rm -i \ + -v "$TMP:/export" \ + --entrypoint /bin/bash \ + parity/staking-miner -c \ + 'cp "$(which staking-miner)" /export' + +echo "Checking binaries we got:" +tree $TMP + +./build-injected.sh $TMP diff --git a/scripts/ci/gitlab/pipeline/build.yml b/scripts/ci/gitlab/pipeline/build.yml index dafca393cd4f..845ac7970108 100644 --- a/scripts/ci/gitlab/pipeline/build.yml +++ b/scripts/ci/gitlab/pipeline/build.yml @@ -39,7 +39,6 @@ build-linux-stable: - echo -n ${CI_JOB_ID} > ./artifacts/BUILD_LINUX_JOB_ID - RELEASE_VERSION=$(./artifacts/polkadot -V | awk '{print $2}'| awk -F "-" '{print $1}') - echo -n "v${RELEASE_VERSION}" > ./artifacts/BUILD_RELEASE_VERSION - - cp -r scripts/* ./artifacts build-test-collators: stage: build @@ -64,7 +63,6 @@ build-test-collators: - echo -n "${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}" > ./artifacts/EXTRATAG - echo "adder-collator version = $(cat ./artifacts/VERSION) (EXTRATAG = $(cat ./artifacts/EXTRATAG))" - echo "undying-collator version = $(cat ./artifacts/VERSION) (EXTRATAG = $(cat ./artifacts/EXTRATAG))" - - cp -r ./scripts/* ./artifacts build-malus: stage: build @@ -88,7 +86,6 @@ build-malus: - echo -n "${CI_COMMIT_REF_NAME}" > ./artifacts/VERSION - echo -n "${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}" > ./artifacts/EXTRATAG - echo "polkadot-test-malus = $(cat ./artifacts/VERSION) (EXTRATAG = $(cat ./artifacts/EXTRATAG))" - - cp -r ./scripts/* ./artifacts build-staking-miner: stage: build @@ -110,7 +107,6 @@ build-staking-miner: - echo -n "${CI_COMMIT_REF_NAME}" > ./artifacts/VERSION - echo -n "${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}" > ./artifacts/EXTRATAG - echo "staking-miner = $(cat ./artifacts/VERSION) (EXTRATAG = $(cat ./artifacts/EXTRATAG))" - - cp -r ./scripts/* ./artifacts build-rustdoc: stage: build diff --git a/scripts/ci/gitlab/pipeline/publish.yml b/scripts/ci/gitlab/pipeline/publish.yml index d9a0dff95767..c224094125e3 100644 --- a/scripts/ci/gitlab/pipeline/publish.yml +++ b/scripts/ci/gitlab/pipeline/publish.yml @@ -19,20 +19,16 @@ publish-polkadot-debug-image: - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs - if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1 variables: - CI_IMAGE: ${BUILDAH_IMAGE} - GIT_STRATEGY: none - DOCKER_USER: ${PARITYPR_USER} - DOCKER_PASS: ${PARITYPR_PASS} - # scripts/ci/dockerfiles/polkadot_injected_debug.Dockerfile - DOCKERFILE: ci/dockerfiles/polkadot_injected_debug.Dockerfile - IMAGE_NAME: docker.io/paritypr/polkadot-debug + IMAGE_NAME: "polkadot-debug" + BINARY: "polkadot,polkadot-execute-worker,polkadot-prepare-worker" needs: - job: build-linux-stable artifacts: true after_script: + - !reference [.build-push-image, after_script] # pass artifacts to the zombienet-tests job # https://docs.gitlab.com/ee/ci/multi_project_pipelines.html#with-variable-inheritance - - echo "PARACHAINS_IMAGE_NAME=${IMAGE_NAME}" > ./artifacts/parachains.env + - echo "PARACHAINS_IMAGE_NAME=${IMAGE}" > ./artifacts/parachains.env - echo "PARACHAINS_IMAGE_TAG=$(cat ./artifacts/EXTRATAG)" >> ./artifacts/parachains.env artifacts: reports: @@ -48,20 +44,15 @@ publish-test-collators-image: - .build-push-image - .zombienet-refs variables: - CI_IMAGE: ${BUILDAH_IMAGE} - GIT_STRATEGY: none - DOCKER_USER: ${PARITYPR_USER} - DOCKER_PASS: ${PARITYPR_PASS} - # scripts/ci/dockerfiles/collator_injected.Dockerfile - DOCKERFILE: ci/dockerfiles/collator_injected.Dockerfile - IMAGE_NAME: docker.io/paritypr/colander + IMAGE_NAME: "colander" + BINARY: "adder-collator,undying-collator" needs: - job: build-test-collators artifacts: true after_script: - - buildah logout --all + - !reference [.build-push-image, after_script] # pass artifacts to the zombienet-tests job - - echo "COLLATOR_IMAGE_NAME=${IMAGE_NAME}" > ./artifacts/collator.env + - echo "COLLATOR_IMAGE_NAME=${IMAGE}" > ./artifacts/collator.env - echo "COLLATOR_IMAGE_TAG=$(cat ./artifacts/EXTRATAG)" >> ./artifacts/collator.env artifacts: reports: @@ -76,20 +67,15 @@ publish-malus-image: - .build-push-image - .zombienet-refs variables: - CI_IMAGE: ${BUILDAH_IMAGE} - GIT_STRATEGY: none - DOCKER_USER: ${PARITYPR_USER} - DOCKER_PASS: ${PARITYPR_PASS} - # scripts/ci/dockerfiles/malus_injected.Dockerfile - DOCKERFILE: ci/dockerfiles/malus_injected.Dockerfile - IMAGE_NAME: docker.io/paritypr/malus + IMAGE_NAME: "malus" + BINARY: "malus,polkadot-execute-worker,polkadot-prepare-worker" needs: - job: build-malus artifacts: true after_script: - - buildah logout "$IMAGE_NAME" + - !reference [.build-push-image, after_script] # pass artifacts to the zombienet-tests job - - echo "MALUS_IMAGE_NAME=${IMAGE_NAME}" > ./artifacts/malus.env + - echo "MALUS_IMAGE_NAME=${IMAGE}" > ./artifacts/malus.env - echo "MALUS_IMAGE_TAG=$(cat ./artifacts/EXTRATAG)" >> ./artifacts/malus.env artifacts: reports: @@ -103,13 +89,11 @@ publish-staking-miner-image: - .build-push-image - .publish-refs variables: - CI_IMAGE: ${BUILDAH_IMAGE} - # scripts/ci/dockerfiles/staking-miner/staking-miner_injected.Dockerfile - DOCKERFILE: ci/dockerfiles/staking-miner/staking-miner_injected.Dockerfile - IMAGE_NAME: docker.io/paritytech/staking-miner - GIT_STRATEGY: none - DOCKER_USER: ${Docker_Hub_User_Parity} - DOCKER_PASS: ${Docker_Hub_Pass_Parity} + IMAGE_NAME: "staking-miner" + BINARY: "staking-miner" + DOCKER_OWNER: "paritytech" + DOCKER_USER: "${Docker_Hub_User_Parity}" + DOCKER_PASS: "${Docker_Hub_Pass_Parity}" needs: - job: build-staking-miner artifacts: true @@ -122,11 +106,11 @@ publish-polkadot-image-description: DOCKER_PASSWORD: ${Docker_Hub_Pass_Parity} DOCKERHUB_REPOSITORY: parity/polkadot SHORT_DESCRIPTION: "Polkadot Official Docker Image" - README_FILEPATH: $CI_PROJECT_DIR/scripts/ci/dockerfiles/polkadot_Dockerfile.README.md + README_FILEPATH: $CI_PROJECT_DIR/scripts/ci/dockerfiles/polkadot/polkadot_Dockerfile.README.md rules: - if: $CI_COMMIT_REF_NAME == "master" changes: - - scripts/ci/dockerfiles/polkadot_Dockerfile.README.md + - scripts/ci/dockerfiles/polkadot/polkadot_Dockerfile.README.md - if: $CI_PIPELINE_SOURCE == "schedule" when: never script: diff --git a/utils/staking-miner/README.md b/utils/staking-miner/README.md index 4148677ee7ca..b7f70de573b0 100644 --- a/utils/staking-miner/README.md +++ b/utils/staking-miner/README.md @@ -28,8 +28,9 @@ There are 2 options to build a staking-miner Docker image: ### Building the injected image First build the binary as documented [above](#building). -You may then inject the binary into a Docker base image from the root of the Polkadot repository: +You may then inject the binary into a Docker base image: `parity/base-bin` (running the command from the root of the Polkadot repository): ``` +TODO: UPDATE THAT docker build -t staking-miner -f scripts/ci/dockerfiles/staking-miner/staking-miner_injected.Dockerfile target/release ``` @@ -39,6 +40,7 @@ Unlike the injected image that requires a Linux pre-built binary, this option do The trade-off however is that it takes a little longer to build and this option is less ideal for CI tasks. You may build the multi-stage image the root of the Polkadot repository with: ``` +TODO: UPDATE THAT docker build -t staking-miner -f scripts/ci/dockerfiles/staking-miner/staking-miner_builder.Dockerfile . ``` @@ -51,7 +53,7 @@ While it won't prevent a malicious actor to read your `SEED` if they gain access # The following line starts with an extra space on purpose: SEED=0x1234... -docker run --rm -it \ +docker run --rm -i \ --name staking-miner \ --read-only \ -e RUST_LOG=info \