From 4b0316cab78c3c6833ebbc558fd8975a93429656 Mon Sep 17 00:00:00 2001 From: lklimek <842586+lklimek@users.noreply.github.com> Date: Wed, 13 Nov 2024 14:02:29 +0100 Subject: [PATCH] ci: prebuilt librocksdb in docker image (#2318) --- .github/actions/docker/action.yaml | 1 + .github/actions/rust/action.yaml | 7 +- .github/workflows/tests-rs-package.yml | 10 +- .github/workflows/tests.yml | 10 +- Cargo.lock | 4 +- Dockerfile | 186 +++++++++++++----- .../v0/v0_methods.rs | 25 ++- 7 files changed, 167 insertions(+), 76 deletions(-) diff --git a/.github/actions/docker/action.yaml b/.github/actions/docker/action.yaml index d41ebf4dcfc..69bc9989d66 100644 --- a/.github/actions/docker/action.yaml +++ b/.github/actions/docker/action.yaml @@ -110,6 +110,7 @@ runs: SCCACHE_S3_KEY_PREFIX=${{ runner.os }}/sccache AWS_ACCESS_KEY_ID=${{ inputs.aws_access_key_id }} AWS_SECRET_ACCESS_KEY=${{ inputs.aws_secret_access_key }} + AWS_REGION=${{ inputs.region }} cache-from: ${{ steps.layer_cache_settings.outputs.cache_from }} cache-to: ${{ steps.layer_cache_settings.outputs.cache_to }} outputs: type=image,name=${{ inputs.image_org }}/${{ inputs.image_name }},push-by-digest=${{ inputs.push_tags != 'true' }},name-canonical=true,push=true diff --git a/.github/actions/rust/action.yaml b/.github/actions/rust/action.yaml index c22923a221b..91544508733 100644 --- a/.github/actions/rust/action.yaml +++ b/.github/actions/rust/action.yaml @@ -85,9 +85,9 @@ runs: export PATH="${PATH}:${HOME}/.local/bin" - name: Run sccache-cache - uses: mozilla-actions/sccache-action@v0.0.3 + uses: mozilla-actions/sccache-action@v0.0.6 with: - version: "v0.7.1" # Must be the same as in Dockerfile + version: "v0.8.2" # Must be the same as in Dockerfile if: inputs.cache == 'true' - name: Hash ref_name @@ -121,5 +121,6 @@ runs: if: runner.os == 'Linux' run: | sudo apt update -qq - sudo apt install -qq --yes clang llvm + # snappy is required by rust rocksdb + sudo apt install -qq --yes clang llvm libsnappy-dev sudo update-alternatives --set cc /usr/bin/clang diff --git a/.github/workflows/tests-rs-package.yml b/.github/workflows/tests-rs-package.yml index a05e68ee6ce..fa038285f2c 100644 --- a/.github/workflows/tests-rs-package.yml +++ b/.github/workflows/tests-rs-package.yml @@ -55,7 +55,8 @@ jobs: SCCACHE_S3_KEY_PREFIX: ${{ runner.os }}/sccache/${{ runner.arch }}/linux-gnu ROCKSDB_STATIC: "/opt/rocksdb/usr/local/lib/librocksdb.a" ROCKSDB_LIB_DIR: "/opt/rocksdb/usr/local/lib" - + SNAPPY_STATIC: "/usr/lib/x86_64-linux-gnu/libsnappy.a" + SNAPPY_LIB_DIR: "/usr/lib/x86_64-linux-gnu" formatting: name: Formatting runs-on: ubuntu-24.04 @@ -119,6 +120,8 @@ jobs: SCCACHE_S3_KEY_PREFIX: ${{ runner.os }}/sccache/${{ runner.arch }}/linux-gnu ROCKSDB_STATIC: "/opt/rocksdb/usr/local/lib/librocksdb.a" ROCKSDB_LIB_DIR: "/opt/rocksdb/usr/local/lib" + SNAPPY_STATIC: "/usr/lib/x86_64-linux-gnu/libsnappy.a" + SNAPPY_LIB_DIR: "/usr/lib/x86_64-linux-gnu" with: args: ${{ steps.crate_info.outputs.cargo_manifest_dir }} @@ -213,7 +216,8 @@ jobs: SCCACHE_S3_KEY_PREFIX: ${{ runner.os }}/sccache/${{ runner.arch }}/linux-gnu ROCKSDB_STATIC: "/opt/rocksdb/usr/local/lib/librocksdb.a" ROCKSDB_LIB_DIR: "/opt/rocksdb/usr/local/lib" - + SNAPPY_STATIC: "/usr/lib/x86_64-linux-gnu/libsnappy.a" + SNAPPY_LIB_DIR: "/usr/lib/x86_64-linux-gnu" check_each_feature: name: Check each feature runs-on: ${{ fromJSON(inputs.test-runner) }} @@ -250,6 +254,8 @@ jobs: SCCACHE_S3_KEY_PREFIX: ${{ runner.os }}/sccache/${{ runner.arch }}/linux-gnu ROCKSDB_STATIC: "/opt/rocksdb/usr/local/lib/librocksdb.a" ROCKSDB_LIB_DIR: "/opt/rocksdb/usr/local/lib" + SNAPPY_STATIC: "/usr/lib/x86_64-linux-gnu/libsnappy.a" + SNAPPY_LIB_DIR: "/usr/lib/x86_64-linux-gnu" run: | echo Verify all features disabled set -ex diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 5f6cec2c08d..46d368b0b2b 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -2,11 +2,6 @@ name: Tests on: workflow_dispatch: - inputs: - rebuild-deps: - description: "Rebuild cached Rust dependencies" - required: false - default: "false" pull_request: types: [opened, synchronize, reopened, ready_for_review] branches: @@ -44,11 +39,14 @@ jobs: build-rust-deps: name: Prebuild and cache some Rust dependencies runs-on: ubuntu-24.04 + needs: + - changes + # run only if any changes were detected, eg. changes job returned non-empty list + if: ${{ needs.changes.outputs.rs-packages != '[]' }} env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} AWS_REGION: ${{ secrets.AWS_REGION }} - steps: - name: Checkout uses: actions/checkout@v4 diff --git a/Cargo.lock b/Cargo.lock index c59d07d2b49..d2e84d05c40 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4327,9 +4327,9 @@ checksum = "d66dc143e6b11c1eddc06d5c423cfc97062865baf299914ab64caa38182078fe" [[package]] name = "simdutf8" -version = "0.1.4" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f27f6278552951f1f2b8cf9da965d10969b2efdea95a6ec47987ab46edfe263a" +checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" [[package]] name = "simple-signer" diff --git a/Dockerfile b/Dockerfile index 23766e2f7dd..1a7273e31c6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,8 +5,10 @@ # This image is divided multiple parts: # - deps-base - includes all base dependencies and some libraries # - deps-sccache - deps image with sccache included +# - deps-compilation - deps image with all compilation dependencies - it's either deps-base or deps-sccache +# - deps-rocksdb - build static rocksdb library # - deps - all deps, including wasm-bindgen-cli; built on top of either deps-base or deps-sccache -# - sources - includes full source code +# - build-planner - image used to prepare build plan for rs-drive-abci # - build-* - actual build process of given image # - drive-abci, dashmate-helper, test-suite, dapi - final images # @@ -15,19 +17,22 @@ # - NODE_ENV - node.js environment name to use to build the library # - RUSTC_WRAPPER - set to `sccache` to enable sccache support and make the following variables available: # - SCCACHE_GHA_ENABLED, ACTIONS_CACHE_URL, ACTIONS_RUNTIME_TOKEN - store sccache caches inside github actions +# - SCCACHE_BUCKET, AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_REGION, SCCACHE_S3_KEY_PREFIX - store caches in S3 # - SCCACHE_MEMCACHED - set to memcache server URI (eg. tcp://172.17.0.1:11211) to enable sccache memcached backend # - ALPINE_VERSION - use different version of Alpine base image; requires also rust:apline... # image to be available # - USERNAME, USER_UID, USER_GID - specification of user used to run the binary # +# +# +# # BUILD PROCESS # # 1. All these --mount... are to cache reusable info between runs. # See https://doc.rust-lang.org/cargo/guide/cargo-home.html#caching-the-cargo-home-in-ci -# 2. We add `--config net.git-fetch-with-cli=true` to address ARM build issue, -# see https://github.com/rust-lang/cargo/issues/10781#issuecomment-1441071052 -# 3. Github Actions have shared networking configured, so we need to set a random -# SCCACHE_SERVER_PORT port to avoid conflicts in case of parallel compilation +# 2. Github Actions have shared networking configured, so we need to set a random SCCACHE_SERVER_PORT port to avoid +# conflicts in case of parallel compilation. +# 3. Configuration variables are shared between runs using /root/env file. ARG ALPINE_VERSION=3.18 ARG PROTOC_VERSION=27.3 @@ -53,6 +58,7 @@ RUN apk add --no-cache \ linux-headers \ llvm-static llvm-dev \ openssl-dev \ + snappy-static snappy-dev \ perl \ python3 \ unzip \ @@ -60,6 +66,13 @@ RUN apk add --no-cache \ xz \ zeromq-dev +# Configure snappy, dependency of librocksdb-sys +RUN <> /root/env +echo "export SNAPPY_LIB_DIR=/usr/lib" >> /root/env +echo "export SNAPPY_INCLUDE_DIR=/usr/include" >> /root/env +EOS + # Configure Node.js RUN npm config set --global audit false @@ -100,9 +113,20 @@ ENV CARGO_BUILD_PROFILE ${CARGO_BUILD_PROFILE} ARG NODE_ENV=production ENV NODE_ENV ${NODE_ENV} +# +# DEPS-SCCACHE stage +# +# This stage is used to install sccache and configure it. +# Later on, one should source /root/env before building to use sccache. + +# Note that, due to security concerns, each stage needs to declare variables containing authentication secrets, like +# ACTIONS_RUNTIME_TOKEN, AWS_SECRET_ACCESS_KEY. It is done using ONBUILD directive, so the secrets are not stored in the +# final image. + FROM deps-base AS deps-sccache -ARG SCCHACHE_VERSION=0.7.1 +# SCCACHE_VERSION must be the same as in github actions, to avoid cache incompatibility +ARG SCCHACHE_VERSION=0.8.2 # Install sccache for caching RUN if [[ "$TARGETARCH" == "arm64" ]] ; then export SCC_ARCH=aarch64; else export SCC_ARCH=x86_64; fi; \ @@ -115,44 +139,115 @@ RUN if [[ "$TARGETARCH" == "arm64" ]] ; then export SCC_ARCH=aarch64; else expor # Configure sccache # ARG RUSTC_WRAPPER -ENV RUSTC_WRAPPER=${RUSTC_WRAPPER} + +# Disable incremental builds, not supported by sccache +RUN echo 'export CARGO_INCREMENTAL=false' >> /root/env # Set args below to use Github Actions cache; see https://github.com/mozilla/sccache/blob/main/docs/GHA.md ARG SCCACHE_GHA_ENABLED -ENV SCCACHE_GHA_ENABLED=${SCCACHE_GHA_ENABLED} - ARG ACTIONS_CACHE_URL -ENV ACTIONS_CACHE_URL=${ACTIONS_CACHE_URL} - -ARG ACTIONS_RUNTIME_TOKEN -ENV ACTIONS_RUNTIME_TOKEN=${ACTIONS_RUNTIME_TOKEN} # Alternative solution is to use memcache ARG SCCACHE_MEMCACHED -ENV SCCACHE_MEMCACHED=${SCCACHE_MEMCACHED} # S3 storage ARG SCCACHE_BUCKET -ENV SCCACHE_BUCKET=${SCCACHE_BUCKET} - +ARG AWS_ACCESS_KEY_ID +ARG AWS_REGION ARG SCCACHE_REGION -ENV SCCACHE_REGION=${SCCACHE_REGION} +ARG SCCACHE_S3_KEY_PREFIX -# Disable incremental buildings, not supported by sccache -ARG CARGO_INCREMENTAL=false -ENV CARGO_INCREMENTAL=${CARGO_INCREMENTAL} +# Generate sccache configuration variables and save them to /root/env +# +# We only enable one cache at a time. Setting env variables belonging to multiple cache backends may fail the build. +RUN <> /root/env + echo "export ACTIONS_CACHE_URL=${ACTIONS_CACHE_URL}" >> /root/env + # ACTIONS_RUNTIME_TOKEN is a secret so we load it using ONBUILD ARG later on + elif [ -n "${SCCACHE_BUCKET}" ]; then + # AWS S3 + if [ -z "${SCCACHE_REGION}" ] ; then + # Default to AWS_REGION if not set + export SCCACHE_REGION=${AWS_REGION} + fi + + echo "export AWS_REGION='${AWS_REGION}'" >> /root/env + echo "export SCCACHE_REGION='${SCCACHE_REGION}'" >> /root/env + echo "export AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID}" >> /root/env + # AWS_SECRET_ACCESS_KEY is a secret so we load it using ONBUILD ARG later on + echo "export SCCACHE_BUCKET='${SCCACHE_BUCKET}'" >> /root/env + echo "export SCCACHE_S3_USE_SSL=true" >> /root/env + echo "export SCCACHE_S3_KEY_PREFIX='${SCCACHE_S3_KEY_PREFIX}/${TARGETARCH}/linux-musl'" >> /root/env + elif [ -n "${SCCACHE_MEMCACHED}" ]; then + # memcached + echo "export SCCACHE_MEMCACHED='${SCCACHE_MEMCACHED}'" >> /root/env + fi + + if [ -n "${RUSTC_WRAPPER}" ]; then + echo "export CXX='${RUSTC_WRAPPER} clang++'" >> /root/env + echo "export CC='${RUSTC_WRAPPER} clang'" >> /root/env + echo "export RUSTC_WRAPPER='${RUSTC_WRAPPER}'" >> /root/env + echo "export SCCACHE_SERVER_PORT=$((RANDOM+1025))" >> /root/env + fi + # for debugging, we display what we generated + cat /root/env +EOS + +# We provide secrets using ONBUILD ARG mechanism, to avoid putting them into a file and potentialy leaking them +# to the final image or to layer cache +ONBUILD ARG ACTIONS_RUNTIME_TOKEN +ONBUILD ARG AWS_SECRET_ACCESS_KEY + +# Image containing compolation dependencies; used to overcome lack of interpolation in COPY --from +FROM deps-${RUSTC_WRAPPER:-base} AS deps-compilation +# Stage intentionally left empty + +# +# BUILD ROCKSDB STATIC LIBRARY +# + +FROM deps-compilation AS deps-rocksdb + +RUN mkdir -p /tmp/rocksdb +WORKDIR /tmp/rocksdb + +RUN <> /root/env +echo "export ROCKSDB_LIB_DIR=/opt/rocksdb/usr/local/lib" >> /root/env +echo "export ROCKSDB_INCLUDE_DIR=/opt/rocksdb/usr/local/include" >> /root/env +EOS -ARG AWS_ACCESS_KEY_ID -ARG AWS_SECRET_ACCESS_KEY # # DEPS: FULL DEPENDENCIES LIST # -# This is separate from `deps` to use sccache for caching -FROM deps-${RUSTC_WRAPPER:-base} AS deps +FROM deps-rocksdb AS deps -ARG SCCACHE_S3_KEY_PREFIX -ENV SCCACHE_S3_KEY_PREFIX=${SCCACHE_S3_KEY_PREFIX}/${TARGETARCH}/linux-musl WORKDIR /platform @@ -162,24 +257,27 @@ RUN --mount=type=cache,sharing=shared,id=cargo_registry_index,target=${CARGO_HOM --mount=type=cache,sharing=shared,id=cargo_registry_cache,target=${CARGO_HOME}/registry/cache \ --mount=type=cache,sharing=shared,id=cargo_git,target=${CARGO_HOME}/git/db \ --mount=type=cache,sharing=shared,id=target_${TARGETARCH},target=/platform/target \ - export SCCACHE_SERVER_PORT=$((RANDOM+1025)) && \ source $HOME/.cargo/env && \ - if [[ -z "${SCCACHE_MEMCACHED}" ]] ; then unset SCCACHE_MEMCACHED ; fi ; \ + source /root/env && \ RUSTFLAGS="-C target-feature=-crt-static" \ CARGO_TARGET_DIR="/platform/target" \ # TODO: Build wasm with build.rs # Meanwhile if you want to update wasm-bindgen you also need to update version in: # - packages/wasm-dpp/Cargo.toml # - packages/wasm-dpp/scripts/build-wasm.sh - cargo install --profile "$CARGO_BUILD_PROFILE" wasm-bindgen-cli@0.2.86 cargo-chef@0.1.67 --locked + cargo install --profile "$CARGO_BUILD_PROFILE" wasm-bindgen-cli@0.2.86 cargo-chef@0.1.67 --locked && \ + if [[ -x /usr/bin/sccache ]]; then sccache --show-stats; fi # # Rust build planner to speed up builds # FROM deps AS build-planner + WORKDIR /platform COPY . . + RUN source $HOME/.cargo/env && \ + source /root/env && \ cargo chef prepare --recipe-path recipe.json # Workaround: as we cache dapi-grpc, its build.rs is not rerun, so we need to touch it @@ -193,8 +291,6 @@ FROM deps AS build-drive-abci SHELL ["/bin/bash", "-o", "pipefail","-e", "-x", "-c"] -ARG SCCACHE_S3_KEY_PREFIX -ENV SCCACHE_S3_KEY_PREFIX=${SCCACHE_S3_KEY_PREFIX}/${TARGETARCH}/linux-musl WORKDIR /platform @@ -206,14 +302,13 @@ RUN --mount=type=cache,sharing=shared,id=cargo_registry_index,target=${CARGO_HOM --mount=type=cache,sharing=shared,id=cargo_git,target=${CARGO_HOME}/git/db \ --mount=type=cache,sharing=shared,id=target_${TARGETARCH},target=/platform/target \ source $HOME/.cargo/env && \ - export SCCACHE_SERVER_PORT=$((RANDOM+1025)) && \ - if [[ -z "${SCCACHE_MEMCACHED}" ]] ; then unset SCCACHE_MEMCACHED ; fi ; \ + source /root/env && \ cargo chef cook \ --recipe-path recipe.json \ --profile "$CARGO_BUILD_PROFILE" \ --package drive-abci \ --locked && \ - if [[ "${RUSTC_WRAPPER}" == "sccache" ]] ; then sccache --show-stats; fi + if [[ -x /usr/bin/sccache ]]; then sccache --show-stats; fi COPY . . @@ -225,32 +320,27 @@ RUN --mount=type=cache,sharing=shared,id=cargo_registry_index,target=${CARGO_HOM --mount=type=cache,sharing=shared,id=cargo_git,target=${CARGO_HOME}/git/db \ --mount=type=cache,sharing=shared,id=target_${TARGETARCH},target=/platform/target \ source $HOME/.cargo/env && \ - export SCCACHE_SERVER_PORT=$((RANDOM+1025)) && \ + source /root/env && \ if [[ "${CARGO_BUILD_PROFILE}" == "release" ]] ; then \ mv .cargo/config-release.toml .cargo/config.toml && \ export OUT_DIRECTORY=release ; \ else \ export FEATURES_FLAG="--features=console,grovedbg" ; \ export OUT_DIRECTORY=debug ; \ - fi && \ - if [[ -z "${SCCACHE_MEMCACHED}" ]] ; then unset SCCACHE_MEMCACHED ; fi ; \ cargo build \ --profile "${CARGO_BUILD_PROFILE}" \ --package drive-abci \ ${FEATURES_FLAG} \ --locked && \ - cp /platform/target/${OUT_DIRECTORY}/drive-abci /artifacts/ && \ - if [[ "${RUSTC_WRAPPER}" == "sccache" ]] ; then sccache --show-stats; fi - + cp /platform/target/${OUT_DIRECTORY}/drive-abci /artifacts/ && \ + if [[ -x /usr/bin/sccache ]]; then sccache --show-stats; fi + # # STAGE: BUILD JAVASCRIPT INTERMEDIATE IMAGE # FROM deps AS build-js -ARG SCCACHE_S3_KEY_PREFIX -ENV SCCACHE_S3_KEY_PREFIX=${SCCACHE_S3_KEY_PREFIX}/wasm/wasm32 - WORKDIR /platform COPY --from=build-planner /platform/recipe.json recipe.json @@ -261,15 +351,14 @@ RUN --mount=type=cache,sharing=shared,id=cargo_registry_index,target=${CARGO_HOM --mount=type=cache,sharing=shared,id=cargo_git,target=${CARGO_HOME}/git/db \ --mount=type=cache,sharing=shared,id=target_${TARGETARCH},target=/platform/target \ source $HOME/.cargo/env && \ - export SCCACHE_SERVER_PORT=$((RANDOM+1025)) && \ - if [[ -z "${SCCACHE_MEMCACHED}" ]] ; then unset SCCACHE_MEMCACHED ; fi ; \ + source /root/env && \ cargo chef cook \ --recipe-path recipe.json \ --profile "$CARGO_BUILD_PROFILE" \ --package wasm-dpp \ --target wasm32-unknown-unknown \ --locked && \ - if [[ "${RUSTC_WRAPPER}" == "sccache" ]] ; then sccache --show-stats; fi + if [[ -x /usr/bin/sccache ]]; then sccache --show-stats; fi COPY . . @@ -279,14 +368,13 @@ RUN --mount=type=cache,sharing=shared,id=cargo_registry_index,target=${CARGO_HOM --mount=type=cache,sharing=shared,id=target_wasm,target=/platform/target \ --mount=type=cache,sharing=shared,id=unplugged_${TARGETARCH},target=/tmp/unplugged \ source $HOME/.cargo/env && \ + source /root/env && \ cp -R /tmp/unplugged /platform/.yarn/ && \ yarn install --inline-builds && \ cp -R /platform/.yarn/unplugged /tmp/ && \ - export SCCACHE_SERVER_PORT=$((RANDOM+1025)) && \ - if [[ -z "${SCCACHE_MEMCACHED}" ]] ; then unset SCCACHE_MEMCACHED ; fi ; \ export SKIP_GRPC_PROTO_BUILD=1 && \ yarn build && \ - if [[ "${RUSTC_WRAPPER}" == "sccache" ]]; then sccache --show-stats; fi + if [[ -x /usr/bin/sccache ]]; then sccache --show-stats; fi # # STAGE: FINAL DRIVE-ABCI IMAGE diff --git a/packages/rs-dpp/src/state_transition/state_transitions/identity/identity_credit_transfer_transition/v0/v0_methods.rs b/packages/rs-dpp/src/state_transition/state_transitions/identity/identity_credit_transfer_transition/v0/v0_methods.rs index e04e061f677..cf376b9b83a 100644 --- a/packages/rs-dpp/src/state_transition/state_transitions/identity/identity_credit_transfer_transition/v0/v0_methods.rs +++ b/packages/rs-dpp/src/state_transition/state_transitions/identity/identity_credit_transfer_transition/v0/v0_methods.rs @@ -52,21 +52,18 @@ impl IdentityCreditTransferTransitionMethodsV0 for IdentityCreditTransferTransit ); } } - None => { - let key = identity - .get_first_public_key_matching( - Purpose::TRANSFER, - SecurityLevel::full_range().into(), - KeyType::all_key_types().into(), - true, + None => identity + .get_first_public_key_matching( + Purpose::TRANSFER, + SecurityLevel::full_range().into(), + KeyType::all_key_types().into(), + true, + ) + .ok_or_else(|| { + ProtocolError::DesiredKeyWithTypePurposeSecurityLevelMissing( + "no transfer public key".to_string(), ) - .ok_or_else(|| { - ProtocolError::DesiredKeyWithTypePurposeSecurityLevelMissing( - "no transfer public key".to_string(), - ) - })?; - key - } + })?, }; transition.sign_external(