From 8fba6944aefc4e0031012346333878f32141ec59 Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Fri, 8 Nov 2024 09:55:26 +0100 Subject: [PATCH 01/27] ci: prebuild librocksdb in docker image --- .github/workflows/tests-build-image.yml | 5 ++-- Dockerfile | 35 ++++++++++++++++++++++++- 2 files changed, 37 insertions(+), 3 deletions(-) diff --git a/.github/workflows/tests-build-image.yml b/.github/workflows/tests-build-image.yml index f8b7c830f42..d21ddd9c132 100644 --- a/.github/workflows/tests-build-image.yml +++ b/.github/workflows/tests-build-image.yml @@ -17,7 +17,8 @@ on: jobs: build-image: name: Build ${{ inputs.name }} image - runs-on: ["self-hosted", "linux", "arm64", "ubuntu-platform"] + # runs-on: ["self-hosted", "linux", "arm64", "ubuntu-platform"] + runs-on: ubuntu-24.04 steps: - name: Check out repo uses: actions/checkout@v4 @@ -41,7 +42,7 @@ jobs: image_name: ${{ inputs.image_name }} image_org: ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.${{ secrets.AWS_REGION }}.amazonaws.com target: ${{ inputs.target }} - platform: linux/arm64 + platform: linux/amd64 push_tags: true dockerhub_username: ${{ secrets.DOCKERHUB_USERNAME }} dockerhub_token: ${{ secrets.DOCKERHUB_TOKEN }} diff --git a/Dockerfile b/Dockerfile index 5039a062b69..59622a7086b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -146,6 +146,20 @@ ARG AWS_ACCESS_KEY_ID ARG AWS_SECRET_ACCESS_KEY # +# BUILD ROCKSDB STATIC LIBRARY +# +FROM deps-${RUSTC_WRAPPER:-base} AS rocksdb + +RUN mkdir -p /tmp/rocksdb +WORKDIR /tmp/rocksdb +RUN git clone https://github.com/facebook/rocksdb.git -b v8.10.2 --depth 1 . && \ + make -j$(nproc) static_lib && \ + mkdir -p /opt/rocksdb/usr/local/lib && \ + cp librocksdb.a /opt/rocksdb/usr/local/lib/ && \ + cp -r include /opt/rocksdb/usr/local/lib/ && \ + cd / && \ + rm -rf /tmp/rocksdb + # DEPS: FULL DEPENDENCIES LIST # # This is separate from `deps` to use sccache for caching @@ -154,6 +168,14 @@ FROM deps-${RUSTC_WRAPPER:-base} AS deps ARG SCCACHE_S3_KEY_PREFIX ENV SCCACHE_S3_KEY_PREFIX=${SCCACHE_S3_KEY_PREFIX}/${TARGETARCH}/linux-musl +# Install prebuilt rocksdb library + +COPY --from=rocksdb /opt/rocksdb /opt/rocksdb +# Set env variables so that Rust's rocksdb-sys will not build rocksdb from scratch + +ENV ROCKSDB_STATIC=/opt/rocksdb/usr/local/lib/librocksdb.a +ENV ROCKSDB_LIB_DIR=/opt/rocksdb/usr/local/lib + WORKDIR /platform # Install wasm-bindgen-cli in the same profile as other components, to sacrifice some performance & disk space to gain @@ -171,12 +193,17 @@ RUN --mount=type=cache,sharing=shared,id=cargo_registry_index,target=${CARGO_HOM # Meanwhile if you want to update wasm-bindgen you also need to update version in: # - packages/wasm-dpp/Cargo.toml # - packages/wasm-dpp/scripts/build-wasm.sh - cargo install --profile "$CARGO_BUILD_PROFILE" wasm-bindgen-cli@0.2.86 cargo-chef@0.1.67 --locked + cargo install --profile "$CARGO_BUILD_PROFILE" wasm-bindgen-cli@0.2.86 cargo-chef@0.1.67 --locked + # # Rust build planner to speed up builds # FROM deps AS build-planner + +ENV ROCKSDB_STATIC=/opt/rocksdb/usr/local/lib/librocksdb.a +ENV ROCKSDB_LIB_DIR=/opt/rocksdb/usr/local/lib + WORKDIR /platform COPY . . RUN source $HOME/.cargo/env && \ @@ -196,6 +223,9 @@ SHELL ["/bin/bash", "-o", "pipefail","-e", "-x", "-c"] ARG SCCACHE_S3_KEY_PREFIX ENV SCCACHE_S3_KEY_PREFIX=${SCCACHE_S3_KEY_PREFIX}/${TARGETARCH}/linux-musl +ENV ROCKSDB_STATIC=/opt/rocksdb/usr/local/lib/librocksdb.a +ENV ROCKSDB_LIB_DIR=/opt/rocksdb/usr/local/lib + WORKDIR /platform COPY --from=build-planner /platform/recipe.json recipe.json @@ -251,6 +281,9 @@ FROM deps AS build-js ARG SCCACHE_S3_KEY_PREFIX ENV SCCACHE_S3_KEY_PREFIX=${SCCACHE_S3_KEY_PREFIX}/wasm/wasm32 +ENV ROCKSDB_STATIC=/opt/rocksdb/usr/local/lib/librocksdb.a +ENV ROCKSDB_LIB_DIR=/opt/rocksdb/usr/local/lib + WORKDIR /platform COPY --from=build-planner /platform/recipe.json recipe.json From e056b9f93029a414bd2b0ea02604a2ef49002203 Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Fri, 8 Nov 2024 09:58:30 +0100 Subject: [PATCH 02/27] chore: fix dockerfile build logic --- .github/actions/docker/action.yaml | 1 + .github/workflows/tests-build-image.yml | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/actions/docker/action.yaml b/.github/actions/docker/action.yaml index d41ebf4dcfc..69bc9989d66 100644 --- a/.github/actions/docker/action.yaml +++ b/.github/actions/docker/action.yaml @@ -110,6 +110,7 @@ runs: SCCACHE_S3_KEY_PREFIX=${{ runner.os }}/sccache AWS_ACCESS_KEY_ID=${{ inputs.aws_access_key_id }} AWS_SECRET_ACCESS_KEY=${{ inputs.aws_secret_access_key }} + AWS_REGION=${{ inputs.region }} cache-from: ${{ steps.layer_cache_settings.outputs.cache_from }} cache-to: ${{ steps.layer_cache_settings.outputs.cache_to }} outputs: type=image,name=${{ inputs.image_org }}/${{ inputs.image_name }},push-by-digest=${{ inputs.push_tags != 'true' }},name-canonical=true,push=true diff --git a/.github/workflows/tests-build-image.yml b/.github/workflows/tests-build-image.yml index d21ddd9c132..737ab87423d 100644 --- a/.github/workflows/tests-build-image.yml +++ b/.github/workflows/tests-build-image.yml @@ -17,8 +17,8 @@ on: jobs: build-image: name: Build ${{ inputs.name }} image - # runs-on: ["self-hosted", "linux", "arm64", "ubuntu-platform"] - runs-on: ubuntu-24.04 + runs-on: ["self-hosted", "linux", "arm64", "ubuntu-platform"] + # runs-on: ubuntu-24.04 steps: - name: Check out repo uses: actions/checkout@v4 @@ -42,7 +42,7 @@ jobs: image_name: ${{ inputs.image_name }} image_org: ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.${{ secrets.AWS_REGION }}.amazonaws.com target: ${{ inputs.target }} - platform: linux/amd64 + platform: linux/arm64 push_tags: true dockerhub_username: ${{ secrets.DOCKERHUB_USERNAME }} dockerhub_token: ${{ secrets.DOCKERHUB_TOKEN }} From 714d9991411877e1824c046930e1efc5854c75a1 Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Fri, 8 Nov 2024 12:36:41 +0100 Subject: [PATCH 03/27] chore: move test image build to gh runners --- .github/workflows/tests-build-image.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/tests-build-image.yml b/.github/workflows/tests-build-image.yml index 737ab87423d..d21ddd9c132 100644 --- a/.github/workflows/tests-build-image.yml +++ b/.github/workflows/tests-build-image.yml @@ -17,8 +17,8 @@ on: jobs: build-image: name: Build ${{ inputs.name }} image - runs-on: ["self-hosted", "linux", "arm64", "ubuntu-platform"] - # runs-on: ubuntu-24.04 + # runs-on: ["self-hosted", "linux", "arm64", "ubuntu-platform"] + runs-on: ubuntu-24.04 steps: - name: Check out repo uses: actions/checkout@v4 @@ -42,7 +42,7 @@ jobs: image_name: ${{ inputs.image_name }} image_org: ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.${{ secrets.AWS_REGION }}.amazonaws.com target: ${{ inputs.target }} - platform: linux/arm64 + platform: linux/amd64 push_tags: true dockerhub_username: ${{ secrets.DOCKERHUB_USERNAME }} dockerhub_token: ${{ secrets.DOCKERHUB_TOKEN }} From 35de1fa0baac0e9da226895d8f6a25378056c21c Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Fri, 8 Nov 2024 12:41:05 +0100 Subject: [PATCH 04/27] chore: don't run build-rust-deps when not needed --- .github/workflows/tests.yml | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 5f6cec2c08d..46d368b0b2b 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -2,11 +2,6 @@ name: Tests on: workflow_dispatch: - inputs: - rebuild-deps: - description: "Rebuild cached Rust dependencies" - required: false - default: "false" pull_request: types: [opened, synchronize, reopened, ready_for_review] branches: @@ -44,11 +39,14 @@ jobs: build-rust-deps: name: Prebuild and cache some Rust dependencies runs-on: ubuntu-24.04 + needs: + - changes + # run only if any changes were detected, eg. changes job returned non-empty list + if: ${{ needs.changes.outputs.rs-packages != '[]' }} env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} AWS_REGION: ${{ secrets.AWS_REGION }} - steps: - name: Checkout uses: actions/checkout@v4 From c8871b5cf6e72cbb06f8b9c88f2cce22e0d2286a Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Fri, 8 Nov 2024 12:43:12 +0100 Subject: [PATCH 05/27] chore: update rust-decimal to trigger workflow --- packages/rs-dpp/Cargo.toml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/rs-dpp/Cargo.toml b/packages/rs-dpp/Cargo.toml index ff377de7480..e2c9ab43392 100644 --- a/packages/rs-dpp/Cargo.toml +++ b/packages/rs-dpp/Cargo.toml @@ -29,7 +29,7 @@ dashcore = { git = "https://github.com/dashpay/rust-dashcore", features = [ "signer", "serde", "bls", - "eddsa" + "eddsa", ], default-features = false, tag = "0.32.0" } env_logger = { version = "0.11" } getrandom = { version = "0.2", features = ["js"] } @@ -58,8 +58,8 @@ platform-serialization = { path = "../rs-platform-serialization" } platform-serialization-derive = { path = "../rs-platform-serialization-derive" } derive_more = { version = "1.0", features = ["from", "display"] } nohash-hasher = "0.2.0" -rust_decimal = "1.29.1" -rust_decimal_macros = "1.29.1" +rust_decimal = { version = "1.36.0", default-features = false } +rust_decimal_macros = "1.36.0" indexmap = { version = "2.0.2", features = ["serde"] } strum = { version = "0.26", features = ["derive"] } json-schema-compatibility-validator = { path = '../rs-json-schema-compatibility-validator' } From efe95f0b0fb3edca298b62e05e84984478e51c4c Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Fri, 8 Nov 2024 12:45:47 +0100 Subject: [PATCH 06/27] chore: update rocsdb in drive-abci --- Cargo.lock | 183 +----------------------------- packages/rs-drive-abci/Cargo.toml | 4 +- 2 files changed, 5 insertions(+), 182 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c59d07d2b49..1733409e87c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -34,17 +34,6 @@ dependencies = [ "cpufeatures", ] -[[package]] -name = "ahash" -version = "0.7.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "891477e0c6a8957309ee5c45a6368af3ae14bb510732d2684ffa19af310920f9" -dependencies = [ - "getrandom", - "once_cell", - "version_check", -] - [[package]] name = "ahash" version = "0.8.11" @@ -533,30 +522,6 @@ dependencies = [ "serde", ] -[[package]] -name = "borsh" -version = "1.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6362ed55def622cddc70a4746a68554d7b687713770de539e59a739b249f8ed" -dependencies = [ - "borsh-derive", - "cfg_aliases", -] - -[[package]] -name = "borsh-derive" -version = "1.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3ef8005764f53cd4dca619f5bf64cafd4664dada50ece25e4d81de54c80cc0b" -dependencies = [ - "once_cell", - "proc-macro-crate 3.1.0", - "proc-macro2", - "quote", - "syn 2.0.75", - "syn_derive", -] - [[package]] name = "bs58" version = "0.5.1" @@ -572,28 +537,6 @@ version = "3.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" -[[package]] -name = "bytecheck" -version = "0.6.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23cdc57ce23ac53c931e88a43d06d070a6fd142f2617be5855eb75efc9beb1c2" -dependencies = [ - "bytecheck_derive", - "ptr_meta", - "simdutf8", -] - -[[package]] -name = "bytecheck_derive" -version = "0.6.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3db406d29fbcd95542e92559bed4d8ad92636d1ca8b3b72ede10b4bcc010e659" -dependencies = [ - "proc-macro2", - "quote", - "syn 1.0.109", -] - [[package]] name = "bytecount" version = "0.6.8" @@ -668,12 +611,6 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" -[[package]] -name = "cfg_aliases" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" - [[package]] name = "check-features" version = "1.5.0" @@ -2154,9 +2091,6 @@ name = "hashbrown" version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" -dependencies = [ - "ahash 0.7.8", -] [[package]] name = "hashbrown" @@ -2164,7 +2098,7 @@ version = "0.14.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" dependencies = [ - "ahash 0.8.11", + "ahash", "allocator-api2", ] @@ -2621,7 +2555,7 @@ name = "jsonschema" version = "0.18.0" source = "git+https://github.com/dashpay/jsonschema-rs?branch=configure_regexp#7b00a2442ce44772e278b468bc4c2adc5e252226" dependencies = [ - "ahash 0.8.11", + "ahash", "anyhow", "base64 0.22.1", "bytecount", @@ -2788,7 +2722,7 @@ version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "884adb57038347dfbaf2d5065887b6cf4312330dc8e94bc30a1a839bd79d3261" dependencies = [ - "ahash 0.8.11", + "ahash", "portable-atomic", ] @@ -3512,29 +3446,6 @@ dependencies = [ "toml_edit 0.21.1", ] -[[package]] -name = "proc-macro-error" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" -dependencies = [ - "proc-macro-error-attr", - "proc-macro2", - "quote", - "version_check", -] - -[[package]] -name = "proc-macro-error-attr" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" -dependencies = [ - "proc-macro2", - "quote", - "version_check", -] - [[package]] name = "proc-macro2" version = "1.0.86" @@ -3597,26 +3508,6 @@ dependencies = [ "prost", ] -[[package]] -name = "ptr_meta" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0738ccf7ea06b608c10564b31debd4f5bc5e197fc8bfe088f68ae5ce81e7a4f1" -dependencies = [ - "ptr_meta_derive", -] - -[[package]] -name = "ptr_meta_derive" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16b845dbfca988fa33db069c0e230574d15a3088f147a87b64c7589eb662c9ac" -dependencies = [ - "proc-macro2", - "quote", - "syn 1.0.109", -] - [[package]] name = "quanta" version = "0.12.3" @@ -3759,15 +3650,6 @@ version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a66a03ae7c801facd77a29370b4faec201768915ac14a721ba36f20bc9c209b" -[[package]] -name = "rend" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "71fe3824f5629716b1589be05dacd749f6aa084c87e00e016714a8cdfccc997c" -dependencies = [ - "bytecheck", -] - [[package]] name = "reopen" version = "1.0.3" @@ -3836,35 +3718,6 @@ dependencies = [ "windows-sys 0.52.0", ] -[[package]] -name = "rkyv" -version = "0.7.45" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9008cd6385b9e161d8229e1f6549dd23c3d022f132a2ea37ac3a10ac4935779b" -dependencies = [ - "bitvec", - "bytecheck", - "bytes", - "hashbrown 0.12.3", - "ptr_meta", - "rend", - "rkyv_derive", - "seahash", - "tinyvec", - "uuid", -] - -[[package]] -name = "rkyv_derive" -version = "0.7.45" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "503d1d27590a2b0a3a4ca4c94755aa2875657196ecbf401a42eff41d7de532c0" -dependencies = [ - "proc-macro2", - "quote", - "syn 1.0.109", -] - [[package]] name = "rocksdb" version = "0.22.0" @@ -3902,13 +3755,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b082d80e3e3cc52b2ed634388d436fe1f4de6af5786cc2de9ba9737527bdf555" dependencies = [ "arrayvec", - "borsh", - "bytes", "num-traits", - "rand", - "rkyv", - "serde", - "serde_json", ] [[package]] @@ -4056,12 +3903,6 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" -[[package]] -name = "seahash" -version = "4.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c107b6f4780854c8b126e228ea8869f4d7b71260f962fefb57b996b8959ba6b" - [[package]] name = "secp256k1" version = "0.30.0" @@ -4325,12 +4166,6 @@ version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d66dc143e6b11c1eddc06d5c423cfc97062865baf299914ab64caa38182078fe" -[[package]] -name = "simdutf8" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f27f6278552951f1f2b8cf9da965d10969b2efdea95a6ec47987ab46edfe263a" - [[package]] name = "simple-signer" version = "1.5.0" @@ -4481,18 +4316,6 @@ dependencies = [ "unicode-ident", ] -[[package]] -name = "syn_derive" -version = "0.1.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1329189c02ff984e9736652b1631330da25eaa6bc639089ed4915d25446cbe7b" -dependencies = [ - "proc-macro-error", - "proc-macro2", - "quote", - "syn 2.0.75", -] - [[package]] name = "sync_wrapper" version = "0.1.2" diff --git a/packages/rs-drive-abci/Cargo.toml b/packages/rs-drive-abci/Cargo.toml index b1762b87060..63a30610a0f 100644 --- a/packages/rs-drive-abci/Cargo.toml +++ b/packages/rs-drive-abci/Cargo.toml @@ -31,8 +31,8 @@ indexmap = { version = "2.2.6", features = ["serde"] } dashcore-rpc = { git = "https://github.com/dashpay/rust-dashcore-rpc", tag = "v0.15.8" } dpp = { path = "../rs-dpp", features = ["abci"] } simple-signer = { path = "../simple-signer" } -rust_decimal = "1.2.5" -rust_decimal_macros = "1.25.0" +rust_decimal = { version = "1.36.0", default-features = false } +rust_decimal_macros = "1.36.0" mockall = { version = "0.13", optional = true } prost = { version = "0.13", default-features = false } tracing = { version = "0.1.37", default-features = false, features = [] } From cb28ec2c4f932aaf0667057ee08e74fefc89669d Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Fri, 8 Nov 2024 12:47:01 +0100 Subject: [PATCH 07/27] chore(dpp): fix some linter warn to trigger ci --- .../v0/v0_methods.rs | 25 ++++++++----------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/packages/rs-dpp/src/state_transition/state_transitions/identity/identity_credit_transfer_transition/v0/v0_methods.rs b/packages/rs-dpp/src/state_transition/state_transitions/identity/identity_credit_transfer_transition/v0/v0_methods.rs index e04e061f677..cf376b9b83a 100644 --- a/packages/rs-dpp/src/state_transition/state_transitions/identity/identity_credit_transfer_transition/v0/v0_methods.rs +++ b/packages/rs-dpp/src/state_transition/state_transitions/identity/identity_credit_transfer_transition/v0/v0_methods.rs @@ -52,21 +52,18 @@ impl IdentityCreditTransferTransitionMethodsV0 for IdentityCreditTransferTransit ); } } - None => { - let key = identity - .get_first_public_key_matching( - Purpose::TRANSFER, - SecurityLevel::full_range().into(), - KeyType::all_key_types().into(), - true, + None => identity + .get_first_public_key_matching( + Purpose::TRANSFER, + SecurityLevel::full_range().into(), + KeyType::all_key_types().into(), + true, + ) + .ok_or_else(|| { + ProtocolError::DesiredKeyWithTypePurposeSecurityLevelMissing( + "no transfer public key".to_string(), ) - .ok_or_else(|| { - ProtocolError::DesiredKeyWithTypePurposeSecurityLevelMissing( - "no transfer public key".to_string(), - ) - })?; - key - } + })?, }; transition.sign_external( From 242836c0075f904700b51b5c22aa136f34581f2b Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Tue, 12 Nov 2024 12:05:12 +0100 Subject: [PATCH 08/27] chore: fix Dockerfile according to coderabbitai feedback --- Dockerfile | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 59622a7086b..7476e267799 100644 --- a/Dockerfile +++ b/Dockerfile @@ -156,7 +156,7 @@ RUN git clone https://github.com/facebook/rocksdb.git -b v8.10.2 --depth 1 . && make -j$(nproc) static_lib && \ mkdir -p /opt/rocksdb/usr/local/lib && \ cp librocksdb.a /opt/rocksdb/usr/local/lib/ && \ - cp -r include /opt/rocksdb/usr/local/lib/ && \ + cp -r include /opt/rocksdb/usr/local/ && \ cd / && \ rm -rf /tmp/rocksdb @@ -175,6 +175,7 @@ COPY --from=rocksdb /opt/rocksdb /opt/rocksdb ENV ROCKSDB_STATIC=/opt/rocksdb/usr/local/lib/librocksdb.a ENV ROCKSDB_LIB_DIR=/opt/rocksdb/usr/local/lib +ENV ROCKSDB_INCLUDE_DIR=/opt/rocksdb/usr/local/include WORKDIR /platform @@ -203,6 +204,7 @@ FROM deps AS build-planner ENV ROCKSDB_STATIC=/opt/rocksdb/usr/local/lib/librocksdb.a ENV ROCKSDB_LIB_DIR=/opt/rocksdb/usr/local/lib +ENV ROCKSDB_INCLUDE_DIR=/opt/rocksdb/usr/local/include WORKDIR /platform COPY . . @@ -225,6 +227,7 @@ ENV SCCACHE_S3_KEY_PREFIX=${SCCACHE_S3_KEY_PREFIX}/${TARGETARCH}/linux-musl ENV ROCKSDB_STATIC=/opt/rocksdb/usr/local/lib/librocksdb.a ENV ROCKSDB_LIB_DIR=/opt/rocksdb/usr/local/lib +ENV ROCKSDB_INCLUDE_DIR=/opt/rocksdb/usr/local/include WORKDIR /platform @@ -283,6 +286,7 @@ ENV SCCACHE_S3_KEY_PREFIX=${SCCACHE_S3_KEY_PREFIX}/wasm/wasm32 ENV ROCKSDB_STATIC=/opt/rocksdb/usr/local/lib/librocksdb.a ENV ROCKSDB_LIB_DIR=/opt/rocksdb/usr/local/lib +ENV ROCKSDB_INCLUDE_DIR=/opt/rocksdb/usr/local/include WORKDIR /platform From a8119adb004f7bb3ac9b5ed5a00984a85751b28d Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Tue, 12 Nov 2024 14:35:45 +0100 Subject: [PATCH 09/27] refactor(Dockerfile): put env vars into /root/env file that is copied between stages --- Dockerfile | 189 ++++++++++++++++++++++++++++++++++------------------- 1 file changed, 123 insertions(+), 66 deletions(-) diff --git a/Dockerfile b/Dockerfile index 7476e267799..b103c1c917f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,8 +5,10 @@ # This image is divided multiple parts: # - deps-base - includes all base dependencies and some libraries # - deps-sccache - deps image with sccache included +# - deps-compilation - deps image with all compilation dependencies - it's either deps-base or deps-sccache # - deps - all deps, including wasm-bindgen-cli; built on top of either deps-base or deps-sccache -# - sources - includes full source code +# - rocksdb - build static rocksdb library +# - build-planner - image used to prepare build plan for rs-drive-abci # - build-* - actual build process of given image # - drive-abci, dashmate-helper, test-suite, dapi - final images # @@ -15,6 +17,7 @@ # - NODE_ENV - node.js environment name to use to build the library # - RUSTC_WRAPPER - set to `sccache` to enable sccache support and make the following variables available: # - SCCACHE_GHA_ENABLED, ACTIONS_CACHE_URL, ACTIONS_RUNTIME_TOKEN - store sccache caches inside github actions +# - SCCACHE_BUCKET, AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_REGION, SCCACHE_S3_KEY_PREFIX - store caches in S3 # - SCCACHE_MEMCACHED - set to memcache server URI (eg. tcp://172.17.0.1:11211) to enable sccache memcached backend # - ALPINE_VERSION - use different version of Alpine base image; requires also rust:apline... # image to be available @@ -100,9 +103,24 @@ ENV CARGO_BUILD_PROFILE ${CARGO_BUILD_PROFILE} ARG NODE_ENV=production ENV NODE_ENV ${NODE_ENV} +# Create generic env file that should be sourced before build +RUN touch /root/env + +# +# DEPS-SCCACHE stage +# +# This stage is used to install sccache and configure it. +# Later on, one should copy /root/env from this image. + +# Note that, due to security concerns, each stage needs to separately declare variables like +# ACTIONS_RUNTIME_TOKEN, AWS_SECRET_ACCESS_KEY: +# +# ARG ACTIONS_RUNTIME_TOKEN +# ARG AWS_SECRET_ACCESS_KEY + FROM deps-base AS deps-sccache -ARG SCCHACHE_VERSION=0.7.1 +ARG SCCHACHE_VERSION=0.8.2 # Install sccache for caching RUN if [[ "$TARGETARCH" == "arm64" ]] ; then export SCC_ARCH=aarch64; else export SCC_ARCH=x86_64; fi; \ @@ -114,80 +132,129 @@ RUN if [[ "$TARGETARCH" == "arm64" ]] ; then export SCC_ARCH=aarch64; else expor # # Configure sccache # -ARG RUSTC_WRAPPER -ENV RUSTC_WRAPPER=${RUSTC_WRAPPER} + +# Disable incremental builds, not supported by sccache +RUN echo 'CARGO_INCREMENTAL=false' >> /root/env # Set args below to use Github Actions cache; see https://github.com/mozilla/sccache/blob/main/docs/GHA.md ARG SCCACHE_GHA_ENABLED -ENV SCCACHE_GHA_ENABLED=${SCCACHE_GHA_ENABLED} - ARG ACTIONS_CACHE_URL -ENV ACTIONS_CACHE_URL=${ACTIONS_CACHE_URL} - -ARG ACTIONS_RUNTIME_TOKEN -ENV ACTIONS_RUNTIME_TOKEN=${ACTIONS_RUNTIME_TOKEN} # Alternative solution is to use memcache ARG SCCACHE_MEMCACHED -ENV SCCACHE_MEMCACHED=${SCCACHE_MEMCACHED} # S3 storage ARG SCCACHE_BUCKET -ENV SCCACHE_BUCKET=${SCCACHE_BUCKET} - +ARG AWS_ACCESS_KEY_ID +ARG AWS_REGION ARG SCCACHE_REGION -ENV SCCACHE_REGION=${SCCACHE_REGION} - -# Disable incremental buildings, not supported by sccache -ARG CARGO_INCREMENTAL=false -ENV CARGO_INCREMENTAL=${CARGO_INCREMENTAL} +ARG SCCACHE_S3_KEY_PREFIX -ARG AWS_ACCESS_KEY_ID -ARG AWS_SECRET_ACCESS_KEY +# Generate sccache configuration +# +# Note this is conditional logic, to only enable one sccache backend at a time +# + +RUN <> /root/env + echo "ACTIONS_CACHE_URL=${ACTIONS_CACHE_URL}" >> /root/env + # ACTIONS_RUNTIME_TOKEN is a secret so we don't put it in the env file + elif [ -n "${SCCACHE_BUCKET}" ]; then + # AWS S3 + if [ -z "${SCCACHE_REGION}" ] ; then + # Default to AWS_REGION if not set + export SCCACHE_REGION=${AWS_REGION} + fi + + echo "AWS_REGION=${AWS_REGION}" >> /root/env + echo "SCCACHE_REGION=${SCCACHE_REGION}" >> /root/env + echo "AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID}" >> /root/env + # AWS_SECRET_ACCESS_KEY is a secret so we don't put it in the env file + echo "SCCACHE_BUCKET=${SCCACHE_BUCKET}" >> /root/env + echo "SCCACHE_S3_USE_SSL=false" >> /root/env + echo "SCCACHE_S3_KEY_PREFIX=${SCCACHE_S3_KEY_PREFIX}/${TARGETARCH}/linux-musl" >> /root/env + elif [ -n "${SCCACHE_MEMCACHED}" ]; then + # memcached + echo "SCCACHE_MEMCACHED=${SCCACHE_MEMCACHED}" >> /root/env + fi + + if [ -n "${RUSTC_WRAPPER}" ]; then + echo CXX="${RUSTC_WRAPPER} clang++" >> /root/env + echo CC="${RUSTC_WRAPPER} clang" >> /root/env + echo RUSTC_WRAPPER="${RUSTC_WRAPPER}" >> /root/env + echo SCCACHE_SERVER_PORT=$((RANDOM+1025)) >> /root/env + fi + + cat /root/env +EOS + +# Image containing compolation dependencies; used to overcome lack of interpolation in COPY --from +FROM deps-${RUSTC_WRAPPER:-base} AS deps-compilation # # BUILD ROCKSDB STATIC LIBRARY # -FROM deps-${RUSTC_WRAPPER:-base} AS rocksdb + +FROM deps-compilation AS rocksdb RUN mkdir -p /tmp/rocksdb WORKDIR /tmp/rocksdb -RUN git clone https://github.com/facebook/rocksdb.git -b v8.10.2 --depth 1 . && \ + +# 1. Clone rocksdb +# 2. Delete source files not needed by librocksdb-sys +# 3. Build static library +# TODO join with code below && \ + +RUN git clone https://github.com/facebook/rocksdb.git -b v8.10.2 --depth 1 . + +ARG ACTIONS_RUNTIME_TOKEN +ARG AWS_SECRET_ACCESS_KEY + +RUN source /root/env && \ make -j$(nproc) static_lib && \ mkdir -p /opt/rocksdb/usr/local/lib && \ cp librocksdb.a /opt/rocksdb/usr/local/lib/ && \ cp -r include /opt/rocksdb/usr/local/ && \ cd / && \ - rm -rf /tmp/rocksdb + rm -rf /tmp/rocksdb && \ + if [[ -x /usr/bin/sccache ]]; then sccache --show-stats; fi + +# Configure sccache +RUN <> /root/env +echo ROCKSDB_LIB_DIR=/opt/rocksdb/usr/local/lib >> /root/env +echo ROCKSDB_INCLUDE_DIR=/opt/rocksdb/usr/local/include >> /root/env +EOS + +# # DEPS: FULL DEPENDENCIES LIST # # This is separate from `deps` to use sccache for caching -FROM deps-${RUSTC_WRAPPER:-base} AS deps - -ARG SCCACHE_S3_KEY_PREFIX -ENV SCCACHE_S3_KEY_PREFIX=${SCCACHE_S3_KEY_PREFIX}/${TARGETARCH}/linux-musl +FROM deps-compilation AS deps # Install prebuilt rocksdb library - COPY --from=rocksdb /opt/rocksdb /opt/rocksdb -# Set env variables so that Rust's rocksdb-sys will not build rocksdb from scratch -ENV ROCKSDB_STATIC=/opt/rocksdb/usr/local/lib/librocksdb.a -ENV ROCKSDB_LIB_DIR=/opt/rocksdb/usr/local/lib -ENV ROCKSDB_INCLUDE_DIR=/opt/rocksdb/usr/local/include WORKDIR /platform +# Configure credentials requied by sccache +ARG ACTIONS_RUNTIME_TOKEN +ARG AWS_SECRET_ACCESS_KEY + # Install wasm-bindgen-cli in the same profile as other components, to sacrifice some performance & disk space to gain # better build caching RUN --mount=type=cache,sharing=shared,id=cargo_registry_index,target=${CARGO_HOME}/registry/index \ --mount=type=cache,sharing=shared,id=cargo_registry_cache,target=${CARGO_HOME}/registry/cache \ --mount=type=cache,sharing=shared,id=cargo_git,target=${CARGO_HOME}/git/db \ --mount=type=cache,sharing=shared,id=target_${TARGETARCH},target=/platform/target \ - export SCCACHE_SERVER_PORT=$((RANDOM+1025)) && \ source $HOME/.cargo/env && \ - if [[ -z "${SCCACHE_MEMCACHED}" ]] ; then unset SCCACHE_MEMCACHED ; fi ; \ + source /root/env && \ RUSTFLAGS="-C target-feature=-crt-static" \ CARGO_TARGET_DIR="/platform/target" \ # TODO: Build wasm with build.rs @@ -201,14 +268,14 @@ RUN --mount=type=cache,sharing=shared,id=cargo_registry_index,target=${CARGO_HOM # Rust build planner to speed up builds # FROM deps AS build-planner - -ENV ROCKSDB_STATIC=/opt/rocksdb/usr/local/lib/librocksdb.a -ENV ROCKSDB_LIB_DIR=/opt/rocksdb/usr/local/lib -ENV ROCKSDB_INCLUDE_DIR=/opt/rocksdb/usr/local/include +# Configure credentials requied by sccache +ARG ACTIONS_RUNTIME_TOKEN +ARG AWS_SECRET_ACCESS_KEY WORKDIR /platform COPY . . RUN source $HOME/.cargo/env && \ + source /root/env && \ cargo chef prepare --recipe-path recipe.json # Workaround: as we cache dapi-grpc, its build.rs is not rerun, so we need to touch it @@ -222,31 +289,28 @@ FROM deps AS build-drive-abci SHELL ["/bin/bash", "-o", "pipefail","-e", "-x", "-c"] -ARG SCCACHE_S3_KEY_PREFIX -ENV SCCACHE_S3_KEY_PREFIX=${SCCACHE_S3_KEY_PREFIX}/${TARGETARCH}/linux-musl - -ENV ROCKSDB_STATIC=/opt/rocksdb/usr/local/lib/librocksdb.a -ENV ROCKSDB_LIB_DIR=/opt/rocksdb/usr/local/lib -ENV ROCKSDB_INCLUDE_DIR=/opt/rocksdb/usr/local/include WORKDIR /platform COPY --from=build-planner /platform/recipe.json recipe.json +# Configure credentials requied by sccache +ARG ACTIONS_RUNTIME_TOKEN +ARG AWS_SECRET_ACCESS_KEY + # Build dependencies - this is the caching Docker layer! RUN --mount=type=cache,sharing=shared,id=cargo_registry_index,target=${CARGO_HOME}/registry/index \ --mount=type=cache,sharing=shared,id=cargo_registry_cache,target=${CARGO_HOME}/registry/cache \ --mount=type=cache,sharing=shared,id=cargo_git,target=${CARGO_HOME}/git/db \ --mount=type=cache,sharing=shared,id=target_${TARGETARCH},target=/platform/target \ source $HOME/.cargo/env && \ - export SCCACHE_SERVER_PORT=$((RANDOM+1025)) && \ - if [[ -z "${SCCACHE_MEMCACHED}" ]] ; then unset SCCACHE_MEMCACHED ; fi ; \ + source /root/env && \ cargo chef cook \ --recipe-path recipe.json \ --profile "$CARGO_BUILD_PROFILE" \ --package drive-abci \ --locked && \ - if [[ "${RUSTC_WRAPPER}" == "sccache" ]] ; then sccache --show-stats; fi + if [[ -x /usr/bin/sccache ]]; then sccache --show-stats; fi COPY . . @@ -258,55 +322,49 @@ RUN --mount=type=cache,sharing=shared,id=cargo_registry_index,target=${CARGO_HOM --mount=type=cache,sharing=shared,id=cargo_git,target=${CARGO_HOME}/git/db \ --mount=type=cache,sharing=shared,id=target_${TARGETARCH},target=/platform/target \ source $HOME/.cargo/env && \ - export SCCACHE_SERVER_PORT=$((RANDOM+1025)) && \ + source /root/env && \ if [[ "${CARGO_BUILD_PROFILE}" == "release" ]] ; then \ mv .cargo/config-release.toml .cargo/config.toml && \ export OUT_DIRECTORY=release ; \ else \ export FEATURES_FLAG="--features=console,grovedbg" ; \ export OUT_DIRECTORY=debug ; \ - fi && \ - if [[ -z "${SCCACHE_MEMCACHED}" ]] ; then unset SCCACHE_MEMCACHED ; fi ; \ cargo build \ --profile "${CARGO_BUILD_PROFILE}" \ --package drive-abci \ ${FEATURES_FLAG} \ --locked && \ - cp /platform/target/${OUT_DIRECTORY}/drive-abci /artifacts/ && \ - if [[ "${RUSTC_WRAPPER}" == "sccache" ]] ; then sccache --show-stats; fi - + cp /platform/target/${OUT_DIRECTORY}/drive-abci /artifacts/ && \ + if [[ -x /usr/bin/sccache ]]; then sccache --show-stats; fi + # # STAGE: BUILD JAVASCRIPT INTERMEDIATE IMAGE # FROM deps AS build-js -ARG SCCACHE_S3_KEY_PREFIX -ENV SCCACHE_S3_KEY_PREFIX=${SCCACHE_S3_KEY_PREFIX}/wasm/wasm32 - -ENV ROCKSDB_STATIC=/opt/rocksdb/usr/local/lib/librocksdb.a -ENV ROCKSDB_LIB_DIR=/opt/rocksdb/usr/local/lib -ENV ROCKSDB_INCLUDE_DIR=/opt/rocksdb/usr/local/include - WORKDIR /platform COPY --from=build-planner /platform/recipe.json recipe.json +# Configure credentials requied by sccache +ARG ACTIONS_RUNTIME_TOKEN +ARG AWS_SECRET_ACCESS_KEY + # Build dependencies - this is the caching Docker layer! RUN --mount=type=cache,sharing=shared,id=cargo_registry_index,target=${CARGO_HOME}/registry/index \ --mount=type=cache,sharing=shared,id=cargo_registry_cache,target=${CARGO_HOME}/registry/cache \ --mount=type=cache,sharing=shared,id=cargo_git,target=${CARGO_HOME}/git/db \ --mount=type=cache,sharing=shared,id=target_${TARGETARCH},target=/platform/target \ source $HOME/.cargo/env && \ - export SCCACHE_SERVER_PORT=$((RANDOM+1025)) && \ - if [[ -z "${SCCACHE_MEMCACHED}" ]] ; then unset SCCACHE_MEMCACHED ; fi ; \ + source /root/env && \ cargo chef cook \ --recipe-path recipe.json \ --profile "$CARGO_BUILD_PROFILE" \ --package wasm-dpp \ --target wasm32-unknown-unknown \ --locked && \ - if [[ "${RUSTC_WRAPPER}" == "sccache" ]] ; then sccache --show-stats; fi + if [[ -x /usr/bin/sccache ]]; then sccache --show-stats; fi COPY . . @@ -316,14 +374,13 @@ RUN --mount=type=cache,sharing=shared,id=cargo_registry_index,target=${CARGO_HOM --mount=type=cache,sharing=shared,id=target_wasm,target=/platform/target \ --mount=type=cache,sharing=shared,id=unplugged_${TARGETARCH},target=/tmp/unplugged \ source $HOME/.cargo/env && \ + source /root/env && \ cp -R /tmp/unplugged /platform/.yarn/ && \ yarn install --inline-builds && \ cp -R /platform/.yarn/unplugged /tmp/ && \ - export SCCACHE_SERVER_PORT=$((RANDOM+1025)) && \ - if [[ -z "${SCCACHE_MEMCACHED}" ]] ; then unset SCCACHE_MEMCACHED ; fi ; \ export SKIP_GRPC_PROTO_BUILD=1 && \ yarn build && \ - if [[ "${RUSTC_WRAPPER}" == "sccache" ]]; then sccache --show-stats; fi + if [[ -x /usr/bin/sccache ]]; then sccache --show-stats; fi # # STAGE: FINAL DRIVE-ABCI IMAGE From d7c3342e2a93d544a5d39e9a498bdc426b315970 Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Tue, 12 Nov 2024 15:09:46 +0100 Subject: [PATCH 10/27] build: update sccache in gha --- .github/actions/rust/action.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/rust/action.yaml b/.github/actions/rust/action.yaml index 88540070133..9fd060c4b36 100644 --- a/.github/actions/rust/action.yaml +++ b/.github/actions/rust/action.yaml @@ -85,9 +85,9 @@ runs: export PATH="${PATH}:${HOME}/.local/bin" - name: Run sccache-cache - uses: mozilla-actions/sccache-action@v0.0.3 + uses: mozilla-actions/sccache-action@v0.0.6 with: - version: "v0.7.1" # Must be the same as in Dockerfile + version: "v0.8.2" # Must be the same as in Dockerfile if: inputs.cache == 'true' - name: Hash ref_name From ccd8426fffda40931a85d63b2e222b7721a8c1b3 Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Tue, 12 Nov 2024 15:13:10 +0100 Subject: [PATCH 11/27] chore: fix sccache --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index b103c1c917f..432c5be7db1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -132,6 +132,7 @@ RUN if [[ "$TARGETARCH" == "arm64" ]] ; then export SCC_ARCH=aarch64; else expor # # Configure sccache # +ARG RUSTC_WRAPPER # Disable incremental builds, not supported by sccache RUN echo 'CARGO_INCREMENTAL=false' >> /root/env From 886a93ddb1166bbea56d49bcde117fb505a6e908 Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Tue, 12 Nov 2024 18:06:03 +0100 Subject: [PATCH 12/27] chore: sccache not working correctly --- Dockerfile | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/Dockerfile b/Dockerfile index 432c5be7db1..c649fa54aa5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -171,23 +171,23 @@ RUN <> /root/env - echo "SCCACHE_REGION=${SCCACHE_REGION}" >> /root/env - echo "AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID}" >> /root/env + echo "export AWS_REGION='${AWS_REGION}'" >> /root/env + echo "export SCCACHE_REGION='${SCCACHE_REGION}'" >> /root/env + echo "export AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID}" >> /root/env # AWS_SECRET_ACCESS_KEY is a secret so we don't put it in the env file - echo "SCCACHE_BUCKET=${SCCACHE_BUCKET}" >> /root/env - echo "SCCACHE_S3_USE_SSL=false" >> /root/env - echo "SCCACHE_S3_KEY_PREFIX=${SCCACHE_S3_KEY_PREFIX}/${TARGETARCH}/linux-musl" >> /root/env + echo "export SCCACHE_BUCKET='${SCCACHE_BUCKET}'" >> /root/env + echo "export SCCACHE_S3_USE_SSL=false" >> /root/env + echo "export SCCACHE_S3_KEY_PREFIX='${SCCACHE_S3_KEY_PREFIX}/${TARGETARCH}/linux-musl'" >> /root/env elif [ -n "${SCCACHE_MEMCACHED}" ]; then # memcached - echo "SCCACHE_MEMCACHED=${SCCACHE_MEMCACHED}" >> /root/env + echo "export SCCACHE_MEMCACHED='${SCCACHE_MEMCACHED}'" >> /root/env fi if [ -n "${RUSTC_WRAPPER}" ]; then - echo CXX="${RUSTC_WRAPPER} clang++" >> /root/env - echo CC="${RUSTC_WRAPPER} clang" >> /root/env - echo RUSTC_WRAPPER="${RUSTC_WRAPPER}" >> /root/env - echo SCCACHE_SERVER_PORT=$((RANDOM+1025)) >> /root/env + echo "export CXX='${RUSTC_WRAPPER} clang++'" >> /root/env + echo "export CC='${RUSTC_WRAPPER} clang'" >> /root/env + echo "export RUSTC_WRAPPER='${RUSTC_WRAPPER}'" >> /root/env + echo "export SCCACHE_SERVER_PORT=$((RANDOM+1025))" >> /root/env fi cat /root/env @@ -210,14 +210,14 @@ WORKDIR /tmp/rocksdb # 3. Build static library # TODO join with code below && \ -RUN git clone https://github.com/facebook/rocksdb.git -b v8.10.2 --depth 1 . - ARG ACTIONS_RUNTIME_TOKEN ARG AWS_SECRET_ACCESS_KEY -RUN source /root/env && \ - make -j$(nproc) static_lib && \ +RUN git clone https://github.com/facebook/rocksdb.git -b v8.10.2 --depth 1 . && \ + source /root/env && \ + PORTABLE=1 make -j$(nproc) static_lib && \ mkdir -p /opt/rocksdb/usr/local/lib && \ + strip librocksdb.a && \ cp librocksdb.a /opt/rocksdb/usr/local/lib/ && \ cp -r include /opt/rocksdb/usr/local/ && \ cd / && \ @@ -227,9 +227,9 @@ RUN source /root/env && \ # Configure sccache RUN <> /root/env -echo ROCKSDB_LIB_DIR=/opt/rocksdb/usr/local/lib >> /root/env -echo ROCKSDB_INCLUDE_DIR=/opt/rocksdb/usr/local/include >> /root/env +echo "export ROCKSDB_STATIC=/opt/rocksdb/usr/local/lib/librocksdb.a" >> /root/env +echo "export ROCKSDB_LIB_DIR=/opt/rocksdb/usr/local/lib" >> /root/env +echo "export ROCKSDB_INCLUDE_DIR=/opt/rocksdb/usr/local/include" >> /root/env EOS # From 6c98a9e19c8d3471ec8d0876e708b66dfa9f0c17 Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Tue, 12 Nov 2024 18:25:58 +0100 Subject: [PATCH 13/27] fix: build fails for some reason --- Dockerfile | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/Dockerfile b/Dockerfile index c649fa54aa5..294892ed1aa 100644 --- a/Dockerfile +++ b/Dockerfile @@ -195,6 +195,7 @@ EOS # Image containing compolation dependencies; used to overcome lack of interpolation in COPY --from FROM deps-${RUSTC_WRAPPER:-base} AS deps-compilation +# Stage intentionally left empty # # BUILD ROCKSDB STATIC LIBRARY @@ -235,11 +236,7 @@ EOS # # DEPS: FULL DEPENDENCIES LIST # -# This is separate from `deps` to use sccache for caching -FROM deps-compilation AS deps - -# Install prebuilt rocksdb library -COPY --from=rocksdb /opt/rocksdb /opt/rocksdb +FROM rocksdb AS deps WORKDIR /platform @@ -270,11 +267,12 @@ RUN --mount=type=cache,sharing=shared,id=cargo_registry_index,target=${CARGO_HOM # FROM deps AS build-planner # Configure credentials requied by sccache +WORKDIR /platform +COPY . . + ARG ACTIONS_RUNTIME_TOKEN ARG AWS_SECRET_ACCESS_KEY -WORKDIR /platform -COPY . . RUN source $HOME/.cargo/env && \ source /root/env && \ cargo chef prepare --recipe-path recipe.json From 5afd8fb74a5b49ee31486fa2542a0557401017bc Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Tue, 12 Nov 2024 18:27:18 +0100 Subject: [PATCH 14/27] chore: minor cleanup --- Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 294892ed1aa..b0c52031db5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,8 +6,8 @@ # - deps-base - includes all base dependencies and some libraries # - deps-sccache - deps image with sccache included # - deps-compilation - deps image with all compilation dependencies - it's either deps-base or deps-sccache +# - deps-rocksdb - build static rocksdb library # - deps - all deps, including wasm-bindgen-cli; built on top of either deps-base or deps-sccache -# - rocksdb - build static rocksdb library # - build-planner - image used to prepare build plan for rs-drive-abci # - build-* - actual build process of given image # - drive-abci, dashmate-helper, test-suite, dapi - final images @@ -201,7 +201,7 @@ FROM deps-${RUSTC_WRAPPER:-base} AS deps-compilation # BUILD ROCKSDB STATIC LIBRARY # -FROM deps-compilation AS rocksdb +FROM deps-compilation AS deps-rocksdb RUN mkdir -p /tmp/rocksdb WORKDIR /tmp/rocksdb @@ -236,7 +236,7 @@ EOS # # DEPS: FULL DEPENDENCIES LIST # -FROM rocksdb AS deps +FROM deps-rocksdb AS deps WORKDIR /platform From 38f94fe32a8cdcbf1be77b23a67ddfdd3859914c Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Tue, 12 Nov 2024 19:26:30 +0100 Subject: [PATCH 15/27] chore: sime fixes + install snappy --- Dockerfile | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index b0c52031db5..b4e98922df0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -56,6 +56,7 @@ RUN apk add --no-cache \ linux-headers \ llvm-static llvm-dev \ openssl-dev \ + snappy-static snappy-dev \ perl \ python3 \ unzip \ @@ -206,11 +207,6 @@ FROM deps-compilation AS deps-rocksdb RUN mkdir -p /tmp/rocksdb WORKDIR /tmp/rocksdb -# 1. Clone rocksdb -# 2. Delete source files not needed by librocksdb-sys -# 3. Build static library -# TODO join with code below && \ - ARG ACTIONS_RUNTIME_TOKEN ARG AWS_SECRET_ACCESS_KEY @@ -226,13 +222,20 @@ RUN git clone https://github.com/facebook/rocksdb.git -b v8.10.2 --depth 1 . && if [[ -x /usr/bin/sccache ]]; then sccache --show-stats; fi -# Configure sccache +# Configure RocksDB env variables RUN <> /root/env echo "export ROCKSDB_LIB_DIR=/opt/rocksdb/usr/local/lib" >> /root/env echo "export ROCKSDB_INCLUDE_DIR=/opt/rocksdb/usr/local/include" >> /root/env EOS +# Configure snappy, dependency of librocksdb-sys +RUN <> /root/env +echo "export SNAPPY_LIB_DIR=/usr/lib" >> /root/env +echo "export SNAPPY_INCLUDE_DIR=/usr/include" >> /root/env +EOS + # # DEPS: FULL DEPENDENCIES LIST # @@ -302,6 +305,7 @@ RUN --mount=type=cache,sharing=shared,id=cargo_registry_index,target=${CARGO_HOM --mount=type=cache,sharing=shared,id=cargo_registry_cache,target=${CARGO_HOME}/registry/cache \ --mount=type=cache,sharing=shared,id=cargo_git,target=${CARGO_HOME}/git/db \ --mount=type=cache,sharing=shared,id=target_${TARGETARCH},target=/platform/target \ + set -ex && \ source $HOME/.cargo/env && \ source /root/env && \ cargo chef cook \ From ecabc9e309cec1029a7d51691b88d7d96ffd77eb Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Tue, 12 Nov 2024 19:46:50 +0100 Subject: [PATCH 16/27] chore: fix --- Dockerfile | 27 ++++++--------------------- 1 file changed, 6 insertions(+), 21 deletions(-) diff --git a/Dockerfile b/Dockerfile index b4e98922df0..fbbdb92ee2a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -185,8 +185,8 @@ RUN <> /root/env - echo "export CC='${RUSTC_WRAPPER} clang'" >> /root/env + echo "export CXX='${RUSTC_WRAPPER} cpp'" >> /root/env + echo "export CC='${RUSTC_WRAPPER} cc'" >> /root/env echo "export RUSTC_WRAPPER='${RUSTC_WRAPPER}'" >> /root/env echo "export SCCACHE_SERVER_PORT=$((RANDOM+1025))" >> /root/env fi @@ -194,6 +194,10 @@ RUN < Date: Tue, 12 Nov 2024 19:49:17 +0100 Subject: [PATCH 17/27] chore: clang --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index fbbdb92ee2a..e5d7525e466 100644 --- a/Dockerfile +++ b/Dockerfile @@ -185,8 +185,8 @@ RUN <> /root/env - echo "export CC='${RUSTC_WRAPPER} cc'" >> /root/env + echo "export CXX='${RUSTC_WRAPPER} clang++'" >> /root/env + echo "export CC='${RUSTC_WRAPPER} clang'" >> /root/env echo "export RUSTC_WRAPPER='${RUSTC_WRAPPER}'" >> /root/env echo "export SCCACHE_SERVER_PORT=$((RANDOM+1025))" >> /root/env fi From bf2161c8546db7e0d89d22f794ca9fd78478642d Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Tue, 12 Nov 2024 20:00:13 +0100 Subject: [PATCH 18/27] chore: minor fixes --- Dockerfile | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index e5d7525e466..9cc07543f7b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -177,7 +177,7 @@ RUN <> /root/env # AWS_SECRET_ACCESS_KEY is a secret so we don't put it in the env file echo "export SCCACHE_BUCKET='${SCCACHE_BUCKET}'" >> /root/env - echo "export SCCACHE_S3_USE_SSL=false" >> /root/env + echo "export SCCACHE_S3_USE_SSL=true" >> /root/env echo "export SCCACHE_S3_KEY_PREFIX='${SCCACHE_S3_KEY_PREFIX}/${TARGETARCH}/linux-musl'" >> /root/env elif [ -n "${SCCACHE_MEMCACHED}" ]; then # memcached @@ -258,8 +258,8 @@ RUN --mount=type=cache,sharing=shared,id=cargo_registry_index,target=${CARGO_HOM # Meanwhile if you want to update wasm-bindgen you also need to update version in: # - packages/wasm-dpp/Cargo.toml # - packages/wasm-dpp/scripts/build-wasm.sh - cargo install --profile "$CARGO_BUILD_PROFILE" wasm-bindgen-cli@0.2.86 cargo-chef@0.1.67 --locked - + cargo install --profile "$CARGO_BUILD_PROFILE" wasm-bindgen-cli@0.2.86 cargo-chef@0.1.67 --locked && \ + if [[ -x /usr/bin/sccache ]]; then sccache --show-stats; fi # # Rust build planner to speed up builds @@ -294,7 +294,6 @@ RUN --mount=type=cache,sharing=shared,id=cargo_registry_index,target=${CARGO_HOM --mount=type=cache,sharing=shared,id=cargo_registry_cache,target=${CARGO_HOME}/registry/cache \ --mount=type=cache,sharing=shared,id=cargo_git,target=${CARGO_HOME}/git/db \ --mount=type=cache,sharing=shared,id=target_${TARGETARCH},target=/platform/target \ - set -ex && \ source $HOME/.cargo/env && \ source /root/env && \ cargo chef cook \ @@ -302,7 +301,7 @@ RUN --mount=type=cache,sharing=shared,id=cargo_registry_index,target=${CARGO_HOM --profile "$CARGO_BUILD_PROFILE" \ --package drive-abci \ --locked && \ - if [[ -x /usr/bin/sccache ]]; then sccache --show-stats; fi + if [[ -x /usr/bin/sccache ]]; then sccache --show-stats; fi COPY . . From 6740cb58aa599f29528a7fabbd76d3905ef8ff19 Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Wed, 13 Nov 2024 10:05:13 +0100 Subject: [PATCH 19/27] build: install snappy lib --- .github/actions/rust/action.yaml | 3 ++- .github/workflows/tests-rs-package.yml | 10 ++++++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/.github/actions/rust/action.yaml b/.github/actions/rust/action.yaml index 9fd060c4b36..20cb5bcd3e1 100644 --- a/.github/actions/rust/action.yaml +++ b/.github/actions/rust/action.yaml @@ -117,5 +117,6 @@ runs: if: runner.os == 'Linux' run: | sudo apt update -qq - sudo apt install -qq --yes clang llvm + # snappy is required by rust rocksdb + sudo apt install -qq --yes clang llvm snappy-dev snappy-static sudo update-alternatives --set cc /usr/bin/clang diff --git a/.github/workflows/tests-rs-package.yml b/.github/workflows/tests-rs-package.yml index a05e68ee6ce..31d1747f694 100644 --- a/.github/workflows/tests-rs-package.yml +++ b/.github/workflows/tests-rs-package.yml @@ -55,7 +55,8 @@ jobs: SCCACHE_S3_KEY_PREFIX: ${{ runner.os }}/sccache/${{ runner.arch }}/linux-gnu ROCKSDB_STATIC: "/opt/rocksdb/usr/local/lib/librocksdb.a" ROCKSDB_LIB_DIR: "/opt/rocksdb/usr/local/lib" - + SNAPPY_STATIC: "/usr/lib/libsnappy.a" + SNAPPY_LIB_DIR: "/usr/lib" formatting: name: Formatting runs-on: ubuntu-24.04 @@ -119,6 +120,8 @@ jobs: SCCACHE_S3_KEY_PREFIX: ${{ runner.os }}/sccache/${{ runner.arch }}/linux-gnu ROCKSDB_STATIC: "/opt/rocksdb/usr/local/lib/librocksdb.a" ROCKSDB_LIB_DIR: "/opt/rocksdb/usr/local/lib" + SNAPPY_STATIC: "/usr/lib/libsnappy.a" + SNAPPY_LIB_DIR: "/usr/lib" with: args: ${{ steps.crate_info.outputs.cargo_manifest_dir }} @@ -213,7 +216,8 @@ jobs: SCCACHE_S3_KEY_PREFIX: ${{ runner.os }}/sccache/${{ runner.arch }}/linux-gnu ROCKSDB_STATIC: "/opt/rocksdb/usr/local/lib/librocksdb.a" ROCKSDB_LIB_DIR: "/opt/rocksdb/usr/local/lib" - + SNAPPY_STATIC: "/usr/lib/libsnappy.a" + SNAPPY_LIB_DIR: "/usr/lib" check_each_feature: name: Check each feature runs-on: ${{ fromJSON(inputs.test-runner) }} @@ -250,6 +254,8 @@ jobs: SCCACHE_S3_KEY_PREFIX: ${{ runner.os }}/sccache/${{ runner.arch }}/linux-gnu ROCKSDB_STATIC: "/opt/rocksdb/usr/local/lib/librocksdb.a" ROCKSDB_LIB_DIR: "/opt/rocksdb/usr/local/lib" + SNAPPY_STATIC: "/usr/lib/libsnappy.a" + SNAPPY_LIB_DIR: "/usr/lib" run: | echo Verify all features disabled set -ex From 19dc0f17014813a6daba8391c3cc21b4acf91215 Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Wed, 13 Nov 2024 10:25:38 +0100 Subject: [PATCH 20/27] chore: self-review and some docs --- Dockerfile | 58 ++++++++++++++++++++++++++---------------------------- 1 file changed, 28 insertions(+), 30 deletions(-) diff --git a/Dockerfile b/Dockerfile index 9cc07543f7b..e5299a51613 100644 --- a/Dockerfile +++ b/Dockerfile @@ -23,14 +23,16 @@ # image to be available # - USERNAME, USER_UID, USER_GID - specification of user used to run the binary # +# +# +# # BUILD PROCESS # # 1. All these --mount... are to cache reusable info between runs. # See https://doc.rust-lang.org/cargo/guide/cargo-home.html#caching-the-cargo-home-in-ci -# 2. We add `--config net.git-fetch-with-cli=true` to address ARM build issue, -# see https://github.com/rust-lang/cargo/issues/10781#issuecomment-1441071052 -# 3. Github Actions have shared networking configured, so we need to set a random -# SCCACHE_SERVER_PORT port to avoid conflicts in case of parallel compilation +# 2. Github Actions have shared networking configured, so we need to set a random SCCACHE_SERVER_PORT port to avoid +# conflicts in case of parallel compilation. +# 3. Configuration variables are shared between runs using /root/env file. ARG ALPINE_VERSION=3.18 ARG PROTOC_VERSION=27.3 @@ -64,6 +66,13 @@ RUN apk add --no-cache \ xz \ zeromq-dev +# Configure snappy, dependency of librocksdb-sys +RUN <> /root/env +echo "export SNAPPY_LIB_DIR=/usr/lib" >> /root/env +echo "export SNAPPY_INCLUDE_DIR=/usr/include" >> /root/env +EOS + # Configure Node.js RUN npm config set --global audit false @@ -104,23 +113,19 @@ ENV CARGO_BUILD_PROFILE ${CARGO_BUILD_PROFILE} ARG NODE_ENV=production ENV NODE_ENV ${NODE_ENV} -# Create generic env file that should be sourced before build -RUN touch /root/env - # # DEPS-SCCACHE stage # # This stage is used to install sccache and configure it. -# Later on, one should copy /root/env from this image. +# Later on, one should source /root/env before building to use sccache. -# Note that, due to security concerns, each stage needs to separately declare variables like -# ACTIONS_RUNTIME_TOKEN, AWS_SECRET_ACCESS_KEY: -# -# ARG ACTIONS_RUNTIME_TOKEN -# ARG AWS_SECRET_ACCESS_KEY +# Note that, due to security concerns, each stage needs to declare variables containing authentication secrets, like +# ACTIONS_RUNTIME_TOKEN, AWS_SECRET_ACCESS_KEY. It is done using ONBUILD directive, so the secrets are not stored in the +# final image. FROM deps-base AS deps-sccache +# SCCACHE_VERSION must be the same as in github actions, to avoid cache incompatibility ARG SCCHACHE_VERSION=0.8.2 # Install sccache for caching @@ -136,7 +141,7 @@ RUN if [[ "$TARGETARCH" == "arm64" ]] ; then export SCC_ARCH=aarch64; else expor ARG RUSTC_WRAPPER # Disable incremental builds, not supported by sccache -RUN echo 'CARGO_INCREMENTAL=false' >> /root/env +RUN echo 'export CARGO_INCREMENTAL=false' >> /root/env # Set args below to use Github Actions cache; see https://github.com/mozilla/sccache/blob/main/docs/GHA.md ARG SCCACHE_GHA_ENABLED @@ -152,19 +157,17 @@ ARG AWS_REGION ARG SCCACHE_REGION ARG SCCACHE_S3_KEY_PREFIX -# Generate sccache configuration +# Generate sccache configuration variables and save them to /root/env # -# Note this is conditional logic, to only enable one sccache backend at a time -# - +# We only enable one cache at a time. Setting env variables belonging to multiple cache backends may fail the build. RUN <> /root/env - echo "ACTIONS_CACHE_URL=${ACTIONS_CACHE_URL}" >> /root/env - # ACTIONS_RUNTIME_TOKEN is a secret so we don't put it in the env file + echo "export SCCACHE_GHA_ENABLED=${SCCACHE_GHA_ENABLED}" >> /root/env + echo "export ACTIONS_CACHE_URL=${ACTIONS_CACHE_URL}" >> /root/env + # ACTIONS_RUNTIME_TOKEN is a secret so we load it using ONBUILD ARG later on elif [ -n "${SCCACHE_BUCKET}" ]; then # AWS S3 if [ -z "${SCCACHE_REGION}" ] ; then @@ -175,7 +178,7 @@ RUN <> /root/env echo "export SCCACHE_REGION='${SCCACHE_REGION}'" >> /root/env echo "export AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID}" >> /root/env - # AWS_SECRET_ACCESS_KEY is a secret so we don't put it in the env file + # AWS_SECRET_ACCESS_KEY is a secret so we load it using ONBUILD ARG later on echo "export SCCACHE_BUCKET='${SCCACHE_BUCKET}'" >> /root/env echo "export SCCACHE_S3_USE_SSL=true" >> /root/env echo "export SCCACHE_S3_KEY_PREFIX='${SCCACHE_S3_KEY_PREFIX}/${TARGETARCH}/linux-musl'" >> /root/env @@ -190,11 +193,12 @@ RUN <> /root/env echo "export SCCACHE_SERVER_PORT=$((RANDOM+1025))" >> /root/env fi - + # for debugging, we display what we generated cat /root/env EOS -# Some security settings +# We provide secrets using ONBUILD ARG mechanism, to avoid putting them into a file and potentialy leaking them +# to the final image or to layer cache ONBUILD ARG ACTIONS_RUNTIME_TOKEN ONBUILD ARG AWS_SECRET_ACCESS_KEY @@ -229,12 +233,6 @@ echo "export ROCKSDB_LIB_DIR=/opt/rocksdb/usr/local/lib" >> /root/env echo "export ROCKSDB_INCLUDE_DIR=/opt/rocksdb/usr/local/include" >> /root/env EOS -# Configure snappy, dependency of librocksdb-sys -RUN <> /root/env -echo "export SNAPPY_LIB_DIR=/usr/lib" >> /root/env -echo "export SNAPPY_INCLUDE_DIR=/usr/include" >> /root/env -EOS # # DEPS: FULL DEPENDENCIES LIST From 54792fd31120ac4f4a5031714f18a9426abeb029 Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Wed, 13 Nov 2024 10:25:57 +0100 Subject: [PATCH 21/27] chore: replace /root/env with /etc/profile --- Dockerfile | 57 ++++++++++++++++++++++++------------------------------ 1 file changed, 25 insertions(+), 32 deletions(-) diff --git a/Dockerfile b/Dockerfile index e5299a51613..da75b192260 100644 --- a/Dockerfile +++ b/Dockerfile @@ -32,7 +32,7 @@ # See https://doc.rust-lang.org/cargo/guide/cargo-home.html#caching-the-cargo-home-in-ci # 2. Github Actions have shared networking configured, so we need to set a random SCCACHE_SERVER_PORT port to avoid # conflicts in case of parallel compilation. -# 3. Configuration variables are shared between runs using /root/env file. +# 3. Configuration variables are shared between runs using /etc/profile file. ARG ALPINE_VERSION=3.18 ARG PROTOC_VERSION=27.3 @@ -68,9 +68,9 @@ RUN apk add --no-cache \ # Configure snappy, dependency of librocksdb-sys RUN <> /root/env -echo "export SNAPPY_LIB_DIR=/usr/lib" >> /root/env -echo "export SNAPPY_INCLUDE_DIR=/usr/include" >> /root/env +echo "export SNAPPY_STATIC=/usr/lib/libsnappy.a" >> /etc/profile +echo "export SNAPPY_LIB_DIR=/usr/lib" >> /etc/profile +echo "export SNAPPY_INCLUDE_DIR=/usr/include" >> /etc/profile EOS # Configure Node.js @@ -117,7 +117,7 @@ ENV NODE_ENV ${NODE_ENV} # DEPS-SCCACHE stage # # This stage is used to install sccache and configure it. -# Later on, one should source /root/env before building to use sccache. +# Later on, one should source /etc/profile before building to use sccache. # Note that, due to security concerns, each stage needs to declare variables containing authentication secrets, like # ACTIONS_RUNTIME_TOKEN, AWS_SECRET_ACCESS_KEY. It is done using ONBUILD directive, so the secrets are not stored in the @@ -141,7 +141,7 @@ RUN if [[ "$TARGETARCH" == "arm64" ]] ; then export SCC_ARCH=aarch64; else expor ARG RUSTC_WRAPPER # Disable incremental builds, not supported by sccache -RUN echo 'export CARGO_INCREMENTAL=false' >> /root/env +RUN echo 'export CARGO_INCREMENTAL=false' >> /etc/profile # Set args below to use Github Actions cache; see https://github.com/mozilla/sccache/blob/main/docs/GHA.md ARG SCCACHE_GHA_ENABLED @@ -157,7 +157,7 @@ ARG AWS_REGION ARG SCCACHE_REGION ARG SCCACHE_S3_KEY_PREFIX -# Generate sccache configuration variables and save them to /root/env +# Generate sccache configuration variables and save them to /etc/profile # # We only enable one cache at a time. Setting env variables belonging to multiple cache backends may fail the build. RUN <> /root/env - echo "export ACTIONS_CACHE_URL=${ACTIONS_CACHE_URL}" >> /root/env + echo "export SCCACHE_GHA_ENABLED=${SCCACHE_GHA_ENABLED}" >> /etc/profile + echo "export ACTIONS_CACHE_URL=${ACTIONS_CACHE_URL}" >> /etc/profile # ACTIONS_RUNTIME_TOKEN is a secret so we load it using ONBUILD ARG later on elif [ -n "${SCCACHE_BUCKET}" ]; then # AWS S3 @@ -175,26 +175,26 @@ RUN <> /root/env - echo "export SCCACHE_REGION='${SCCACHE_REGION}'" >> /root/env - echo "export AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID}" >> /root/env + echo "export AWS_REGION='${AWS_REGION}'" >> /etc/profile + echo "export SCCACHE_REGION='${SCCACHE_REGION}'" >> /etc/profile + echo "export AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID}" >> /etc/profile # AWS_SECRET_ACCESS_KEY is a secret so we load it using ONBUILD ARG later on - echo "export SCCACHE_BUCKET='${SCCACHE_BUCKET}'" >> /root/env - echo "export SCCACHE_S3_USE_SSL=true" >> /root/env - echo "export SCCACHE_S3_KEY_PREFIX='${SCCACHE_S3_KEY_PREFIX}/${TARGETARCH}/linux-musl'" >> /root/env + echo "export SCCACHE_BUCKET='${SCCACHE_BUCKET}'" >> /etc/profile + echo "export SCCACHE_S3_USE_SSL=true" >> /etc/profile + echo "export SCCACHE_S3_KEY_PREFIX='${SCCACHE_S3_KEY_PREFIX}/${TARGETARCH}/linux-musl'" >> /etc/profile elif [ -n "${SCCACHE_MEMCACHED}" ]; then # memcached - echo "export SCCACHE_MEMCACHED='${SCCACHE_MEMCACHED}'" >> /root/env + echo "export SCCACHE_MEMCACHED='${SCCACHE_MEMCACHED}'" >> /etc/profile fi if [ -n "${RUSTC_WRAPPER}" ]; then - echo "export CXX='${RUSTC_WRAPPER} clang++'" >> /root/env - echo "export CC='${RUSTC_WRAPPER} clang'" >> /root/env - echo "export RUSTC_WRAPPER='${RUSTC_WRAPPER}'" >> /root/env - echo "export SCCACHE_SERVER_PORT=$((RANDOM+1025))" >> /root/env + echo "export CXX='${RUSTC_WRAPPER} clang++'" >> /etc/profile + echo "export CC='${RUSTC_WRAPPER} clang'" >> /etc/profile + echo "export RUSTC_WRAPPER='${RUSTC_WRAPPER}'" >> /etc/profile + echo "export SCCACHE_SERVER_PORT=$((RANDOM+1025))" >> /etc/profile fi # for debugging, we display what we generated - cat /root/env + cat /etc/profile EOS # We provide secrets using ONBUILD ARG mechanism, to avoid putting them into a file and potentialy leaking them @@ -216,7 +216,6 @@ RUN mkdir -p /tmp/rocksdb WORKDIR /tmp/rocksdb RUN git clone https://github.com/facebook/rocksdb.git -b v8.10.2 --depth 1 . && \ - source /root/env && \ PORTABLE=1 make -j$(nproc) static_lib && \ mkdir -p /opt/rocksdb/usr/local/lib && \ cp librocksdb.a /opt/rocksdb/usr/local/lib/ && \ @@ -228,9 +227,9 @@ RUN git clone https://github.com/facebook/rocksdb.git -b v8.10.2 --depth 1 . && # Configure RocksDB env variables RUN <> /root/env -echo "export ROCKSDB_LIB_DIR=/opt/rocksdb/usr/local/lib" >> /root/env -echo "export ROCKSDB_INCLUDE_DIR=/opt/rocksdb/usr/local/include" >> /root/env +echo "export ROCKSDB_STATIC=/opt/rocksdb/usr/local/lib/librocksdb.a" >> /etc/profile +echo "export ROCKSDB_LIB_DIR=/opt/rocksdb/usr/local/lib" >> /etc/profile +echo "export ROCKSDB_INCLUDE_DIR=/opt/rocksdb/usr/local/include" >> /etc/profile EOS @@ -249,7 +248,6 @@ RUN --mount=type=cache,sharing=shared,id=cargo_registry_index,target=${CARGO_HOM --mount=type=cache,sharing=shared,id=cargo_git,target=${CARGO_HOME}/git/db \ --mount=type=cache,sharing=shared,id=target_${TARGETARCH},target=/platform/target \ source $HOME/.cargo/env && \ - source /root/env && \ RUSTFLAGS="-C target-feature=-crt-static" \ CARGO_TARGET_DIR="/platform/target" \ # TODO: Build wasm with build.rs @@ -263,12 +261,11 @@ RUN --mount=type=cache,sharing=shared,id=cargo_registry_index,target=${CARGO_HOM # Rust build planner to speed up builds # FROM deps AS build-planner -# Configure credentials requied by sccache + WORKDIR /platform COPY . . RUN source $HOME/.cargo/env && \ - source /root/env && \ cargo chef prepare --recipe-path recipe.json # Workaround: as we cache dapi-grpc, its build.rs is not rerun, so we need to touch it @@ -293,7 +290,6 @@ RUN --mount=type=cache,sharing=shared,id=cargo_registry_index,target=${CARGO_HOM --mount=type=cache,sharing=shared,id=cargo_git,target=${CARGO_HOME}/git/db \ --mount=type=cache,sharing=shared,id=target_${TARGETARCH},target=/platform/target \ source $HOME/.cargo/env && \ - source /root/env && \ cargo chef cook \ --recipe-path recipe.json \ --profile "$CARGO_BUILD_PROFILE" \ @@ -311,7 +307,6 @@ RUN --mount=type=cache,sharing=shared,id=cargo_registry_index,target=${CARGO_HOM --mount=type=cache,sharing=shared,id=cargo_git,target=${CARGO_HOME}/git/db \ --mount=type=cache,sharing=shared,id=target_${TARGETARCH},target=/platform/target \ source $HOME/.cargo/env && \ - source /root/env && \ if [[ "${CARGO_BUILD_PROFILE}" == "release" ]] ; then \ mv .cargo/config-release.toml .cargo/config.toml && \ export OUT_DIRECTORY=release ; \ @@ -342,7 +337,6 @@ RUN --mount=type=cache,sharing=shared,id=cargo_registry_index,target=${CARGO_HOM --mount=type=cache,sharing=shared,id=cargo_git,target=${CARGO_HOME}/git/db \ --mount=type=cache,sharing=shared,id=target_${TARGETARCH},target=/platform/target \ source $HOME/.cargo/env && \ - source /root/env && \ cargo chef cook \ --recipe-path recipe.json \ --profile "$CARGO_BUILD_PROFILE" \ @@ -359,7 +353,6 @@ RUN --mount=type=cache,sharing=shared,id=cargo_registry_index,target=${CARGO_HOM --mount=type=cache,sharing=shared,id=target_wasm,target=/platform/target \ --mount=type=cache,sharing=shared,id=unplugged_${TARGETARCH},target=/tmp/unplugged \ source $HOME/.cargo/env && \ - source /root/env && \ cp -R /tmp/unplugged /platform/.yarn/ && \ yarn install --inline-builds && \ cp -R /platform/.yarn/unplugged /tmp/ && \ From c1abdb05cc884eb2d12b00c2f9a5e5b06d99dfe3 Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Wed, 13 Nov 2024 10:26:43 +0100 Subject: [PATCH 22/27] chore: revert some cargo.toml changes that are out of scope --- Cargo.lock | 183 +++++++++++++++++++++++++++++- packages/rs-dpp/Cargo.toml | 6 +- packages/rs-drive-abci/Cargo.toml | 4 +- 3 files changed, 185 insertions(+), 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1733409e87c..d2e84d05c40 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -34,6 +34,17 @@ dependencies = [ "cpufeatures", ] +[[package]] +name = "ahash" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "891477e0c6a8957309ee5c45a6368af3ae14bb510732d2684ffa19af310920f9" +dependencies = [ + "getrandom", + "once_cell", + "version_check", +] + [[package]] name = "ahash" version = "0.8.11" @@ -522,6 +533,30 @@ dependencies = [ "serde", ] +[[package]] +name = "borsh" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6362ed55def622cddc70a4746a68554d7b687713770de539e59a739b249f8ed" +dependencies = [ + "borsh-derive", + "cfg_aliases", +] + +[[package]] +name = "borsh-derive" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3ef8005764f53cd4dca619f5bf64cafd4664dada50ece25e4d81de54c80cc0b" +dependencies = [ + "once_cell", + "proc-macro-crate 3.1.0", + "proc-macro2", + "quote", + "syn 2.0.75", + "syn_derive", +] + [[package]] name = "bs58" version = "0.5.1" @@ -537,6 +572,28 @@ version = "3.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" +[[package]] +name = "bytecheck" +version = "0.6.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23cdc57ce23ac53c931e88a43d06d070a6fd142f2617be5855eb75efc9beb1c2" +dependencies = [ + "bytecheck_derive", + "ptr_meta", + "simdutf8", +] + +[[package]] +name = "bytecheck_derive" +version = "0.6.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3db406d29fbcd95542e92559bed4d8ad92636d1ca8b3b72ede10b4bcc010e659" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "bytecount" version = "0.6.8" @@ -611,6 +668,12 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "cfg_aliases" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" + [[package]] name = "check-features" version = "1.5.0" @@ -2091,6 +2154,9 @@ name = "hashbrown" version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" +dependencies = [ + "ahash 0.7.8", +] [[package]] name = "hashbrown" @@ -2098,7 +2164,7 @@ version = "0.14.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" dependencies = [ - "ahash", + "ahash 0.8.11", "allocator-api2", ] @@ -2555,7 +2621,7 @@ name = "jsonschema" version = "0.18.0" source = "git+https://github.com/dashpay/jsonschema-rs?branch=configure_regexp#7b00a2442ce44772e278b468bc4c2adc5e252226" dependencies = [ - "ahash", + "ahash 0.8.11", "anyhow", "base64 0.22.1", "bytecount", @@ -2722,7 +2788,7 @@ version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "884adb57038347dfbaf2d5065887b6cf4312330dc8e94bc30a1a839bd79d3261" dependencies = [ - "ahash", + "ahash 0.8.11", "portable-atomic", ] @@ -3446,6 +3512,29 @@ dependencies = [ "toml_edit 0.21.1", ] +[[package]] +name = "proc-macro-error" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" +dependencies = [ + "proc-macro-error-attr", + "proc-macro2", + "quote", + "version_check", +] + +[[package]] +name = "proc-macro-error-attr" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" +dependencies = [ + "proc-macro2", + "quote", + "version_check", +] + [[package]] name = "proc-macro2" version = "1.0.86" @@ -3508,6 +3597,26 @@ dependencies = [ "prost", ] +[[package]] +name = "ptr_meta" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0738ccf7ea06b608c10564b31debd4f5bc5e197fc8bfe088f68ae5ce81e7a4f1" +dependencies = [ + "ptr_meta_derive", +] + +[[package]] +name = "ptr_meta_derive" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16b845dbfca988fa33db069c0e230574d15a3088f147a87b64c7589eb662c9ac" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "quanta" version = "0.12.3" @@ -3650,6 +3759,15 @@ version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a66a03ae7c801facd77a29370b4faec201768915ac14a721ba36f20bc9c209b" +[[package]] +name = "rend" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71fe3824f5629716b1589be05dacd749f6aa084c87e00e016714a8cdfccc997c" +dependencies = [ + "bytecheck", +] + [[package]] name = "reopen" version = "1.0.3" @@ -3718,6 +3836,35 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "rkyv" +version = "0.7.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9008cd6385b9e161d8229e1f6549dd23c3d022f132a2ea37ac3a10ac4935779b" +dependencies = [ + "bitvec", + "bytecheck", + "bytes", + "hashbrown 0.12.3", + "ptr_meta", + "rend", + "rkyv_derive", + "seahash", + "tinyvec", + "uuid", +] + +[[package]] +name = "rkyv_derive" +version = "0.7.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "503d1d27590a2b0a3a4ca4c94755aa2875657196ecbf401a42eff41d7de532c0" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "rocksdb" version = "0.22.0" @@ -3755,7 +3902,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b082d80e3e3cc52b2ed634388d436fe1f4de6af5786cc2de9ba9737527bdf555" dependencies = [ "arrayvec", + "borsh", + "bytes", "num-traits", + "rand", + "rkyv", + "serde", + "serde_json", ] [[package]] @@ -3903,6 +4056,12 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "seahash" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c107b6f4780854c8b126e228ea8869f4d7b71260f962fefb57b996b8959ba6b" + [[package]] name = "secp256k1" version = "0.30.0" @@ -4166,6 +4325,12 @@ version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d66dc143e6b11c1eddc06d5c423cfc97062865baf299914ab64caa38182078fe" +[[package]] +name = "simdutf8" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" + [[package]] name = "simple-signer" version = "1.5.0" @@ -4316,6 +4481,18 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "syn_derive" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1329189c02ff984e9736652b1631330da25eaa6bc639089ed4915d25446cbe7b" +dependencies = [ + "proc-macro-error", + "proc-macro2", + "quote", + "syn 2.0.75", +] + [[package]] name = "sync_wrapper" version = "0.1.2" diff --git a/packages/rs-dpp/Cargo.toml b/packages/rs-dpp/Cargo.toml index e2c9ab43392..ff377de7480 100644 --- a/packages/rs-dpp/Cargo.toml +++ b/packages/rs-dpp/Cargo.toml @@ -29,7 +29,7 @@ dashcore = { git = "https://github.com/dashpay/rust-dashcore", features = [ "signer", "serde", "bls", - "eddsa", + "eddsa" ], default-features = false, tag = "0.32.0" } env_logger = { version = "0.11" } getrandom = { version = "0.2", features = ["js"] } @@ -58,8 +58,8 @@ platform-serialization = { path = "../rs-platform-serialization" } platform-serialization-derive = { path = "../rs-platform-serialization-derive" } derive_more = { version = "1.0", features = ["from", "display"] } nohash-hasher = "0.2.0" -rust_decimal = { version = "1.36.0", default-features = false } -rust_decimal_macros = "1.36.0" +rust_decimal = "1.29.1" +rust_decimal_macros = "1.29.1" indexmap = { version = "2.0.2", features = ["serde"] } strum = { version = "0.26", features = ["derive"] } json-schema-compatibility-validator = { path = '../rs-json-schema-compatibility-validator' } diff --git a/packages/rs-drive-abci/Cargo.toml b/packages/rs-drive-abci/Cargo.toml index 63a30610a0f..b1762b87060 100644 --- a/packages/rs-drive-abci/Cargo.toml +++ b/packages/rs-drive-abci/Cargo.toml @@ -31,8 +31,8 @@ indexmap = { version = "2.2.6", features = ["serde"] } dashcore-rpc = { git = "https://github.com/dashpay/rust-dashcore-rpc", tag = "v0.15.8" } dpp = { path = "../rs-dpp", features = ["abci"] } simple-signer = { path = "../simple-signer" } -rust_decimal = { version = "1.36.0", default-features = false } -rust_decimal_macros = "1.36.0" +rust_decimal = "1.2.5" +rust_decimal_macros = "1.25.0" mockall = { version = "0.13", optional = true } prost = { version = "0.13", default-features = false } tracing = { version = "0.1.37", default-features = false, features = [] } From f491026bfa73e1e639f666e38c8d0fcf7e7326d5 Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Wed, 13 Nov 2024 10:32:41 +0100 Subject: [PATCH 23/27] chore: fix snappy in ubuntu x86-64 --- .github/actions/rust/action.yaml | 2 +- .github/workflows/tests-rs-package.yml | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/actions/rust/action.yaml b/.github/actions/rust/action.yaml index 20cb5bcd3e1..1f737e74087 100644 --- a/.github/actions/rust/action.yaml +++ b/.github/actions/rust/action.yaml @@ -118,5 +118,5 @@ runs: run: | sudo apt update -qq # snappy is required by rust rocksdb - sudo apt install -qq --yes clang llvm snappy-dev snappy-static + sudo apt install -qq --yes clang llvm libsnappy-dev sudo update-alternatives --set cc /usr/bin/clang diff --git a/.github/workflows/tests-rs-package.yml b/.github/workflows/tests-rs-package.yml index 31d1747f694..fa038285f2c 100644 --- a/.github/workflows/tests-rs-package.yml +++ b/.github/workflows/tests-rs-package.yml @@ -55,8 +55,8 @@ jobs: SCCACHE_S3_KEY_PREFIX: ${{ runner.os }}/sccache/${{ runner.arch }}/linux-gnu ROCKSDB_STATIC: "/opt/rocksdb/usr/local/lib/librocksdb.a" ROCKSDB_LIB_DIR: "/opt/rocksdb/usr/local/lib" - SNAPPY_STATIC: "/usr/lib/libsnappy.a" - SNAPPY_LIB_DIR: "/usr/lib" + SNAPPY_STATIC: "/usr/lib/x86_64-linux-gnu/libsnappy.a" + SNAPPY_LIB_DIR: "/usr/lib/x86_64-linux-gnu" formatting: name: Formatting runs-on: ubuntu-24.04 @@ -120,8 +120,8 @@ jobs: SCCACHE_S3_KEY_PREFIX: ${{ runner.os }}/sccache/${{ runner.arch }}/linux-gnu ROCKSDB_STATIC: "/opt/rocksdb/usr/local/lib/librocksdb.a" ROCKSDB_LIB_DIR: "/opt/rocksdb/usr/local/lib" - SNAPPY_STATIC: "/usr/lib/libsnappy.a" - SNAPPY_LIB_DIR: "/usr/lib" + SNAPPY_STATIC: "/usr/lib/x86_64-linux-gnu/libsnappy.a" + SNAPPY_LIB_DIR: "/usr/lib/x86_64-linux-gnu" with: args: ${{ steps.crate_info.outputs.cargo_manifest_dir }} @@ -216,8 +216,8 @@ jobs: SCCACHE_S3_KEY_PREFIX: ${{ runner.os }}/sccache/${{ runner.arch }}/linux-gnu ROCKSDB_STATIC: "/opt/rocksdb/usr/local/lib/librocksdb.a" ROCKSDB_LIB_DIR: "/opt/rocksdb/usr/local/lib" - SNAPPY_STATIC: "/usr/lib/libsnappy.a" - SNAPPY_LIB_DIR: "/usr/lib" + SNAPPY_STATIC: "/usr/lib/x86_64-linux-gnu/libsnappy.a" + SNAPPY_LIB_DIR: "/usr/lib/x86_64-linux-gnu" check_each_feature: name: Check each feature runs-on: ${{ fromJSON(inputs.test-runner) }} @@ -254,8 +254,8 @@ jobs: SCCACHE_S3_KEY_PREFIX: ${{ runner.os }}/sccache/${{ runner.arch }}/linux-gnu ROCKSDB_STATIC: "/opt/rocksdb/usr/local/lib/librocksdb.a" ROCKSDB_LIB_DIR: "/opt/rocksdb/usr/local/lib" - SNAPPY_STATIC: "/usr/lib/libsnappy.a" - SNAPPY_LIB_DIR: "/usr/lib" + SNAPPY_STATIC: "/usr/lib/x86_64-linux-gnu/libsnappy.a" + SNAPPY_LIB_DIR: "/usr/lib/x86_64-linux-gnu" run: | echo Verify all features disabled set -ex From 650103f5aa85a75aa5fe7976c43ce0bd66962449 Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Wed, 13 Nov 2024 11:17:08 +0100 Subject: [PATCH 24/27] Revert "chore: replace /root/env with /etc/profile" This reverts commit 54792fd31120ac4f4a5031714f18a9426abeb029. --- Dockerfile | 55 ++++++++++++++++++++++++++++++------------------------ 1 file changed, 31 insertions(+), 24 deletions(-) diff --git a/Dockerfile b/Dockerfile index da75b192260..b6def24025e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -32,7 +32,7 @@ # See https://doc.rust-lang.org/cargo/guide/cargo-home.html#caching-the-cargo-home-in-ci # 2. Github Actions have shared networking configured, so we need to set a random SCCACHE_SERVER_PORT port to avoid # conflicts in case of parallel compilation. -# 3. Configuration variables are shared between runs using /etc/profile file. +# 3. Configuration variables are shared between runs using /root/env file. ARG ALPINE_VERSION=3.18 ARG PROTOC_VERSION=27.3 @@ -68,9 +68,9 @@ RUN apk add --no-cache \ # Configure snappy, dependency of librocksdb-sys RUN <> /etc/profile -echo "export SNAPPY_LIB_DIR=/usr/lib" >> /etc/profile -echo "export SNAPPY_INCLUDE_DIR=/usr/include" >> /etc/profile +echo "export SNAPPY_STATIC=/usr/lib/libsnappy.a" >> /root/env +echo "export SNAPPY_LIB_DIR=/usr/lib" >> /root/env +echo "export SNAPPY_INCLUDE_DIR=/usr/include" >> /root/env EOS # Configure Node.js @@ -117,7 +117,7 @@ ENV NODE_ENV ${NODE_ENV} # DEPS-SCCACHE stage # # This stage is used to install sccache and configure it. -# Later on, one should source /etc/profile before building to use sccache. +# Later on, one should source /root/env before building to use sccache. # Note that, due to security concerns, each stage needs to declare variables containing authentication secrets, like # ACTIONS_RUNTIME_TOKEN, AWS_SECRET_ACCESS_KEY. It is done using ONBUILD directive, so the secrets are not stored in the @@ -141,7 +141,7 @@ RUN if [[ "$TARGETARCH" == "arm64" ]] ; then export SCC_ARCH=aarch64; else expor ARG RUSTC_WRAPPER # Disable incremental builds, not supported by sccache -RUN echo 'export CARGO_INCREMENTAL=false' >> /etc/profile +RUN echo 'export CARGO_INCREMENTAL=false' >> /root/env # Set args below to use Github Actions cache; see https://github.com/mozilla/sccache/blob/main/docs/GHA.md ARG SCCACHE_GHA_ENABLED @@ -157,7 +157,7 @@ ARG AWS_REGION ARG SCCACHE_REGION ARG SCCACHE_S3_KEY_PREFIX -# Generate sccache configuration variables and save them to /etc/profile +# Generate sccache configuration variables and save them to /root/env # # We only enable one cache at a time. Setting env variables belonging to multiple cache backends may fail the build. RUN <> /etc/profile - echo "export ACTIONS_CACHE_URL=${ACTIONS_CACHE_URL}" >> /etc/profile + echo "export SCCACHE_GHA_ENABLED=${SCCACHE_GHA_ENABLED}" >> /root/env + echo "export ACTIONS_CACHE_URL=${ACTIONS_CACHE_URL}" >> /root/env # ACTIONS_RUNTIME_TOKEN is a secret so we load it using ONBUILD ARG later on elif [ -n "${SCCACHE_BUCKET}" ]; then # AWS S3 @@ -175,26 +175,26 @@ RUN <> /etc/profile - echo "export SCCACHE_REGION='${SCCACHE_REGION}'" >> /etc/profile - echo "export AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID}" >> /etc/profile + echo "export AWS_REGION='${AWS_REGION}'" >> /root/env + echo "export SCCACHE_REGION='${SCCACHE_REGION}'" >> /root/env + echo "export AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID}" >> /root/env # AWS_SECRET_ACCESS_KEY is a secret so we load it using ONBUILD ARG later on - echo "export SCCACHE_BUCKET='${SCCACHE_BUCKET}'" >> /etc/profile - echo "export SCCACHE_S3_USE_SSL=true" >> /etc/profile - echo "export SCCACHE_S3_KEY_PREFIX='${SCCACHE_S3_KEY_PREFIX}/${TARGETARCH}/linux-musl'" >> /etc/profile + echo "export SCCACHE_BUCKET='${SCCACHE_BUCKET}'" >> /root/env + echo "export SCCACHE_S3_USE_SSL=true" >> /root/env + echo "export SCCACHE_S3_KEY_PREFIX='${SCCACHE_S3_KEY_PREFIX}/${TARGETARCH}/linux-musl'" >> /root/env elif [ -n "${SCCACHE_MEMCACHED}" ]; then # memcached - echo "export SCCACHE_MEMCACHED='${SCCACHE_MEMCACHED}'" >> /etc/profile + echo "export SCCACHE_MEMCACHED='${SCCACHE_MEMCACHED}'" >> /root/env fi if [ -n "${RUSTC_WRAPPER}" ]; then - echo "export CXX='${RUSTC_WRAPPER} clang++'" >> /etc/profile - echo "export CC='${RUSTC_WRAPPER} clang'" >> /etc/profile - echo "export RUSTC_WRAPPER='${RUSTC_WRAPPER}'" >> /etc/profile - echo "export SCCACHE_SERVER_PORT=$((RANDOM+1025))" >> /etc/profile + echo "export CXX='${RUSTC_WRAPPER} clang++'" >> /root/env + echo "export CC='${RUSTC_WRAPPER} clang'" >> /root/env + echo "export RUSTC_WRAPPER='${RUSTC_WRAPPER}'" >> /root/env + echo "export SCCACHE_SERVER_PORT=$((RANDOM+1025))" >> /root/env fi # for debugging, we display what we generated - cat /etc/profile + cat /root/env EOS # We provide secrets using ONBUILD ARG mechanism, to avoid putting them into a file and potentialy leaking them @@ -216,6 +216,7 @@ RUN mkdir -p /tmp/rocksdb WORKDIR /tmp/rocksdb RUN git clone https://github.com/facebook/rocksdb.git -b v8.10.2 --depth 1 . && \ + source /root/env && \ PORTABLE=1 make -j$(nproc) static_lib && \ mkdir -p /opt/rocksdb/usr/local/lib && \ cp librocksdb.a /opt/rocksdb/usr/local/lib/ && \ @@ -227,9 +228,9 @@ RUN git clone https://github.com/facebook/rocksdb.git -b v8.10.2 --depth 1 . && # Configure RocksDB env variables RUN <> /etc/profile -echo "export ROCKSDB_LIB_DIR=/opt/rocksdb/usr/local/lib" >> /etc/profile -echo "export ROCKSDB_INCLUDE_DIR=/opt/rocksdb/usr/local/include" >> /etc/profile +echo "export ROCKSDB_STATIC=/opt/rocksdb/usr/local/lib/librocksdb.a" >> /root/env +echo "export ROCKSDB_LIB_DIR=/opt/rocksdb/usr/local/lib" >> /root/env +echo "export ROCKSDB_INCLUDE_DIR=/opt/rocksdb/usr/local/include" >> /root/env EOS @@ -248,6 +249,7 @@ RUN --mount=type=cache,sharing=shared,id=cargo_registry_index,target=${CARGO_HOM --mount=type=cache,sharing=shared,id=cargo_git,target=${CARGO_HOME}/git/db \ --mount=type=cache,sharing=shared,id=target_${TARGETARCH},target=/platform/target \ source $HOME/.cargo/env && \ + source /root/env && \ RUSTFLAGS="-C target-feature=-crt-static" \ CARGO_TARGET_DIR="/platform/target" \ # TODO: Build wasm with build.rs @@ -266,6 +268,7 @@ WORKDIR /platform COPY . . RUN source $HOME/.cargo/env && \ + source /root/env && \ cargo chef prepare --recipe-path recipe.json # Workaround: as we cache dapi-grpc, its build.rs is not rerun, so we need to touch it @@ -290,6 +293,7 @@ RUN --mount=type=cache,sharing=shared,id=cargo_registry_index,target=${CARGO_HOM --mount=type=cache,sharing=shared,id=cargo_git,target=${CARGO_HOME}/git/db \ --mount=type=cache,sharing=shared,id=target_${TARGETARCH},target=/platform/target \ source $HOME/.cargo/env && \ + source /root/env && \ cargo chef cook \ --recipe-path recipe.json \ --profile "$CARGO_BUILD_PROFILE" \ @@ -307,6 +311,7 @@ RUN --mount=type=cache,sharing=shared,id=cargo_registry_index,target=${CARGO_HOM --mount=type=cache,sharing=shared,id=cargo_git,target=${CARGO_HOME}/git/db \ --mount=type=cache,sharing=shared,id=target_${TARGETARCH},target=/platform/target \ source $HOME/.cargo/env && \ + source /root/env && \ if [[ "${CARGO_BUILD_PROFILE}" == "release" ]] ; then \ mv .cargo/config-release.toml .cargo/config.toml && \ export OUT_DIRECTORY=release ; \ @@ -337,6 +342,7 @@ RUN --mount=type=cache,sharing=shared,id=cargo_registry_index,target=${CARGO_HOM --mount=type=cache,sharing=shared,id=cargo_git,target=${CARGO_HOME}/git/db \ --mount=type=cache,sharing=shared,id=target_${TARGETARCH},target=/platform/target \ source $HOME/.cargo/env && \ + source /root/env && \ cargo chef cook \ --recipe-path recipe.json \ --profile "$CARGO_BUILD_PROFILE" \ @@ -353,6 +359,7 @@ RUN --mount=type=cache,sharing=shared,id=cargo_registry_index,target=${CARGO_HOM --mount=type=cache,sharing=shared,id=target_wasm,target=/platform/target \ --mount=type=cache,sharing=shared,id=unplugged_${TARGETARCH},target=/tmp/unplugged \ source $HOME/.cargo/env && \ + source /root/env && \ cp -R /tmp/unplugged /platform/.yarn/ && \ yarn install --inline-builds && \ cp -R /platform/.yarn/unplugged /tmp/ && \ From 58ac034829c27d3ac78db24d33a0e50583f87d1e Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Wed, 13 Nov 2024 11:58:42 +0100 Subject: [PATCH 25/27] build: move to github-hosted runners --- .github/workflows/tests-build-js.yml | 3 ++- .github/workflows/tests-dashmate.yml | 4 ++-- .github/workflows/tests-packges-functional.yml | 3 ++- .github/workflows/tests-rs-package.yml | 5 +++-- .github/workflows/tests-test-suite.yml | 7 ++++--- 5 files changed, 13 insertions(+), 9 deletions(-) diff --git a/.github/workflows/tests-build-js.yml b/.github/workflows/tests-build-js.yml index a367af55fc8..efe6597f599 100644 --- a/.github/workflows/tests-build-js.yml +++ b/.github/workflows/tests-build-js.yml @@ -4,7 +4,8 @@ on: jobs: build-js: name: Build JS - runs-on: ["self-hosted", "linux", "arm64", "ubuntu-platform"] + # runs-on: ["self-hosted", "linux", "arm64", "ubuntu-platform"] + runs-on: ubuntu-24.04 steps: - name: Configure AWS credentials and bucket region uses: aws-actions/configure-aws-credentials@v4 diff --git a/.github/workflows/tests-dashmate.yml b/.github/workflows/tests-dashmate.yml index 369c72e06a1..67197c40a20 100644 --- a/.github/workflows/tests-dashmate.yml +++ b/.github/workflows/tests-dashmate.yml @@ -18,7 +18,8 @@ jobs: dashmate-test: name: Run ${{ inputs.name }} tests # TODO: Try with Github Runner, probably it will be the same time - runs-on: [ "self-hosted", "linux", "arm64", "ubuntu-platform" ] + # runs-on: [ "self-hosted", "linux", "arm64", "ubuntu-platform" ] + runs-on: ubuntu-24.04 timeout-minutes: 15 steps: - name: Check out repo @@ -118,4 +119,3 @@ jobs: - name: Show Docker logs if: ${{ failure() }} uses: jwalton/gh-docker-logs@v2 - diff --git a/.github/workflows/tests-packges-functional.yml b/.github/workflows/tests-packges-functional.yml index 57f735f0d5e..9506b7556b6 100644 --- a/.github/workflows/tests-packges-functional.yml +++ b/.github/workflows/tests-packges-functional.yml @@ -4,7 +4,8 @@ on: jobs: test-functional: name: Run functional tests - runs-on: [ "self-hosted", "linux", "arm64", "ubuntu-platform" ] + # runs-on: [ "self-hosted", "linux", "arm64", "ubuntu-platform" ] + runs-on: ubuntu-24.04 timeout-minutes: 15 env: CHROME_BIN: /usr/bin/brave-browser diff --git a/.github/workflows/tests-rs-package.yml b/.github/workflows/tests-rs-package.yml index fa038285f2c..e08f16f8dfe 100644 --- a/.github/workflows/tests-rs-package.yml +++ b/.github/workflows/tests-rs-package.yml @@ -8,11 +8,12 @@ on: test-runner: description: Runner for tests. Must be JSON valid string. type: string - default: '[ "self-hosted", "linux", "arm64", "ubuntu-platform" ]' + # default: '[ "self-hosted", "linux", "arm64", "ubuntu-platform" ]' + default: '[ "ubuntu-24.04" ]' lint-runner: description: Runner for linting. Must be JSON valid string. type: string - default: '"ubuntu-24.04"' + default: '[ "ubuntu-24.04" ]' check-each-feature: description: If true, try to build each individual feature for this crate type: boolean diff --git a/.github/workflows/tests-test-suite.yml b/.github/workflows/tests-test-suite.yml index f1d0ac16212..505378dc404 100644 --- a/.github/workflows/tests-test-suite.yml +++ b/.github/workflows/tests-test-suite.yml @@ -22,7 +22,8 @@ on: jobs: test-suite: name: Run ${{ inputs.name }} - runs-on: [ "self-hosted", "linux", "arm64", "ubuntu-platform" ] + # runs-on: [ "self-hosted", "linux", "arm64", "ubuntu-platform" ] + runs-on: ubuntu-24.04 timeout-minutes: 15 env: CHROME_BIN: /usr/bin/brave-browser @@ -57,8 +58,8 @@ jobs: - name: Run test suite run: yarn workspace @dashevo/platform-test-suite ${{ inputs.command }} env: - BROWSER_TEST_BATCH_INDEX: ${{ inputs.batch_index }} - BROWSER_TEST_BATCH_TOTAL: ${{ inputs.batch_total }} + BROWSER_TEST_BATCH_INDEX: ${{ inputs.batch_index }} + BROWSER_TEST_BATCH_TOTAL: ${{ inputs.batch_total }} - name: Show Docker logs if: ${{ failure() }} From cd55eaadc14a9f13d189c950e17b4364e09b4ddf Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Wed, 13 Nov 2024 12:02:52 +0100 Subject: [PATCH 26/27] feat(Dockerfile): on amd64,. build rocksdb optimized for haswell cpu --- Dockerfile | 29 +++++++++++++++++++---------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/Dockerfile b/Dockerfile index b6def24025e..1a7273e31c6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -215,16 +215,25 @@ FROM deps-compilation AS deps-rocksdb RUN mkdir -p /tmp/rocksdb WORKDIR /tmp/rocksdb -RUN git clone https://github.com/facebook/rocksdb.git -b v8.10.2 --depth 1 . && \ - source /root/env && \ - PORTABLE=1 make -j$(nproc) static_lib && \ - mkdir -p /opt/rocksdb/usr/local/lib && \ - cp librocksdb.a /opt/rocksdb/usr/local/lib/ && \ - cp -r include /opt/rocksdb/usr/local/ && \ - cd / && \ - rm -rf /tmp/rocksdb && \ - if [[ -x /usr/bin/sccache ]]; then sccache --show-stats; fi - +RUN < Date: Wed, 13 Nov 2024 12:13:26 +0100 Subject: [PATCH 27/27] Revert "build: move to github-hosted runners" This reverts commit 58ac034829c27d3ac78db24d33a0e50583f87d1e. --- .github/workflows/tests-build-js.yml | 3 +-- .github/workflows/tests-dashmate.yml | 4 ++-- .github/workflows/tests-packges-functional.yml | 3 +-- .github/workflows/tests-rs-package.yml | 5 ++--- .github/workflows/tests-test-suite.yml | 7 +++---- 5 files changed, 9 insertions(+), 13 deletions(-) diff --git a/.github/workflows/tests-build-js.yml b/.github/workflows/tests-build-js.yml index efe6597f599..a367af55fc8 100644 --- a/.github/workflows/tests-build-js.yml +++ b/.github/workflows/tests-build-js.yml @@ -4,8 +4,7 @@ on: jobs: build-js: name: Build JS - # runs-on: ["self-hosted", "linux", "arm64", "ubuntu-platform"] - runs-on: ubuntu-24.04 + runs-on: ["self-hosted", "linux", "arm64", "ubuntu-platform"] steps: - name: Configure AWS credentials and bucket region uses: aws-actions/configure-aws-credentials@v4 diff --git a/.github/workflows/tests-dashmate.yml b/.github/workflows/tests-dashmate.yml index 67197c40a20..369c72e06a1 100644 --- a/.github/workflows/tests-dashmate.yml +++ b/.github/workflows/tests-dashmate.yml @@ -18,8 +18,7 @@ jobs: dashmate-test: name: Run ${{ inputs.name }} tests # TODO: Try with Github Runner, probably it will be the same time - # runs-on: [ "self-hosted", "linux", "arm64", "ubuntu-platform" ] - runs-on: ubuntu-24.04 + runs-on: [ "self-hosted", "linux", "arm64", "ubuntu-platform" ] timeout-minutes: 15 steps: - name: Check out repo @@ -119,3 +118,4 @@ jobs: - name: Show Docker logs if: ${{ failure() }} uses: jwalton/gh-docker-logs@v2 + diff --git a/.github/workflows/tests-packges-functional.yml b/.github/workflows/tests-packges-functional.yml index 9506b7556b6..57f735f0d5e 100644 --- a/.github/workflows/tests-packges-functional.yml +++ b/.github/workflows/tests-packges-functional.yml @@ -4,8 +4,7 @@ on: jobs: test-functional: name: Run functional tests - # runs-on: [ "self-hosted", "linux", "arm64", "ubuntu-platform" ] - runs-on: ubuntu-24.04 + runs-on: [ "self-hosted", "linux", "arm64", "ubuntu-platform" ] timeout-minutes: 15 env: CHROME_BIN: /usr/bin/brave-browser diff --git a/.github/workflows/tests-rs-package.yml b/.github/workflows/tests-rs-package.yml index e08f16f8dfe..fa038285f2c 100644 --- a/.github/workflows/tests-rs-package.yml +++ b/.github/workflows/tests-rs-package.yml @@ -8,12 +8,11 @@ on: test-runner: description: Runner for tests. Must be JSON valid string. type: string - # default: '[ "self-hosted", "linux", "arm64", "ubuntu-platform" ]' - default: '[ "ubuntu-24.04" ]' + default: '[ "self-hosted", "linux", "arm64", "ubuntu-platform" ]' lint-runner: description: Runner for linting. Must be JSON valid string. type: string - default: '[ "ubuntu-24.04" ]' + default: '"ubuntu-24.04"' check-each-feature: description: If true, try to build each individual feature for this crate type: boolean diff --git a/.github/workflows/tests-test-suite.yml b/.github/workflows/tests-test-suite.yml index 505378dc404..f1d0ac16212 100644 --- a/.github/workflows/tests-test-suite.yml +++ b/.github/workflows/tests-test-suite.yml @@ -22,8 +22,7 @@ on: jobs: test-suite: name: Run ${{ inputs.name }} - # runs-on: [ "self-hosted", "linux", "arm64", "ubuntu-platform" ] - runs-on: ubuntu-24.04 + runs-on: [ "self-hosted", "linux", "arm64", "ubuntu-platform" ] timeout-minutes: 15 env: CHROME_BIN: /usr/bin/brave-browser @@ -58,8 +57,8 @@ jobs: - name: Run test suite run: yarn workspace @dashevo/platform-test-suite ${{ inputs.command }} env: - BROWSER_TEST_BATCH_INDEX: ${{ inputs.batch_index }} - BROWSER_TEST_BATCH_TOTAL: ${{ inputs.batch_total }} + BROWSER_TEST_BATCH_INDEX: ${{ inputs.batch_index }} + BROWSER_TEST_BATCH_TOTAL: ${{ inputs.batch_total }} - name: Show Docker logs if: ${{ failure() }}