From 356ea8a96045bf8d8cb3e3773361051a71957218 Mon Sep 17 00:00:00 2001 From: Vid Kersic Date: Thu, 6 Jun 2024 19:48:41 +0200 Subject: [PATCH] feat: introduce cross for cross-platform docker image --- .github/workflows/book.yml | 5 +- .github/workflows/ci.yml | 8 +-- .github/workflows/docker.yml | 72 +++++++++---------- Cargo.lock | 26 ++++--- Cargo.toml | 2 +- Cross.toml | 10 +++ Dockerfile | 30 ++++---- Dockerfile.cross | 13 ++++ Makefile | 35 +++++++++ README.md | 2 +- crates/grpc/build.rs | 10 ++- .../simulation_trace/storage_access.rs | 4 +- crates/p2p/src/rpc/protocol.rs | 2 +- crates/rpc/Cargo.toml | 2 +- rust-toolchain.toml | 2 +- 15 files changed, 149 insertions(+), 74 deletions(-) create mode 100644 Cross.toml create mode 100644 Dockerfile.cross diff --git a/.github/workflows/book.yml b/.github/workflows/book.yml index 3a1f8a5d..b59c254f 100644 --- a/.github/workflows/book.yml +++ b/.github/workflows/book.yml @@ -7,11 +7,13 @@ name: book and docs jobs: build: + name: build book and docs runs-on: ubuntu-latest - steps: - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@nightly + - name: Install mdbook run: | mkdir mdbook @@ -96,6 +98,7 @@ jobs: if-no-files-found: error deploy: + name: deploy book and docs runs-on: ubuntu-latest needs: [build] diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fe44ab23..531610bf 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,7 +8,7 @@ name: ci jobs: build_and_test: - name: Silius - ERC-4337 bundler in Rust + name: build and test Silius runs-on: ubuntu-latest steps: # github action worker got about 14GB available space for building which is not enough @@ -22,8 +22,8 @@ jobs: - uses: actions/checkout@v4 - - name: Setup Rust toolchain (stable) - uses: dtolnay/rust-toolchain@1.76.0 + - name: Setup Rust toolchain + uses: dtolnay/rust-toolchain@1.78.0 with: components: clippy @@ -94,7 +94,7 @@ jobs: make lint bundler_spec_tests: - name: Run bundler spec tests + name: bundler spec tests runs-on: ubuntu-latest needs: build_and_test steps: diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 0320706e..11699603 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -3,25 +3,43 @@ on: branches: - main -name: Create and publish a Docker image +name: docker env: REGISTRY: ghcr.io IMAGE_NAME: ${{ github.repository }} jobs: - build_and_push_docker_image: + build: + name: build and push docker image runs-on: ubuntu-latest permissions: - contents: read packages: write - + contents: read steps: - - name: Checkout repository - uses: actions/checkout@v4 + # github action worker got about 14GB available space for building which is not enough + # remove some unused binary in the image to get more spaces + - name: Make more space to compilation + run: | + sudo rm -rf /usr/share/dotnet + sudo rm -rf /opt/ghc + sudo rm -rf "/usr/local/share/boost" + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + + - uses: actions/checkout@v4 with: fetch-depth: 0 + - name: Setup Rust toolchain + uses: dtolnay/rust-toolchain@1.78.0 + + - name: Setup Rust cache + uses: Swatinem/rust-cache@v2 + with: + cache-on-failure: true + + - uses: taiki-e/install-action@cross + - name: Log in to the Container registry uses: docker/login-action@v3 with: @@ -29,51 +47,31 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Get Git Tag - run: echo "tags=$(git describe --tags)" >> $GITHUB_OUTPUT - id: tag - - - name: Extract metadata (tags, labels) for Docker - id: meta - uses: docker/metadata-action@v5 - with: - images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - - name: Fetch third-party dependencies run: | make fetch-thirdparty - + - name: Setup Yarn cache - submodule "thirdparty/account-abstraction" uses: actions/setup-node@v3 with: node-version: '16.17' cache: 'yarn' cache-dependency-path: crates/contracts/thirdparty/account-abstraction - + - name: Setup Yarn cache - submodule "thirdparty/bundler" uses: actions/setup-node@v3 with: node-version: '16.17' cache: 'yarn' cache-dependency-path: tests/thirdparty/bundler - - - name: Setup third-party dependencies - run: | - make setup-thirdparty - - name: Setup Docker Buildx - uses: docker/setup-buildx-action@v3 - with: - platforms: linux/amd64,linux/arm64 + - name: Install solc + run: .github/scripts/install_solc.sh - - name: Build and push Docker image - uses: docker/build-push-action@v5 - with: - context: . - push: true - platforms: linux/amd64,linux/arm64 - provenance: false - cache-from: type=gha - cache-to: type=gha,mode=max - tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.tag.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} + - name: Set up Docker builder + run: | + docker run --privileged --rm tonistiigi/binfmt --install arm64,amd64 + docker buildx create --use --name cross-builder + + - name: Build and push Silius image + run: make docker-build-push diff --git a/Cargo.lock b/Cargo.lock index 0655c900..d9a7b81b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4978,6 +4978,15 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" +[[package]] +name = "openssl-src" +version = "300.3.1+3.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7259953d42a81bf137fbbd73bd30a8e1914d6dce43c2b90ed575783a22608b91" +dependencies = [ + "cc", +] + [[package]] name = "openssl-sys" version = "0.9.102" @@ -4986,6 +4995,7 @@ checksum = "c597637d56fbc83893a35eb0dd04b2b8e7a50c91e64e9493e398b5df4fb45fa2" dependencies = [ "cc", "libc", + "openssl-src", "pkg-config", "vcpkg", ] @@ -5550,9 +5560,9 @@ dependencies = [ [[package]] name = "prost" -version = "0.12.4" +version = "0.12.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0f5d036824e4761737860779c906171497f6d55681139d8312388f8fe398922" +checksum = "deb1435c188b76130da55f17a466d252ff7b1418b2ad3e037d127b94e3411f29" dependencies = [ "bytes", "prost-derive", @@ -5560,9 +5570,9 @@ dependencies = [ [[package]] name = "prost-build" -version = "0.12.4" +version = "0.12.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80b776a1b2dc779f5ee0641f8ade0125bc1298dd41a9a0c16d8bd57b42d222b1" +checksum = "22505a5c94da8e3b7c2996394d1c933236c4d743e81a410bcca4e6989fc066a4" dependencies = [ "bytes", "heck 0.5.0", @@ -5581,9 +5591,9 @@ dependencies = [ [[package]] name = "prost-derive" -version = "0.12.5" +version = "0.12.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9554e3ab233f0a932403704f1a1d08c30d5ccd931adfdfa1e8b5a19b52c1d55a" +checksum = "81bddcdb20abf9501610992b6759a4c888aef7d1a7247ef75e2404275ac24af1" dependencies = [ "anyhow", "itertools 0.12.1", @@ -5594,9 +5604,9 @@ dependencies = [ [[package]] name = "prost-types" -version = "0.12.4" +version = "0.12.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3235c33eb02c1f1e212abdbe34c78b264b038fb58ca612664343271e36e55ffe" +checksum = "9091c90b0a32608e984ff2fa4091273cbdd755d54935c51d520887f4a1dbd5b0" dependencies = [ "prost", ] diff --git a/Cargo.toml b/Cargo.toml index 4123bc35..9610326d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,7 +20,7 @@ authors = ["Vid Kersic "] edition = "2021" license = "MIT OR Apache-2.0" repository = "https://github.com/silius-rs/silius" -rust-version = "1.76.0" +rust-version = "1.78.0" keywords = [ "silius", "primitives", diff --git a/Cross.toml b/Cross.toml new file mode 100644 index 00000000..c65b3608 --- /dev/null +++ b/Cross.toml @@ -0,0 +1,10 @@ +[build] +pre-build = [ + "apt-get update && apt-get -y upgrade && apt-get install -y wget pkg-config llvm-dev libclang-6.0-dev clang-6.0 libssl-dev", + "wget -c https://github.com/ethereum/solidity/releases/download/v0.8.20/solc-static-linux && mv solc-static-linux /usr/local/bin/solc && chmod a+x /usr/local/bin/solc" +] + +[build.env] +passthrough = [ + "JEMALLOC_SYS_WITH_LG_PAGE", +] diff --git a/Dockerfile b/Dockerfile index a8d0ebe7..973f5b50 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,10 +1,18 @@ -FROM lukemathwalker/cargo-chef:latest-rust-1.76.0 AS chef +FROM lukemathwalker/cargo-chef:latest-rust-1.78.0 AS chef WORKDIR /app LABEL org.opencontainers.image.source=https://github.com/silius-rs/silius LABEL org.opencontainers.image.description="Silius - ERC-4337 (Account Abstraction) bundler implementation in Rust." LABEL org.opencontainers.image.licenses="MIT OR Apache-2.0" +# Install system dependencies +RUN apt-get update && apt-get -y upgrade && apt-get install -y wget pkg-config libclang-dev libssl-dev + +# Install solc +RUN wget -c "https://github.com/ethereum/solidity/releases/download/v0.8.20/solc-static-linux" +RUN mv solc-static-linux /usr/local/bin/solc +RUN chmod a+x /usr/local/bin/solc + # build cargo-chef plan FROM chef AS planner COPY . . @@ -17,30 +25,22 @@ COPY --from=planner /app/recipe.json recipe.json ARG BUILD_PROFILE=release ENV BUILD_PROFILE $BUILD_PROFILE -# Set the build target platform -ARG TARGETPLATFORM - -# Install system dependencies -RUN apt-get update && apt-get -y upgrade && apt-get install -y pkg-config libclang-dev libssl-dev - -# Install solc -RUN wget -c "https://github.com/ethereum/solidity/releases/download/v0.8.20/solc-static-linux" -RUN mv solc-static-linux /usr/local/bin/solc -RUN chmod a+x /usr/local/bin/solc +# Extra Cargo features +ARG FEATURES="" +ENV FEATURES $FEATURES # Builds dependencies -RUN cargo chef cook --profile $BUILD_PROFILE --recipe-path recipe.json +RUN cargo chef cook --profile $BUILD_PROFILE --features "$FEATURES" --recipe-path recipe.json # Build application COPY . . -RUN if [ "$TARGETPLATFORM" = "linux/arm64" ]; then JEMALLOC_SYS_WITH_LG_PAGE=16 ; fi && \ - cargo build --profile $BUILD_PROFILE --locked +RUN cargo build --profile $BUILD_PROFILE --features "$FEATURES" --locked # Copy application RUN cp /app/target/$BUILD_PROFILE/silius /app/silius # Use ubuntu as a runtime image -FROM --platform=$TARGETPLATFORM ubuntu AS runtime +FROM ubuntu:22.04 AS runtime # Create data folder RUN mkdir -p /data/silius diff --git a/Dockerfile.cross b/Dockerfile.cross new file mode 100644 index 00000000..553135ae --- /dev/null +++ b/Dockerfile.cross @@ -0,0 +1,13 @@ +FROM --platform=$TARGETPLATFORM ubuntu:22.04 + +LABEL org.opencontainers.image.source=https://github.com/silius-rs/silius +LABEL org.opencontainers.image.description="Silius - ERC-4337 (Account Abstraction) bundler implementation in Rust." +LABEL org.opencontainers.image.licenses="MIT OR Apache-2.0" + +ARG TARGETARCH + +COPY ./dist/bin/$TARGETARCH/silius /usr/local/bin/silius + +EXPOSE 3000 3001 + +ENTRYPOINT ["/usr/local/bin/silius"] diff --git a/Makefile b/Makefile index 570bc465..a15388bf 100644 --- a/Makefile +++ b/Makefile @@ -71,3 +71,38 @@ clean: cd crates/contracts/thirdparty/account-abstraction && yarn clean && cd ../.. cd tests/thirdparty/bundler && yarn clear && cd ../.. cargo clean + +# Heavily inspired by Lighthouse: https://github.com/sigp/lighthouse/blob/693886b94176faa4cb450f024696cb69cda2fe58/Makefile +# docker image cross-platform build + +GIT_TAG = $(shell git describe --tags) +BIN_DIR = dist/bin +BUILD_PATH = target +PROFILE = release + +DOCKER_IMAGE_NAME = ghcr.io/silius-rs/silius + +docker-build-push: + $(call docker_build_push) + +define docker_build_push + $(MAKE) fetch-thirdparty + $(MAKE) setup-thirdparty + + RUSTFLAGS="-C link-arg=-lgcc -Clink-arg=-static-libgcc" \ + cross build --target x86_64-unknown-linux-gnu --features "" --profile "$(PROFILE)" + mkdir -p $(BIN_DIR)/amd64 + cp $(BUILD_PATH)/x86_64-unknown-linux-gnu/$(PROFILE)/silius $(BIN_DIR)/amd64/silius + + RUSTFLAGS="-C link-arg=-lgcc -Clink-arg=-static-libgcc" JEMALLOC_SYS_WITH_LG_PAGE=16 \ + cross build --target aarch64-unknown-linux-gnu --features "" --profile "$(PROFILE)" + mkdir -p $(BIN_DIR)/arm64 + cp $(BUILD_PATH)/aarch64-unknown-linux-gnu/$(PROFILE)/silius $(BIN_DIR)/arm64/silius + + docker buildx build --file ./Dockerfile.cross . \ + --platform linux/amd64,linux/arm64 \ + --tag $(DOCKER_IMAGE_NAME):$(GIT_TAG) \ + --tag $(DOCKER_IMAGE_NAME):$(GIT_TAG) \ + --provenance=false \ + --push +endef diff --git a/README.md b/README.md index 922ae34c..68e8e911 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@ For more information: **Prerequisites:** -Rust version: 1.76.0 +Rust version: 1.78.0 1. `libclang-dev`, `pkg-config` and `libssl-dev` on Debian/Ubuntu. 2. Ethereum execution client JSON-RPC API with enabled [`debug_traceCall`](https://geth.ethereum.org/docs/interacting-with-geth/rpc/ns-debug#debug_tracecall). For production, you can use [Geth](https://github.com/ethereum/go-ethereum) or [Erigon](https://github.com/ledgerwatch/erigon). For testing, we are using Geth dev mode (tested with [v1.12.0](https://github.com/ethereum/go-ethereum/releases/tag/v1.12.0)); so you need to install [Geth](https://geth.ethereum.org/docs/getting-started/installing-geth) for running tests. diff --git a/crates/grpc/build.rs b/crates/grpc/build.rs index f09192e9..a8522c83 100644 --- a/crates/grpc/build.rs +++ b/crates/grpc/build.rs @@ -9,8 +9,14 @@ fn config() -> prost_build::Config { fn make_protos(protos: &[&str]) { let out_dir = PathBuf::from(env::var("OUT_DIR").expect("OUT_DIT not set")); tonic_build::configure() - .server_mod_attribute("uopool", r#"#[allow(clippy::unwrap_used)]"#) - .server_mod_attribute("bundler", r#"#[allow(clippy::unwrap_used)]"#) + .server_mod_attribute( + "uopool", + r#"#[allow(clippy::unwrap_used, clippy::mixed_attributes_style)]"#, + ) + .server_mod_attribute( + "bundler", + r#"#[allow(clippy::unwrap_used, clippy::mixed_attributes_style)]"#, + ) .file_descriptor_set_path(out_dir.join("descriptor.bin")) .compile_with_config(config(), protos, &["./src/protos"]) .expect("Failed to compile protos."); diff --git a/crates/mempool/src/validate/simulation_trace/storage_access.rs b/crates/mempool/src/validate/simulation_trace/storage_access.rs index 732e74de..be4fadad 100644 --- a/crates/mempool/src/validate/simulation_trace/storage_access.rs +++ b/crates/mempool/src/validate/simulation_trace/storage_access.rs @@ -153,7 +153,7 @@ impl SimulationTraceCheck for StorageAccess { uo.sender == stake_info_l.address && stake_info[FACTORY_LEVEL].is_staked()) { - slot_staked = slot.clone(); + slot_staked.clone_from(&slot); } } else if *addr == stake_info_l.address // [STO-031] - access the entity's own storage (if entity staked) || self.associated_with_slot(&stake_info_l.address, &slot, &slots)? // [STO-032] - read/write Access to storage slots that is associated with the entity, in any non-entity contract (if entity staked) @@ -161,7 +161,7 @@ impl SimulationTraceCheck for StorageAccess { // [STO-033] - read-only access to any storage in non-entity contract (if // entity staked) { - slot_staked = slot.clone(); + slot_staked.clone_from(&slot); } else { return Err(SimulationError::StorageAccess { slot }); } diff --git a/crates/p2p/src/rpc/protocol.rs b/crates/p2p/src/rpc/protocol.rs index 95f38189..c45de6d6 100644 --- a/crates/p2p/src/rpc/protocol.rs +++ b/crates/p2p/src/rpc/protocol.rs @@ -57,7 +57,7 @@ pub struct ProtocolId { /// The encoding of the RPC. pub encoding: Encoding, - /// + /// The protocol id. protocol_id: String, } diff --git a/crates/rpc/Cargo.toml b/crates/rpc/Cargo.toml index 6158d5e3..5e9c8b56 100644 --- a/crates/rpc/Cargo.toml +++ b/crates/rpc/Cargo.toml @@ -23,7 +23,7 @@ ethers = { workspace = true } # rpc hyper = { version = "0.14.20" } -hyper-tls = { version = "0.5.0" } +hyper-tls = { version = "0.5.0", features = ["vendored"] } jsonrpsee = { workspace = true } tower = { version = "0.4.13" } tower-http = { version = "0.4.0", features = ["cors"] } diff --git a/rust-toolchain.toml b/rust-toolchain.toml index 624eb0ea..51985806 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -1,2 +1,2 @@ [toolchain] -channel = "1.76.0" +channel = "1.78.0"