From a4f2b118c28c4370103e358b15ad594c53e026e8 Mon Sep 17 00:00:00 2001 From: Vid Kersic Date: Thu, 6 Jun 2024 19:48:41 +0200 Subject: [PATCH] feat: introduce cross for cross-platform docker image --- .github/workflows/book.yml | 5 ++- .github/workflows/ci.yml | 8 ++-- .github/workflows/docker.yml | 71 +++++++++++++++------------------- Cargo.lock | 10 +++++ Cargo.toml | 2 +- Cross.toml | 10 +++++ Dockerfile | 30 +++++++------- Dockerfile.cross | 13 +++++++ Makefile | 35 +++++++++++++++++ README.md | 2 +- crates/p2p/src/rpc/protocol.rs | 2 +- crates/rpc/Cargo.toml | 2 +- rust-toolchain.toml | 2 +- 13 files changed, 128 insertions(+), 64 deletions(-) create mode 100644 Cross.toml create mode 100644 Dockerfile.cross diff --git a/.github/workflows/book.yml b/.github/workflows/book.yml index 3a1f8a5d..b59c254f 100644 --- a/.github/workflows/book.yml +++ b/.github/workflows/book.yml @@ -7,11 +7,13 @@ name: book and docs jobs: build: + name: build book and docs runs-on: ubuntu-latest - steps: - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@nightly + - name: Install mdbook run: | mkdir mdbook @@ -96,6 +98,7 @@ jobs: if-no-files-found: error deploy: + name: deploy book and docs runs-on: ubuntu-latest needs: [build] diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fe44ab23..531610bf 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,7 +8,7 @@ name: ci jobs: build_and_test: - name: Silius - ERC-4337 bundler in Rust + name: build and test Silius runs-on: ubuntu-latest steps: # github action worker got about 14GB available space for building which is not enough @@ -22,8 +22,8 @@ jobs: - uses: actions/checkout@v4 - - name: Setup Rust toolchain (stable) - uses: dtolnay/rust-toolchain@1.76.0 + - name: Setup Rust toolchain + uses: dtolnay/rust-toolchain@1.78.0 with: components: clippy @@ -94,7 +94,7 @@ jobs: make lint bundler_spec_tests: - name: Run bundler spec tests + name: bundler spec tests runs-on: ubuntu-latest needs: build_and_test steps: diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 0320706e..0c3e92e2 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -3,24 +3,40 @@ on: branches: - main -name: Create and publish a Docker image +name: docker env: REGISTRY: ghcr.io IMAGE_NAME: ${{ github.repository }} jobs: - build_and_push_docker_image: + build: + name: build and push docker image runs-on: ubuntu-latest permissions: - contents: read packages: write - + contents: read steps: - - name: Checkout repository - uses: actions/checkout@v4 + # github action worker got about 14GB available space for building which is not enough + # remove some unused binary in the image to get more spaces + - name: Make more space to compilation + run: | + sudo rm -rf /usr/share/dotnet + sudo rm -rf /opt/ghc + sudo rm -rf "/usr/local/share/boost" + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + + - uses: actions/checkout@v4 + + - name: Setup Rust toolchain + uses: dtolnay/rust-toolchain@1.78.0 + + - name: Setup Rust cache + uses: Swatinem/rust-cache@v2 with: - fetch-depth: 0 + cache-on-failure: true + + - uses: taiki-e/install-action@cross - name: Log in to the Container registry uses: docker/login-action@v3 @@ -29,20 +45,17 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Get Git Tag - run: echo "tags=$(git describe --tags)" >> $GITHUB_OUTPUT - id: tag - - - name: Extract metadata (tags, labels) for Docker - id: meta - uses: docker/metadata-action@v5 - with: - images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + - name: Install solc + run: .github/scripts/install_solc.sh - - name: Fetch third-party dependencies + - name: Set up Docker builder run: | - make fetch-thirdparty - + docker run --privileged --rm tonistiigi/binfmt --install arm64,amd64 + docker buildx create --use --name cross-builder + + - name: Build and push Silius image + run: make docker-build-push + - name: Setup Yarn cache - submodule "thirdparty/account-abstraction" uses: actions/setup-node@v3 with: @@ -57,23 +70,3 @@ jobs: cache: 'yarn' cache-dependency-path: tests/thirdparty/bundler - - name: Setup third-party dependencies - run: | - make setup-thirdparty - - - name: Setup Docker Buildx - uses: docker/setup-buildx-action@v3 - with: - platforms: linux/amd64,linux/arm64 - - - name: Build and push Docker image - uses: docker/build-push-action@v5 - with: - context: . - push: true - platforms: linux/amd64,linux/arm64 - provenance: false - cache-from: type=gha - cache-to: type=gha,mode=max - tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.tag.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} diff --git a/Cargo.lock b/Cargo.lock index 0655c900..641f7bd2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4978,6 +4978,15 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" +[[package]] +name = "openssl-src" +version = "300.3.1+3.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7259953d42a81bf137fbbd73bd30a8e1914d6dce43c2b90ed575783a22608b91" +dependencies = [ + "cc", +] + [[package]] name = "openssl-sys" version = "0.9.102" @@ -4986,6 +4995,7 @@ checksum = "c597637d56fbc83893a35eb0dd04b2b8e7a50c91e64e9493e398b5df4fb45fa2" dependencies = [ "cc", "libc", + "openssl-src", "pkg-config", "vcpkg", ] diff --git a/Cargo.toml b/Cargo.toml index 4123bc35..9610326d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,7 +20,7 @@ authors = ["Vid Kersic "] edition = "2021" license = "MIT OR Apache-2.0" repository = "https://github.com/silius-rs/silius" -rust-version = "1.76.0" +rust-version = "1.78.0" keywords = [ "silius", "primitives", diff --git a/Cross.toml b/Cross.toml new file mode 100644 index 00000000..c65b3608 --- /dev/null +++ b/Cross.toml @@ -0,0 +1,10 @@ +[build] +pre-build = [ + "apt-get update && apt-get -y upgrade && apt-get install -y wget pkg-config llvm-dev libclang-6.0-dev clang-6.0 libssl-dev", + "wget -c https://github.com/ethereum/solidity/releases/download/v0.8.20/solc-static-linux && mv solc-static-linux /usr/local/bin/solc && chmod a+x /usr/local/bin/solc" +] + +[build.env] +passthrough = [ + "JEMALLOC_SYS_WITH_LG_PAGE", +] diff --git a/Dockerfile b/Dockerfile index a8d0ebe7..973f5b50 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,10 +1,18 @@ -FROM lukemathwalker/cargo-chef:latest-rust-1.76.0 AS chef +FROM lukemathwalker/cargo-chef:latest-rust-1.78.0 AS chef WORKDIR /app LABEL org.opencontainers.image.source=https://github.com/silius-rs/silius LABEL org.opencontainers.image.description="Silius - ERC-4337 (Account Abstraction) bundler implementation in Rust." LABEL org.opencontainers.image.licenses="MIT OR Apache-2.0" +# Install system dependencies +RUN apt-get update && apt-get -y upgrade && apt-get install -y wget pkg-config libclang-dev libssl-dev + +# Install solc +RUN wget -c "https://github.com/ethereum/solidity/releases/download/v0.8.20/solc-static-linux" +RUN mv solc-static-linux /usr/local/bin/solc +RUN chmod a+x /usr/local/bin/solc + # build cargo-chef plan FROM chef AS planner COPY . . @@ -17,30 +25,22 @@ COPY --from=planner /app/recipe.json recipe.json ARG BUILD_PROFILE=release ENV BUILD_PROFILE $BUILD_PROFILE -# Set the build target platform -ARG TARGETPLATFORM - -# Install system dependencies -RUN apt-get update && apt-get -y upgrade && apt-get install -y pkg-config libclang-dev libssl-dev - -# Install solc -RUN wget -c "https://github.com/ethereum/solidity/releases/download/v0.8.20/solc-static-linux" -RUN mv solc-static-linux /usr/local/bin/solc -RUN chmod a+x /usr/local/bin/solc +# Extra Cargo features +ARG FEATURES="" +ENV FEATURES $FEATURES # Builds dependencies -RUN cargo chef cook --profile $BUILD_PROFILE --recipe-path recipe.json +RUN cargo chef cook --profile $BUILD_PROFILE --features "$FEATURES" --recipe-path recipe.json # Build application COPY . . -RUN if [ "$TARGETPLATFORM" = "linux/arm64" ]; then JEMALLOC_SYS_WITH_LG_PAGE=16 ; fi && \ - cargo build --profile $BUILD_PROFILE --locked +RUN cargo build --profile $BUILD_PROFILE --features "$FEATURES" --locked # Copy application RUN cp /app/target/$BUILD_PROFILE/silius /app/silius # Use ubuntu as a runtime image -FROM --platform=$TARGETPLATFORM ubuntu AS runtime +FROM ubuntu:22.04 AS runtime # Create data folder RUN mkdir -p /data/silius diff --git a/Dockerfile.cross b/Dockerfile.cross new file mode 100644 index 00000000..553135ae --- /dev/null +++ b/Dockerfile.cross @@ -0,0 +1,13 @@ +FROM --platform=$TARGETPLATFORM ubuntu:22.04 + +LABEL org.opencontainers.image.source=https://github.com/silius-rs/silius +LABEL org.opencontainers.image.description="Silius - ERC-4337 (Account Abstraction) bundler implementation in Rust." +LABEL org.opencontainers.image.licenses="MIT OR Apache-2.0" + +ARG TARGETARCH + +COPY ./dist/bin/$TARGETARCH/silius /usr/local/bin/silius + +EXPOSE 3000 3001 + +ENTRYPOINT ["/usr/local/bin/silius"] diff --git a/Makefile b/Makefile index 570bc465..532b780f 100644 --- a/Makefile +++ b/Makefile @@ -71,3 +71,38 @@ clean: cd crates/contracts/thirdparty/account-abstraction && yarn clean && cd ../.. cd tests/thirdparty/bundler && yarn clear && cd ../.. cargo clean + +# Heavily inspired by Lighthouse: https://github.com/sigp/lighthouse/blob/693886b94176faa4cb450f024696cb69cda2fe58/Makefile +# docker image cross-platform build + +GIT_TAG = $(shell git describe --tags) +BIN_DIR = "dist/bin" +BUILD_PATH = "target" +PROFILE = release + +DOCKER_IMAGE_NAME = ghcr.io/silius-rs/silius + +docker-build-push: + $(call docker_build_push,$(GIT_TAG),$(GIT_TAG)) + +define docker_build_push + $(MAKE) fetch-thirdparty + $(MAKE) setup-thirdparty + + RUSTFLAGS="-C link-arg=-lgcc -Clink-arg=-static-libgcc" \ + cross build --target x86_64-unknown-linux-gnu --features "" --profile "$(PROFILE)" + mkdir -p $(BIN_DIR)/amd64 + cp $(BUILD_PATH)/x86_64-unknown-linux-gnu/$(PROFILE)/silius $(BIN_DIR)/amd64/silius + + RUSTFLAGS="-C link-arg=-lgcc -Clink-arg=-static-libgcc" JEMALLOC_SYS_WITH_LG_PAGE=16 \ + cross build --target aarch64-unknown-linux-gnu --features "" --profile "$(PROFILE)" + mkdir -p $(BIN_DIR)/arm64 + cp $(BUILD_PATH)/aarch64-unknown-linux-gnu/$(PROFILE)/silius $(BIN_DIR)/arm64/silius + + docker buildx build --file ./Dockerfile.cross . \ + --platform linux/amd64,linux/arm64 \ + --tag $(DOCKER_IMAGE_NAME):$(1) \ + --tag $(DOCKER_IMAGE_NAME):$(2) \ + --provenance=false \ + --push +endef diff --git a/README.md b/README.md index 922ae34c..68e8e911 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@ For more information: **Prerequisites:** -Rust version: 1.76.0 +Rust version: 1.78.0 1. `libclang-dev`, `pkg-config` and `libssl-dev` on Debian/Ubuntu. 2. Ethereum execution client JSON-RPC API with enabled [`debug_traceCall`](https://geth.ethereum.org/docs/interacting-with-geth/rpc/ns-debug#debug_tracecall). For production, you can use [Geth](https://github.com/ethereum/go-ethereum) or [Erigon](https://github.com/ledgerwatch/erigon). For testing, we are using Geth dev mode (tested with [v1.12.0](https://github.com/ethereum/go-ethereum/releases/tag/v1.12.0)); so you need to install [Geth](https://geth.ethereum.org/docs/getting-started/installing-geth) for running tests. diff --git a/crates/p2p/src/rpc/protocol.rs b/crates/p2p/src/rpc/protocol.rs index 95f38189..c45de6d6 100644 --- a/crates/p2p/src/rpc/protocol.rs +++ b/crates/p2p/src/rpc/protocol.rs @@ -57,7 +57,7 @@ pub struct ProtocolId { /// The encoding of the RPC. pub encoding: Encoding, - /// + /// The protocol id. protocol_id: String, } diff --git a/crates/rpc/Cargo.toml b/crates/rpc/Cargo.toml index 6158d5e3..5e9c8b56 100644 --- a/crates/rpc/Cargo.toml +++ b/crates/rpc/Cargo.toml @@ -23,7 +23,7 @@ ethers = { workspace = true } # rpc hyper = { version = "0.14.20" } -hyper-tls = { version = "0.5.0" } +hyper-tls = { version = "0.5.0", features = ["vendored"] } jsonrpsee = { workspace = true } tower = { version = "0.4.13" } tower-http = { version = "0.4.0", features = ["cors"] } diff --git a/rust-toolchain.toml b/rust-toolchain.toml index 624eb0ea..51985806 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -1,2 +1,2 @@ [toolchain] -channel = "1.76.0" +channel = "1.78.0"