From b804aa205a9a46c332ec5a000cf67b9b35025040 Mon Sep 17 00:00:00 2001 From: Federico Borello <156438142+fborello-lambda@users.noreply.github.com> Date: Thu, 14 Nov 2024 14:21:37 -0300 Subject: [PATCH] feat(l2): verify proof on chain (#1115) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit **Motivation** After sending the commitment, we should generate the zkProof and then verify it on chain. **Description** Using the Risc0Groth16Verifier in Sepolia. Steps: Use the `.env.example` but change the following variables: ``` RISC0_DEV_MODE=0 ETH_RPC_URL= DEPLOYER_CONTRACT_VERIFIER=0xd9b0d07CeCd808a8172F21fA7C97992168f045CA (`Risc0Groth16Verifier`) ``` 1. cd `~/lambda_ethereum_rust/crates/l2` 2. run `make deploy-l1` -- if it fails change the `SALT` in `crates/l2/contracts/deployer.rs` 1. Copy the `OnChainProposer` address given in the .env file (`PROPOSER_ON_CHAIN_PROPOSER_ADDRESS`) 2. Copy the `Bridge address` given in the .env file (`L1_WATCHER_BRIDGE_ADDRESS`) 3. rm the libmdbx files `rm -rf ~/.local/share/ethereum_rust` 4. run the proposer/sequencer: `make init-l2` 5. In a new tab/window run the prover: `make init-l2-prover-gpu` --------- Co-authored-by: Manuel Iรฑaki Bilbao Co-authored-by: Ivan Litteri <67517699+ilitteri@users.noreply.github.com> --- cmd/ethereum_rust/ethereum_rust.rs | 17 +- crates/l2/.env.example | 21 +- crates/l2/Makefile | 35 +- crates/l2/contracts/deployer.rs | 44 +- .../l2/contracts/src/l1/OnChainProposer.sol | 55 +- .../src/l1/interfaces/IOnChainProposer.sol | 12 +- .../src/l1/interfaces/IRiscZeroVerifier.sol | 49 ++ crates/l2/docs/prover.md | 87 ++- crates/l2/proposer/errors.rs | 32 +- crates/l2/proposer/l1_committer.rs | 503 ++++++++++++++++++ crates/l2/proposer/mod.rs | 390 +------------- crates/l2/proposer/prover_server.rs | 380 ++++++++++--- crates/l2/prover/src/main.rs | 3 +- crates/l2/prover/src/prover.rs | 2 +- crates/l2/prover/src/prover_client.rs | 102 ++-- crates/l2/utils/config/committer.rs | 23 + crates/l2/utils/config/mod.rs | 5 +- crates/l2/utils/config/proposer.rs | 7 - crates/l2/utils/config/prover_client.rs | 1 + crates/l2/utils/config/prover_server.rs | 12 +- crates/l2/utils/eth_client/eth_sender.rs | 4 +- crates/l2/utils/eth_client/mod.rs | 22 +- 22 files changed, 1220 insertions(+), 586 deletions(-) create mode 100644 crates/l2/contracts/src/l1/interfaces/IRiscZeroVerifier.sol create mode 100644 crates/l2/proposer/l1_committer.rs create mode 100644 crates/l2/utils/config/committer.rs diff --git a/cmd/ethereum_rust/ethereum_rust.rs b/cmd/ethereum_rust/ethereum_rust.rs index e5c8a86f2..d9e1702fe 100644 --- a/cmd/ethereum_rust/ethereum_rust.rs +++ b/cmd/ethereum_rust/ethereum_rust.rs @@ -26,15 +26,15 @@ use tracing_subscriber::{EnvFilter, FmtSubscriber}; mod cli; mod decode; +const DEFAULT_DATADIR: &str = "ethereum_rust"; #[tokio::main] async fn main() { let matches = cli::cli().get_matches(); if let Some(matches) = matches.subcommand_matches("removedb") { - let default_datadir = get_default_datadir(); let data_dir = matches .get_one::("datadir") - .unwrap_or(&default_datadir); + .map_or(set_datadir(DEFAULT_DATADIR), |datadir| set_datadir(datadir)); let path = Path::new(&data_dir); if path.exists() { std::fs::remove_dir_all(path).expect("Failed to remove data directory"); @@ -110,11 +110,11 @@ async fn main() { let tcp_socket_addr = parse_socket_addr(tcp_addr, tcp_port).expect("Failed to parse addr and port"); - let default_datadir = get_default_datadir(); let data_dir = matches .get_one::("datadir") - .unwrap_or(&default_datadir); - let store = Store::new(data_dir, EngineType::Libmdbx).expect("Failed to create Store"); + .map_or(set_datadir(DEFAULT_DATADIR), |datadir| set_datadir(datadir)); + + let store = Store::new(&data_dir, EngineType::Libmdbx).expect("Failed to create Store"); let genesis = read_genesis_file(genesis_file_path); store @@ -204,7 +204,7 @@ async fn main() { // We do not want to start the networking module if the l2 feature is enabled. cfg_if::cfg_if! { if #[cfg(feature = "l2")] { - let l2_proposer = ethereum_rust_l2::start_proposer(store.clone()).into_future(); + let l2_proposer = ethereum_rust_l2::start_proposer(store).into_future(); tracker.spawn(l2_proposer); } else if #[cfg(feature = "dev")] { use ethereum_rust_dev; @@ -283,9 +283,8 @@ fn parse_socket_addr(addr: &str, port: &str) -> io::Result { )) } -fn get_default_datadir() -> String { - let project_dir = - ProjectDirs::from("", "", "ethereum_rust").expect("Couldn't find home directory"); +fn set_datadir(datadir: &str) -> String { + let project_dir = ProjectDirs::from("", "", datadir).expect("Couldn't find home directory"); project_dir .data_local_dir() .to_str() diff --git a/crates/l2/.env.example b/crates/l2/.env.example index 3a8ffed0f..a90392b37 100644 --- a/crates/l2/.env.example +++ b/crates/l2/.env.example @@ -1,4 +1,9 @@ ETH_RPC_URL=http://localhost:8545 +# If set to 0xAA skip proof verification. +# Only use in dev mode. +DEPLOYER_CONTRACT_VERIFIER=0x00000000000000000000000000000000000000AA +# Risc0Groth16Verifier Sepolia Address +# DEPLOYER_CONTRACT_VERIFIER=0xd9b0d07CeCd808a8172F21fA7C97992168f045CA DEPLOYER_ADDRESS=0x3d1e15a1a55578f7c920884a9943b3b35d0d885b DEPLOYER_PRIVATE_KEY=0x385c546456b6a603a1cfcaa9ec9494ba4832da08dd6bcf4de9a71e4a01b74924 # If set to false, the salt will be randomized. @@ -9,16 +14,22 @@ L1_WATCHER_CHECK_INTERVAL_MS=5000 L1_WATCHER_MAX_BLOCK_STEP=5000 L1_WATCHER_L2_PROPOSER_PRIVATE_KEY=0x385c546456b6a603a1cfcaa9ec9494ba4832da08dd6bcf4de9a71e4a01b74924 L1_WATCHER_L2_PROPOSER_ADDRESS=0x3d1e15a1a55578f7c920884a9943b3b35d0d885b -ENGINE_API_RPC_URL=http://localhost:8551 +ENGINE_API_RPC_URL=http://localhost:8552 ENGINE_API_JWT_PATH=./jwt.hex PROVER_SERVER_LISTEN_IP=127.0.0.1 PROVER_SERVER_LISTEN_PORT=3000 +# Not the same account as the COMMITTER_L1 Account +# The proposer is in charge of blob commitments. +# The prover_server is in charge of verifying the zkProofs. +PROVER_SERVER_VERIFIER_ADDRESS=0xE25583099BA105D9ec0A67f5Ae86D90e50036425 +PROVER_SERVER_VERIFIER_PRIVATE_KEY=0x39725efee3fb28614de3bacaffe4cc4bd8c436257e2c8bb887c4b5c4be45e76d PROVER_CLIENT_PROVER_SERVER_ENDPOINT=localhost:3000 -PROPOSER_ON_CHAIN_PROPOSER_ADDRESS=0xe9927d77c931f8648da4cc6751ef4e5e2ce74608 -PROPOSER_L1_ADDRESS=0x3d1e15a1a55578f7c920884a9943b3b35d0d885b -PROPOSER_L1_PRIVATE_KEY=0x385c546456b6a603a1cfcaa9ec9494ba4832da08dd6bcf4de9a71e4a01b74924 +COMMITTER_ON_CHAIN_PROPOSER_ADDRESS=0xe9927d77c931f8648da4cc6751ef4e5e2ce74608 +COMMITTER_L1_ADDRESS=0x3d1e15a1a55578f7c920884a9943b3b35d0d885b +COMMITTER_L1_PRIVATE_KEY=0x385c546456b6a603a1cfcaa9ec9494ba4832da08dd6bcf4de9a71e4a01b74924 +COMMITTER_INTERVAL_MS=1000 PROPOSER_INTERVAL_MS=5000 # https://dev.risczero.com/api/generating-proofs/dev-mode # 1/true means fake proofs +# The RISC0_DEV_MODE=1 should only be used with DEPLOYER_CONTRACT_VERIFIER=0xAA RISC0_DEV_MODE=1 -RUST_LOG="[executor]=info" diff --git a/crates/l2/Makefile b/crates/l2/Makefile index 08cf945cf..e6ca8a2e0 100644 --- a/crates/l2/Makefile +++ b/crates/l2/Makefile @@ -3,6 +3,7 @@ .PHONY: help init down clean init-local-l1 down-local-l1 clean-local-l1 init-l2 down-l2 deploy-l1 deploy-block-executor deploy-inbox setup-prover test L2_GENESIS_FILE_PATH=../../test_data/genesis-l2.json +L1_GENESIS_FILE_PATH=../../test_data/genesis-l1.json help: ## ๐Ÿ“š Show help for each of the Makefile recipes @grep -E '^[a-zA-Z0-9_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' @@ -29,10 +30,26 @@ ETHEREUM_RUST_L2_CONTRACTS_PATH=./contracts L1_RPC_URL=http://localhost:8545 L1_PRIVATE_KEY=0x385c546456b6a603a1cfcaa9ec9494ba4832da08dd6bcf4de9a71e4a01b74924 +ETHEREUM_RUST_L2_DEV_LIBMDBX=dev_ethereum_rust_l2 +ETHEREUM_RUST_L1_DEV_LIBMDBX=dev_ethereum_rust_l1 +L1_PORT=8545 +L2_PORT=1729 +L1_AUTH_PORT=8551 +# Used in the .env file. Ensure the same port is used for `ENGINE_API_RPC_URL`. +L2_AUTH_PORT=8552 + # Local L1 -init-local-l1: ## ๐Ÿš€ Initializes an L1 Lambda Ethereum Rust Client +init-local-l1: ## ๐Ÿš€ Initializes an L1 Lambda Ethereum Rust Client with Docker (Used with make init) docker compose -f ${ETHEREUM_RUST_DEV_DOCKER_COMPOSE_PATH} up -d + +init-l1: ## ๐Ÿš€ Initializes an L1 Lambda Ethereum Rust Client + cargo run --release --manifest-path ../../Cargo.toml --bin ethereum_rust --features dev -- \ + --network ${L1_GENESIS_FILE_PATH} \ + --http.port ${L1_PORT} \ + --http.addr 0.0.0.0 \ + --authrpc.port ${L1_AUTH_PORT} \ + --datadir ${ETHEREUM_RUST_L1_DEV_LIBMDBX} down-local-l1: ## ๐Ÿ›‘ Shuts down the L1 Lambda Ethereum Rust Client docker compose -f ${ETHEREUM_RUST_DEV_DOCKER_COMPOSE_PATH} down @@ -40,6 +57,9 @@ down-local-l1: ## ๐Ÿ›‘ Shuts down the L1 Lambda Ethereum Rust Client restart-local-l1: down-local-l1 init-local-l1 ## ๐Ÿ”„ Restarts the L1 Lambda Ethereum Rust Client +rm_dev_libmdbx_l1: ## ๐Ÿ›‘ Removes the Libmdbx DB used by the L1 + cargo run --release --manifest-path ../../Cargo.toml --bin ethereum_rust -- removedb --datadir ${ETHEREUM_RUST_L1_DEV_LIBMDBX} + # Contracts clean-contract-deps: ## ๐Ÿงน Cleans the dependencies for the L1 contracts. @@ -54,7 +74,12 @@ deploy-l1: ## ๐Ÿ“œ Deploys the L1 contracts # L2 init-l2: ## ๐Ÿš€ Initializes an L2 Lambda Ethereum Rust Client - cargo run --release --manifest-path ../../Cargo.toml --bin ethereum_rust --features l2 -- --network ${L2_GENESIS_FILE_PATH} --http.port 1729 + cargo run --release --manifest-path ../../Cargo.toml --bin ethereum_rust --features l2 -- \ + --network ${L2_GENESIS_FILE_PATH} \ + --http.port ${L2_PORT} \ + --http.addr 0.0.0.0 \ + --authrpc.port ${L2_AUTH_PORT} \ + --datadir ${ETHEREUM_RUST_L2_DEV_LIBMDBX} down-l2: ## ๐Ÿ›‘ Shuts down the L2 Lambda Ethereum Rust Client pkill -f ethereum_rust || exit 0 @@ -65,7 +90,11 @@ init-l2-prover: ## ๐Ÿš€ Initializes the Prover cargo run --release --features build_zkvm --manifest-path ../../Cargo.toml --bin ethereum_rust_prover init-l2-prover-gpu: ## ๐Ÿš€ Initializes the Prover with GPU support - cargo run --release --features "build_zkvm,cuda" --manifest-path ../../Cargo.toml --bin ethereum_rust_prover + cargo run --release --features "build_zkvm,gpu" --manifest-path ../../Cargo.toml --bin ethereum_rust_prover + +rm_dev_libmdbx_l2: ## ๐Ÿ›‘ Removes the Libmdbx DB used by the L2 + cargo run --release --manifest-path ../../Cargo.toml --bin ethereum_rust -- removedb --datadir ${ETHEREUM_RUST_L2_DEV_LIBMDBX} + # CI Testing diff --git a/crates/l2/contracts/deployer.rs b/crates/l2/contracts/deployer.rs index c8dcad59b..602bd21b3 100644 --- a/crates/l2/contracts/deployer.rs +++ b/crates/l2/contracts/deployer.rs @@ -28,7 +28,8 @@ lazy_static::lazy_static! { #[tokio::main] async fn main() { - let (deployer, deployer_private_key, eth_client, contracts_path) = setup(); + let (deployer, deployer_private_key, contract_verifier_address, eth_client, contracts_path) = + setup(); download_contract_deps(&contracts_path); compile_contracts(&contracts_path); let (on_chain_proposer, bridge_address) = @@ -38,6 +39,7 @@ async fn main() { deployer_private_key, on_chain_proposer, bridge_address, + contract_verifier_address, ð_client, ) .await; @@ -50,7 +52,7 @@ async fn main() { if let Some(eq) = line.find('=') { let (envar, _) = line.split_at(eq); line = match envar { - "PROPOSER_ON_CHAIN_PROPOSER_ADDRESS" => { + "COMMITTER_ON_CHAIN_PROPOSER_ADDRESS" => { format!("{envar}={on_chain_proposer:#x}") } "L1_WATCHER_BRIDGE_ADDRESS" => { @@ -64,7 +66,7 @@ async fn main() { write_env(wr_lines).expect("Failed to write changes to the .env file."); } -fn setup() -> (Address, SecretKey, EthClient, PathBuf) { +fn setup() -> (Address, SecretKey, Address, EthClient, PathBuf) { if let Err(e) = read_env_file() { warn!("Failed to read .env file: {e}"); } @@ -99,12 +101,20 @@ fn setup() -> (Address, SecretKey, EthClient, PathBuf) { "false" | "0" => { let mut salt = SALT.lock().unwrap(); *salt = H256::random(); - println!("SALT: {salt:?}"); } _ => panic!("Invalid boolean string: {input}"), }; - - (deployer, deployer_private_key, eth_client, contracts_path) + let contract_verifier_address = std::env::var("DEPLOYER_CONTRACT_VERIFIER") + .expect("DEPLOYER_CONTRACT_VERIFIER not set") + .parse() + .expect("Malformed DEPLOYER_CONTRACT_VERIFIER"); + ( + deployer, + deployer_private_key, + contract_verifier_address, + eth_client, + contracts_path, + ) } fn download_contract_deps(contracts_path: &Path) { @@ -177,9 +187,15 @@ async fn deploy_contracts( eth_client: &EthClient, contracts_path: &Path, ) -> (Address, Address) { + let gas_price = if eth_client.url.contains("localhost:8545") { + Some(1_000_000_000) + } else { + Some(eth_client.get_gas_price().await.unwrap().as_u64() * 2) + }; + let overrides = Overrides { gas_limit: Some(GAS_LIMIT_MINIMUM * GAS_LIMIT_ADJUSTMENT_FACTOR), - gas_price: Some(1_000_000_000), + gas_price, ..Default::default() }; @@ -307,6 +323,7 @@ async fn create2_deploy( ) .await .expect("Failed to build create2 deploy tx"); + let deploy_tx_hash = eth_client .send_eip1559_transaction(deploy_tx, &deployer_private_key) .await @@ -341,6 +358,7 @@ async fn initialize_contracts( deployer_private_key: SecretKey, on_chain_proposer: Address, bridge: Address, + contract_verifier_address: Address, eth_client: &EthClient, ) { let initialize_frames = spinner!(["๐Ÿช„โฑโฑ", "โฑ๐Ÿช„โฑ", "โฑโฑ๐Ÿช„"], 200); @@ -354,6 +372,7 @@ async fn initialize_contracts( let initialize_tx_hash = initialize_on_chain_proposer( on_chain_proposer, bridge, + contract_verifier_address, deployer, deployer_private_key, eth_client, @@ -388,11 +407,12 @@ async fn initialize_contracts( async fn initialize_on_chain_proposer( on_chain_proposer: Address, bridge: Address, + contract_verifier_address: Address, deployer: Address, deployer_private_key: SecretKey, eth_client: &EthClient, ) -> H256 { - let on_chain_proposer_initialize_selector = keccak(b"initialize(address)") + let on_chain_proposer_initialize_selector = keccak(b"initialize(address,address)") .as_bytes() .get(..4) .expect("Failed to get initialize selector") @@ -404,10 +424,18 @@ async fn initialize_on_chain_proposer( encoded_bridge }; + let encoded_contract_verifier = { + let offset = 32 - contract_verifier_address.as_bytes().len() % 32; + let mut encoded_contract_verifier = vec![0; offset]; + encoded_contract_verifier.extend_from_slice(contract_verifier_address.as_bytes()); + encoded_contract_verifier + }; + let mut on_chain_proposer_initialization_calldata = Vec::new(); on_chain_proposer_initialization_calldata .extend_from_slice(&on_chain_proposer_initialize_selector); on_chain_proposer_initialization_calldata.extend_from_slice(&encoded_bridge); + on_chain_proposer_initialization_calldata.extend_from_slice(&encoded_contract_verifier); let initialize_tx = eth_client .build_eip1559_transaction( diff --git a/crates/l2/contracts/src/l1/OnChainProposer.sol b/crates/l2/contracts/src/l1/OnChainProposer.sol index 0cc1fcfd2..303c558cb 100644 --- a/crates/l2/contracts/src/l1/OnChainProposer.sol +++ b/crates/l2/contracts/src/l1/OnChainProposer.sol @@ -6,6 +6,7 @@ import "../../lib/openzeppelin-contracts/contracts/utils/ReentrancyGuard.sol"; import "./interfaces/IOnChainProposer.sol"; import {CommonBridge} from "./CommonBridge.sol"; import {ICommonBridge} from "./interfaces/ICommonBridge.sol"; +import {IRiscZeroVerifier} from "./interfaces/IRiscZeroVerifier.sol"; /// @title OnChainProposer contract. /// @author LambdaClass @@ -28,10 +29,27 @@ contract OnChainProposer is IOnChainProposer, ReentrancyGuard { /// @dev This is crucial for ensuring that only valid and confirmed blocks are processed in the contract. uint256 public lastVerifiedBlock; + /// @notice The latest committed block number. + /// @dev This variable holds the block number of the most recently committed block. + /// @dev All blocks with a block number less than or equal to `lastCommittedBlock` are considered committed. + /// @dev Blocks with a block number greater than `lastCommittedBlock` have not been committed yet. + /// @dev This is crucial for ensuring that only subsequents blocks are committed in the contract. + /// @dev In the initialize function, `lastCommittedBlock` is set to u64::MAX == 0xFFFFFFFFFFFFFFFF, this value is used to allow the block 0 to be committed. + uint256 public lastCommittedBlock; + address public BRIDGE; + address public R0VERIFIER; + + /// @notice Address used to avoid the verification process. + /// @dev If the `R0VERIFIER` contract address is set to this address, the verification process will not happen. + /// @dev Used only in dev mode. + address public constant DEV_MODE = address(0xAA); /// @inheritdoc IOnChainProposer - function initialize(address bridge) public nonReentrant { + function initialize( + address bridge, + address r0verifier + ) public nonReentrant { require( BRIDGE == address(0), "OnChainProposer: contract already initialized" @@ -45,6 +63,22 @@ contract OnChainProposer is IOnChainProposer, ReentrancyGuard { "OnChainProposer: bridge is the contract address" ); BRIDGE = bridge; + + require( + R0VERIFIER == address(0), + "OnChainProposer: contract already initialized" + ); + require( + r0verifier != address(0), + "OnChainProposer: r0verifier is the zero address" + ); + require( + r0verifier != address(this), + "OnChainProposer: r0verifier is the contract address" + ); + R0VERIFIER = r0verifier; + + lastCommittedBlock = 0xFFFFFFFFFFFFFFFF; } /// @inheritdoc IOnChainProposer @@ -55,8 +89,9 @@ contract OnChainProposer is IOnChainProposer, ReentrancyGuard { bytes32 depositLogs ) external override { require( - blockNumber == lastVerifiedBlock + 1, - "OnChainProposer: block already verified" + blockNumber == lastCommittedBlock + 1 || + (blockNumber == 0 && lastCommittedBlock == 0xFFFFFFFFFFFFFFFF), + "OnChainProposer: blockNumber is not the immediate succesor of lastCommittedBlock" ); require( blockCommitments[blockNumber].commitmentHash == bytes32(0), @@ -82,13 +117,16 @@ contract OnChainProposer is IOnChainProposer, ReentrancyGuard { commitment, depositLogs ); + lastCommittedBlock = blockNumber; emit BlockCommitted(commitment); } /// @inheritdoc IOnChainProposer function verify( uint256 blockNumber, - bytes calldata // blockProof + bytes calldata blockProof, + bytes32 imageId, + bytes32 journalDigest ) external override { require( blockCommitments[blockNumber].commitmentHash != bytes32(0), @@ -99,6 +137,15 @@ contract OnChainProposer is IOnChainProposer, ReentrancyGuard { "OnChainProposer: block already verified" ); + if (R0VERIFIER != DEV_MODE) { + // If the verification fails, it will revert. + IRiscZeroVerifier(R0VERIFIER).verify( + blockProof, + imageId, + journalDigest + ); + } + lastVerifiedBlock = blockNumber; ICommonBridge(BRIDGE).removeDepositLogs( // The first 2 bytes are the number of deposits. diff --git a/crates/l2/contracts/src/l1/interfaces/IOnChainProposer.sol b/crates/l2/contracts/src/l1/interfaces/IOnChainProposer.sol index b60a15da3..01013ec98 100644 --- a/crates/l2/contracts/src/l1/interfaces/IOnChainProposer.sol +++ b/crates/l2/contracts/src/l1/interfaces/IOnChainProposer.sol @@ -21,7 +21,8 @@ interface IOnChainProposer { /// @dev This method is called only once after the contract is deployed. /// @dev It sets the bridge address. /// @param bridge the address of the bridge contract. - function initialize(address bridge) external; + /// @param r0verifier the address of the risc0 groth16 verifier. + function initialize(address bridge, address r0verifier) external; /// @notice Commits to an L2 block. /// @dev Committing to an L2 block means to store the block's commitment @@ -43,5 +44,12 @@ interface IOnChainProposer { /// verified (this is after proved). /// @param blockNumber is the number of the block to be verified. /// @param blockProof is the proof of the block to be verified. - function verify(uint256 blockNumber, bytes calldata blockProof) external; + /// @param imageId Digest of the zkVM imageid. + /// @param journalDigest Digest of the public_inputs aka journal + function verify( + uint256 blockNumber, + bytes calldata blockProof, + bytes32 imageId, + bytes32 journalDigest + ) external; } diff --git a/crates/l2/contracts/src/l1/interfaces/IRiscZeroVerifier.sol b/crates/l2/contracts/src/l1/interfaces/IRiscZeroVerifier.sol new file mode 100644 index 000000000..2d92013a6 --- /dev/null +++ b/crates/l2/contracts/src/l1/interfaces/IRiscZeroVerifier.sol @@ -0,0 +1,49 @@ +// Copyright 2024 RISC Zero, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 +// +// NOTICE: +// Modified from the original file. +// Making use of the IRiscZeroVerifier interface and nothing else. + +pragma solidity ^0.8.9; + +/// @notice A receipt attesting to the execution of a guest program. +/// @dev A receipt contains two parts: a seal and a claim. The seal is a zero-knowledge proof +/// attesting to knowledge of a zkVM execution resulting in the claim. The claim is a set of public +/// outputs for the execution. Crucially, the claim includes the journal and the image ID. The +/// image ID identifies the program that was executed, and the journal is the public data written +/// by the program. Note that this struct only contains the claim digest, as can be obtained with +/// the `digest()` function on `ReceiptClaimLib`. +struct Receipt { + bytes seal; + bytes32 claimDigest; +} + +/// @notice Error raised when cryptographic verification of the zero-knowledge proof fails. +error VerificationFailed(); + +/// @notice Verifier interface for RISC Zero receipts of execution. +interface IRiscZeroVerifier { + /// @notice Verify that the given seal is a valid RISC Zero proof of execution with the + /// given image ID and journal digest. Reverts on failure. + /// @dev This method additionally ensures that the input hash is all-zeros (i.e. no + /// committed input), the exit code is (Halted, 0), and there are no assumptions (i.e. the + /// receipt is unconditional). + /// @param seal The encoded cryptographic proof (i.e. SNARK). + /// @param imageId The identifier for the guest program. + /// @param journalDigest The SHA-256 digest of the journal bytes. + function verify(bytes calldata seal, bytes32 imageId, bytes32 journalDigest) external view; +} diff --git a/crates/l2/docs/prover.md b/crates/l2/docs/prover.md index df3c45060..9726c88a3 100644 --- a/crates/l2/docs/prover.md +++ b/crates/l2/docs/prover.md @@ -8,12 +8,14 @@ - [How](#how) - [Dev Mode](#dev-mode) - [Quick Test](#quick-test) + - [Run the whole system with the prover](#run-the-whole-system-with-the-prover) - [GPU mode](#gpu-mode) - [Proving Process Test](#proving-process-test) + - [Run the whole system with the prover in Sepolia](#run-the-whole-system-with-the-prover-in-sepolia) - [Configuration](#configuration) >[!NOTE] -> The shipping/deploying process and the `Prover` itself is under development. +> The shipping/deploying process and the `Prover` itself are under development. ## What @@ -28,13 +30,13 @@ The `Prover Server` monitors requests for new jobs from the `Prover Client`, whi ```mermaid sequenceDiagram - participant Prover + participant zkVM participant ProverClient participant ProverServer ProverClient->>+ProverServer: ProofData::Request ProverServer-->>-ProverClient: ProofData::Response(block_number, ProverInputs) - ProverClient->>+Prover: Prove(block_number, ProverInputs) - Prover-->>-ProverClient: Creates zkProof + ProverClient->>+zkVM: Prove(ProverInputs) + zkVM-->>-ProverClient: Creates zkProof ProverClient->>+ProverServer: ProofData::Submit(block_number, zkProof) ProverServer-->>-ProverClient: ProofData::SubmitAck(block_number) ``` @@ -65,26 +67,38 @@ cd crates/l2/prover make perf_test_proving ``` -### GPU mode +#### Run the whole system with the prover -**Dependencies (based on the Docker CUDA image):** +1. `cd crates/l2` +2. `make rm_dev_libmdbx_l2 && make down` + - It will remove any old database, if present, stored in your computer. The absolute path of libmdbx is defined by [data_dir](https://docs.rs/dirs/latest/dirs/fn.data_dir.html). +3. `cp .env.example .env` → check if you want to change any config. +4. `make init` + - Init the L1 in a docker container on port `8545`. + - Deploy the needed contracts for the L2 on the L1. + - Start the L2 locally on port `1729`. +5. In a new terminal → `make init-l2-prover`. ->[!NOTE] -> If you don't want to run it inside a Docker container based on the NVIDIA CUDA image, [the following steps from RISC0](https://dev.risczero.com/api/generating-proofs/local-proving) may be helpful. +After this initialization the system has to be running in `dev-mode` → No proof verification. -- [Rust](https://www.rust-lang.org/tools/install) -- [RISC0](https://dev.risczero.com/api/zkvm/install) +### GPU mode -Next, install the following packages: +**Steps for Ubuntu 22.04 with Nvidia A4000:** -```sh -sudo apt-get install libssl-dev pkg-config libclang-dev clang -``` - -To start the `prover_client`, use the following command: +1. Install `docker` → using the [Ubuntu apt repository](https://docs.docker.com/engine/install/ubuntu/#install-using-the-repository) + - Add the `user` you are using to the `docker` group → command: `sudo usermod -aG docker $USER`. (needs reboot, doing it after CUDA installation) + - `id -nG` after reboot to check if the user is in the group. +2. Install [Rust](https://www.rust-lang.org/tools/install) +3. Install [RISC0](https://dev.risczero.com/api/zkvm/install) +4. Install [CUDA for Ubuntu](https://developer.nvidia.com/cuda-downloads?target_os=Linux&target_arch=x86_64&Distribution=Ubuntu&target_version=22.04&target_type=deb_local) + - Install `CUDA Toolkit Installer` first. Then the `nvidia-open` drivers. +5. Reboot +6. Run the following commands: ```sh -make init-l2-prover-gpu +sudo apt-get install libssl-dev pkg-config libclang-dev clang +echo 'export PATH=/usr/local/cuda/bin:$PATH' >> ~/.bashrc +echo 'export LD_LIBRARY_PATH=/usr/local/cuda/lib64:$LD_LIBRARY_PATH' >> ~/.bashrc ``` #### Proving Process Test @@ -96,6 +110,40 @@ cd crates/l2/prover make perf_gpu ``` +#### Run the whole system with the prover in Sepolia + +Two servers are required: one for the `prover` and another for the `proposer`. If you run both components on the same machine, the `prover` may consume all available resources, leading to potential stuttering or performance issues for the `proposer`/`node`. + +1. `prover`/`zkvm` → prover with gpu, make sure to have all the required dependencies described at the beginning of [Gpu Mode](#gpu-mode) section. + 1. `cd lambda_ethereum_rust/crates/l2` + 2. `cp .example.env` and change the `PROVER_CLIENT_PROVER_SERVER_ENDPOINT` with the ip of the other server. + +The env variables needed are: + +```sh +PROVER_CLIENT_PROVER_SERVER_ENDPOINT=:3000 +RISC0_DEV_MODE=0 +``` + +Finally, to start the `prover_client`/`zkvm`, run: + +- `make init-l2-prover-gpu` + +2. ย `proposer` → this server just needs rust installed. + 1. `cd lambda_ethereum_rust/crates/l2` + 2. `cp .example.env` and change the addresses and the following fields: + - `PROVER_SERVER_LISTEN_IP=0.0.0.0`ย → used to handle the tcp communication with the other server. + - The `COMMITTER` and `PROVER_SERVER_VERIFIER` must be different accounts, the `DEPLOYER_ADDRESS` as well as the `L1_WATCHER` may be the same account used by the `COMMITTER` + - `DEPLOYER_CONTRACT_VERIFIER=0xd9b0d07CeCd808a8172F21fA7C97992168f045CA`ย → risc0โ€™s verifier contract deployed on Sepolia. + - Set the `ETH_RPC_URL` to any Sepolia's endpoint. + +>[!NOTE] +> Make sure to have funds, if you want to perform a quick test `0.2[ether]` on each account should be enough. + +Finally, to start the `proposer`/`l2 node`, run: + - `make rm_dev_libmdbx_l2 && make down` + - `make init` + ## Configuration The following environment variables are available to configure the prover: @@ -103,3 +151,8 @@ The following environment variables are available to configure the prover: - `PROVER_SERVER_LISTEN_IP`: IP used to start the Server. - `PROVER_SERVER_LISTEN_PORT`: Port used to start the Server. - `PROVER_CLIENT_PROVER_SERVER_ENDPOINT`: Prover Server's Endpoint used to connect the Client to the Server. +- `PROVER_SERVER_VERIFIER_ADDRESS`: The address of the account that sends the zkProofs on-chain and interacts with the `OnChainProposer` `verify()` function. +- `PROVER_SERVER_VERIFIER_PRIVATE_KEY`: The private key of the account that sends the zkProofs on-chain and interacts with the `OnChainProposer` `verify()` function. + +>[!NOTE] +> The `PROVER_SERVER_VERIFIER` account must differ from the `COMMITTER_L1` account. diff --git a/crates/l2/proposer/errors.rs b/crates/l2/proposer/errors.rs index 2ff47bc4b..df37f122c 100644 --- a/crates/l2/proposer/errors.rs +++ b/crates/l2/proposer/errors.rs @@ -1,6 +1,8 @@ use crate::utils::{config::errors::ConfigError, eth_client::errors::EthClientError}; use ethereum_rust_dev::utils::engine_client::errors::EngineClientError; +use ethereum_rust_storage::error::StoreError; use ethereum_rust_vm::EvmError; +use ethereum_types::FromStrRadixErr; #[derive(Debug, thiserror::Error)] pub enum L1WatcherError { @@ -22,28 +24,42 @@ pub enum L1WatcherError { pub enum ProverServerError { #[error("ProverServer connection failed: {0}")] ConnectionError(#[from] std::io::Error), + #[error("ProverServer failed because of an EthClient error: {0}")] + EthClientError(#[from] EthClientError), + #[error("ProverServer failed to send transaction: {0}")] + FailedToVerifyProofOnChain(String), } #[derive(Debug, thiserror::Error)] pub enum ProposerError { - #[error("Proposer failed because of an EthClient error: {0}")] - EthClientError(#[from] EthClientError), #[error("Proposer failed because of an EngineClient error: {0}")] EngineClientError(#[from] EngineClientError), #[error("Proposer failed to produce block: {0}")] FailedToProduceBlock(String), #[error("Proposer failed to prepare PayloadAttributes timestamp: {0}")] FailedToGetSystemTime(#[from] std::time::SystemTimeError), - #[error("Proposer failed to serialize block: {0}")] - FailedToRetrieveBlockFromStorage(String), - #[error("Proposer failed to encode state diff: {0}")] +} + +#[derive(Debug, thiserror::Error)] +pub enum CommitterError { + #[error("Committer failed because of an EthClient error: {0}")] + EthClientError(#[from] EthClientError), + #[error("Committer failed to {0}")] + FailedToParseLastCommittedBlock(#[from] FromStrRadixErr), + #[error("Committer failed retrieve block from storage: {0}")] + FailedToRetrieveBlockFromStorage(#[from] StoreError), + #[error("Committer failed to get information from storage")] + FailedToGetInformationFromStorage(String), + #[error("Committer failed to encode state diff: {0}")] FailedToEncodeStateDiff(#[from] StateDiffError), - #[error("Proposer failed to open Points file: {0}")] + #[error("Committer failed to open Points file: {0}")] FailedToOpenPointsFile(#[from] std::io::Error), - #[error("Proposer failed to re-execute block: {0}")] + #[error("Committer failed to re-execute block: {0}")] FailedToReExecuteBlock(#[from] EvmError), - #[error("Proposer failed to make KZG operations: {0}")] + #[error("Committer failed to make KZG operations: {0}")] KZGError(#[from] c_kzg::Error), + #[error("Committer failed to send transaction: {0}")] + FailedToSendCommitment(String), } #[derive(Debug, thiserror::Error)] diff --git a/crates/l2/proposer/l1_committer.rs b/crates/l2/proposer/l1_committer.rs new file mode 100644 index 000000000..c803d90c4 --- /dev/null +++ b/crates/l2/proposer/l1_committer.rs @@ -0,0 +1,503 @@ +use crate::{ + proposer::{ + errors::CommitterError, + state_diff::{AccountStateDiff, DepositLog, StateDiff, WithdrawalLog}, + }, + utils::{ + config::{committer::CommitterConfig, eth::EthConfig}, + eth_client::{ + errors::EthClientError, eth_sender::Overrides, transaction::blob_from_bytes, EthClient, + }, + merkle_tree::merkelize, + }, +}; +use bytes::Bytes; +use c_kzg::{Bytes48, KzgSettings}; +use ethereum_rust_blockchain::constants::TX_GAS_COST; +use ethereum_rust_core::{ + types::{ + BlobsBundle, Block, EIP1559Transaction, GenericTransaction, PrivilegedL2Transaction, + PrivilegedTxType, Transaction, TxKind, BYTES_PER_BLOB, + }, + Address, H256, U256, +}; +use ethereum_rust_rpc::types::transaction::WrappedEIP4844Transaction; +use ethereum_rust_storage::Store; +use ethereum_rust_vm::{evm_state, execute_block, get_state_transitions}; +use keccak_hash::keccak; +use secp256k1::SecretKey; +use sha2::{Digest, Sha256}; +use std::ops::Div; +use std::{collections::HashMap, time::Duration}; +use tokio::time::sleep; +use tracing::{error, info, warn}; + +const COMMIT_FUNCTION_SELECTOR: [u8; 4] = [132, 97, 12, 179]; + +pub struct Committer { + eth_client: EthClient, + on_chain_proposer_address: Address, + store: Store, + l1_address: Address, + l1_private_key: SecretKey, + interval_ms: u64, + kzg_settings: &'static KzgSettings, +} + +pub async fn start_l1_commiter(store: Store) { + let eth_config = EthConfig::from_env().expect("EthConfig::from_env()"); + let committer_config = CommitterConfig::from_env().expect("CommitterConfig::from_env"); + let committer = Committer::new_from_config(&committer_config, eth_config, store); + committer.start().await.expect("committer.start()"); +} + +impl Committer { + pub fn new_from_config( + committer_config: &CommitterConfig, + eth_config: EthConfig, + store: Store, + ) -> Self { + Self { + eth_client: EthClient::new(ð_config.rpc_url), + on_chain_proposer_address: committer_config.on_chain_proposer_address, + store, + l1_address: committer_config.l1_address, + l1_private_key: committer_config.l1_private_key, + interval_ms: committer_config.interval_ms, + kzg_settings: c_kzg::ethereum_kzg_settings(), + } + } + + pub async fn start(&self) -> Result<(), CommitterError> { + loop { + let last_committed_block = get_last_committed_block( + &self.eth_client, + self.on_chain_proposer_address, + Overrides::default(), + ) + .await?; + + let last_committed_block = last_committed_block + .strip_prefix("0x") + .expect("Couldn't strip prefix from last_committed_block."); + + if last_committed_block.is_empty() { + error!("Failed to fetch last_committed_block"); + panic!("Failed to fetch last_committed_block. Manual intervention required"); + } + + let last_committed_block = U256::from_str_radix(last_committed_block, 16) + .map_err(CommitterError::from)? + .as_u64(); + + let block_number_to_fetch = if last_committed_block == u64::MAX { + 0 + } else { + last_committed_block + 1 + }; + + if let Some(block_to_commit_body) = self + .store + .get_block_body(block_number_to_fetch) + .map_err(CommitterError::from)? + { + let block_to_commit_header = self + .store + .get_block_header(block_number_to_fetch) + .map_err(CommitterError::from)? + .ok_or(CommitterError::FailedToGetInformationFromStorage( + "Failed to get_block_header() after get_block_body()".to_owned(), + ))?; + + let block_to_commit = Block::new(block_to_commit_header, block_to_commit_body); + + let withdrawals = self.get_block_withdrawals(&block_to_commit)?; + let deposits = self.get_block_deposits(&block_to_commit)?; + + let withdrawal_logs_merkle_root = self.get_withdrawals_merkle_root( + withdrawals.iter().map(|(hash, _tx)| *hash).collect(), + ); + let deposit_logs_hash = self.get_deposit_hash( + deposits + .iter() + .filter_map(|tx| tx.get_deposit_hash()) + .collect(), + ); + + let state_diff = self.prepare_state_diff( + &block_to_commit, + self.store.clone(), + withdrawals, + deposits, + )?; + + let (blob_commitment, blob_proof) = + self.prepare_blob_commitment(state_diff.clone())?; + + let head_block_hash = block_to_commit.hash(); + match self + .send_commitment( + block_to_commit.header.number, + withdrawal_logs_merkle_root, + deposit_logs_hash, + blob_commitment, + blob_proof, + state_diff.encode()?, + ) + .await + { + Ok(commit_tx_hash) => { + info!( + "Sent commitment to block {head_block_hash:#x}, with transaction hash {commit_tx_hash:#x}" + ); + } + Err(error) => { + error!("Failed to send commitment to block {head_block_hash:#x}. Manual intervention required: {error}"); + panic!("Failed to send commitment to block {head_block_hash:#x}. Manual intervention required: {error}"); + } + } + } + + sleep(Duration::from_millis(self.interval_ms)).await; + } + } + + pub fn get_block_withdrawals( + &self, + block: &Block, + ) -> Result, CommitterError> { + let withdrawals = block + .body + .transactions + .iter() + .filter_map(|tx| match tx { + Transaction::PrivilegedL2Transaction(priv_tx) + if priv_tx.tx_type == PrivilegedTxType::Withdrawal => + { + Some((tx.compute_hash(), priv_tx.clone())) + } + _ => None, + }) + .collect(); + + Ok(withdrawals) + } + + pub fn get_withdrawals_merkle_root(&self, withdrawals_hashes: Vec) -> H256 { + if !withdrawals_hashes.is_empty() { + merkelize(withdrawals_hashes) + } else { + H256::zero() + } + } + + pub fn get_block_deposits( + &self, + block: &Block, + ) -> Result, CommitterError> { + let deposits = block + .body + .transactions + .iter() + .filter_map(|tx| match tx { + Transaction::PrivilegedL2Transaction(tx) + if tx.tx_type == PrivilegedTxType::Deposit => + { + Some(tx.clone()) + } + _ => None, + }) + .collect(); + + Ok(deposits) + } + + pub fn get_deposit_hash(&self, deposit_hashes: Vec) -> H256 { + if !deposit_hashes.is_empty() { + H256::from_slice( + [ + &(deposit_hashes.len() as u16).to_be_bytes(), + &keccak( + deposit_hashes + .iter() + .map(H256::as_bytes) + .collect::>() + .concat(), + ) + .as_bytes()[2..32], + ] + .concat() + .as_slice(), + ) + } else { + H256::zero() + } + } + /// Prepare the state diff for the block. + pub fn prepare_state_diff( + &self, + block: &Block, + store: Store, + withdrawals: Vec<(H256, PrivilegedL2Transaction)>, + deposits: Vec, + ) -> Result { + info!("Preparing state diff for block {}", block.header.number); + + let mut state = evm_state(store.clone(), block.header.parent_hash); + execute_block(block, &mut state).map_err(CommitterError::from)?; + let account_updates = get_state_transitions(&mut state); + + let mut modified_accounts = HashMap::new(); + account_updates.iter().for_each(|account_update| { + modified_accounts.insert( + account_update.address, + AccountStateDiff { + new_balance: account_update.info.clone().map(|info| info.balance), + nonce_diff: account_update.info.clone().map(|info| info.nonce as u16), + storage: account_update.added_storage.clone().into_iter().collect(), + bytecode: account_update.code.clone(), + bytecode_hash: None, + }, + ); + }); + + let state_diff = StateDiff { + modified_accounts, + version: StateDiff::default().version, + withdrawal_logs: withdrawals + .iter() + .map(|(hash, tx)| WithdrawalLog { + address: match tx.to { + TxKind::Call(address) => address, + TxKind::Create => Address::zero(), + }, + amount: tx.value, + tx_hash: *hash, + }) + .collect(), + deposit_logs: deposits + .iter() + .map(|tx| DepositLog { + address: match tx.to { + TxKind::Call(address) => address, + TxKind::Create => Address::zero(), + }, + amount: tx.value, + }) + .collect(), + }; + + Ok(state_diff) + } + + /// Generate the KZG commitment and proof for the blob. This commitment can then be used + /// to calculate the blob versioned hash, necessary for the EIP-4844 transaction. + pub fn prepare_blob_commitment( + &self, + state_diff: StateDiff, + ) -> Result<([u8; 48], [u8; 48]), CommitterError> { + let blob_data = state_diff.encode().map_err(CommitterError::from)?; + + let blob = blob_from_bytes(blob_data).map_err(CommitterError::from)?; + + let commitment = c_kzg::KzgCommitment::blob_to_kzg_commitment(&blob, self.kzg_settings) + .map_err(CommitterError::from)?; + let commitment_bytes = + Bytes48::from_bytes(commitment.as_slice()).map_err(CommitterError::from)?; + let proof = + c_kzg::KzgProof::compute_blob_kzg_proof(&blob, &commitment_bytes, self.kzg_settings) + .map_err(CommitterError::from)?; + + let mut commitment_bytes = [0u8; 48]; + commitment_bytes.copy_from_slice(commitment.as_slice()); + let mut proof_bytes = [0u8; 48]; + proof_bytes.copy_from_slice(proof.as_slice()); + + Ok((commitment_bytes, proof_bytes)) + } + + pub async fn send_commitment( + &self, + block_number: u64, + withdrawal_logs_merkle_root: H256, + deposit_logs_hash: H256, + commitment: [u8; 48], + proof: [u8; 48], + blob_data: Bytes, + ) -> Result { + info!("Sending commitment for block {block_number}"); + + let mut hasher = Sha256::new(); + hasher.update(commitment); + let mut blob_versioned_hash = hasher.finalize(); + blob_versioned_hash[0] = 0x01; // EIP-4844 versioning + + let mut calldata = Vec::with_capacity(132); + calldata.extend(COMMIT_FUNCTION_SELECTOR); + let mut block_number_bytes = [0_u8; 32]; + U256::from(block_number).to_big_endian(&mut block_number_bytes); + calldata.extend(block_number_bytes); + calldata.extend(blob_versioned_hash); + calldata.extend(withdrawal_logs_merkle_root.0); + calldata.extend(deposit_logs_hash.0); + + let mut buf = [0u8; BYTES_PER_BLOB]; + buf.copy_from_slice( + blob_from_bytes(blob_data) + .map_err(CommitterError::from)? + .iter() + .as_slice(), + ); + + let blobs_bundle = BlobsBundle { + blobs: vec![buf], + commitments: vec![commitment], + proofs: vec![proof], + }; + let wrapped_tx = self + .eth_client + .build_eip4844_transaction( + self.on_chain_proposer_address, + Bytes::from(calldata), + Overrides { + from: Some(self.l1_address), + gas_price_per_blob: Some(U256::from_dec_str("100000000000").unwrap()), + ..Default::default() + }, + blobs_bundle, + ) + .await + .map_err(CommitterError::from)?; + + let commit_tx_hash = self + .eth_client + .send_eip4844_transaction(wrapped_tx.clone(), &self.l1_private_key) + .await + .map_err(CommitterError::from)?; + + let commit_tx_hash = wrapped_eip4844_transaction_handler( + &self.eth_client, + &wrapped_tx, + &self.l1_private_key, + commit_tx_hash, + 10, + ) + .await?; + + info!("Commitment sent: {commit_tx_hash:#x}"); + + Ok(commit_tx_hash) + } +} + +pub async fn send_transaction_with_calldata( + eth_client: &EthClient, + l1_address: Address, + l1_private_key: SecretKey, + to: Address, + nonce: Option, + calldata: Bytes, +) -> Result { + let mut tx = EIP1559Transaction { + to: TxKind::Call(to), + data: calldata, + max_fee_per_gas: eth_client.get_gas_price().await?.as_u64() * 2, + nonce: nonce.unwrap_or(eth_client.get_nonce(l1_address).await?), + chain_id: eth_client.get_chain_id().await?.as_u64(), + // Should the max_priority_fee_per_gas be dynamic? + max_priority_fee_per_gas: 10u64, + ..Default::default() + }; + + let mut generic_tx = GenericTransaction::from(tx.clone()); + generic_tx.from = l1_address; + + tx.gas_limit = eth_client + .estimate_gas(generic_tx) + .await? + .saturating_add(TX_GAS_COST); + + eth_client + .send_eip1559_transaction(tx, &l1_private_key) + .await +} + +async fn get_last_committed_block( + eth_client: &EthClient, + contract_address: Address, + overrides: Overrides, +) -> Result { + let selector = keccak(b"lastCommittedBlock()") + .as_bytes() + .get(..4) + .expect("Failed to get initialize selector") + .to_vec(); + + let mut calldata = Vec::new(); + calldata.extend_from_slice(&selector); + + let leading_zeros = 32 - ((calldata.len() - 4) % 32); + calldata.extend(vec![0; leading_zeros]); + + eth_client + .call(contract_address, calldata.into(), overrides) + .await +} + +async fn wrapped_eip4844_transaction_handler( + eth_client: &EthClient, + wrapped_eip4844: &WrappedEIP4844Transaction, + l1_private_key: &SecretKey, + commit_tx_hash: H256, + max_retries: u32, +) -> Result { + let mut retries = 0; + let max_receipt_retries: u32 = 60 * 2; // 2 minutes + let mut commit_tx_hash = commit_tx_hash; + let mut wrapped_tx = wrapped_eip4844.clone(); + + while retries < max_retries { + if (eth_client.get_transaction_receipt(commit_tx_hash).await?).is_some() { + // If the tx_receipt was found, return the tx_hash. + return Ok(commit_tx_hash); + } else { + // Else, wait for receipt and send again if necessary. + let mut receipt_retries = 0; + + // Try for 2 minutes with an interval of 1 second to get the tx_receipt. + while receipt_retries < max_receipt_retries { + match eth_client.get_transaction_receipt(commit_tx_hash).await? { + Some(_) => return Ok(commit_tx_hash), + None => { + receipt_retries += 1; + sleep(Duration::from_secs(1)).await; + } + } + } + + // If receipt was not found, send the same tx(same nonce) but with more gas. + // Sometimes the penalty is a 100% + warn!("Transaction not confirmed, resending with 110% more gas..."); + // Increase max fee per gas by 110% (set it to 210% of the original) + wrapped_tx.tx.max_fee_per_gas = + (wrapped_tx.tx.max_fee_per_gas as f64 * 2.1).round() as u64; + wrapped_tx.tx.max_priority_fee_per_gas = + (wrapped_tx.tx.max_priority_fee_per_gas as f64 * 2.1).round() as u64; + wrapped_tx.tx.max_fee_per_blob_gas = wrapped_tx + .tx + .max_fee_per_blob_gas + .saturating_mul(U256::from(20)) + .div(10); + + commit_tx_hash = eth_client + .send_eip4844_transaction(wrapped_tx.clone(), l1_private_key) + .await + .map_err(CommitterError::from)?; + + retries += 1; + } + } + Err(CommitterError::FailedToSendCommitment( + "Error handling eip4844".to_owned(), + )) +} diff --git a/crates/l2/proposer/mod.rs b/crates/l2/proposer/mod.rs index 056539aa9..ec5d1aa22 100644 --- a/crates/l2/proposer/mod.rs +++ b/crates/l2/proposer/mod.rs @@ -1,48 +1,23 @@ -use crate::utils::{ - config::{eth::EthConfig, proposer::ProposerConfig, read_env_file}, - eth_client::{eth_sender::Overrides, transaction::blob_from_bytes, EthClient}, - merkle_tree::merkelize, -}; -use bytes::Bytes; -use c_kzg::{Bytes48, KzgSettings}; +use crate::utils::config::{proposer::ProposerConfig, read_env_file}; use errors::ProposerError; -use ethereum_rust_core::types::{ - BlobsBundle, Block, PrivilegedL2Transaction, PrivilegedTxType, Transaction, TxKind, - BYTES_PER_BLOB, -}; use ethereum_rust_dev::utils::engine_client::{config::EngineApiConfig, EngineClient}; use ethereum_rust_rpc::types::fork_choice::{ForkChoiceState, PayloadAttributesV3}; use ethereum_rust_storage::Store; -use ethereum_rust_vm::{evm_state, execute_block, get_state_transitions}; -use ethereum_types::{Address, H256, U256}; -use keccak_hash::keccak; -use secp256k1::SecretKey; -use sha2::{Digest, Sha256}; -use state_diff::{AccountStateDiff, DepositLog, StateDiff, WithdrawalLog}; -use std::{ - collections::HashMap, - time::{Duration, SystemTime, UNIX_EPOCH}, -}; +use ethereum_types::{Address, H256}; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; use tokio::time::sleep; use tracing::{error, info, warn}; +pub mod l1_committer; pub mod l1_watcher; pub mod prover_server; pub mod state_diff; pub mod errors; -const COMMIT_FUNCTION_SELECTOR: [u8; 4] = [132, 97, 12, 179]; -const VERIFY_FUNCTION_SELECTOR: [u8; 4] = [133, 133, 44, 228]; - pub struct Proposer { - eth_client: EthClient, engine_client: EngineClient, - on_chain_proposer_address: Address, - l1_address: Address, - l1_private_key: SecretKey, block_production_interval: Duration, - kzg_settings: &'static KzgSettings, } pub async fn start_proposer(store: Store) { @@ -53,12 +28,12 @@ pub async fn start_proposer(store: Store) { } let l1_watcher = tokio::spawn(l1_watcher::start_l1_watcher(store.clone())); + let l1_committer = tokio::spawn(l1_committer::start_l1_commiter(store.clone())); let prover_server = tokio::spawn(prover_server::start_prover_server(store.clone())); let proposer = tokio::spawn(async move { - let eth_config = EthConfig::from_env().expect("EthConfig::from_env"); let proposer_config = ProposerConfig::from_env().expect("ProposerConfig::from_env"); let engine_config = EngineApiConfig::from_env().expect("EngineApiConfig::from_env"); - let proposer = Proposer::new_from_config(&proposer_config, eth_config, engine_config) + let proposer = Proposer::new_from_config(&proposer_config, engine_config) .expect("Proposer::new_from_config"); let head_block_hash = { let current_block_number = store @@ -71,31 +46,25 @@ pub async fn start_proposer(store: Store) { .expect("store.get_canonical_block_hash returned None") }; proposer - .start(head_block_hash, store) + .start(head_block_hash) .await .expect("Proposer::start"); }); - tokio::try_join!(l1_watcher, prover_server, proposer).expect("tokio::try_join"); + tokio::try_join!(l1_watcher, l1_committer, prover_server, proposer).expect("tokio::try_join"); } impl Proposer { pub fn new_from_config( proposer_config: &ProposerConfig, - eth_config: EthConfig, engine_config: EngineApiConfig, ) -> Result { Ok(Self { - eth_client: EthClient::new(ð_config.rpc_url), engine_client: EngineClient::new_from_config(engine_config)?, - on_chain_proposer_address: proposer_config.on_chain_proposer_address, - l1_address: proposer_config.l1_address, - l1_private_key: proposer_config.l1_private_key, block_production_interval: Duration::from_millis(proposer_config.interval_ms), - kzg_settings: c_kzg::ethereum_kzg_settings(), }) } - pub async fn start(&self, head_block_hash: H256, store: Store) -> Result<(), ProposerError> { + pub async fn start(&self, head_block_hash: H256) -> Result<(), ProposerError> { let mut head_block_hash = head_block_hash; loop { head_block_hash = self.produce_block(head_block_hash).await?; @@ -106,72 +75,6 @@ impl Proposer { continue; } - let block = store - .get_block_by_hash(head_block_hash) - .map_err(|error| { - ProposerError::FailedToRetrieveBlockFromStorage(error.to_string()) - })? - .ok_or(ProposerError::FailedToProduceBlock( - "Failed to get block by hash from storage".to_string(), - ))?; - - let withdrawals = self.get_block_withdrawals(&block)?; - let deposits = self.get_block_deposits(&block)?; - - let withdrawal_logs_merkle_root = self.get_withdrawals_merkle_root( - withdrawals - .iter() - .map(|(_hash, tx)| tx.get_withdrawal_hash().unwrap()) - .collect(), - ); - let deposit_logs_hash = self.get_deposit_hash( - deposits - .iter() - .filter_map(|tx| tx.get_deposit_hash()) - .collect(), - ); - - let state_diff = - self.prepare_state_diff(&block, store.clone(), withdrawals, deposits)?; - - let (blob_commitment, blob_proof) = self.prepare_blob_commitment(state_diff.clone())?; - - match self - .send_commitment( - block.header.number, - withdrawal_logs_merkle_root, - deposit_logs_hash, - blob_commitment, - blob_proof, - state_diff.encode()?, - ) - .await - { - Ok(commit_tx_hash) => { - info!( - "Sent commitment to block {head_block_hash:#x}, with transaction hash {commit_tx_hash:#x}" - ); - } - Err(error) => { - error!("Failed to send commitment to block {head_block_hash:#x}. Manual intervention required: {error}"); - panic!("Failed to send commitment to block {head_block_hash:#x}. Manual intervention required: {error}"); - } - } - - let proof = Vec::new(); - - match self.send_proof(block.header.number, &proof).await { - Ok(verify_tx_hash) => { - info!( - "Sent proof for block {head_block_hash}, with transaction hash {verify_tx_hash:#x}" - ); - } - Err(error) => { - error!("Failed to send proof to block {head_block_hash:#x}. Manual intervention required: {error}"); - panic!("Failed to send proof to block {head_block_hash:#x}. Manual intervention required: {error}"); - } - } - sleep(self.block_production_interval).await; } } @@ -247,279 +150,4 @@ impl Proposer { info!("Produced block {produced_block_hash:#x}"); Ok(produced_block_hash) } - - pub fn get_block_withdrawals( - &self, - block: &Block, - ) -> Result, ProposerError> { - let withdrawals = block - .body - .transactions - .iter() - .filter_map(|tx| match tx { - Transaction::PrivilegedL2Transaction(priv_tx) - if priv_tx.tx_type == PrivilegedTxType::Withdrawal => - { - Some((tx.compute_hash(), priv_tx.clone())) - } - _ => None, - }) - .collect(); - - Ok(withdrawals) - } - - pub fn get_withdrawals_merkle_root(&self, withdrawals_hashes: Vec) -> H256 { - if !withdrawals_hashes.is_empty() { - merkelize(withdrawals_hashes) - } else { - H256::zero() - } - } - - pub fn get_block_deposits( - &self, - block: &Block, - ) -> Result, ProposerError> { - let deposits = block - .body - .transactions - .iter() - .filter_map(|tx| match tx { - Transaction::PrivilegedL2Transaction(tx) - if tx.tx_type == PrivilegedTxType::Deposit => - { - Some(tx.clone()) - } - _ => None, - }) - .collect(); - - Ok(deposits) - } - - pub fn get_deposit_hash(&self, deposit_hashes: Vec) -> H256 { - if !deposit_hashes.is_empty() { - H256::from_slice( - [ - &(deposit_hashes.len() as u16).to_be_bytes(), - &keccak( - deposit_hashes - .iter() - .map(H256::as_bytes) - .collect::>() - .concat(), - ) - .as_bytes()[2..32], - ] - .concat() - .as_slice(), - ) - } else { - H256::zero() - } - } - - /// Prepare the state diff for the block. - pub fn prepare_state_diff( - &self, - block: &Block, - store: Store, - withdrawals: Vec<(H256, PrivilegedL2Transaction)>, - deposits: Vec, - ) -> Result { - info!("Preparing state diff for block {}", block.header.number); - - let mut state = evm_state(store.clone(), block.header.parent_hash); - execute_block(block, &mut state).map_err(ProposerError::from)?; - let account_updates = get_state_transitions(&mut state); - - let mut modified_accounts = HashMap::new(); - account_updates.iter().for_each(|account_update| { - modified_accounts.insert( - account_update.address, - AccountStateDiff { - new_balance: account_update.info.clone().map(|info| info.balance), - nonce_diff: account_update.info.clone().map(|info| info.nonce as u16), - storage: account_update.added_storage.clone().into_iter().collect(), - bytecode: account_update.code.clone(), - bytecode_hash: None, - }, - ); - }); - - let state_diff = StateDiff { - modified_accounts, - version: StateDiff::default().version, - withdrawal_logs: withdrawals - .iter() - .map(|(hash, tx)| WithdrawalLog { - address: match tx.to { - TxKind::Call(address) => address, - TxKind::Create => Address::zero(), - }, - amount: tx.value, - tx_hash: *hash, - }) - .collect(), - deposit_logs: deposits - .iter() - .map(|tx| DepositLog { - address: match tx.to { - TxKind::Call(address) => address, - TxKind::Create => Address::zero(), - }, - amount: tx.value, - }) - .collect(), - }; - - Ok(state_diff) - } - - /// Generate the KZG commitment and proof for the blob. This commitment can then be used - /// to calculate the blob versioned hash, necessary for the EIP-4844 transaction. - pub fn prepare_blob_commitment( - &self, - state_diff: StateDiff, - ) -> Result<([u8; 48], [u8; 48]), ProposerError> { - let blob_data = state_diff.encode().map_err(ProposerError::from)?; - - let blob = blob_from_bytes(blob_data).map_err(ProposerError::from)?; - - let commitment = c_kzg::KzgCommitment::blob_to_kzg_commitment(&blob, self.kzg_settings) - .map_err(ProposerError::from)?; - let commitment_bytes = - Bytes48::from_bytes(commitment.as_slice()).map_err(ProposerError::from)?; - let proof = - c_kzg::KzgProof::compute_blob_kzg_proof(&blob, &commitment_bytes, self.kzg_settings) - .map_err(ProposerError::from)?; - - let mut commitment_bytes = [0u8; 48]; - commitment_bytes.copy_from_slice(commitment.as_slice()); - let mut proof_bytes = [0u8; 48]; - proof_bytes.copy_from_slice(proof.as_slice()); - - Ok((commitment_bytes, proof_bytes)) - } - - pub async fn send_commitment( - &self, - block_number: u64, - withdrawal_logs_merkle_root: H256, - deposit_logs_hash: H256, - commitment: [u8; 48], - proof: [u8; 48], - blob_data: Bytes, - ) -> Result { - info!("Sending commitment for block {block_number}"); - - let mut hasher = Sha256::new(); - hasher.update(commitment); - let mut blob_versioned_hash = hasher.finalize(); - blob_versioned_hash[0] = 0x01; // EIP-4844 versioning - - let mut calldata = Vec::with_capacity(132); - calldata.extend(COMMIT_FUNCTION_SELECTOR); - let mut block_number_bytes = [0_u8; 32]; - U256::from(block_number).to_big_endian(&mut block_number_bytes); - calldata.extend(block_number_bytes); - calldata.extend(blob_versioned_hash); - calldata.extend(withdrawal_logs_merkle_root.0); - calldata.extend(deposit_logs_hash.0); - - let mut buf = [0u8; BYTES_PER_BLOB]; - buf.copy_from_slice( - blob_from_bytes(blob_data) - .map_err(ProposerError::from)? - .iter() - .as_slice(), - ); - - let blobs_bundle = BlobsBundle { - blobs: vec![buf], - commitments: vec![commitment], - proofs: vec![proof], - }; - let wrapped_tx = self - .eth_client - .build_eip4844_transaction( - self.on_chain_proposer_address, - Bytes::from(calldata), - Overrides { - from: Some(self.l1_address), - gas_price_per_blob: Some(U256::from_dec_str("100000000000000").unwrap()), - ..Default::default() - }, - blobs_bundle, - ) - .await - .map_err(ProposerError::from)?; - - let commit_tx_hash = self - .eth_client - .send_eip4844_transaction(wrapped_tx, &self.l1_private_key) - .await - .map_err(ProposerError::from)?; - - info!("Commitment sent: {commit_tx_hash:#x}"); - - while self - .eth_client - .get_transaction_receipt(commit_tx_hash) - .await? - .is_none() - { - sleep(Duration::from_secs(1)).await; - } - - Ok(commit_tx_hash) - } - - pub async fn send_proof( - &self, - block_number: u64, - block_proof: &[u8], - ) -> Result { - info!("Sending proof"); - let mut calldata = Vec::new(); - calldata.extend(VERIFY_FUNCTION_SELECTOR); - let mut block_number_bytes = [0_u8; 32]; - U256::from(block_number).to_big_endian(&mut block_number_bytes); - calldata.extend(block_number_bytes); - calldata.extend(H256::from_low_u64_be(64).as_bytes()); - calldata.extend(H256::from_low_u64_be(block_proof.len() as u64).as_bytes()); - calldata.extend(block_proof); - let leading_zeros = 32 - ((calldata.len() - 4) % 32); - calldata.extend(vec![0; leading_zeros]); - - let verify_tx = self - .eth_client - .build_eip1559_transaction( - self.on_chain_proposer_address, - calldata.into(), - Overrides { - from: Some(self.l1_address), - ..Default::default() - }, - ) - .await?; - let verify_tx_hash = self - .eth_client - .send_eip1559_transaction(verify_tx, &self.l1_private_key) - .await?; - - info!("Proof sent: {verify_tx_hash:#x}"); - - while self - .eth_client - .get_transaction_receipt(verify_tx_hash) - .await? - .is_none() - { - sleep(Duration::from_secs(1)).await; - } - - Ok(verify_tx_hash) - } } diff --git a/crates/l2/proposer/prover_server.rs b/crates/l2/proposer/prover_server.rs index 1c752337b..8522af1b6 100644 --- a/crates/l2/proposer/prover_server.rs +++ b/crates/l2/proposer/prover_server.rs @@ -1,17 +1,26 @@ -use crate::utils::eth_client::RpcResponse; use ethereum_rust_storage::Store; use ethereum_rust_vm::execution_db::ExecutionDB; -use reqwest::Client; +use keccak_hash::keccak; +use secp256k1::SecretKey; use serde::{Deserialize, Serialize}; use std::{ io::{BufReader, BufWriter}, net::{IpAddr, Shutdown, TcpListener, TcpStream}, sync::mpsc::{self, Receiver}, + time::Duration, +}; +use tokio::{ + signal::unix::{signal, SignalKind}, + time::sleep, +}; +use tracing::{debug, error, info, warn}; + +use ethereum_rust_core::{ + types::{Block, BlockHeader, EIP1559Transaction}, + Address, H256, }; -use tokio::signal::unix::{signal, SignalKind}; -use tracing::{debug, info, warn}; -use ethereum_rust_core::types::{Block, BlockHeader}; +use risc0_zkvm::sha::{Digest, Digestible}; #[derive(Debug, Serialize, Deserialize, Default)] pub struct ProverInputData { @@ -20,13 +29,19 @@ pub struct ProverInputData { pub parent_header: BlockHeader, } -use crate::utils::config::prover_server::ProverServerConfig; +use crate::utils::{ + config::{committer::CommitterConfig, eth::EthConfig, prover_server::ProverServerConfig}, + eth_client::{eth_sender::Overrides, EthClient}, +}; use super::errors::ProverServerError; pub async fn start_prover_server(store: Store) { - let config = ProverServerConfig::from_env().expect("ProverServerConfig::from_env()"); - let prover_server = ProverServer::new_from_config(config.clone(), store); + let server_config = ProverServerConfig::from_env().expect("ProverServerConfig::from_env()"); + let eth_config = EthConfig::from_env().expect("EthConfig::from_env()"); + let proposer_config = CommitterConfig::from_env().expect("CommitterConfig::from_env()"); + let mut prover_server = + ProverServer::new_from_config(server_config.clone(), &proposer_config, eth_config, store); let (tx, rx) = mpsc::channel(); @@ -37,40 +52,68 @@ pub async fn start_prover_server(store: Store) { .expect("prover_server.start()") }); - ProverServer::handle_sigint(tx, config).await; + ProverServer::handle_sigint(tx, server_config).await; tokio::try_join!(server).expect("tokio::try_join!()"); } +/// Enum for the ProverServer <--> ProverClient Communication Protocol. #[derive(Debug, Serialize, Deserialize)] pub enum ProofData { - Request {}, + /// 1. + /// The Client initiates the connection with a Request. + /// Asking for the ProverInputData the prover_server considers/needs. + Request, + + /// 2. + /// The Server responds with a Response containing the ProverInputData. + /// If the Response will is ProofData::Response{None, None}, the Client knows that the Request couldn't be performed. Response { block_number: Option, - input: ProverInputData, + input: Option, }, + + /// 3. + /// The Client submits the zk Proof generated by the prover + /// for the specified block. Submit { block_number: u64, // zk Proof - receipt: Box, - }, - SubmitAck { - block_number: u64, + receipt: Box<(risc0_zkvm::Receipt, Vec)>, }, + + /// 4. + /// The Server acknowledges the receipt of the proof and updates its state, + SubmitAck { block_number: u64 }, } struct ProverServer { ip: IpAddr, port: u16, store: Store, + eth_client: EthClient, + on_chain_proposer_address: Address, + verifier_address: Address, + verifier_private_key: SecretKey, + latest_proven_block: u64, } impl ProverServer { - pub fn new_from_config(config: ProverServerConfig, store: Store) -> Self { + pub fn new_from_config( + config: ProverServerConfig, + committer_config: &CommitterConfig, + eth_config: EthConfig, + store: Store, + ) -> Self { Self { ip: config.listen_ip, port: config.listen_port, store, + eth_client: EthClient::new(ð_config.rpc_url), + on_chain_proposer_address: committer_config.on_chain_proposer_address, + verifier_address: config.verifier_address, + verifier_private_key: config.verifier_private_key, + latest_proven_block: 0, } } @@ -84,11 +127,9 @@ impl ProverServer { .expect("TcpStream::shutdown()"); } - pub async fn start(&self, rx: Receiver<()>) -> Result<(), ProverServerError> { + pub async fn start(&mut self, rx: Receiver<()>) -> Result<(), ProverServerError> { let listener = TcpListener::bind(format!("{}:{}", self.ip, self.port))?; - let mut last_proved_block = 0; - info!("Starting TCP server at {}:{}", self.ip, self.port); for stream in listener.incoming() { if let Ok(()) = rx.try_recv() { @@ -97,19 +138,21 @@ impl ProverServer { } debug!("Connection established!"); - self.handle_connection(stream?, &mut last_proved_block) - .await; + self.handle_connection(stream?).await?; } Ok(()) } - async fn handle_connection(&self, mut stream: TcpStream, last_proved_block: &mut u64) { + async fn handle_connection(&mut self, mut stream: TcpStream) -> Result<(), ProverServerError> { let buf_reader = BufReader::new(&stream); let data: Result = serde_json::de::from_reader(buf_reader); match data { - Ok(ProofData::Request {}) => { - if let Err(e) = self.handle_request(&mut stream, *last_proved_block).await { + Ok(ProofData::Request) => { + if let Err(e) = self + .handle_request(&mut stream, self.latest_proven_block + 1) + .await + { warn!("Failed to handle request: {e}"); } } @@ -117,10 +160,15 @@ impl ProverServer { block_number, receipt, }) => { - if let Err(e) = self.handle_submit(&mut stream, block_number, receipt) { - warn!("Failed to handle submit: {e}"); + if let Err(e) = self.handle_submit(&mut stream, block_number) { + error!("Failed to handle submit_ack: {e}"); + panic!("Failed to handle submit_ack: {e}"); } - *last_proved_block += 1; + // Seems to be stopping the prover_server <--> prover_client + self.handle_proof_submission(block_number, receipt).await?; + + assert!(block_number == (self.latest_proven_block + 1), "Prover Client submitted an invalid block_number: {block_number}. The last_proved_block is: {}", self.latest_proven_block); + self.latest_proven_block = block_number; } Err(e) => { warn!("Failed to parse request: {e}"); @@ -131,84 +179,122 @@ impl ProverServer { } debug!("Connection closed"); - } - - async fn _get_last_block_number() -> Result { - let response = Client::new() - .post("http://localhost:8551") - .header("content-type", "application/json") - .body( - r#"{ - "jsonrpc": "2.0", - "method": "eth_blockNumber", - "params": [], - "id": 1 - }"#, - ) - .send() - .await - .map_err(|e| e.to_string())? - .json::() - .await - .map_err(|e| e.to_string())?; - - if let RpcResponse::Success(r) = response { - u64::from_str_radix( - r.result - .as_str() - .ok_or("Response format error".to_string())? - .strip_prefix("0x") - .ok_or("Response format error".to_string())?, - 16, - ) - .map_err(|e| e.to_string()) - } else { - Err("Failed to get last block number".to_string()) - } + Ok(()) } async fn handle_request( &self, stream: &mut TcpStream, - last_proved_block: u64, + block_number: u64, ) -> Result<(), String> { debug!("Request received"); - let last_block_number = self + let latest_block_number = self .store .get_latest_block_number() .map_err(|e| e.to_string())? - .ok_or("missing latest block number".to_string())?; - let input = self.create_prover_input(last_block_number)?; + .unwrap(); - let response = if last_block_number > last_proved_block { - ProofData::Response { - block_number: Some(last_block_number), - input, - } - } else { - ProofData::Response { + let response = if block_number > latest_block_number { + let response = ProofData::Response { block_number: None, - input, - } + input: None, + }; + warn!("Didn't send response"); + response + } else { + let input = self.create_prover_input(block_number)?; + let response = ProofData::Response { + block_number: Some(block_number), + input: Some(input), + }; + info!("Sent Response for block_number: {block_number}"); + response }; + let writer = BufWriter::new(stream); serde_json::to_writer(writer, &response).map_err(|e| e.to_string()) } - fn handle_submit( - &self, - stream: &mut TcpStream, - block_number: u64, - receipt: Box, - ) -> Result<(), String> { - debug!("Submit received. ID: {block_number}, proof: {:?}", receipt); + fn handle_submit(&self, stream: &mut TcpStream, block_number: u64) -> Result<(), String> { + debug!("Submit received for BlockNumber: {block_number}"); let response = ProofData::SubmitAck { block_number }; let writer = BufWriter::new(stream); serde_json::to_writer(writer, &response).map_err(|e| e.to_string()) } + async fn handle_proof_submission( + &self, + block_number: u64, + receipt: Box<(risc0_zkvm::Receipt, Vec)>, + ) -> Result<(), ProverServerError> { + // Send Tx + // If we run the prover_client with RISC0_DEV_MODE=0 we will have a groth16 proof + // Else, we will have a fake proof. + // + // The RISC0_DEV_MODE=1 should only be used with DEPLOYER_CONTRACT_VERIFIER=0xAA + let seal = match receipt.0.inner.groth16() { + Ok(inner) => { + // The SELECTOR is used to perform an extra check inside the groth16 verifier contract. + let mut selector = + hex::encode(inner.verifier_parameters.as_bytes().get(..4).unwrap()); + let seal = hex::encode(inner.clone().seal); + selector.push_str(&seal); + hex::decode(selector).unwrap() + } + Err(_) => vec![32; 0], + }; + + let mut image_id: [u32; 8] = [0; 8]; + for (i, b) in image_id.iter_mut().enumerate() { + *b = *receipt.1.get(i).unwrap(); + } + + let image_id: risc0_zkvm::sha::Digest = image_id.into(); + + let journal_digest = Digestible::digest(&receipt.0.journal); + + // Retry proof verification, the transaction may fail if the blobs commited were not included. + // The error message is `address already reserved`. Retrying 100 times, if there is another error it panics. + let mut attempts = 0; + let max_retries = 100; + let retry_secs = std::time::Duration::from_secs(5); + while attempts < max_retries { + match self + .send_proof(block_number, &seal, image_id, journal_digest) + .await + { + Ok(tx_hash) => { + info!( + "Sent proof for block {block_number}, with transaction hash {tx_hash:#x}" + ); + break; // Exit the while loop + } + + Err(e) => { + warn!("Failed to send proof to block {block_number:#x}. Error: {e}"); + let eth_client_error = format!("{e}"); + if eth_client_error.contains("block not committed") { + attempts += 1; + if attempts < max_retries { + warn!("Retrying... Attempt {}/{}", attempts, max_retries); + sleep(retry_secs).await; // Wait before retrying + } else { + error!("Max retries reached. Giving up on sending proof for block {block_number:#x}."); + panic!("Failed to send proof after {} attempts.", max_retries); + } + } else { + error!("Failed to send proof to block {block_number:#x}. Manual intervention required: {e}"); + panic!("Failed to send proof to block {block_number:#x}. Manual intervention required: {e}"); + } + } + } + } + + Ok(()) + } + fn create_prover_input(&self, block_number: u64) -> Result { let header = self .store @@ -239,4 +325,136 @@ impl ProverServer { parent_header, }) } + + pub async fn send_proof( + &self, + block_number: u64, + seal: &[u8], + image_id: Digest, + journal_digest: Digest, + ) -> Result { + info!("Sending proof"); + let mut calldata = Vec::new(); + + // IOnChainProposer + // function verify(uint256,bytes,bytes32,bytes32) + // Verifier + // function verify(bytes,bytes32,bytes32) + // blockNumber, seal, imageId, journalDigest + // From crates/l2/contracts/l1/interfaces/IOnChainProposer.sol + let verify_proof_selector = keccak(b"verify(uint256,bytes,bytes32,bytes32)") + .as_bytes() + .get(..4) + .expect("Failed to get initialize selector") + .to_vec(); + calldata.extend(verify_proof_selector); + + // The calldata has to be structured in the following way: + // block_number + // size in bytes + // image_id digest + // journal digest + // size of seal + // seal + + // extend with block_number + calldata.extend(H256::from_low_u64_be(block_number).as_bytes()); + + // extend with size in bytes + // 4 u256 goes after this field so: 0x80 == 128bytes == 32bytes * 4 + calldata.extend(H256::from_low_u64_be(4 * 32).as_bytes()); + + // extend with image_id + calldata.extend(image_id.as_bytes()); + + // extend with journal_digest + calldata.extend(journal_digest.as_bytes()); + + // extend with size of seal + calldata.extend(H256::from_low_u64_be(seal.len() as u64).as_bytes()); + // extend with seal + calldata.extend(seal); + // extend with zero padding + let leading_zeros = 32 - ((calldata.len() - 4) % 32); + calldata.extend(vec![0; leading_zeros]); + + let verify_tx = self + .eth_client + .build_eip1559_transaction( + self.on_chain_proposer_address, + calldata.into(), + Overrides { + from: Some(self.verifier_address), + ..Default::default() + }, + ) + .await?; + let verify_tx_hash = self + .eth_client + .send_eip1559_transaction(verify_tx.clone(), &self.verifier_private_key) + .await?; + + eip1559_transaction_handler( + &self.eth_client, + &verify_tx, + &self.verifier_private_key, + verify_tx_hash, + 20, + ) + .await?; + + Ok(verify_tx_hash) + } +} + +async fn eip1559_transaction_handler( + eth_client: &EthClient, + eip1559: &EIP1559Transaction, + l1_private_key: &SecretKey, + verify_tx_hash: H256, + max_retries: u32, +) -> Result { + let mut retries = 0; + let max_receipt_retries: u32 = 60 * 2; // 2 minutes + let mut verify_tx_hash = verify_tx_hash; + let mut tx = eip1559.clone(); + + while retries < max_retries { + if (eth_client.get_transaction_receipt(verify_tx_hash).await?).is_some() { + // If the tx_receipt was found, return the tx_hash. + return Ok(verify_tx_hash); + } else { + // Else, wait for receipt and send again if necessary. + let mut receipt_retries = 0; + + // Try for 2 minutes with an interval of 1 second to get the tx_receipt. + while receipt_retries < max_receipt_retries { + match eth_client.get_transaction_receipt(verify_tx_hash).await? { + Some(_) => return Ok(verify_tx_hash), + None => { + receipt_retries += 1; + sleep(Duration::from_secs(1)).await; + } + } + } + + // If receipt was not found, send the same tx(same nonce) but with more gas. + // Sometimes the penalty is a 100% + warn!("Transaction not confirmed, resending with 110% more gas..."); + // Increase max fee per gas by 110% (set it to 210% of the original) + tx.max_fee_per_gas = (tx.max_fee_per_gas as f64 * 2.1).round() as u64; + tx.max_priority_fee_per_gas += + (tx.max_priority_fee_per_gas as f64 * 2.1).round() as u64; + + verify_tx_hash = eth_client + .send_eip1559_transaction(tx.clone(), l1_private_key) + .await + .map_err(ProverServerError::from)?; + + retries += 1; + } + } + Err(ProverServerError::FailedToVerifyProofOnChain( + "Error handling eip1559".to_owned(), + )) } diff --git a/crates/l2/prover/src/main.rs b/crates/l2/prover/src/main.rs index 9333fb753..403c65d04 100644 --- a/crates/l2/prover/src/main.rs +++ b/crates/l2/prover/src/main.rs @@ -6,7 +6,8 @@ use tracing::{self, debug, warn, Level}; #[tokio::main] async fn main() { let subscriber = tracing_subscriber::FmtSubscriber::builder() - .with_max_level(Level::DEBUG) + // Hiding debug!() logs. + .with_max_level(Level::INFO) .finish(); tracing::subscriber::set_global_default(subscriber).expect("setting default subscriber failed"); diff --git a/crates/l2/prover/src/prover.rs b/crates/l2/prover/src/prover.rs index 1840d2b25..05e8fc90b 100644 --- a/crates/l2/prover/src/prover.rs +++ b/crates/l2/prover/src/prover.rs @@ -28,7 +28,7 @@ pub struct ProverOutputData { pub struct Prover<'a> { env_builder: ExecutorEnvBuilder<'a>, elf: &'a [u8], - id: [u32; 8], + pub id: [u32; 8], } impl<'a> Default for Prover<'a> { diff --git a/crates/l2/prover/src/prover_client.rs b/crates/l2/prover/src/prover_client.rs index 93c5a1e2d..21626b9ac 100644 --- a/crates/l2/prover/src/prover_client.rs +++ b/crates/l2/prover/src/prover_client.rs @@ -15,18 +15,20 @@ use ethereum_rust_l2::{ use super::prover::Prover; pub async fn start_proof_data_client(config: ProverClientConfig) { - let proof_data_client = ProverClient::new(config.prover_server_endpoint.clone()); + let proof_data_client = ProverClient::new(config); proof_data_client.start().await; } struct ProverClient { prover_server_endpoint: String, + interval_ms: u64, } impl ProverClient { - pub fn new(prover_server_endpoint: String) -> Self { + pub fn new(config: ProverClientConfig) -> Self { Self { - prover_server_endpoint, + prover_server_endpoint: config.prover_server_endpoint, + interval_ms: config.interval_ms, } } @@ -34,78 +36,86 @@ impl ProverClient { let mut prover = Prover::new(); loop { - match self.request_new_data() { - Ok((Some(block_number), input)) => { + match self.request_new_input() { + Ok((block_number, input)) => { match prover.set_input(input).prove() { Ok(proof) => { - if let Err(e) = self.submit_proof(block_number, proof) { - // TODO: Retry + if let Err(e) = + self.submit_proof(block_number, proof, prover.id.to_vec()) + { + // TODO: Retry? warn!("Failed to submit proof: {e}"); } } Err(e) => error!(e), }; } - Ok((None, _)) => sleep(Duration::from_secs(10)).await, Err(e) => { - sleep(Duration::from_secs(10)).await; + sleep(Duration::from_millis(self.interval_ms)).await; warn!("Failed to request new data: {e}"); } } } } - fn request_new_data(&self) -> Result<(Option, ProverInputData), String> { - let stream = TcpStream::connect(&self.prover_server_endpoint) - .map_err(|e| format!("Failed to connect to Prover Server: {e}"))?; - let buf_writer = BufWriter::new(&stream); - - debug!("Connection established!"); - - let request = ProofData::Request {}; - serde_json::ser::to_writer(buf_writer, &request).map_err(|e| e.to_string())?; - stream - .shutdown(std::net::Shutdown::Write) - .map_err(|e| e.to_string())?; - - let buf_reader = BufReader::new(&stream); - let response: ProofData = serde_json::de::from_reader(buf_reader) - .map_err(|e| format!("Invalid response format: {e}"))?; + fn request_new_input(&self) -> Result<(u64, ProverInputData), String> { + // Request the input with the correct block_number + let request = ProofData::Request; + let response = connect_to_prover_server_wr(&self.prover_server_endpoint, &request) + .map_err(|e| format!("Failed to get Response: {e}"))?; match response { ProofData::Response { block_number, input, - } => Ok((block_number, input)), - _ => Err(format!("Unexpected response {response:?}")), + } => match (block_number, input) { + (Some(n), Some(i)) => { + info!("Received Response for block_number: {n}"); + Ok((n, i)) + } + _ => Err( + "Received Empty Response, meaning that the ProverServer doesn't have blocks to prove.\nThe Prover may be advancing faster than the Proposer." + .to_owned(), + ), + }, + _ => Err(format!("Expecting ProofData::Response {response:?}")), } } - fn submit_proof(&self, block_number: u64, receipt: risc0_zkvm::Receipt) -> Result<(), String> { - let stream = TcpStream::connect(&self.prover_server_endpoint) - .map_err(|e| format!("Failed to connect to Prover Server: {e}"))?; - let buf_writer = BufWriter::new(&stream); - + fn submit_proof( + &self, + block_number: u64, + receipt: risc0_zkvm::Receipt, + prover_id: Vec, + ) -> Result<(), String> { let submit = ProofData::Submit { block_number, - receipt: Box::new(receipt), + receipt: Box::new((receipt, prover_id)), }; - serde_json::ser::to_writer(buf_writer, &submit).map_err(|e| e.to_string())?; - stream - .shutdown(std::net::Shutdown::Write) - .map_err(|e| e.to_string())?; + let submit_ack = connect_to_prover_server_wr(&self.prover_server_endpoint, &submit) + .map_err(|e| format!("Failed to get SubmitAck: {e}"))?; - let buf_reader = BufReader::new(&stream); - let response: ProofData = serde_json::de::from_reader(buf_reader) - .map_err(|e| format!("Invalid response format: {e}"))?; - match response { - ProofData::SubmitAck { - block_number: res_id, - } => { - info!("Received submit ack: {res_id}"); + match submit_ack { + ProofData::SubmitAck { block_number } => { + info!("Received submit ack for block_number: {}", block_number); Ok(()) } - _ => Err(format!("Unexpected response {response:?}")), + _ => Err(format!("Expecting ProofData::SubmitAck {submit_ack:?}")), } } } + +fn connect_to_prover_server_wr( + addr: &str, + write: &ProofData, +) -> Result> { + let stream = TcpStream::connect(addr)?; + let buf_writer = BufWriter::new(&stream); + debug!("Connection established!"); + serde_json::ser::to_writer(buf_writer, write)?; + stream.shutdown(std::net::Shutdown::Write)?; + + let buf_reader = BufReader::new(&stream); + let response: ProofData = serde_json::de::from_reader(buf_reader)?; + Ok(response) +} diff --git a/crates/l2/utils/config/committer.rs b/crates/l2/utils/config/committer.rs new file mode 100644 index 000000000..2cd653b3f --- /dev/null +++ b/crates/l2/utils/config/committer.rs @@ -0,0 +1,23 @@ +use crate::utils::secret_key_deserializer; +use ethereum_types::Address; +use secp256k1::SecretKey; +use serde::Deserialize; + +use super::errors::ConfigError; + +#[derive(Deserialize)] +pub struct CommitterConfig { + pub on_chain_proposer_address: Address, + pub l1_address: Address, + #[serde(deserialize_with = "secret_key_deserializer")] + pub l1_private_key: SecretKey, + pub interval_ms: u64, +} + +impl CommitterConfig { + pub fn from_env() -> Result { + envy::prefixed("COMMITTER_") + .from_env::() + .map_err(ConfigError::from) + } +} diff --git a/crates/l2/utils/config/mod.rs b/crates/l2/utils/config/mod.rs index 5786ddb3b..6cce03aa0 100644 --- a/crates/l2/utils/config/mod.rs +++ b/crates/l2/utils/config/mod.rs @@ -2,6 +2,7 @@ use std::io::{BufRead, Write}; use tracing::debug; +pub mod committer; pub mod eth; pub mod l1_watcher; pub mod proposer; @@ -11,7 +12,7 @@ pub mod prover_server; pub mod errors; pub fn read_env_file() -> Result<(), errors::ConfigError> { - let env_file_name = std::env::var("ENV_FILE").unwrap_or_else(|_| ".env".to_string()); + let env_file_name = std::env::var("ENV_FILE").unwrap_or(".env".to_string()); let env_file = std::fs::File::open(env_file_name)?; let reader = std::io::BufReader::new(env_file); @@ -49,7 +50,7 @@ pub fn read_env_as_lines( } pub fn write_env(lines: Vec) -> Result<(), errors::ConfigError> { - let env_file_name = std::env::var("ENV_FILE").unwrap_or_else(|_| ".env".to_string()); + let env_file_name = std::env::var("ENV_FILE").unwrap_or(".env".to_string()); let file = std::fs::OpenOptions::new() .write(true) diff --git a/crates/l2/utils/config/proposer.rs b/crates/l2/utils/config/proposer.rs index e61bb5848..068ab33ff 100644 --- a/crates/l2/utils/config/proposer.rs +++ b/crates/l2/utils/config/proposer.rs @@ -1,16 +1,9 @@ -use crate::utils::secret_key_deserializer; -use ethereum_types::Address; -use secp256k1::SecretKey; use serde::Deserialize; use super::errors::ConfigError; #[derive(Deserialize)] pub struct ProposerConfig { - pub on_chain_proposer_address: Address, - pub l1_address: Address, - #[serde(deserialize_with = "secret_key_deserializer")] - pub l1_private_key: SecretKey, pub interval_ms: u64, } diff --git a/crates/l2/utils/config/prover_client.rs b/crates/l2/utils/config/prover_client.rs index 36b460efe..c37e2be18 100644 --- a/crates/l2/utils/config/prover_client.rs +++ b/crates/l2/utils/config/prover_client.rs @@ -5,6 +5,7 @@ use super::errors::ConfigError; #[derive(Deserialize, Debug)] pub struct ProverClientConfig { pub prover_server_endpoint: String, + pub interval_ms: u64, } impl ProverClientConfig { diff --git a/crates/l2/utils/config/prover_server.rs b/crates/l2/utils/config/prover_server.rs index 60fb2e2ee..2fd9acbf4 100644 --- a/crates/l2/utils/config/prover_server.rs +++ b/crates/l2/utils/config/prover_server.rs @@ -1,13 +1,17 @@ -use std::net::IpAddr; - -use serde::Deserialize; - use super::errors::ConfigError; +use crate::utils::secret_key_deserializer; +use ethereum_types::Address; +use secp256k1::SecretKey; +use serde::Deserialize; +use std::net::IpAddr; #[derive(Clone, Deserialize)] pub struct ProverServerConfig { pub listen_ip: IpAddr, pub listen_port: u16, + pub verifier_address: Address, + #[serde(deserialize_with = "secret_key_deserializer")] + pub verifier_private_key: SecretKey, } impl ProverServerConfig { diff --git a/crates/l2/utils/eth_client/eth_sender.rs b/crates/l2/utils/eth_client/eth_sender.rs index e7b6a89c2..16969aeb6 100644 --- a/crates/l2/utils/eth_client/eth_sender.rs +++ b/crates/l2/utils/eth_client/eth_sender.rs @@ -53,8 +53,8 @@ impl EthClient { TxKind::Call(addr) => format!("{addr:#x}"), TxKind::Create => format!("{:#x}", Address::zero()), }, - "input": format!("{:#x}", tx.input), - "value": format!("{}", tx.value), + "input": format!("0x{:#x}", tx.input), + "value": format!("{:#x}", tx.value), "from": format!("{:#x}", tx.from), }), json!("latest"), diff --git a/crates/l2/utils/eth_client/mod.rs b/crates/l2/utils/eth_client/mod.rs index 7462fd1f3..30be0d3d8 100644 --- a/crates/l2/utils/eth_client/mod.rs +++ b/crates/l2/utils/eth_client/mod.rs @@ -39,7 +39,7 @@ pub enum RpcResponse { pub struct EthClient { client: Client, - url: String, + pub url: String, } // 0x08c379a0 == Error(String) @@ -142,7 +142,7 @@ impl EthClient { }; let mut data = json!({ "to": format!("{to:#x}"), - "input": format!("{:#x}", transaction.input), + "input": format!("0x{:#x}", transaction.input), "from": format!("{:#x}", transaction.from), "value": format!("{:#x}", transaction.value), }); @@ -405,7 +405,11 @@ impl EthClient { self.get_chain_id().await?.as_u64() }, nonce: self.get_nonce_from_overrides(&overrides).await?, - max_priority_fee_per_gas: overrides.priority_gas_price.unwrap_or_default(), + max_priority_fee_per_gas: if let Some(gas_price) = overrides.priority_gas_price { + gas_price + } else { + self.get_gas_price().await?.as_u64() + }, max_fee_per_gas: if let Some(gas_price) = overrides.gas_price { gas_price } else { @@ -462,7 +466,11 @@ impl EthClient { self.get_chain_id().await?.as_u64() }, nonce: self.get_nonce_from_overrides(&overrides).await?, - max_priority_fee_per_gas: overrides.priority_gas_price.unwrap_or_default(), + max_priority_fee_per_gas: if let Some(gas_price) = overrides.priority_gas_price { + gas_price + } else { + self.get_gas_price().await?.as_u64() + }, max_fee_per_gas: if let Some(gas_price) = overrides.gas_price { gas_price } else { @@ -510,7 +518,11 @@ impl EthClient { self.get_chain_id().await?.as_u64() }, nonce: self.get_nonce_from_overrides(&overrides).await?, - max_priority_fee_per_gas: overrides.priority_gas_price.unwrap_or_default(), + max_priority_fee_per_gas: if let Some(gas_price) = overrides.priority_gas_price { + gas_price + } else { + self.get_gas_price().await?.as_u64() + }, max_fee_per_gas: if let Some(gas_price) = overrides.gas_price { gas_price } else {