From 991e53a2b681fabb762f61650ee098c5a4e9b2dd Mon Sep 17 00:00:00 2001 From: Joseph Livesey Date: Thu, 28 Nov 2024 11:04:24 -0500 Subject: [PATCH] chore: extract post-merge gRPC support to separate repos --- Cargo.lock | 32 +- Cargo.toml | 3 + crates/firehose-protos/Cargo.toml | 1 + crates/firehose-protos/build.rs | 6 +- crates/firehose-protos/protos/firehose.proto | 150 ----- .../src/ethereum_v2/eth_block.rs | 6 +- crates/firehose-protos/src/firehose_v2/mod.rs | 6 - .../src/firehose_v2/request.rs | 16 - crates/firehose-protos/src/lib.rs | 20 - crates/forrestrie-examples/Cargo.toml | 3 +- .../examples/fetch_and_verify_block.rs | 2 +- .../examples/match_ethereum_to_beacon.rs | 10 +- .../examples/single_execution_block.rs | 8 +- .../examples/verify-era.rs | 10 +- .../examples/verify_block_inclusion_proof.rs | 8 +- crates/forrestrie/Cargo.toml | 11 - crates/forrestrie/build.rs | 23 - crates/forrestrie/protos/README.md | 5 - crates/forrestrie/protos/type.proto | 270 --------- crates/forrestrie/src/beacon_v1.rs | 528 ------------------ crates/forrestrie/src/lib.rs | 1 - 21 files changed, 49 insertions(+), 1070 deletions(-) delete mode 100644 crates/firehose-protos/protos/firehose.proto delete mode 100644 crates/firehose-protos/src/firehose_v2/mod.rs delete mode 100644 crates/firehose-protos/src/firehose_v2/request.rs delete mode 100644 crates/forrestrie/build.rs delete mode 100644 crates/forrestrie/protos/README.md delete mode 100644 crates/forrestrie/protos/type.proto delete mode 100644 crates/forrestrie/src/beacon_v1.rs diff --git a/Cargo.lock b/Cargo.lock index 549131c4..aee83724 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -841,6 +841,27 @@ version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b" +[[package]] +name = "beacon-protos" +version = "0.1.1" +source = "git+https://github.com/semiotic-ai/beacon-protos.git?branch=main#ce68a3b7cef376887d0583dcc9e2234ffc5abb7f" +dependencies = [ + "bls", + "firehose-rs", + "primitive-types", + "prost", + "prost-build", + "prost-wkt", + "prost-wkt-types", + "serde", + "ssz_types 0.6.0", + "thiserror 2.0.3", + "tonic", + "tonic-build", + "tree_hash 0.6.0", + "types", +] + [[package]] name = "bimap" version = "0.6.3" @@ -2166,6 +2187,7 @@ dependencies = [ "alloy-primitives 0.8.13", "alloy-rlp", "ethportal-api", + "firehose-rs", "hex", "primitive-types", "prost", @@ -2301,23 +2323,14 @@ dependencies = [ "alloy-primitives 0.8.13", "alloy-rlp", "bls", - "ethportal-api", "fake", - "firehose-protos", - "futures", "merkle_proof", "primitive-types", - "prost", - "prost-build", - "prost-wkt", - "prost-wkt-types", "rand", "reth-primitives", "reth-trie-common", "serde", "ssz_types 0.6.0", - "tonic", - "tonic-build", "tree_hash 0.6.0", "types", ] @@ -2326,6 +2339,7 @@ dependencies = [ name = "forrestrie-examples" version = "0.1.1" dependencies = [ + "beacon-protos", "bls", "ethereum_ssz 0.7.1", "ethereum_ssz_derive 0.7.1", diff --git a/Cargo.toml b/Cargo.toml index 197c9bbb..4903da96 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,6 +8,7 @@ alloy-consensus = "0.4.2" alloy-eip2930 = "0.1.0" alloy-rlp = "0.3.9" base64 = "0.21.7" +beacon-protos = { git = "https://github.com/semiotic-ai/beacon-protos.git", branch = "main" } bincode = "1.3.3" bls = { git = "https://github.com/semiotic-ai/lighthouse.git", branch = "stable" } clap = { version = "4.4.10", features = ["derive"] } @@ -16,6 +17,8 @@ dotenvy = "0.15.7" ethereum-types = "=0.14.1" ethportal-api = { git = "https://github.com/ethereum/trin.git", version = "0.2.2", tag = "v0.1.0-alpha.51" } fake = "2.10.0" +firehose-client = { git = "https://github.com/semiotic-ai/firehose-client.git", branch = "main" } +firehose-rs = { git = "https://github.com/semiotic-ai/firehose-rs.git", branch = "main" } futures = "0.3.31" hex = "0.4.3" http = "1.1.0" diff --git a/crates/firehose-protos/Cargo.toml b/crates/firehose-protos/Cargo.toml index 14c56e28..194c7bfd 100644 --- a/crates/firehose-protos/Cargo.toml +++ b/crates/firehose-protos/Cargo.toml @@ -14,6 +14,7 @@ alloy-eip2930.workspace = true alloy-primitives.workspace = true alloy-rlp.workspace = true ethportal-api.workspace = true +firehose-rs.workspace = true primitive-types.workspace = true prost.workspace = true prost-wkt.workspace = true diff --git a/crates/firehose-protos/build.rs b/crates/firehose-protos/build.rs index 2c7ce5bd..b9691cd1 100644 --- a/crates/firehose-protos/build.rs +++ b/crates/firehose-protos/build.rs @@ -22,11 +22,7 @@ fn main() { .file_descriptor_set_path(out_dir.join("descriptors.bin")) .compile_protos_with_config( config, - &[ - "protos/block.proto", - "protos/bstream.proto", - "protos/firehose.proto", - ], + &["protos/block.proto", "protos/bstream.proto"], &["protos/"], ) .unwrap(); diff --git a/crates/firehose-protos/protos/firehose.proto b/crates/firehose-protos/protos/firehose.proto deleted file mode 100644 index 5299bd8d..00000000 --- a/crates/firehose-protos/protos/firehose.proto +++ /dev/null @@ -1,150 +0,0 @@ -syntax = "proto3"; - -package sf.firehose.v2; - -import "google/protobuf/any.proto"; -import "google/protobuf/timestamp.proto"; - -option go_package = "github.com/streamingfast/pbgo/sf/firehose/v2;pbfirehose"; - -service Stream { - rpc Blocks(Request) returns (stream Response); -} - -service Fetch { - rpc Block(SingleBlockRequest) returns (SingleBlockResponse); -} - -message SingleBlockRequest { - - // Get the current known canonical version of a block at with this number - message BlockNumber{ - uint64 num=1; - } - - // Get the current block with specific hash and number - message BlockHashAndNumber{ - uint64 num=1; - string hash=2; - } - - // Get the block that generated a specific cursor - message Cursor{ - string cursor=1; - } - - oneof reference{ - BlockNumber block_number=3; - BlockHashAndNumber block_hash_and_number=4; - Cursor cursor=5; - } - - repeated google.protobuf.Any transforms = 6; -} - -message SingleBlockResponse { - google.protobuf.Any block = 1; - - // Metadata about the block, added in some Firehose version, so consumer - // should be ready to handle the absence of this field. - BlockMetadata metadata = 2; -} - -message Request { - - // Controls where the stream of blocks will start. - // - // The stream will start **inclusively** at the requested block num. - // - // When not provided, starts at first streamable block of the chain. Not all - // chain starts at the same block number, so you might get an higher block than - // requested when using default value of 0. - // - // Can be negative, will be resolved relative to the chain head block, assuming - // a chain at head block #100, then using `-50` as the value will start at block - // #50. If it resolves before first streamable block of chain, we assume start - // of chain. - // - // If `start_cursor` is given, this value is ignored and the stream instead starts - // immediately after the Block pointed by the opaque `start_cursor` value. - int64 start_block_num = 1; - - // Controls where the stream of blocks will start which will be immediately after - // the Block pointed by this opaque cursor. - // - // Obtain this value from a previously received `Response.cursor`. - // - // This value takes precedence over `start_block_num`. - string cursor = 2; - - // When non-zero, controls where the stream of blocks will stop. - // - // The stream will close **after** that block has passed so the boundary is - // **inclusive**. - uint64 stop_block_num = 3; - - // With final_block_only, you only receive blocks with STEP_FINAL - // Default behavior will send blocks as STEP_NEW, with occasional STEP_UNDO - bool final_blocks_only = 4; - - repeated google.protobuf.Any transforms = 10; -} - -message Response { - // Chain specific block payload, ex: - // - sf.eosio.type.v1.Block - // - sf.ethereum.type.v1.Block - // - sf.near.type.v1.Block - google.protobuf.Any block = 1; - ForkStep step = 6; - string cursor = 10; - - // Metadata about the block, added in some Firehose version, so consumer - // should be ready to handle the absence of this field. - BlockMetadata metadata = 12; -} - -message BlockMetadata { - // Num is the block number of this response's block. - uint64 num = 1; - // ID is the block ID of this response's block. The ID actual representation is chain specific. - // - Antelope & Ethereum uses hex. - // - NEAR & Solana uses base58. - // - // Refer to the chain documentation for more details. - string id = 2; - - // ParentNum is the block number of the parent of this response's block - uint64 parent_num = 3; - - // ParentID is the block ID of the parent of this response's block. If this response is the genesis block, - // this field is empty. - // - // The ID actual representation is chain specific. - // - Antelope & Ethereum uses hex. - // - NEAR & Solana uses base58. - // - // Refer to the chain documentation for more details. - string parent_id = 4; - - // LibNum is the block number of the last irreversible block (a.k.a last finalized block) at the time of this - // response' block. It determines the finality of the block. - uint64 lib_num = 5; - - // Time is the time at which the block was produced. - google.protobuf.Timestamp time = 6; -} - -enum ForkStep { - STEP_UNSET = 0; - - // Incoming block - STEP_NEW = 1; - - // A reorg caused this specific block to be excluded from the chain - STEP_UNDO = 2; - - // Block is now final and can be committed (finality is chain specific, - // see chain documentation for more details) - STEP_FINAL = 3; -} diff --git a/crates/firehose-protos/src/ethereum_v2/eth_block.rs b/crates/firehose-protos/src/ethereum_v2/eth_block.rs index 39135f78..b806d33b 100644 --- a/crates/firehose-protos/src/ethereum_v2/eth_block.rs +++ b/crates/firehose-protos/src/ethereum_v2/eth_block.rs @@ -5,6 +5,7 @@ use super::{Block, BlockHeader, TransactionReceipt, TransactionTrace}; use alloy_primitives::{hex, Address, Bloom, FixedBytes, Uint, B256}; use alloy_rlp::{Encodable, Header as RlpHeader}; use ethportal_api::types::execution::header::Header; +use firehose_rs::{Response, SingleBlockResponse}; use prost::Message; use prost_wkt_types::Any; use reth_primitives::{ @@ -13,10 +14,7 @@ use reth_primitives::{ use reth_trie_common::root::ordered_trie_root_with_encoder; use tracing::error; -use crate::{ - error::ProtosError, - firehose_v2::{Response, SingleBlockResponse}, -}; +use crate::error::ProtosError; impl TryFrom<&Block> for Header { type Error = ProtosError; diff --git a/crates/firehose-protos/src/firehose_v2/mod.rs b/crates/firehose-protos/src/firehose_v2/mod.rs deleted file mode 100644 index 03ab512a..00000000 --- a/crates/firehose-protos/src/firehose_v2/mod.rs +++ /dev/null @@ -1,6 +0,0 @@ -// Copyright 2024-, Semiotic AI, Inc. -// SPDX-License-Identifier: Apache-2.0 - -pub mod request; - -tonic::include_proto!("sf.firehose.v2"); diff --git a/crates/firehose-protos/src/firehose_v2/request.rs b/crates/firehose-protos/src/firehose_v2/request.rs deleted file mode 100644 index e84e086d..00000000 --- a/crates/firehose-protos/src/firehose_v2/request.rs +++ /dev/null @@ -1,16 +0,0 @@ -// Copyright 2024-, Semiotic AI, Inc. -// SPDX-License-Identifier: Apache-2.0 - -use crate::BlockNumber; - -use super::{single_block_request::Reference, SingleBlockRequest}; - -impl SingleBlockRequest { - /// Create a Firehose [`SingleBlockRequest`] for the given *block number*. - pub fn new(num: u64) -> SingleBlockRequest { - SingleBlockRequest { - reference: Some(Reference::BlockNumber(BlockNumber { num })), - ..Default::default() - } - } -} diff --git a/crates/firehose-protos/src/lib.rs b/crates/firehose-protos/src/lib.rs index 35a96b4c..1ca3ae41 100644 --- a/crates/firehose-protos/src/lib.rs +++ b/crates/firehose-protos/src/lib.rs @@ -19,7 +19,6 @@ mod error; mod ethereum_v2; -mod firehose_v2; mod bstream { pub mod v1 { @@ -30,22 +29,3 @@ mod bstream { pub use bstream::v1::Block as BstreamBlock; pub use error::ProtosError; pub use ethereum_v2::{eth_block::FullReceipt, Block as EthBlock, BlockHeader}; -pub(crate) use firehose_v2::single_block_request::BlockNumber; - -/// Interact programatically with the Firehose v2 Fetch API. -pub use firehose_v2::fetch_client::FetchClient; - -/// Create Firehose API fetch requests. -pub use firehose_v2::Request; - -/// Work with Firehose API streaming responses. -pub use firehose_v2::Response; - -/// Create Firehose API streaming requests. -pub use firehose_v2::SingleBlockRequest; - -/// Receive Firehose API fetch responses. -pub use firehose_v2::SingleBlockResponse; - -/// Work with the Firehose v2 Stream API. -pub use firehose_v2::stream_client::StreamClient; diff --git a/crates/forrestrie-examples/Cargo.toml b/crates/forrestrie-examples/Cargo.toml index 0de5795b..faf71d80 100644 --- a/crates/forrestrie-examples/Cargo.toml +++ b/crates/forrestrie-examples/Cargo.toml @@ -4,9 +4,10 @@ version = "0.1.1" edition = "2021" [dev-dependencies] +beacon-protos.workspace = true bls.workspace = true ethportal-api.workspace = true -firehose-client = { git = "https://github.com/semiotic-ai/firehose-client.git", branch = "main" } +firehose-client.workspace = true firehose-protos = { path = "../firehose-protos" } forrestrie = { path = "../forrestrie" } futures.workspace = true diff --git a/crates/forrestrie-examples/examples/fetch_and_verify_block.rs b/crates/forrestrie-examples/examples/fetch_and_verify_block.rs index 7b2ca56d..59067700 100644 --- a/crates/forrestrie-examples/examples/fetch_and_verify_block.rs +++ b/crates/forrestrie-examples/examples/fetch_and_verify_block.rs @@ -27,8 +27,8 @@ //! assert_eq!(block_root, block_header_root); //! ``` //! +use beacon_protos::Block as FirehoseBeaconBlock; use firehose_client::{Chain, FirehoseClient}; -use forrestrie::beacon_v1::Block as FirehoseBeaconBlock; use tree_hash::TreeHash; use types::{BeaconBlock, MainnetEthSpec}; diff --git a/crates/forrestrie-examples/examples/match_ethereum_to_beacon.rs b/crates/forrestrie-examples/examples/match_ethereum_to_beacon.rs index 7fba4d71..063b8d74 100644 --- a/crates/forrestrie-examples/examples/match_ethereum_to_beacon.rs +++ b/crates/forrestrie-examples/examples/match_ethereum_to_beacon.rs @@ -32,11 +32,9 @@ //! comparison until the correct Beacon slot is found. //! +use beacon_protos::{Block as FirehoseBeaconBlock, Body}; use firehose_client::{Chain, FirehoseClient}; -use forrestrie::{ - beacon_state::ETHEREUM_BEACON_DENEB_OFFSET, - beacon_v1::{block, Block as FirehoseBeaconBlock}, -}; +use forrestrie::beacon_state::ETHEREUM_BEACON_DENEB_OFFSET; use std::cmp::Ordering::*; use tracing::info; use tracing_subscriber::FmtSubscriber; @@ -74,7 +72,7 @@ async fn main() { let response = beacon_client.fetch_block(mid).await.unwrap().unwrap(); let block = FirehoseBeaconBlock::try_from(response.into_inner()).unwrap(); - let Some(block::Body::Deneb(body)) = &block.body else { + let Some(Body::Deneb(body)) = &block.body else { panic!("Unsupported block version!"); }; @@ -113,7 +111,7 @@ async fn try_final_fetches(low: u64, high: u64, client: &mut FirehoseClient) -> let block = FirehoseBeaconBlock::try_from(response.into_inner()).unwrap(); - let Some(block::Body::Deneb(body)) = &block.body else { + let Some(Body::Deneb(body)) = &block.body else { return None; }; diff --git a/crates/forrestrie-examples/examples/single_execution_block.rs b/crates/forrestrie-examples/examples/single_execution_block.rs index 62843842..cbebc77d 100644 --- a/crates/forrestrie-examples/examples/single_execution_block.rs +++ b/crates/forrestrie-examples/examples/single_execution_block.rs @@ -25,6 +25,7 @@ //! the block roots tree hash root, which can then be compared to the tree hash root in the historical summary //! for the era. //! +use beacon_protos::{Block, BlockRoot, Body}; use ethportal_api::Header; use firehose_client::{Chain, FirehoseClient}; use firehose_protos::EthBlock; @@ -36,7 +37,6 @@ use forrestrie::{ compute_block_roots_proof_only, HeadState, CAPELLA_START_ERA, HISTORY_TREE_DEPTH, SLOTS_PER_HISTORICAL_ROOT, }, - beacon_v1::{self, BlockRoot}, }; use futures::StreamExt; use merkle_proof::verify_merkle_proof; @@ -85,7 +85,7 @@ async fn main() { .await .unwrap() .unwrap(); - let beacon_block = beacon_v1::Block::try_from(response.into_inner()).unwrap(); + let beacon_block = Block::try_from(response.into_inner()).unwrap(); assert_eq!(beacon_block.slot, BEACON_SLOT_NUMBER); // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -111,7 +111,7 @@ async fn main() { lighthouse_beacon_block_root.as_bytes(), beacon_block.root.as_slice() ); - let Some(beacon_v1::block::Body::Deneb(body)) = beacon_block.body else { + let Some(Body::Deneb(body)) = beacon_block.body else { panic!("Unsupported block version!"); }; let block_body: BeaconBlockBodyDeneb = body.try_into().unwrap(); @@ -155,7 +155,7 @@ async fn main() { println!("Requesting 8192 blocks for the era... (this takes a while)"); let num_blocks = SLOTS_PER_HISTORICAL_ROOT as u64; let mut stream = beacon_client - .stream_beacon_with_retry((era * SLOTS_PER_HISTORICAL_ROOT) as u64, num_blocks) + .stream_blocks::((era * SLOTS_PER_HISTORICAL_ROOT) as u64, num_blocks) .await .unwrap(); let mut block_roots: Vec = Vec::with_capacity(SLOTS_PER_HISTORICAL_ROOT); diff --git a/crates/forrestrie-examples/examples/verify-era.rs b/crates/forrestrie-examples/examples/verify-era.rs index 76d3b57b..92f29020 100644 --- a/crates/forrestrie-examples/examples/verify-era.rs +++ b/crates/forrestrie-examples/examples/verify-era.rs @@ -8,12 +8,12 @@ //! blocks using the extracted block numbers and verify the execution block data against the //! extracted block headers. +use beacon_protos::Block; use ethportal_api::Header; use firehose_client::{Chain, FirehoseClient}; use firehose_protos::EthBlock; -use forrestrie::{ - beacon_state::{HeadState, CAPELLA_START_ERA, HISTORY_TREE_DEPTH, SLOTS_PER_HISTORICAL_ROOT}, - beacon_v1::{self}, +use forrestrie::beacon_state::{ + HeadState, CAPELLA_START_ERA, HISTORY_TREE_DEPTH, SLOTS_PER_HISTORICAL_ROOT, }; use futures::StreamExt; use tree_hash::TreeHash; @@ -55,7 +55,7 @@ async fn main() { println!("Requesting 8192 blocks for the era... (this takes a while)"); let num_blocks = SLOTS_PER_HISTORICAL_ROOT as u64; let mut stream = beacon_client - .stream_beacon_with_retry((era * SLOTS_PER_HISTORICAL_ROOT) as u64, num_blocks) + .stream_blocks::((era * SLOTS_PER_HISTORICAL_ROOT) as u64, num_blocks) .await .unwrap(); @@ -72,7 +72,7 @@ async fn main() { // Get the exeuction block number and blockhash. let lighthouse_beacon_block = BeaconBlock::::try_from(block.clone()) .expect("Failed to convert Beacon block to Lighthouse BeaconBlock"); - let Some(beacon_v1::block::Body::Deneb(body)) = block.body else { + let Some(beacon_protos::Body::Deneb(body)) = block.body else { panic!("Unsupported block version!"); }; let block_body: BeaconBlockBodyDeneb = body.try_into().unwrap(); diff --git a/crates/forrestrie-examples/examples/verify_block_inclusion_proof.rs b/crates/forrestrie-examples/examples/verify_block_inclusion_proof.rs index 791e3bfe..a2ae87fd 100644 --- a/crates/forrestrie-examples/examples/verify_block_inclusion_proof.rs +++ b/crates/forrestrie-examples/examples/verify_block_inclusion_proof.rs @@ -27,12 +27,10 @@ //! let proof = body.compute_merkle_proof(EXECUTION_PAYLOAD_INDEX).unwrap(); //! ``` //! +use beacon_protos::Block as FirehoseBeaconBlock; use firehose_client::{Chain, FirehoseClient}; -use forrestrie::{ - beacon_block::{ - HistoricalDataProofs, BEACON_BLOCK_BODY_PROOF_DEPTH, EXECUTION_PAYLOAD_FIELD_INDEX, - }, - beacon_v1::Block as FirehoseBeaconBlock, +use forrestrie::beacon_block::{ + HistoricalDataProofs, BEACON_BLOCK_BODY_PROOF_DEPTH, EXECUTION_PAYLOAD_FIELD_INDEX, }; use merkle_proof::verify_merkle_proof; use tree_hash::TreeHash; diff --git a/crates/forrestrie/Cargo.toml b/crates/forrestrie/Cargo.toml index b1b22a5d..578dde6f 100644 --- a/crates/forrestrie/Cargo.toml +++ b/crates/forrestrie/Cargo.toml @@ -11,26 +11,15 @@ path = "src/lib.rs" alloy-rlp.workspace = true alloy-primitives.workspace = true bls.workspace = true -ethportal-api.workspace = true -firehose-protos = { path = "../firehose-protos" } -futures.workspace = true merkle_proof.workspace = true primitive-types.workspace = true -prost.workspace = true -prost-wkt.workspace = true -prost-wkt-types.workspace = true reth-trie-common.workspace = true reth-primitives.workspace = true serde = { workspace = true, features = ["derive"] } ssz_types.workspace = true -tonic.workspace = true tree_hash = "0.6.0" types.workspace = true [dev-dependencies] rand.workspace = true fake.workspace = true - -[build-dependencies] -prost-build.workspace = true -tonic-build.workspace = true diff --git a/crates/forrestrie/build.rs b/crates/forrestrie/build.rs deleted file mode 100644 index 0bbb1db8..00000000 --- a/crates/forrestrie/build.rs +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright 2024-, Semiotic AI, Inc. -// SPDX-License-Identifier: Apache-2.0 - -use prost_build::Config; -use std::{env, path::PathBuf}; - -fn main() { - let out_dir = PathBuf::from(env::var("OUT_DIR").unwrap()); - - // Configure prost to derive serde traits on specific types - let mut config = Config::new(); - config.type_attribute(".", "#[derive(serde::Serialize, serde::Deserialize)]"); - - // Map Google protobuf types to prost_wkt_types - config.extern_path(".google.protobuf.Any", "::prost_wkt_types::Any"); - config.extern_path(".google.protobuf.Timestamp", "::prost_wkt_types::Timestamp"); - - tonic_build::configure() - .build_client(true) - .file_descriptor_set_path(out_dir.join("descriptors.bin")) - .compile_protos_with_config(config, &["protos/type.proto"], &["protos/"]) - .unwrap(); -} diff --git a/crates/forrestrie/protos/README.md b/crates/forrestrie/protos/README.md deleted file mode 100644 index b9aa605b..00000000 --- a/crates/forrestrie/protos/README.md +++ /dev/null @@ -1,5 +0,0 @@ -# Protobuffer definitions - -## [`type.proto`](https://github.com/pinax-network/firehose-beacon/blob/main/proto/sf/beacon/type/v1/type.proto) - -Pinax's Firehose Beacon `Block` implementation. diff --git a/crates/forrestrie/protos/type.proto b/crates/forrestrie/protos/type.proto deleted file mode 100644 index 302fdb1c..00000000 --- a/crates/forrestrie/protos/type.proto +++ /dev/null @@ -1,270 +0,0 @@ -syntax = "proto3"; - -option go_package = "github.com/pinax-network/firehose-beacon/pb/sf/beacon/type/v1;pbbeacon"; - -package sf.beacon.type.v1; - -import "google/protobuf/timestamp.proto"; - -enum Spec { - UNSPECIFIED = 0; - PHASE0 = 1; - ALTAIR = 2; - BELLATRIX = 3; - CAPELLA = 4; - DENEB = 5; -} - -message Block { - - uint32 version = 1; - - Spec spec = 2; - uint64 slot = 3; - uint64 parent_slot = 4; - bytes root = 5; - bytes parent_root = 6; - bytes state_root = 7; - uint64 proposer_index = 8; - bytes body_root = 9; - - oneof Body { - Phase0Body phase0 = 20; - AltairBody altair = 21; - BellatrixBody bellatrix = 22; - CapellaBody capella = 23; - DenebBody deneb = 24; - } - - bytes signature = 30; - google.protobuf.Timestamp timestamp = 31; -} - -message Phase0Body { - bytes rando_reveal = 1; - Eth1Data eth1_data = 2; - bytes graffiti = 3; - repeated ProposerSlashing proposer_slashings = 4; - repeated AttesterSlashing attester_slashings = 5; - repeated Attestation attestations = 6; - repeated Deposit deposits = 7; - repeated SignedVoluntaryExit voluntary_exits = 8; -} - -message AltairBody { - bytes rando_reveal = 1; - Eth1Data eth1_data = 2; - bytes graffiti = 3; - repeated ProposerSlashing proposer_slashings = 4; - repeated AttesterSlashing attester_slashings = 5; - repeated Attestation attestations = 6; - repeated Deposit deposits = 7; - repeated SignedVoluntaryExit voluntary_exits = 8; - SyncAggregate sync_aggregate = 9; -} - -message BellatrixBody { - bytes rando_reveal = 1; - Eth1Data eth1_data = 2; - bytes graffiti = 3; - repeated ProposerSlashing proposer_slashings = 4; - repeated AttesterSlashing attester_slashings = 5; - repeated Attestation attestations = 6; - repeated Deposit deposits = 7; - repeated SignedVoluntaryExit voluntary_exits = 8; - SyncAggregate sync_aggregate = 9; - BellatrixExecutionPayload execution_payload = 10; -} - -message CapellaBody { - bytes rando_reveal = 1; - Eth1Data eth1_data = 2; - bytes graffiti = 3; - repeated ProposerSlashing proposer_slashings = 4; - repeated AttesterSlashing attester_slashings = 5; - repeated Attestation attestations = 6; - repeated Deposit deposits = 7; - repeated SignedVoluntaryExit voluntary_exits = 8; - SyncAggregate sync_aggregate = 9; - CapellaExecutionPayload execution_payload = 10; -} - -message DenebBody { - bytes rando_reveal = 1; - Eth1Data eth1_data = 2; - bytes graffiti = 3; - repeated ProposerSlashing proposer_slashings = 4; - repeated AttesterSlashing attester_slashings = 5; - repeated Attestation attestations = 6; - repeated Deposit deposits = 7; - repeated SignedVoluntaryExit voluntary_exits = 8; - SyncAggregate sync_aggregate = 9; - DenebExecutionPayload execution_payload = 10; - repeated SignedBLSToExecutionChange bls_to_execution_changes = 11; - repeated bytes blob_kzg_commitments = 12; - - repeated Blob embedded_blobs = 20; -} - -message Eth1Data { - bytes deposit_root = 1; - uint64 deposit_count = 2; - bytes block_hash = 3; -} - -message ProposerSlashing { - SignedBeaconBlockHeader signed_header_1 = 1; - SignedBeaconBlockHeader signed_header_2 = 2; -} - -message AttesterSlashing { - IndexedAttestation attestation_1 = 1; - IndexedAttestation attestation_2 = 2; -} - -message Attestation { - bytes aggregation_bits = 1; - AttestationData data = 2; - bytes signature = 3; -} - -message Deposit { - repeated bytes proof = 1; - DepositData data = 2; -} - -message SignedVoluntaryExit { - VoluntaryExit message = 1; - bytes signature = 2; -} - -message SyncAggregate { - bytes sync_commitee_bits = 1; - bytes sync_comittee_signature = 2; -} - -message BellatrixExecutionPayload { - bytes parent_hash = 1; - bytes fee_recipient = 2; - bytes state_root = 3; - bytes receipts_root = 4; - bytes logs_bloom = 5; - bytes prev_randao = 6; - uint64 block_number = 7; - uint64 gas_limit = 8; - uint64 gas_used = 9; - google.protobuf.Timestamp timestamp = 10; - bytes extra_data = 11; - bytes base_fee_per_gas = 12; - bytes block_hash = 13; - repeated bytes transactions = 14; -} - - -message CapellaExecutionPayload { - bytes parent_hash = 1; - bytes fee_recipient = 2; - bytes state_root = 3; - bytes receipts_root = 4; - bytes logs_bloom = 5; - bytes prev_randao = 6; - uint64 block_number = 7; - uint64 gas_limit = 8; - uint64 gas_used = 9; - google.protobuf.Timestamp timestamp = 10; - bytes extra_data = 11; - bytes base_fee_per_gas = 12; - bytes block_hash = 13; - repeated bytes transactions = 14; - repeated Withdrawal withdrawals = 15; -} - -message DenebExecutionPayload { - bytes parent_hash = 1; - bytes fee_recipient = 2; - bytes state_root = 3; - bytes receipts_root = 4; - bytes logs_bloom = 5; - bytes prev_randao = 6; - uint64 block_number = 7; - uint64 gas_limit = 8; - uint64 gas_used = 9; - google.protobuf.Timestamp timestamp = 10; - bytes extra_data = 11; - bytes base_fee_per_gas = 12; - bytes block_hash = 13; - repeated bytes transactions = 14; - repeated Withdrawal withdrawals = 15; - uint64 blob_gas_used = 16; - uint64 excess_blob_gas = 17; -} - -message SignedBLSToExecutionChange { - BLSToExecutionChange message = 1; - bytes signature = 2; -} - -message BLSToExecutionChange { - uint64 validator_index = 1; - bytes from_bls_pub_key = 2; - bytes to_execution_address = 3; -} - -message Withdrawal { - uint64 withdrawal_index = 1; - uint64 validator_index = 2; - bytes address = 3; - uint64 gwei = 4; -} - -message VoluntaryExit { - uint64 epoch = 1; - uint64 validator_index = 2; -} - -message DepositData { - bytes public_key = 1; - bytes withdrawal_credentials = 2; - uint64 gwei = 3; - bytes signature = 4; -} - -message IndexedAttestation { - repeated uint64 attesting_indices = 1; - AttestationData data = 2; - bytes signature = 3; -} - -message AttestationData { - uint64 slot = 1; - uint64 committee_index = 2; - bytes beacon_block_root = 3; - Checkpoint source = 4; - Checkpoint target = 5; -} - -message Checkpoint { - uint64 epoch = 1; - bytes root = 2; -} - -message SignedBeaconBlockHeader { - BeaconBlockHeader message = 1; - bytes Signature = 2; -} - -message BeaconBlockHeader { - uint64 slot = 1; - uint64 proposer_index = 2; - bytes parent_root = 3; - bytes state_root = 4; - bytes body_root = 5; -} - -message Blob { - uint64 index = 1; - bytes blob = 2; - bytes kzg_commitment = 3; - bytes kzg_proof = 4; - repeated bytes kzg_commitment_inclusion_proof = 5; -} \ No newline at end of file diff --git a/crates/forrestrie/src/beacon_v1.rs b/crates/forrestrie/src/beacon_v1.rs deleted file mode 100644 index 65c32d0d..00000000 --- a/crates/forrestrie/src/beacon_v1.rs +++ /dev/null @@ -1,528 +0,0 @@ -// Copyright 2024-, Semiotic AI, Inc. -// SPDX-License-Identifier: Apache-2.0 - -//! Firehose Beacon-related data structures and operations. -//! See the protobuffer definitions section of the README for more information. -//! -use firehose_protos::{ProtosError, Response, SingleBlockResponse}; -use primitive_types::{H256, U256}; -use prost::Message; -use ssz_types::{length::Fixed, Bitfield, FixedVector}; -use types::{ - Address, BeaconBlock, BeaconBlockBodyDeneb, BitList, EthSpec, ExecutionBlockHash, Graffiti, - IndexedAttestationBase, MainnetEthSpec, GRAFFITI_BYTES_LEN, -}; - -tonic::include_proto!("sf.beacon.r#type.v1"); - -impl TryFrom for types::AttestationBase { - type Error = ProtosError; - - fn try_from( - Attestation { - aggregation_bits, - data, - signature, - }: Attestation, - ) -> Result { - Ok(Self { - aggregation_bits: BitList::from_bytes(aggregation_bits.as_slice().into()) - .map_err(|e| ProtosError::SszTypesError(format!("{:?}", e)))?, - data: data - .ok_or(ProtosError::AttestationDataMissing)? - .try_into()?, - signature: bls::generics::GenericAggregateSignature::deserialize(signature.as_slice()) - .map_err(|e| ProtosError::Bls(format!("{:?}", e)))?, - }) - } -} - -impl TryFrom for types::AttestationData { - type Error = ProtosError; - - fn try_from( - AttestationData { - slot, - committee_index, - beacon_block_root, - source, - target, - }: AttestationData, - ) -> Result { - Ok(Self { - slot: slot.into(), - index: committee_index, - beacon_block_root: H256::from_slice(beacon_block_root.as_slice()), - source: source.ok_or(ProtosError::CheckpointMissing)?.into(), - target: target.ok_or(ProtosError::CheckpointMissing)?.into(), - }) - } -} - -impl TryFrom for types::AttesterSlashingBase { - type Error = ProtosError; - - fn try_from( - AttesterSlashing { - attestation_1, - attestation_2, - }: AttesterSlashing, - ) -> Result { - let attestation_1 = attestation_1.ok_or(ProtosError::SignerMissing)?; - let attestation_2 = attestation_2.ok_or(ProtosError::SignerMissing)?; - - Ok(Self { - attestation_1: attestation_1.try_into()?, - attestation_2: attestation_2.try_into()?, - }) - } -} - -impl From for types::BeaconBlockHeader { - fn from( - BeaconBlockHeader { - slot, - proposer_index, - parent_root, - state_root, - body_root, - }: BeaconBlockHeader, - ) -> Self { - Self { - slot: slot.into(), - proposer_index, - parent_root: H256::from_slice(parent_root.as_slice()), - state_root: H256::from_slice(state_root.as_slice()), - body_root: H256::from_slice(body_root.as_slice()), - } - } -} - -impl TryFrom for types::BlsToExecutionChange { - type Error = ProtosError; - - fn try_from( - BlsToExecutionChange { - validator_index, - from_bls_pub_key, - to_execution_address, - }: BlsToExecutionChange, - ) -> Result { - Ok(Self { - validator_index, - from_bls_pubkey: bls::generics::GenericPublicKeyBytes::deserialize( - from_bls_pub_key.as_slice(), - ) - .map_err(|e| ProtosError::Bls(format!("{e:?}")))?, - to_execution_address: Address::from_slice(to_execution_address.as_slice()), - }) - } -} - -impl From for types::Checkpoint { - fn from(Checkpoint { epoch, root }: Checkpoint) -> Self { - Self { - epoch: epoch.into(), - root: H256::from_slice(root.as_slice()), - } - } -} - -impl TryFrom for types::ExecutionPayloadDeneb { - type Error = ProtosError; - - fn try_from( - DenebExecutionPayload { - parent_hash, - fee_recipient, - state_root, - receipts_root, - logs_bloom, - prev_randao, - block_number, - gas_limit, - gas_used, - timestamp, - extra_data, - base_fee_per_gas, - block_hash, - transactions, - withdrawals, - blob_gas_used, - excess_blob_gas, - }: DenebExecutionPayload, - ) -> Result { - Ok(Self { - parent_hash: ExecutionBlockHash::from_root(H256::from_slice(parent_hash.as_slice())), - fee_recipient: Address::from_slice(fee_recipient.as_slice()), - state_root: H256::from_slice(state_root.as_slice()), - receipts_root: H256::from_slice(receipts_root.as_slice()), - logs_bloom: FixedVector::from(logs_bloom), - prev_randao: H256::from_slice(prev_randao.as_slice()), - block_number, - gas_limit, - gas_used, - timestamp: timestamp - .as_ref() - .ok_or(ProtosError::BlockConversionError)? - .seconds as u64, - extra_data: extra_data.into(), - base_fee_per_gas: U256::from_big_endian(base_fee_per_gas.as_slice()), - block_hash: ExecutionBlockHash(H256::from_slice(block_hash.as_slice())), - transactions: transactions - .into_iter() - .map(Into::into) - .collect::>() - .into(), - withdrawals: withdrawals - .into_iter() - .map(Into::into) - .collect::>() - .into(), - blob_gas_used, - excess_blob_gas, - }) - } -} - -impl TryFrom for types::Deposit { - type Error = ProtosError; - - fn try_from(Deposit { proof, data }: Deposit) -> Result { - Ok(Self { - proof: proof - .into_iter() - .map(|v| H256::from_slice(v.as_slice())) - .collect::>() - .into(), - data: data.ok_or(ProtosError::DepositDataMissing)?.try_into()?, - }) - } -} - -impl TryFrom for types::DepositData { - type Error = ProtosError; - - fn try_from( - DepositData { - public_key, - withdrawal_credentials, - gwei, - signature, - }: DepositData, - ) -> Result { - Ok(Self { - pubkey: bls::generics::GenericPublicKeyBytes::deserialize(public_key.as_slice()) - .map_err(|e| ProtosError::Bls(format!("{:?}", e)))?, - withdrawal_credentials: H256::from_slice(withdrawal_credentials.as_slice()), - amount: gwei, - signature: bls::generics::GenericSignatureBytes::deserialize(signature.as_slice()) - .map_err(|e| ProtosError::Bls(format!("{:?}", e)))?, - }) - } -} - -impl From for types::Eth1Data { - fn from( - Eth1Data { - deposit_root, - deposit_count, - block_hash, - }: Eth1Data, - ) -> Self { - Self { - deposit_root: H256::from_slice(deposit_root.as_slice()), - deposit_count, - block_hash: H256::from_slice(block_hash.as_slice()), - } - } -} - -impl TryFrom for types::IndexedAttestationBase { - type Error = ProtosError; - - fn try_from( - IndexedAttestation { - attesting_indices, - data, - signature, - }: IndexedAttestation, - ) -> Result { - Ok(IndexedAttestationBase { - attesting_indices: attesting_indices.into(), - data: data - .ok_or(ProtosError::IndexedAttestationDataMissing)? - .try_into()?, - signature: bls::generics::GenericAggregateSignature::deserialize(signature.as_slice()) - .map_err(|e| ProtosError::Bls(format!("{:?}", e)))?, - }) - } -} - -impl TryFrom for types::ProposerSlashing { - type Error = ProtosError; - - fn try_from( - ProposerSlashing { - signed_header_1, - signed_header_2, - }: ProposerSlashing, - ) -> Result { - Ok(Self { - signed_header_1: signed_header_1 - .ok_or(ProtosError::SignerMissing)? - .try_into()?, - signed_header_2: signed_header_2 - .ok_or(ProtosError::SignerMissing)? - .try_into()?, - }) - } -} - -impl TryFrom for types::SignedBeaconBlockHeader { - type Error = ProtosError; - - fn try_from( - SignedBeaconBlockHeader { message, signature }: SignedBeaconBlockHeader, - ) -> Result { - Ok(Self { - message: message - .ok_or(ProtosError::SignedBeaconBlockHeaderMessageMissing)? - .into(), - signature: bls::generics::GenericSignature::deserialize(signature.as_slice()) - .map_err(|e| ProtosError::Bls(format!("{:?}", e)))?, - }) - } -} - -impl TryFrom for types::SignedBlsToExecutionChange { - type Error = ProtosError; - - fn try_from( - SignedBlsToExecutionChange { message, signature }: SignedBlsToExecutionChange, - ) -> Result { - Ok(Self { - message: message - .ok_or(ProtosError::BlsToExecutionChangeMissing)? - .try_into()?, - signature: bls::generics::GenericSignature::deserialize(signature.as_slice()) - .expect("Failed to deserialize signature"), - }) - } -} - -impl TryFrom for types::SignedVoluntaryExit { - type Error = ProtosError; - - fn try_from( - SignedVoluntaryExit { message, signature }: SignedVoluntaryExit, - ) -> Result { - Ok(Self { - message: message.ok_or(ProtosError::VoluntaryExitMissing)?.into(), - signature: bls::generics::GenericSignature::deserialize(signature.as_slice()) - .map_err(|e| ProtosError::Bls(format!("{:?}", e)))?, - }) - } -} - -impl TryFrom for Block { - type Error = ProtosError; - - fn try_from(response: SingleBlockResponse) -> Result { - let any = response.block.ok_or(ProtosError::BlockMissingInResponse)?; - let block = Block::decode(any.value.as_ref())?; - Ok(block) - } -} - -impl TryFrom for Block { - type Error = ProtosError; - - fn try_from(response: Response) -> Result { - let any = response.block.ok_or(ProtosError::BlockMissingInResponse)?; - let block = Block::decode(any.value.as_ref())?; - Ok(block) - } -} - -impl TryFrom for types::SyncAggregate { - type Error = ProtosError; - - fn try_from( - SyncAggregate { - sync_commitee_bits, - sync_comittee_signature, - }: SyncAggregate, - ) -> Result { - Ok(Self { - sync_committee_bits: Bitfield::::SyncCommitteeSize>>::from_bytes( - sync_commitee_bits.as_slice().into(), - ) - .map_err(|e| ProtosError::SszTypesError(format!("{:?}", e)))?, - sync_committee_signature: bls::generics::GenericAggregateSignature::deserialize( - sync_comittee_signature.as_slice(), - ) - .map_err(|e| ProtosError::Bls(format!("{:?}", e)))?, - }) - } -} - -impl From for types::VoluntaryExit { - fn from( - VoluntaryExit { - epoch, - validator_index, - }: VoluntaryExit, - ) -> Self { - Self { - epoch: epoch.into(), - validator_index, - } - } -} - -impl From for types::Withdrawal { - fn from( - Withdrawal { - withdrawal_index, - validator_index, - address, - gwei, - }: Withdrawal, - ) -> Self { - Self { - index: withdrawal_index, - validator_index, - address: Address::from_slice(address.as_slice()), - amount: gwei, - } - } -} - -impl TryFrom for types::BeaconBlockBodyDeneb { - type Error = ProtosError; - - fn try_from( - DenebBody { - rando_reveal, - eth1_data, - graffiti, - proposer_slashings, - attester_slashings, - attestations, - deposits, - voluntary_exits, - sync_aggregate, - execution_payload, - bls_to_execution_changes, - blob_kzg_commitments, - // Blobs not included. - .. - }: DenebBody, - ) -> Result { - let beacon_block_body = BeaconBlockBodyDeneb { - randao_reveal: bls::generics::GenericSignature::deserialize(&rando_reveal) - .map_err(|e| ProtosError::Bls(format!("{:?}", e)))?, - eth1_data: eth1_data - .map(|eth1_data| eth1_data.into()) - .unwrap_or_default(), - graffiti: Graffiti::from( - <[u8; GRAFFITI_BYTES_LEN]>::try_from(graffiti.as_slice()) - .map_err(|_| ProtosError::GraffitiInvalid)?, - ), - proposer_slashings: proposer_slashings - .into_iter() - .map(|proposer_slashing| proposer_slashing.try_into()) - .collect::, _>>()? - .into(), - attester_slashings: attester_slashings - .into_iter() - .map(|attester_slashing| attester_slashing.try_into()) - .collect::, _>>()? - .into(), - attestations: attestations - .into_iter() - .map(|attestation| attestation.try_into()) - .collect::, _>>()? - .into(), - deposits: deposits - .into_iter() - .map(|deposit| deposit.try_into()) - .collect::, _>>()? - .into(), - voluntary_exits: voluntary_exits - .into_iter() - .map(|voluntary_exit| voluntary_exit.try_into()) - .collect::, _>>()? - .into(), - sync_aggregate: sync_aggregate - .map(|sync_aggregate| sync_aggregate.try_into()) - .transpose()? - .unwrap_or_else(types::SyncAggregate::new), - execution_payload: execution_payload - .ok_or(ProtosError::ExecutionPayloadMissing) - .and_then(types::ExecutionPayloadDeneb::try_from)? - .into(), - bls_to_execution_changes: bls_to_execution_changes - .into_iter() - .map(|bls_to_execution_change| bls_to_execution_change.try_into()) - .collect::, _>>()? - .into(), - blob_kzg_commitments: blob_kzg_commitments - .into_iter() - .map(|blob_kzg_commitment| { - <[u8; 48]>::try_from(blob_kzg_commitment.as_slice()) - .map(types::KzgCommitment) - .map_err(|_| ProtosError::KzgCommitmentInvalid) - }) - .collect::, _>>()? - .into(), - }; - Ok(beacon_block_body) - } -} - -impl TryFrom for types::BeaconBlockBodyDeneb { - type Error = ProtosError; - - fn try_from(body: crate::beacon_v1::block::Body) -> Result { - match body { - crate::beacon_v1::block::Body::Deneb(deneb) => Ok(deneb.try_into()?), - _ => panic!("Invalid body type"), - } - } -} - -impl TryFrom for types::BeaconBlock { - type Error = ProtosError; - - fn try_from( - Block { - slot, - proposer_index, - parent_root, - state_root, - body, - .. - }: Block, - ) -> Result { - Ok(Self::Deneb(types::BeaconBlockDeneb { - slot: slot.into(), - proposer_index, - parent_root: H256::from_slice(parent_root.as_slice()), - state_root: H256::from_slice(state_root.as_slice()), - body: body.ok_or(ProtosError::BlockConversionError)?.try_into()?, - })) - } -} - -pub struct BlockRoot(pub H256); - -impl TryFrom for BlockRoot { - type Error = ProtosError; - - fn try_from(beacon_block: Block) -> Result { - let lighthouse_beacon_block = BeaconBlock::::try_from(beacon_block)?; - Ok(Self(lighthouse_beacon_block.canonical_root())) - } -} diff --git a/crates/forrestrie/src/lib.rs b/crates/forrestrie/src/lib.rs index 39c9f68e..19785755 100644 --- a/crates/forrestrie/src/lib.rs +++ b/crates/forrestrie/src/lib.rs @@ -3,5 +3,4 @@ pub mod beacon_block; pub mod beacon_state; -pub mod beacon_v1; pub mod execution_layer;