fix: create forrestrie-examples crate to fix circular dependency

semiotic-ai · Oct 23, 2024 · b4d0c75 · b4d0c75
1 parent d7b2eb1
commit b4d0c75
Show file tree

Hide file tree

Showing 20 changed files with 132 additions and 65 deletions.
diff --git a/.release-please-manifest.json b/.release-please-manifest.json
@@ -3,6 +3,7 @@
   "crates/flat-files-decoder": "0.2.0",
   "crates/flat-head": "0.1.0",
   "crates/forrestrie": "0.1.1",
+  "crates/forrestrie-examples": "0.1.1",
   "crates/header-accumulator": "0.2.0",
   "crates/firehose-protos": "0.1.0"
 }
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/README.md b/README.md
@@ -1,51 +1,13 @@
 # Veemon
 
-## Overview
-
-Verifying Ethereum data.
+Semiotic Labs monorepo for Verifiable Extraction of blockchain data.
 
 ## Crates
 
-### Forrestrie
-
-Library of types and methods for verifying post-merge Ethereum data.
-
-### Firehose Client
-
-Support for interfacing programmatically with Firehose gRPC endpoints.
-
-For more information see the
-[`firehose-client/README`](./crates/firehose-client/README.md).
-
-## Documentation
-
-- Notion doc on
-[Post-merge Header Record Data Structure](https://www.notion.so/semiotic/Post-merge-header_record-data-structure-7290d03d356946188bdb9ac29366f510?pvs=4).
-- [Beacon Chain `BeaconState` spec](https://github.com/ethereum/consensus-specs/blob/dev/specs/capella/beacon-chain.md#beaconstate)
-- [Beacon Chain `BeaconBlockBody` spec](https://github.com/ethereum/consensus-specs/blob/dev/specs/deneb/beacon-chain.md#beaconblockbody)
-- The [fork of `sigp/lighthouse`](https://github.com/semiotic-ai/lighthouse) we've been spiking.
-- [Google Drive shared resources](https://drive.google.com/drive/folders/19QBMHZFAV7uo_Cu4RwLPTwGpBcQMd-hy),
-including `head-state.json` used in `beacon_state.rs` tests.
-
-## Examples
-
-Here's an example of how to run one of the examples in the `forrestrie` crate:
-
-```terminal
-cd crates/forrestrie && cargo run -- --examples historical_state_roots_proof
-```
-
-Use environment variables to provide Firehose Ethereum and Firehose Beacon providers of
-your choice.
-
-To do so, place a `.env` file in the root of the crate you want to run examples for. 
-Your `.env` file should look like something this, depending on your requirements:
-
-```shell
-FIREHOSE_ETHEREUM_URL=<YOUR-FIREHOSE-ETHEREUM-URL>
-FIREHOSE_ETHEREUM_PORT=<YOUR-FIREHOSE-ETHEREUM-PORT>
-FIREHOSE_BEACON_URL=<YOUR-FIREHOSE-BEACON-URL>
-FIREHOSE_BEACON_PORT=<YOUR-FIREHOSE-BEACON-PORT>
-BEACON_API_KEY=<YOUR-API-KEY>
-ETHEREUM_API_KEY=<YOUR-API-KEY>
-```
+- [firehose-client](./crates/firehose-client/README.md)
+- [firehose-protos](./crates/firehose-protos/README.md)
+- [flat-files-decoder](./crates/flat-files-decoder/README.md)
+- [flat-head](./crates/flat-head/README.md)
+- [forrestrie](./crates/forrestrie/README.md)
+- [forrestrie-examples](./crates/forrestrie-examples/README.md)
+- [header-accumulator](./crates/header-accumulator/README.md)
diff --git a/crates/firehose-client/README.md b/crates/firehose-client/README.md
@@ -19,7 +19,7 @@ BEACON_API_KEY=<YOUR-API-KEY>
 ETHEREUM_API_KEY=<YOUR-API-KEY>
 ```
 
-## firehose-ethereum Execution Layer gRPC
+## firehose-ethereum and firehose-beacon gRPC
 
 ### proto files
 
@@ -29,34 +29,53 @@ We use the following protobuffers developed by Streamingfast via our
 See the [`firehose-protos` docs](./../firehose-protos/README.md) for information
 on these protobuffers and links to the different repos that they've come from.
 
-### gRPC service examples
+Also check out [`forrestrie`](./../forrestrie/protos/README.md) for 
+Rust-compiled protobuffer Beacon block implementations.
 
-Below is the description output for the Firehose gRPC server using `grpcurl`.
+### gRPC Service Examples
 
-```terminal
+If you're looking to quickly explore what the Firehose API offers,
+you can use the `grpcurl` tool to test your connection, verify your
+API token, and get a list of available gRPC services. This is a great
+way to interact with the API without writing any code.
+
+To get started, you can run the following command to retrieve the
+service descriptions from the Firehose gRPC server:
+
+```bash
 grpcurl -plaintext <your-grpc-service>:<port> describe
+```
+
+Below is an example of the service descriptions you might see when
+querying the Firehose server:
 
+```terminal
 grpc.health.v1.Health is a service:
 service Health {
   rpc Check ( .grpc.health.v1.HealthCheckRequest ) returns ( .grpc.health.v1.HealthCheckResponse );
   rpc Watch ( .grpc.health.v1.HealthCheckRequest ) returns ( stream .grpc.health.v1.HealthCheckResponse );
 }
+
 grpc.reflection.v1.ServerReflection is a service:
 service ServerReflection {
   rpc ServerReflectionInfo ( stream .grpc.reflection.v1.ServerReflectionRequest ) returns ( stream .grpc.reflection.v1.ServerReflectionResponse );
 }
+
 grpc.reflection.v1alpha.ServerReflection is a service:
 service ServerReflection {
   rpc ServerReflectionInfo ( stream .grpc.reflection.v1alpha.ServerReflectionRequest ) returns ( stream .grpc.reflection.v1alpha.ServerReflectionResponse );
 }
+
 sf.firehose.v1.Stream is a service:
 service Stream {
   rpc Blocks ( .sf.firehose.v1.Request ) returns ( stream .sf.firehose.v1.Response );
 }
+
 sf.firehose.v2.Fetch is a service:
 service Fetch {
   rpc Block ( .sf.firehose.v2.SingleBlockRequest ) returns ( .sf.firehose.v2.SingleBlockResponse );
 }
+
 sf.firehose.v2.Stream is a service:
 service Stream {
   rpc Blocks ( .sf.firehose.v2.Request ) returns ( stream .sf.firehose.v2.Response );

diff --git a/crates/forrestrie-examples/Cargo.toml b/crates/forrestrie-examples/Cargo.toml
@@ -0,0 +1,28 @@
+[package]
+name = "forrestrie-examples"
+version = "0.1.1"
+edition = "2021"
+
+[dev-dependencies]
+bls.workspace = true
+ethportal-api.workspace = true
+firehose-client = { path = "../firehose-client" }
+firehose-protos = { path = "../firehose-protos" }
+forrestrie = { path = "../forrestrie" }
+futures.workspace = true
+insta.workspace = true
+merkle_proof.workspace = true
+primitive-types.workspace = true
+prost.workspace = true
+prost-wkt.workspace = true
+prost-wkt-types.workspace = true
+reqwest = { workspace = true, features = ["json"] }
+serde = { workspace = true, features = ["derive"] }
+serde_json.workspace = true
+ssz_types.workspace = true
+tokio = { workspace = true, features = ["macros", "rt-multi-thread"] }
+tonic.workspace = true
+tracing.workspace = true
+tracing-subscriber = "0.3"
+tree_hash = "0.6.0"
+types.workspace = true
diff --git a/crates/forrestrie-examples/README.md b/crates/forrestrie-examples/README.md
@@ -0,0 +1,27 @@
+# Forrestrie Examples
+
+Here's an example of how to run one of the examples in the `forrestrie-examples` crate:
+
+```terminal
+cd crates/forrestrie-examples && cargo run -- --examples historical_state_roots_proof
+```
+
+Use environment variables to provide Firehose Ethereum and Firehose
+Beacon providers of your choice.
+
+To do this, place a `.env` file in the root of `veemon`. 
+
+Your `.env` file should look like something this, depending on your
+requirements:
+
+```shell
+FIREHOSE_ETHEREUM_URL=<YOUR-FIREHOSE-ETHEREUM-URL>
+FIREHOSE_ETHEREUM_PORT=<YOUR-FIREHOSE-ETHEREUM-PORT>
+FIREHOSE_BEACON_URL=<YOUR-FIREHOSE-BEACON-URL>
+FIREHOSE_BEACON_PORT=<YOUR-FIREHOSE-BEACON-PORT>
+BEACON_API_KEY=<YOUR-API-KEY>
+ETHEREUM_API_KEY=<YOUR-API-KEY>
+```
+
+Pinax is a great provider for blockchain data, so if you need an API key,
+you might want to check them out.
diff --git a/...estrie/examples/block_roots_only_proof.rs → ...amples/examples/block_roots_only_proof.rs b/...estrie/examples/block_roots_only_proof.rs → ...amples/examples/block_roots_only_proof.rs
diff --git a/...forrestrie/examples/block_roots_proofs.rs → ...e-examples/examples/block_roots_proofs.rs b/...forrestrie/examples/block_roots_proofs.rs → ...e-examples/examples/block_roots_proofs.rs
diff --git a/.../forrestrie/examples/empty_slot_hashes.rs → ...ie-examples/examples/empty_slot_hashes.rs b/.../forrestrie/examples/empty_slot_hashes.rs → ...ie-examples/examples/empty_slot_hashes.rs
diff --git a/...estrie/examples/fetch_and_verify_block.rs → ...amples/examples/fetch_and_verify_block.rs b/...estrie/examples/fetch_and_verify_block.rs → ...amples/examples/fetch_and_verify_block.rs
diff --git a/.../examples/historical_state_roots_proof.rs → .../examples/historical_state_roots_proof.rs b/.../examples/historical_state_roots_proof.rs → .../examples/historical_state_roots_proof.rs
diff --git a/...trie/examples/historical_summary_proof.rs → ...ples/examples/historical_summary_proof.rs b/...trie/examples/historical_summary_proof.rs → ...ples/examples/historical_summary_proof.rs
diff --git a/...trie/examples/match_ethereum_to_beacon.rs → ...ples/examples/match_ethereum_to_beacon.rs b/...trie/examples/match_ethereum_to_beacon.rs → ...ples/examples/match_ethereum_to_beacon.rs
@@ -30,8 +30,10 @@
 //!
 
 use firehose_client::client::{Chain, FirehoseClient};
-use firehose_protos::beacon_v1::{block, Block as FirehoseBeaconBlock};
-use forrestrie::beacon_state::ETHEREUM_BEACON_DENEB_OFFSET;
+use forrestrie::{
+    beacon_state::ETHEREUM_BEACON_DENEB_OFFSET,
+    beacon_v1::{block, Block as FirehoseBeaconBlock},
+};
 use std::cmp::Ordering::*;
 use tracing::info;
 use tracing_subscriber::FmtSubscriber;

diff --git a/...estrie/examples/single_execution_block.rs → ...amples/examples/single_execution_block.rs b/...estrie/examples/single_execution_block.rs → ...amples/examples/single_execution_block.rs
diff --git a/.../examples/verify_block_inclusion_proof.rs → .../examples/verify_block_inclusion_proof.rs b/.../examples/verify_block_inclusion_proof.rs → .../examples/verify_block_inclusion_proof.rs
diff --git a/crates/forrestrie-examples/src/lib.rs b/crates/forrestrie-examples/src/lib.rs
diff --git a/crates/forrestrie/Cargo.toml b/crates/forrestrie/Cargo.toml
@@ -18,20 +18,11 @@ prost.workspace = true
 prost-wkt.workspace = true
 prost-wkt-types.workspace = true
 serde = { workspace = true, features = ["derive"] }
-tracing.workspace = true
 ssz_types.workspace = true
 tonic.workspace = true
 tree_hash = "0.6.0"
 types.workspace = true
 
-[dev-dependencies]
-firehose-client = { path = "../firehose-client" }
-insta.workspace = true
-reqwest = { workspace = true, features = ["json"] }
-serde_json.workspace = true
-tokio = { workspace = true, features = ["macros", "rt-multi-thread"] }
-tracing-subscriber = "0.3"
-
 [build-dependencies]
 prost-build.workspace = true
 tonic-build.workspace = true
diff --git a/crates/forrestrie/README.md b/crates/forrestrie/README.md
@@ -0,0 +1,17 @@
+# Forrestrie
+
+Library of types and methods for verifying post-merge Ethereum data.
+
+## Documentation
+
+- Notion doc on
+[Post-merge Header Record Data Structure](https://www.notion.so/semiotic/Post-merge-header_record-data-structure-7290d03d356946188bdb9ac29366f510?pvs=4).
+- [Beacon Chain `BeaconState` spec](https://github.com/ethereum/consensus-specs/blob/dev/specs/capella/beacon-chain.md#beaconstate)
+- [Beacon Chain `BeaconBlockBody` spec](https://github.com/ethereum/consensus-specs/blob/dev/specs/deneb/beacon-chain.md#beaconblockbody)
+- The [fork of `sigp/lighthouse`](https://github.com/semiotic-ai/lighthouse) we've been spiking.
+- [Google Drive shared resources](https://drive.google.com/drive/folders/19QBMHZFAV7uo_Cu4RwLPTwGpBcQMd-hy),
+including `head-state.json` used in `beacon_state.rs` tests.
+
+## Examples
+
+See [forrestrie-examples](./../forrestrie-examples/README.md)!
diff --git a/crates/header-accumulator/Cargo.toml b/crates/header-accumulator/Cargo.toml
@@ -9,7 +9,6 @@ edition = "2021"
 alloy-primitives.workspace = true
 base64.workspace = true
 clap.workspace = true
-env_logger.workspace = true
 ethportal-api.workspace = true
 firehose-protos = { path = "../firehose-protos" }
 log.workspace = true

diff --git a/release-please-config.json b/release-please-config.json
@@ -8,6 +8,7 @@
     "crates/flat-files-decoder": {},
     "crates/flat-head": {},
     "crates/forrestrie": {},
+    "crates/forrestrie-examples": {},
     "crates/header-accumulator": {},
     "crates/firehose-protos": {}
   }