Merge pull request #70 from semiotic-ai/joseph/back-179-bring-example…

…s-to-fore-in-vee-crate-documentation

docs: bring examples to fore in vee crate documentation

crates/firehose-protos-examples/README.md

@@ -1,14 +1,26 @@
 # Firehose Protos Examples
 
-Here's an example of how to run one of the examples in the `firehose-protos-examples` crate:
+Examples that use methods implemented on the Rust-compiled Firehose
+protobuffer types from [Firehose Protos](../firehose_protos/index.html).
+
+## Running Examples
+
+To run the examples, you need access to a Firehose provider for each chain from which you want
+to extract data - an endpoint and an API key (if the latter is required).
+
+If you need access to a Firehose provider, we suggest using [Pinax](https://app.pinax.network/).
+
+Add your endpoint and API key to a `.env` file in the root of the repository. See `.env.examples` for
+a configuration template.
+
+To run individual examples, use the following command:
 
 ```terminal
-cd crates/firehose-protos-examples && cargo run -- --examples receipt_root
+cargo run -p firehose-protos-examples --example <example_name>
 ```
 
-Use environment variables to provide Firehose Ethereum and Firehose
-Beacon providers of your choice.
+So, for example, to run the `receipt_root` example:
 
-To do this, place a `.env` file in the root of `veemon`. See the
-`.env.example` file in the root of this repository for what you'll need,
-depending on your requirements.
+```terminal
+cargo run -p firehose-protos-examples --example receipt_root
+```

crates/firehose-protos-examples/src/lib.rs

@@ -1,32 +1,5 @@
 // Copyright 2024-, Semiotic AI, Inc.
 // SPDX-License-Identifier: Apache-2.0
 
-//! # Firehose Protos Examples
-//!
-//! Examples that use methods implemented on the Rust-compiled Firehose
-//! protobuffer types from [Firehose Protos](../firehose_protos/index.html).
-//!
-//! ## Running Examples
-//!
-//! To run the examples, you need access to a Firehose provider for each chain from which you want
-//! to extract data - an endpoint and an API key (if the latter is required).
-//!
-//! If you need access to a Firehose provider, we suggest using [Pinax](https://app.pinax.network/).
-//!
-//! Add your endpoint and API key to a `.env` file in the root of the repository. See `.env.examples` for
-//! a configuration template.
-//!
-//! To run individual examples, use the following command:
-//!
-//! ```terminal
-//! cargo run -p firehose-protos-examples --example <example_name>
-//! ```
-//!
-//! So, for example, to run the `receipt_root` example:
-//!
-//! ```terminal
-//! cargo run -p firehose-protos-examples --example receipt_root
-//! ```
-//!
-
 #![deny(missing_docs)]
+#![doc = include_str!("../README.md")]

crates/firehose-protos/README.md

@@ -1,8 +1,7 @@
-# `firehose-protos`
+# Verifiable Extraction Protocol Buffers in Rust
 
-StreamingFast's Firehose protocol buffers compiled to Rust,
-used in [header-accumulator](./../header-accumulator/Readme.md)
-and [flat-files-decoder](./../flat-files-decoder/Readme.md).
+This module provides Rust implementations of StreamingFast's Firehose protocol buffer
+definitions, enabling efficient encoding and decoding of data for Ethereum blockchains via StreamingFast’s framework for streaming blockchain data.
 
 ## Protobuffer definitions
 
@@ -13,3 +12,11 @@ Representation of the tracing of a block in the Ethereum blockchain.
 ### [`bstream.proto`](https://github.com/streamingfast/bstream/blob/develop/proto/sf/bstream/v1/bstream.proto)
 
 `Block` type from the Streamingfast block streaming Handlers library. Lower level building block of dfuse.
+
+## Usage
+
+To ingest these block types from flat files, check out
+[`veemon/crates/flat-files-decoder`](../flat-files-decoder/index.html).
+
+For a high-level Rust client to use with Firehose endpoint providers like Pinax or StremaingFast,
+check out [`semiotic-ai/firehose-client`](https://github.com/semiotic-ai/firehose-client).

crates/firehose-protos/src/lib.rs

@@ -1,21 +1,8 @@
 // Copyright 2024-, Semiotic AI, Inc.
 // SPDX-License-Identifier: Apache-2.0
 
-//! # Verifiable Extraction Protocol Buffers in Rust
-//!
-//! This module provides Rust implementations of StreamingFast's protocol buffer definitions,
-//! enabling efficient encoding and decoding of data for Ethereum blockchain services and Firehose,
-//! StreamingFast’s framework for streaming blockchain data.
-//!
-//! ## Usage
-//!
-//! Check out [`veemon/firehose-client`](../firehose_client/index.html) for a high-level client
-//! that you can use with chain data endpoint providers like Pinax or StremaingFast.
-//!
-//! Alternatively, for tools for ingesting these block types from flat files, check out
-//! [`veemon/flat-files-decoder`](../flat-files-decoder/index.html).
-
 #![deny(missing_docs)]
+#![doc = include_str!("../README.md")]
 
 mod error;
 mod ethereum_v2;

crates/flat-files-decoder/README.md

@@ -1,5 +1,7 @@
 # Flat Files Decoder
 
+Read, decode, and verify blockchain block flat files.
+
 ## Overview
 
 Check out the crate documentation in your browser by running, from

crates/flat-files-decoder/src/lib.rs

@@ -1,11 +1,8 @@
 // Copyright 2024-, Semiotic AI, Inc.
 // SPDX-License-Identifier: Apache-2.0
 
-//! # Flat Files Decoder
-//!
-//! Read, decode, and verify blockchain block flat files.
-
 #![deny(missing_docs)]
+#![doc = include_str!("../README.md")]
 
 mod dbin;
 mod decoder;

crates/header-accumulator/README.md

@@ -25,19 +25,19 @@ cd crates/header-accumulator && cargo doc --open
 ## Features
 
 - **`era_validate`**: Validates entire ERAs of flat files against
-Header Accumulators. Use this command to ensure data integrity across
-multiple ERAs.
+  Header Accumulators. Use this command to ensure data integrity across
+  multiple ERAs.
 - **`generate_inclusion_proof`**: Generates inclusion proofs for a
-specified range of blocks, useful for verifying the presence of
-specific blocks within a dataset.
+  specified range of blocks, useful for verifying the presence of
+  specific blocks within a dataset.
 - **`verify_inclusion_proof`**: Verifies inclusion proofs for a 
-specified range of blocks. Use this to confirm the accuracy of
-inclusion proofs.
+  specified range of blocks. Use this to confirm the accuracy of
+  inclusion proofs.
 
 ### Options
 
 - `-h, --help`: Displays a help message that includes usage 
-information, commands, and options.
+  information, commands, and options.
 
 ## Goals
 
@@ -48,10 +48,124 @@ to offer a comprehensive solution.
 
 ## Testing
 
-Some tests depend on [flat-files-decoder](https://github.com/semiotic-ai/flat-files-decoder) as a development dependency.
+Some tests depend on [flat-files-decoder](../flat-files-decoder/README.md) as a development dependency.
 
 Run all tests with:
 
 ```terminal
 cargo test
 ```
+
+## Examples
+
+### Inclusion proof
+
+```rust,no_run
+use std::{fs::File, io::BufReader};
+use flat_files_decoder::{read_blocks_from_reader, Compression};
+use header_accumulator::{
+    generate_inclusion_proofs, verify_inclusion_proofs, Epoch, EraValidateError, ExtHeaderRecord,
+};
+
+fn main() -> Result<(), EraValidateError> {
+   let mut headers: Vec<ExtHeaderRecord> = Vec::new();
+
+    for flat_file_number in (0..=8200).step_by(100) {
+        let file = format!(
+            "your_files/ethereum_firehose_first_8200/{:010}.dbin",
+            flat_file_number
+        );
+        match read_blocks_from_reader(
+            BufReader::new(File::open(&file).unwrap()),
+            Compression::None,
+        ) {
+            Ok(blocks) => {
+                headers.extend(
+                    blocks
+                        .iter()
+                        .map(|block| ExtHeaderRecord::try_from(block).unwrap())
+                        .collect::<Vec<ExtHeaderRecord>>(),
+                );
+            }
+            Err(e) => {
+                eprintln!("error: {:?}", e);
+                break;
+            }
+        }
+    }
+
+    let start_block = 301;
+    let end_block = 402;
+    let headers_to_prove: Vec<_> = headers[start_block..end_block]
+        .iter()
+        .map(|ext| ext.full_header.as_ref().unwrap().clone())
+        .collect();
+    let epoch: Epoch = headers.try_into().unwrap();
+
+    let inclusion_proof = generate_inclusion_proofs(vec![epoch], headers_to_prove.clone())
+        .unwrap_or_else(|e| {
+            println!("Error occurred: {}", e);
+            std::process::exit(1);
+        });
+    assert_eq!(inclusion_proof.len(), headers_to_prove.len());
+
+    let proof_headers = headers_to_prove
+        .into_iter()
+        .zip(inclusion_proof)
+        .map(|(header, proof)| proof.with_header(header))
+        .collect::<Result<Vec<_>, _>>()?;
+
+    // Verify inclusion proof
+    assert!(verify_inclusion_proofs(None, proof_headers).is_ok());
+
+    println!("Inclusion proof verified successfully!");
+
+    Ok(())
+}    
+```
+
+### Era validator
+
+```rust,no_run
+use std::{fs::File, io::BufReader};
+
+use flat_files_decoder::{read_blocks_from_reader, Compression};
+use header_accumulator::{Epoch, EraValidateError, EraValidator, ExtHeaderRecord};
+use tree_hash::Hash256;
+
+fn create_test_reader(path: &str) -> BufReader<File> {
+    BufReader::new(File::open(path).unwrap())
+}
+
+fn main() -> Result<(), EraValidateError> {
+    let mut headers: Vec<ExtHeaderRecord> = Vec::new();
+    for number in (0..=8200).step_by(100) {
+        let file_name = format!(
+            "your-test-assets/ethereum_firehose_first_8200/{:010}.dbin",
+            number
+        );
+        let reader = create_test_reader(&file_name);
+        let blocks = read_blocks_from_reader(reader, Compression::None).unwrap();
+        let successful_headers = blocks
+            .iter()
+            .cloned()
+            .map(|block| ExtHeaderRecord::try_from(&block))
+            .collect::<Result<Vec<_>, _>>()?;
+        headers.extend(successful_headers);
+    }
+    assert_eq!(headers.len(), 8300);
+    assert_eq!(headers[0].block_number, 0);
+    let era_verifier = EraValidator::default();
+    let epoch: Epoch = headers.try_into().unwrap();
+    let result = era_verifier.validate_era(&epoch)?;
+    let expected = Hash256::new([
+        94, 193, 255, 184, 195, 177, 70, 244, 38, 6, 199, 76, 237, 151, 61, 193, 110, 197, 161, 7,
+        192, 52, 88, 88, 195, 67, 252, 148, 120, 11, 66, 24,
+    ]);
+    assert_eq!(result, expected);
+
+    println!("Era validated successfully!");
+
+    Ok(())
+}
+```