From ece136658163665b7e8577598f9ad9626d700d0c Mon Sep 17 00:00:00 2001 From: Joseph Livesey Date: Sat, 30 Nov 2024 15:51:14 -0500 Subject: [PATCH] docs(vee): bring accumulator doc examples to the top level --- Cargo.lock | 144 +++++++++--------- crates/firehose-protos-examples/README.md | 26 +++- crates/firehose-protos-examples/src/lib.rs | 29 +--- crates/firehose-protos/README.md | 15 +- crates/firehose-protos/src/lib.rs | 15 +- crates/flat-files-decoder/README.md | 2 + .../flat-files-decoder/input_files/.gitkeep | 0 .../flat-files-decoder/output_files/.gitkeep | 0 crates/flat-files-decoder/src/lib.rs | 5 +- crates/header-accumulator/README.md | 130 +++++++++++++++- .../examples/era_validator.rs | 49 ------ .../examples/inclusion_proof.rs | 66 -------- .../header-accumulator/src/era_validator.rs | 6 + crates/header-accumulator/src/lib.rs | 7 +- crates/vee/Cargo.toml | 3 + crates/vee/README.md | 120 ++++++++++++++- crates/vee/src/lib.rs | 1 + 17 files changed, 357 insertions(+), 261 deletions(-) delete mode 100644 crates/flat-files-decoder/input_files/.gitkeep delete mode 100644 crates/flat-files-decoder/output_files/.gitkeep delete mode 100644 crates/header-accumulator/examples/era_validator.rs delete mode 100644 crates/header-accumulator/examples/inclusion_proof.rs diff --git a/Cargo.lock b/Cargo.lock index 7c6cb318..be53f74e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -252,7 +252,7 @@ dependencies = [ "proptest", "rand", "ruint", - "rustc-hash 2.0.0", + "rustc-hash 2.1.0", "serde", "sha3 0.10.8", "tiny-keccak", @@ -277,7 +277,7 @@ checksum = "2b09cae092c27b6f1bde952653a22708691802e57bfef4a2973b80bea21efd3f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.89", + "syn 2.0.90", ] [[package]] @@ -333,7 +333,7 @@ dependencies = [ "proc-macro-error2", "proc-macro2", "quote", - "syn 2.0.89", + "syn 2.0.90", ] [[package]] @@ -349,7 +349,7 @@ dependencies = [ "proc-macro-error2", "proc-macro2", "quote", - "syn 2.0.89", + "syn 2.0.90", "syn-solidity", "tiny-keccak", ] @@ -365,7 +365,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.89", + "syn 2.0.90", "syn-solidity", ] @@ -635,7 +635,7 @@ checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.89", + "syn 2.0.90", ] [[package]] @@ -646,7 +646,7 @@ checksum = "721cae7de5c34fbb2acd27e21e6d2cf7b886dce0c27388d46c4e6c47ea4318dd" dependencies = [ "proc-macro2", "quote", - "syn 2.0.89", + "syn 2.0.90", ] [[package]] @@ -674,7 +674,7 @@ checksum = "3c87f3f15e7794432337fc718554eaa4dc8f04c9677a950ffe366f20a162ae42" dependencies = [ "proc-macro2", "quote", - "syn 2.0.89", + "syn 2.0.90", ] [[package]] @@ -835,7 +835,7 @@ dependencies = [ "regex", "rustc-hash 1.1.0", "shlex", - "syn 2.0.89", + "syn 2.0.90", "which", ] @@ -1093,7 +1093,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.89", + "syn 2.0.90", ] [[package]] @@ -1372,7 +1372,7 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.89", + "syn 2.0.90", ] [[package]] @@ -1420,7 +1420,7 @@ dependencies = [ "proc-macro2", "quote", "strsim 0.11.1", - "syn 2.0.89", + "syn 2.0.90", ] [[package]] @@ -1442,7 +1442,7 @@ checksum = "d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806" dependencies = [ "darling_core 0.20.10", "quote", - "syn 2.0.89", + "syn 2.0.90", ] [[package]] @@ -1503,7 +1503,7 @@ dependencies = [ "convert_case", "proc-macro2", "quote", - "syn 2.0.89", + "syn 2.0.90", "unicode-xid", ] @@ -1586,7 +1586,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.89", + "syn 2.0.90", ] [[package]] @@ -1698,7 +1698,7 @@ checksum = "2f9ed6b3789237c8a0c1c505af1c7eb2c560df6186f01b098c3a1064ea532f38" dependencies = [ "proc-macro2", "quote", - "syn 2.0.89", + "syn 2.0.90", ] [[package]] @@ -1794,7 +1794,7 @@ dependencies = [ "darling 0.20.10", "proc-macro2", "quote", - "syn 2.0.89", + "syn 2.0.90", ] [[package]] @@ -1882,7 +1882,7 @@ checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d" [[package]] name = "firehose-client" version = "0.1.1" -source = "git+https://github.com/semiotic-ai/firehose-client.git?branch=main#edf275a0eb9ddca94fea70f6f06be41627636ac1" +source = "git+https://github.com/semiotic-ai/firehose-client.git?branch=main#7591ec9a05ec8691de338683724c1abcaaddcdc5" dependencies = [ "dotenvy", "firehose-rs", @@ -2087,7 +2087,7 @@ checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", - "syn 2.0.89", + "syn 2.0.90", ] [[package]] @@ -2655,7 +2655,7 @@ checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.89", + "syn 2.0.90", ] [[package]] @@ -2719,7 +2719,7 @@ checksum = "a0eb5a3343abf848c0984fe4604b2b105da9539376e24fc0a3b0007411ae4fd9" dependencies = [ "proc-macro2", "quote", - "syn 2.0.89", + "syn 2.0.90", ] [[package]] @@ -2845,10 +2845,11 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.73" +version = "0.3.74" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb15147158e79fd8b8afd0252522769c4f48725460b37338544d8379d94fc8f9" +checksum = "a865e038f7f6ed956f788f0d7d60c541fff74c7bd74272c5d4cf15c63743e705" dependencies = [ + "once_cell", "wasm-bindgen", ] @@ -2912,7 +2913,7 @@ dependencies = [ "parking_lot 0.12.3", "pin-project", "rand", - "rustc-hash 2.0.0", + "rustc-hash 2.1.0", "serde", "serde_json", "thiserror 1.0.69", @@ -2957,7 +2958,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.89", + "syn 2.0.90", ] [[package]] @@ -3366,7 +3367,7 @@ checksum = "af1844ef2428cc3e1cb900be36181049ef3d3193c63e43026cfe202983b27a56" dependencies = [ "proc-macro2", "quote", - "syn 2.0.89", + "syn 2.0.90", ] [[package]] @@ -3487,7 +3488,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.89", + "syn 2.0.90", ] [[package]] @@ -3588,7 +3589,7 @@ checksum = "3c0f5fad0874fc7abcd4d750e76917eaebbecaa2c20bde22e1dbeeba8beb758c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.89", + "syn 2.0.90", ] [[package]] @@ -3681,7 +3682,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "64d1ec885c64d0457d564db4ec299b2dae3f9c02808b8ad9c3a089c591b18033" dependencies = [ "proc-macro2", - "syn 2.0.89", + "syn 2.0.90", ] [[package]] @@ -3747,7 +3748,7 @@ dependencies = [ "proc-macro-error-attr2", "proc-macro2", "quote", - "syn 2.0.89", + "syn 2.0.90", ] [[package]] @@ -3806,7 +3807,7 @@ dependencies = [ "prost", "prost-types", "regex", - "syn 2.0.89", + "syn 2.0.90", "tempfile", ] @@ -3820,7 +3821,7 @@ dependencies = [ "itertools 0.13.0", "proc-macro2", "quote", - "syn 2.0.89", + "syn 2.0.90", ] [[package]] @@ -4066,7 +4067,7 @@ dependencies = [ "convert_case", "proc-macro2", "quote", - "syn 2.0.89", + "syn 2.0.90", ] [[package]] @@ -4081,7 +4082,7 @@ dependencies = [ "crc", "dyn-clone", "once_cell", - "rustc-hash 2.0.0", + "rustc-hash 2.1.0", "serde", "thiserror-no-std", ] @@ -4297,7 +4298,7 @@ dependencies = [ "proc-macro2", "quote", "rust-embed-utils", - "syn 2.0.89", + "syn 2.0.90", "walkdir", ] @@ -4325,9 +4326,9 @@ checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" [[package]] name = "rustc-hash" -version = "2.0.0" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "583034fd73374156e66797ed8e5b0d5690409c9226b22d87cb7f19821c05d152" +checksum = "c7fb8039b3032c191086b10f11f319a6e99e1e82889c5cc6046f515c9db1d497" dependencies = [ "rand", ] @@ -4638,7 +4639,7 @@ checksum = "ad1e866f866923f252f05c889987993144fb74e722403468a4ebd70c3cd756c0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.89", + "syn 2.0.90", ] [[package]] @@ -4887,7 +4888,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.89", + "syn 2.0.90", ] [[package]] @@ -4923,9 +4924,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.89" +version = "2.0.90" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44d46482f1c1c87acd84dea20c1bf5ebff4c757009ed6bf19cfd36fb10e92c4e" +checksum = "919d3b74a5dd0ccd15aeb8f93e7006bd9e14c295087c9896a110f490752bcf31" dependencies = [ "proc-macro2", "quote", @@ -4941,7 +4942,7 @@ dependencies = [ "paste", "proc-macro2", "quote", - "syn 2.0.89", + "syn 2.0.90", ] [[package]] @@ -4964,7 +4965,7 @@ checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" dependencies = [ "proc-macro2", "quote", - "syn 2.0.89", + "syn 2.0.90", ] [[package]] @@ -5012,7 +5013,7 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.89", + "syn 2.0.90", ] [[package]] @@ -5023,7 +5024,7 @@ checksum = "f077553d607adc1caf65430528a576c757a71ed73944b66ebb58ef2bbd243568" dependencies = [ "proc-macro2", "quote", - "syn 2.0.89", + "syn 2.0.90", ] [[package]] @@ -5168,7 +5169,7 @@ checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752" dependencies = [ "proc-macro2", "quote", - "syn 2.0.89", + "syn 2.0.90", ] [[package]] @@ -5270,7 +5271,7 @@ dependencies = [ "prost-build", "prost-types", "quote", - "syn 2.0.89", + "syn 2.0.90", ] [[package]] @@ -5339,7 +5340,7 @@ checksum = "395ae124c09f9e6918a2310af6038fba074bcf474ac352496d5910dd59a2226d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.89", + "syn 2.0.90", ] [[package]] @@ -5414,7 +5415,7 @@ dependencies = [ "darling 0.20.10", "proc-macro2", "quote", - "syn 2.0.89", + "syn 2.0.90", ] [[package]] @@ -5492,7 +5493,7 @@ checksum = "70b20a22c42c8f1cd23ce5e34f165d4d37038f5b663ad20fb6adbdf029172483" dependencies = [ "proc-macro2", "quote", - "syn 2.0.89", + "syn 2.0.90", ] [[package]] @@ -5702,6 +5703,7 @@ dependencies = [ "firehose-protos", "flat-files-decoder", "header-accumulator", + "tree_hash", ] [[package]] @@ -5746,9 +5748,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.96" +version = "0.2.97" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21d3b25c3ea1126a2ad5f4f9068483c2af1e64168f847abe863a526b8dbfe00b" +checksum = "d15e63b4482863c109d70a7b8706c1e364eb6ea449b201a76c5b89cedcec2d5c" dependencies = [ "cfg-if", "once_cell", @@ -5757,24 +5759,24 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.96" +version = "0.2.97" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52857d4c32e496dc6537646b5b117081e71fd2ff06de792e3577a150627db283" +checksum = "8d36ef12e3aaca16ddd3f67922bc63e48e953f126de60bd33ccc0101ef9998cd" dependencies = [ "bumpalo", "log", "once_cell", "proc-macro2", "quote", - "syn 2.0.89", + "syn 2.0.90", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-futures" -version = "0.4.46" +version = "0.4.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "951fe82312ed48443ac78b66fa43eded9999f738f6022e67aead7b708659e49a" +checksum = "9dfaf8f50e5f293737ee323940c7d8b08a66a95a419223d9f41610ca08b0833d" dependencies = [ "cfg-if", "js-sys", @@ -5785,9 +5787,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.96" +version = "0.2.97" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "920b0ffe069571ebbfc9ddc0b36ba305ef65577c94b06262ed793716a1afd981" +checksum = "705440e08b42d3e4b36de7d66c944be628d579796b8090bfa3471478a2260051" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -5795,28 +5797,28 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.96" +version = "0.2.97" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf59002391099644be3524e23b781fa43d2be0c5aa0719a18c0731b9d195cab6" +checksum = "98c9ae5a76e46f4deecd0f0255cc223cfa18dc9b261213b8aa0c7b36f61b3f1d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.89", + "syn 2.0.90", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.96" +version = "0.2.97" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5047c5392700766601942795a436d7d2599af60dcc3cc1248c9120bfb0827b0" +checksum = "6ee99da9c5ba11bd675621338ef6fa52296b76b83305e9b6e5c77d4c286d6d49" [[package]] name = "web-sys" -version = "0.3.73" +version = "0.3.74" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "476364ff87d0ae6bfb661053a9104ab312542658c3d8f963b7ace80b6f9b26b9" +checksum = "a98bc3c33f0fe7e59ad7cd041b89034fa82a7c2d4365ca538dda6cdaf513863c" dependencies = [ "js-sys", "wasm-bindgen", @@ -6015,7 +6017,7 @@ checksum = "2380878cad4ac9aac1e2435f3eb4020e8374b5f13c296cb75b4620ff8e229154" dependencies = [ "proc-macro2", "quote", - "syn 2.0.89", + "syn 2.0.90", "synstructure", ] @@ -6037,7 +6039,7 @@ checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.89", + "syn 2.0.90", ] [[package]] @@ -6057,7 +6059,7 @@ checksum = "595eed982f7d355beb85837f651fa22e90b3c044842dc7f2c2842c086f295808" dependencies = [ "proc-macro2", "quote", - "syn 2.0.89", + "syn 2.0.90", "synstructure", ] @@ -6078,7 +6080,7 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" dependencies = [ "proc-macro2", "quote", - "syn 2.0.89", + "syn 2.0.90", ] [[package]] @@ -6100,7 +6102,7 @@ checksum = "6eafa6dfb17584ea3e2bd6e76e0cc15ad7af12b09abdd1ca55961bed9b1063c6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.89", + "syn 2.0.90", ] [[package]] diff --git a/crates/firehose-protos-examples/README.md b/crates/firehose-protos-examples/README.md index f70dc2a9..27cd97f1 100644 --- a/crates/firehose-protos-examples/README.md +++ b/crates/firehose-protos-examples/README.md @@ -1,14 +1,26 @@ # Firehose Protos Examples -Here's an example of how to run one of the examples in the `firehose-protos-examples` crate: +Examples that use methods implemented on the Rust-compiled Firehose +protobuffer types from [Firehose Protos](../firehose_protos/index.html). + +## Running Examples + +To run the examples, you need access to a Firehose provider for each chain from which you want +to extract data - an endpoint and an API key (if the latter is required). + +If you need access to a Firehose provider, we suggest using [Pinax](https://app.pinax.network/). + +Add your endpoint and API key to a `.env` file in the root of the repository. See `.env.examples` for +a configuration template. + +To run individual examples, use the following command: ```terminal -cd crates/firehose-protos-examples && cargo run -- --examples receipt_root +cargo run -p firehose-protos-examples --example ``` -Use environment variables to provide Firehose Ethereum and Firehose -Beacon providers of your choice. +So, for example, to run the `receipt_root` example: -To do this, place a `.env` file in the root of `veemon`. See the -`.env.example` file in the root of this repository for what you'll need, -depending on your requirements. +```terminal +cargo run -p firehose-protos-examples --example receipt_root +``` diff --git a/crates/firehose-protos-examples/src/lib.rs b/crates/firehose-protos-examples/src/lib.rs index 2ab561c3..16d6626d 100644 --- a/crates/firehose-protos-examples/src/lib.rs +++ b/crates/firehose-protos-examples/src/lib.rs @@ -1,32 +1,5 @@ // Copyright 2024-, Semiotic AI, Inc. // SPDX-License-Identifier: Apache-2.0 -//! # Firehose Protos Examples -//! -//! Examples that use methods implemented on the Rust-compiled Firehose -//! protobuffer types from [Firehose Protos](../firehose_protos/index.html). -//! -//! ## Running Examples -//! -//! To run the examples, you need access to a Firehose provider for each chain from which you want -//! to extract data - an endpoint and an API key (if the latter is required). -//! -//! If you need access to a Firehose provider, we suggest using [Pinax](https://app.pinax.network/). -//! -//! Add your endpoint and API key to a `.env` file in the root of the repository. See `.env.examples` for -//! a configuration template. -//! -//! To run individual examples, use the following command: -//! -//! ```terminal -//! cargo run -p firehose-protos-examples --example -//! ``` -//! -//! So, for example, to run the `receipt_root` example: -//! -//! ```terminal -//! cargo run -p firehose-protos-examples --example receipt_root -//! ``` -//! - #![deny(missing_docs)] +#![doc = include_str!("../README.md")] diff --git a/crates/firehose-protos/README.md b/crates/firehose-protos/README.md index 71928635..bbb8fe38 100644 --- a/crates/firehose-protos/README.md +++ b/crates/firehose-protos/README.md @@ -1,8 +1,7 @@ -# `firehose-protos` +# Verifiable Extraction Protocol Buffers in Rust -StreamingFast's Firehose protocol buffers compiled to Rust, -used in [header-accumulator](./../header-accumulator/Readme.md) -and [flat-files-decoder](./../flat-files-decoder/Readme.md). +This module provides Rust implementations of StreamingFast's Firehose protocol buffer +definitions, enabling efficient encoding and decoding of data for Ethereum blockchains via StreamingFast’s framework for streaming blockchain data. ## Protobuffer definitions @@ -13,3 +12,11 @@ Representation of the tracing of a block in the Ethereum blockchain. ### [`bstream.proto`](https://github.com/streamingfast/bstream/blob/develop/proto/sf/bstream/v1/bstream.proto) `Block` type from the Streamingfast block streaming Handlers library. Lower level building block of dfuse. + +## Usage + +To ingest these block types from flat files, check out +[`veemon/crates/flat-files-decoder`](../flat-files-decoder/index.html). + +For a high-level Rust client to use with Firehose endpoint providers like Pinax or StremaingFast, +check out [`semiotic-ai/firehose-client`](https://github.com/semiotic-ai/firehose-client). diff --git a/crates/firehose-protos/src/lib.rs b/crates/firehose-protos/src/lib.rs index 1ca3ae41..b6736427 100644 --- a/crates/firehose-protos/src/lib.rs +++ b/crates/firehose-protos/src/lib.rs @@ -1,21 +1,8 @@ // Copyright 2024-, Semiotic AI, Inc. // SPDX-License-Identifier: Apache-2.0 -//! # Verifiable Extraction Protocol Buffers in Rust -//! -//! This module provides Rust implementations of StreamingFast's protocol buffer definitions, -//! enabling efficient encoding and decoding of data for Ethereum blockchain services and Firehose, -//! StreamingFast’s framework for streaming blockchain data. -//! -//! ## Usage -//! -//! Check out [`veemon/firehose-client`](../firehose_client/index.html) for a high-level client -//! that you can use with chain data endpoint providers like Pinax or StremaingFast. -//! -//! Alternatively, for tools for ingesting these block types from flat files, check out -//! [`veemon/flat-files-decoder`](../flat-files-decoder/index.html). - #![deny(missing_docs)] +#![doc = include_str!("../README.md")] mod error; mod ethereum_v2; diff --git a/crates/flat-files-decoder/README.md b/crates/flat-files-decoder/README.md index 8edc28b0..cfe6067b 100644 --- a/crates/flat-files-decoder/README.md +++ b/crates/flat-files-decoder/README.md @@ -1,5 +1,7 @@ # Flat Files Decoder +Read, decode, and verify blockchain block flat files. + ## Overview Check out the crate documentation in your browser by running, from diff --git a/crates/flat-files-decoder/input_files/.gitkeep b/crates/flat-files-decoder/input_files/.gitkeep deleted file mode 100644 index e69de29b..00000000 diff --git a/crates/flat-files-decoder/output_files/.gitkeep b/crates/flat-files-decoder/output_files/.gitkeep deleted file mode 100644 index e69de29b..00000000 diff --git a/crates/flat-files-decoder/src/lib.rs b/crates/flat-files-decoder/src/lib.rs index 458ff0b9..d02f3688 100644 --- a/crates/flat-files-decoder/src/lib.rs +++ b/crates/flat-files-decoder/src/lib.rs @@ -1,11 +1,8 @@ // Copyright 2024-, Semiotic AI, Inc. // SPDX-License-Identifier: Apache-2.0 -//! # Flat Files Decoder -//! -//! Read, decode, and verify blockchain block flat files. - #![deny(missing_docs)] +#![doc = include_str!("../README.md")] mod dbin; mod decoder; diff --git a/crates/header-accumulator/README.md b/crates/header-accumulator/README.md index 84ac1e81..bdc8955f 100644 --- a/crates/header-accumulator/README.md +++ b/crates/header-accumulator/README.md @@ -25,19 +25,19 @@ cd crates/header-accumulator && cargo doc --open ## Features - **`era_validate`**: Validates entire ERAs of flat files against -Header Accumulators. Use this command to ensure data integrity across -multiple ERAs. + Header Accumulators. Use this command to ensure data integrity across + multiple ERAs. - **`generate_inclusion_proof`**: Generates inclusion proofs for a -specified range of blocks, useful for verifying the presence of -specific blocks within a dataset. + specified range of blocks, useful for verifying the presence of + specific blocks within a dataset. - **`verify_inclusion_proof`**: Verifies inclusion proofs for a -specified range of blocks. Use this to confirm the accuracy of -inclusion proofs. + specified range of blocks. Use this to confirm the accuracy of + inclusion proofs. ### Options - `-h, --help`: Displays a help message that includes usage -information, commands, and options. + information, commands, and options. ## Goals @@ -48,10 +48,124 @@ to offer a comprehensive solution. ## Testing -Some tests depend on [flat-files-decoder](https://github.com/semiotic-ai/flat-files-decoder) as a development dependency. +Some tests depend on [flat-files-decoder](../flat-files-decoder/README.md) as a development dependency. Run all tests with: ```terminal cargo test ``` + +## Examples + +### Inclusion proof + +```rust,no_run +use std::{fs::File, io::BufReader}; +use flat_files_decoder::{read_blocks_from_reader, Compression}; +use header_accumulator::{ + generate_inclusion_proofs, verify_inclusion_proofs, Epoch, EraValidateError, ExtHeaderRecord, +}; + +fn main() -> Result<(), EraValidateError> { + let mut headers: Vec = Vec::new(); + + for flat_file_number in (0..=8200).step_by(100) { + let file = format!( + "your_files/ethereum_firehose_first_8200/{:010}.dbin", + flat_file_number + ); + match read_blocks_from_reader( + BufReader::new(File::open(&file).unwrap()), + Compression::None, + ) { + Ok(blocks) => { + headers.extend( + blocks + .iter() + .map(|block| ExtHeaderRecord::try_from(block).unwrap()) + .collect::>(), + ); + } + Err(e) => { + eprintln!("error: {:?}", e); + break; + } + } + } + + let start_block = 301; + let end_block = 402; + let headers_to_prove: Vec<_> = headers[start_block..end_block] + .iter() + .map(|ext| ext.full_header.as_ref().unwrap().clone()) + .collect(); + let epoch: Epoch = headers.try_into().unwrap(); + + let inclusion_proof = generate_inclusion_proofs(vec![epoch], headers_to_prove.clone()) + .unwrap_or_else(|e| { + println!("Error occurred: {}", e); + std::process::exit(1); + }); + assert_eq!(inclusion_proof.len(), headers_to_prove.len()); + + let proof_headers = headers_to_prove + .into_iter() + .zip(inclusion_proof) + .map(|(header, proof)| proof.with_header(header)) + .collect::, _>>()?; + + // Verify inclusion proof + assert!(verify_inclusion_proofs(None, proof_headers).is_ok()); + + println!("Inclusion proof verified successfully!"); + + Ok(()) +} +``` + +### Era validator + +```rust,no_run +use std::{fs::File, io::BufReader}; + +use flat_files_decoder::{read_blocks_from_reader, Compression}; +use header_accumulator::{Epoch, EraValidateError, EraValidator, ExtHeaderRecord}; +use tree_hash::Hash256; + +fn create_test_reader(path: &str) -> BufReader { + BufReader::new(File::open(path).unwrap()) +} + +fn main() -> Result<(), EraValidateError> { + let mut headers: Vec = Vec::new(); + for number in (0..=8200).step_by(100) { + let file_name = format!( + "your-test-assets/ethereum_firehose_first_8200/{:010}.dbin", + number + ); + let reader = create_test_reader(&file_name); + let blocks = read_blocks_from_reader(reader, Compression::None).unwrap(); + let successful_headers = blocks + .iter() + .cloned() + .map(|block| ExtHeaderRecord::try_from(&block)) + .collect::, _>>()?; + headers.extend(successful_headers); + } + assert_eq!(headers.len(), 8300); + assert_eq!(headers[0].block_number, 0); + let era_verifier = EraValidator::default(); + let epoch: Epoch = headers.try_into().unwrap(); + let result = era_verifier.validate_era(&epoch)?; + let expected = Hash256::new([ + 94, 193, 255, 184, 195, 177, 70, 244, 38, 6, 199, 76, 237, 151, 61, 193, 110, 197, 161, 7, + 192, 52, 88, 88, 195, 67, 252, 148, 120, 11, 66, 24, + ]); + assert_eq!(result, expected); + + println!("Era validated successfully!"); + + Ok(()) +} +``` \ No newline at end of file diff --git a/crates/header-accumulator/examples/era_validator.rs b/crates/header-accumulator/examples/era_validator.rs deleted file mode 100644 index 9878d3a7..00000000 --- a/crates/header-accumulator/examples/era_validator.rs +++ /dev/null @@ -1,49 +0,0 @@ -// Copyright 2024-, Semiotic AI, Inc. -// SPDX-License-Identifier: Apache-2.0 - -use std::{fs::File, io::BufReader}; - -use flat_files_decoder::{read_blocks_from_reader, Compression}; -use header_accumulator::{Epoch, EraValidateError, EraValidator, ExtHeaderRecord}; -use tree_hash::Hash256; -use trin_validation::accumulator::PreMergeAccumulator; - -fn create_test_reader(path: &str) -> BufReader { - BufReader::new(File::open(path).unwrap()) -} - -fn main() -> Result<(), EraValidateError> { - let mut headers: Vec = Vec::new(); - - for number in (0..=8200).step_by(100) { - let file_name = format!( - "your-test-assets/ethereum_firehose_first_8200/{:010}.dbin", - number - ); - let reader = create_test_reader(&file_name); - let blocks = read_blocks_from_reader(reader, Compression::None).unwrap(); - let successful_headers = blocks - .iter() - .cloned() - .map(|block| ExtHeaderRecord::try_from(&block)) - .collect::, _>>()?; - headers.extend(successful_headers); - } - - assert_eq!(headers.len(), 8300); - assert_eq!(headers[0].block_number, 0); - - let premerge_accumulator: EraValidator = PreMergeAccumulator::default().into(); - let epoch: Epoch = headers.try_into().unwrap(); - - let result = premerge_accumulator.validate_era(&epoch)?; - let expected = Hash256::new([ - 94, 193, 255, 184, 195, 177, 70, 244, 38, 6, 199, 76, 237, 151, 61, 193, 110, 197, 161, 7, - 192, 52, 88, 88, 195, 67, 252, 148, 120, 11, 66, 24, - ]); - assert_eq!(result, expected); - - println!("Era validated successfully!"); - - Ok(()) -} diff --git a/crates/header-accumulator/examples/inclusion_proof.rs b/crates/header-accumulator/examples/inclusion_proof.rs deleted file mode 100644 index bcfe6683..00000000 --- a/crates/header-accumulator/examples/inclusion_proof.rs +++ /dev/null @@ -1,66 +0,0 @@ -// Copyright 2024-, Semiotic AI, Inc. -// SPDX-License-Identifier: Apache-2.0 - -use std::{fs::File, io::BufReader}; - -use flat_files_decoder::{read_blocks_from_reader, Compression}; -use header_accumulator::{ - generate_inclusion_proofs, verify_inclusion_proofs, Epoch, EraValidateError, ExtHeaderRecord, -}; - -fn create_test_reader(path: &str) -> BufReader { - BufReader::new(File::open(path).unwrap()) -} - -fn main() -> Result<(), EraValidateError> { - let mut headers: Vec = Vec::new(); - - for flat_file_number in (0..=8200).step_by(100) { - let file = format!( - "your-test-assets/ethereum_firehose_first_8200/{:010}.dbin", - flat_file_number - ); - match read_blocks_from_reader(create_test_reader(&file), Compression::None) { - Ok(blocks) => { - headers.extend( - blocks - .iter() - .map(|block| ExtHeaderRecord::try_from(block).unwrap()) - .collect::>(), - ); - } - Err(e) => { - eprintln!("error: {:?}", e); - break; - } - } - } - - let start_block = 301; - let end_block = 402; - let headers_to_prove: Vec<_> = headers[start_block..end_block] - .iter() - .map(|ext| ext.full_header.as_ref().unwrap().clone()) - .collect(); - let epoch: Epoch = headers.try_into().unwrap(); - - let inclusion_proof = generate_inclusion_proofs(vec![epoch], headers_to_prove.clone()) - .unwrap_or_else(|e| { - println!("Error occurred: {}", e); - std::process::exit(1); - }); - assert_eq!(inclusion_proof.len(), headers_to_prove.len()); - - let proof_headers = headers_to_prove - .into_iter() - .zip(inclusion_proof) - .map(|(header, proof)| proof.with_header(header)) - .collect::, _>>()?; - - // Verify inclusion proof - assert!(verify_inclusion_proofs(None, proof_headers).is_ok()); - - println!("Inclusion proof verified successfully!"); - - Ok(()) -} diff --git a/crates/header-accumulator/src/era_validator.rs b/crates/header-accumulator/src/era_validator.rs index 93b2bd61..61bf2669 100644 --- a/crates/header-accumulator/src/era_validator.rs +++ b/crates/header-accumulator/src/era_validator.rs @@ -15,6 +15,12 @@ pub struct EraValidator { historical_epochs: HistoricalEpochRoots, } +impl Default for EraValidator { + fn default() -> Self { + PreMergeAccumulator::default().into() + } +} + impl From for EraValidator { fn from(value: PreMergeAccumulator) -> Self { Self { diff --git a/crates/header-accumulator/src/lib.rs b/crates/header-accumulator/src/lib.rs index 0d884b99..763e8b6b 100644 --- a/crates/header-accumulator/src/lib.rs +++ b/crates/header-accumulator/src/lib.rs @@ -1,13 +1,8 @@ // Copyright 2024-, Semiotic AI, Inc. // SPDX-License-Identifier: Apache-2.0 -//! # Header accumulator -//! -//! This crate is used to accumulate block headers and compare them -//! against header accumulators. This process is used to verify the -//! authenticity of these blocks. - #![deny(missing_docs)] +#![doc = include_str!("../README.md")] mod epoch; mod era_validator; diff --git a/crates/vee/Cargo.toml b/crates/vee/Cargo.toml index f8957dc4..0a336deb 100644 --- a/crates/vee/Cargo.toml +++ b/crates/vee/Cargo.toml @@ -11,3 +11,6 @@ name = "vee" firehose-protos.workspace = true flat-files-decoder.workspace = true header-accumulator.workspace = true + +[dev-dependencies] +tree_hash = "0.8.0" diff --git a/crates/vee/README.md b/crates/vee/README.md index 648e57fa..66aff05d 100644 --- a/crates/vee/README.md +++ b/crates/vee/README.md @@ -2,8 +2,120 @@ Verifiable Extraction for Blockchain. -## Re-exports +## Examples -- [firehose-protos](./crates/firehose-protos/README.md) -- [flat-files-decoder](./crates/flat-files-decoder/README.md) -- [header-accumulator](./crates/header-accumulator/README.md) +### Inclusion proof + +```rust,no_run +use std::{fs::File, io::BufReader}; +use vee::{ + generate_inclusion_proofs, read_blocks_from_reader, verify_inclusion_proofs, Compression, + Epoch, EraValidateError, ExtHeaderRecord, +}; + +fn main() -> Result<(), EraValidateError> { + let mut headers: Vec = Vec::new(); + + for flat_file_number in (0..=8200).step_by(100) { + let file = format!( + "your_files/ethereum_firehose_first_8200/{:010}.dbin", + flat_file_number + ); + match read_blocks_from_reader( + BufReader::new(File::open(&file).unwrap()), + Compression::None, + ) { + Ok(blocks) => { + headers.extend( + blocks + .iter() + .map(|block| ExtHeaderRecord::try_from(block).unwrap()) + .collect::>(), + ); + } + Err(e) => { + eprintln!("error: {:?}", e); + break; + } + } + } + + let start_block = 301; + let end_block = 402; + let headers_to_prove: Vec<_> = headers[start_block..end_block] + .iter() + .map(|ext| ext.full_header.as_ref().unwrap().clone()) + .collect(); + let epoch: Epoch = headers.try_into().unwrap(); + + let inclusion_proof = generate_inclusion_proofs(vec![epoch], headers_to_prove.clone()) + .unwrap_or_else(|e| { + println!("Error occurred: {}", e); + std::process::exit(1); + }); + assert_eq!(inclusion_proof.len(), headers_to_prove.len()); + + let proof_headers = headers_to_prove + .into_iter() + .zip(inclusion_proof) + .map(|(header, proof)| proof.with_header(header)) + .collect::, _>>()?; + + // Verify inclusion proof + assert!(verify_inclusion_proofs(None, proof_headers).is_ok()); + + println!("Inclusion proof verified successfully!"); + + Ok(()) +} +``` + +### Era validator + +```rust,no_run +use std::{fs::File, io::BufReader}; +use tree_hash::Hash256; +use vee::{ + read_blocks_from_reader, Compression, Epoch, EraValidateError, EraValidator, + ExtHeaderRecord, +}; + +fn create_test_reader(path: &str) -> BufReader { + BufReader::new(File::open(path).unwrap()) +} + +fn main() -> Result<(), EraValidateError> { + let mut headers: Vec = Vec::new(); + + for number in (0..=8200).step_by(100) { + let file_name = format!( + "your-test-assets/ethereum_firehose_first_8200/{:010}.dbin", + number + ); + let reader = create_test_reader(&file_name); + let blocks = read_blocks_from_reader(reader, Compression::None).unwrap(); + let successful_headers = blocks + .iter() + .cloned() + .map(|block| ExtHeaderRecord::try_from(&block)) + .collect::, _>>()?; + headers.extend(successful_headers); + } + + assert_eq!(headers.len(), 8300); + assert_eq!(headers[0].block_number, 0); + + let era_verifier = EraValidator::default(); + let epoch: Epoch = headers.try_into().unwrap(); + let result = era_verifier.validate_era(&epoch)?; + let expected = Hash256::new([ + 94, 193, 255, 184, 195, 177, 70, 244, 38, 6, 199, 76, 237, 151, 61, 193, 110, 197, 161, 7, + 192, 52, 88, 88, 195, 67, 252, 148, 120, 11, 66, 24, + ]); + assert_eq!(result, expected); + + println!("Era validated successfully!"); + + Ok(()) +} +``` diff --git a/crates/vee/src/lib.rs b/crates/vee/src/lib.rs index 07ce1ce1..9bda12f1 100644 --- a/crates/vee/src/lib.rs +++ b/crates/vee/src/lib.rs @@ -1,6 +1,7 @@ // Copyright 2024-, Semiotic AI, Inc. // SPDX-License-Identifier: Apache-2.0 +#![deny(missing_docs)] #![doc = include_str!("../README.md")] // 🚀✨ Main Re-exports ✨🚀