add test_vectors to optimism

.github/workflows/compact.yml

@@ -1,4 +1,4 @@
-# Ensures that `Compact` codec changes are backwards compatible.
+# Workflow to ensure `Compact` codec changes are backwards compatible
 
 on:
   pull_request:
@@ -14,6 +14,11 @@ jobs:
   compact-codec:
     runs-on:
       group: Reth
+    strategy:
+      matrix:
+        bin:
+          - cargo run --bin reth --features "dev"
+          - cargo run --bin op-reth --features "optimism dev" --manifest-path crates/optimism/bin/Cargo.toml
     steps:
       - uses: dtolnay/rust-toolchain@stable
       - uses: Swatinem/rust-cache@v2
@@ -25,11 +30,12 @@ jobs:
           ref: ${{ github.base_ref || 'main' }}
       # On `main` branch, generates test vectors and serializes them to disk using `serde-json`.
       - name: Generate compact vectors
-        run: cargo run --bin reth --features dev -- test-vectors compact --write
+        run: ${{ matrix.bin }} -- test-vectors compact --write
       - name: Checkout PR
         uses: actions/checkout@v4
         with:
           clean: false
       # On incoming merge try to read and decode previously generated vectors 
       - name: Read vectors
-        run: cargo run --bin reth --features dev -- test-vectors compact --read
+        run: ${{ matrix.bin }} -- test-vectors compact --read
+

crates/cli/commands/src/test_vectors/compact.rs

@@ -31,33 +31,34 @@ use reth_stages_types::{
 };
 use reth_trie::{hash_builder::HashBuilderValue, TrieMask};
 use reth_trie_common::{hash_builder::HashBuilderState, StoredNibbles, StoredNibblesSubKey};
-use std::{collections::HashSet, fs::File, io::BufReader, sync::LazyLock};
+use std::{fs::File, io::BufReader};
 
-const VECTORS_FOLDER: &str = "testdata/micro/compact";
-const VECTOR_SIZE: usize = 100;
+pub const VECTORS_FOLDER: &str = "testdata/micro/compact";
+pub const VECTOR_SIZE: usize = 100;
 
+#[macro_export]
 macro_rules! compact_types {
     (regular: [$($regular_ty:ident),*], identifier: [$($id_ty:ident),*]) => {
-        const GENERATE_VECTORS: &[fn(&mut TestRunner) -> Result<()>] = &[
+        pub const GENERATE_VECTORS: &[fn(&mut TestRunner) -> eyre::Result<()>] = &[
             $(
-                generate_vector::<$regular_ty> as fn(&mut TestRunner) -> Result<()>,
+                generate_vector::<$regular_ty> as fn(&mut TestRunner) -> eyre::Result<()>,
             )*
             $(
-                generate_vector::<$id_ty> as fn(&mut TestRunner) -> Result<()>,
+                generate_vector::<$id_ty> as fn(&mut TestRunner) -> eyre::Result<()>,
             )*
         ];
 
-        const READ_VECTORS: &[fn() -> Result<()>] = &[
+        pub const READ_VECTORS: &[fn() -> eyre::Result<()>] = &[
             $(
-                read_vector::<$regular_ty> as fn() -> Result<()>,
+                read_vector::<$regular_ty> as fn() -> eyre::Result<()>,
             )*
             $(
-                read_vector::<$id_ty> as fn() -> Result<()>,
+                read_vector::<$id_ty> as fn() -> eyre::Result<()>,
             )*
         ];
 
-        static IDENTIFIER_TYPE: LazyLock<HashSet<String>> = LazyLock::new(|| {
-            let mut map = HashSet::new();
+        pub static IDENTIFIER_TYPE: std::sync::LazyLock<std::collections::HashSet<String>> = std::sync::LazyLock::new(|| {
+            let mut map = std::collections::HashSet::new();
             $(
                 map.insert(type_name::<$id_ty>());
             )*
@@ -129,7 +130,15 @@ compact_types!(
     ]
 );
 
-pub(crate) fn generate_vectors() -> Result<()> {
+pub fn generate_vectors() -> Result<()> {
+    generate_vectors_with(GENERATE_VECTORS)
+}
+
+pub fn read_vectors() -> Result<()> {
+    read_vectors_with(READ_VECTORS)
+}
+
+pub fn generate_vectors_with(gen: &[fn(&mut TestRunner) -> eyre::Result<()>]) -> Result<()> {
     // Prepare random seed for test (same method as used by proptest)
     let mut seed = [0u8; 32];
     getrandom(&mut seed)?;
@@ -142,25 +151,25 @@ pub(crate) fn generate_vectors() -> Result<()> {
 
     fs::create_dir_all(VECTORS_FOLDER)?;
 
-    for generate_fn in GENERATE_VECTORS {
+    for generate_fn in gen {
         generate_fn(&mut runner)?;
     }
 
     Ok(())
 }
 
-pub fn read_vectors() -> Result<()> {
+pub fn read_vectors_with(read: &[fn() -> eyre::Result<()>]) -> Result<()> {
     fs::create_dir_all(VECTORS_FOLDER)?;
 
-    for read_fn in READ_VECTORS {
+    for read_fn in read {
         read_fn()?;
     }
 
     Ok(())
 }
 
 /// Generates test vectors for a specific type `T`
-fn generate_vector<T>(runner: &mut TestRunner) -> Result<()>
+pub fn generate_vector<T>(runner: &mut TestRunner) -> Result<()>
 where
     T: for<'a> Arbitrary<'a>
         + reth_codecs::Compact
@@ -202,7 +211,7 @@ where
 
 /// Reads vectors from the file and compares the original T with the one reconstructed using
 /// T::from_compact.
-fn read_vector<T>() -> Result<()>
+pub fn read_vector<T>() -> Result<()>
 where
     T: serde::de::DeserializeOwned + reth_codecs::Compact + PartialEq + Clone + std::fmt::Debug,
 {
@@ -243,6 +252,6 @@ where
     Ok(())
 }
 
-fn type_name<T>() -> String {
+pub fn type_name<T>() -> String {
     std::any::type_name::<T>().replace("::", "__")
 }

crates/cli/commands/src/test_vectors/mod.rs

@@ -2,8 +2,8 @@
 
 use clap::{Parser, Subcommand};
 
-mod compact;
-mod tables;
+pub mod compact;
+pub mod tables;
 
 /// Generate test-vectors for different data types.
 #[derive(Debug, Parser)]

crates/cli/commands/src/test_vectors/tables.rs

@@ -17,7 +17,7 @@ const VECTORS_FOLDER: &str = "testdata/micro/db";
 const PER_TABLE: usize = 1000;
 
 /// Generates test vectors for specified `tables`. If list is empty, then generate for all tables.
-pub(crate) fn generate_vectors(mut tables: Vec<String>) -> Result<()> {
+pub fn generate_vectors(mut tables: Vec<String>) -> Result<()> {
     // Prepare random seed for test (same method as used by proptest)
     let mut seed = [0u8; 32];
     getrandom(&mut seed)?;

crates/optimism/bin/Cargo.toml

@@ -47,6 +47,10 @@ optimism = [
 	"reth-provider/optimism"
 ]
 
+dev = [
+	"reth-optimism-cli/dev"
+]
+
 min-error-logs = ["tracing/release_max_level_error"]
 min-warn-logs = ["tracing/release_max_level_warn"]
 min-info-logs = ["tracing/release_max_level_info"]

crates/optimism/cli/Cargo.toml

@@ -65,6 +65,13 @@ tokio-util = { workspace = true, features = ["codec"] }
 tracing.workspace = true
 eyre.workspace = true
 
+# reth test-vectors
+proptest = { workspace = true, optional = true }
+op-alloy-consensus = { workspace = true, features = [
+    "arbitrary",
+], optional = true }
+
+
 [dev-dependencies]
 tempfile.workspace = true
 reth-stages = { workspace = true, features = ["test-utils"] }
@@ -94,3 +101,9 @@ jemalloc = [
     "reth-node-core/jemalloc",
     "reth-node-metrics/jemalloc"
 ]
+
+dev = [
+    "dep:proptest",
+    "reth-cli-commands/arbitrary",
+    "op-alloy-consensus"
+]

crates/optimism/cli/src/commands/mod.rs

@@ -15,6 +15,7 @@ mod build_pipeline;
 pub mod import;
 pub mod import_receipts;
 pub mod init_state;
+pub mod test_vectors;
 
 /// Commands to be executed
 #[derive(Debug, Subcommand)]
@@ -55,4 +56,8 @@ pub enum Commands<Spec: ChainSpecParser = OpChainSpecParser, Ext: clap::Args + f
     /// Prune according to the configuration without any limits
     #[command(name = "prune")]
     Prune(prune::PruneCommand<Spec>),
+    /// Generate Test Vectors
+    #[cfg(feature = "dev")]
+    #[command(name = "test-vectors")]
+    TestVectors(test_vectors::Command),
 }

crates/optimism/cli/src/commands/test_vectors.rs

@@ -0,0 +1,72 @@
+//! Command for generating test vectors.
+
+use clap::{Parser, Subcommand};
+use op_alloy_consensus::TxDeposit;
+use proptest::test_runner::TestRunner;
+use reth_cli_commands::{
+    compact_types,
+    test_vectors::{
+        compact,
+        compact::{
+            generate_vector, read_vector, GENERATE_VECTORS as ETH_GENERATE_VECTORS,
+            READ_VECTORS as ETH_READ_VECTORS,
+        },
+        tables,
+    },
+};
+
+/// Generate test-vectors for different data types.
+#[derive(Debug, Parser)]
+pub struct Command {
+    #[command(subcommand)]
+    command: Subcommands,
+}
+
+#[derive(Subcommand, Debug)]
+/// `reth test-vectors` subcommands
+pub enum Subcommands {
+    /// Generates test vectors for specified tables. If no table is specified, generate for all.
+    Tables {
+        /// List of table names. Case-sensitive.
+        names: Vec<String>,
+    },
+    /// Generates test vectors for `Compact` types with `--write`. Reads and checks generated
+    /// vectors with `--read`.
+    #[group(multiple = false, required = true)]
+    Compact {
+        /// Write test vectors to a file.
+        #[arg(long)]
+        write: bool,
+
+        /// Read test vectors from a file.
+        #[arg(long)]
+        read: bool,
+    },
+}
+
+impl Command {
+    /// Execute the command
+    pub async fn execute(self) -> eyre::Result<()> {
+        match self.command {
+            Subcommands::Tables { names } => {
+                tables::generate_vectors(names)?;
+            }
+            Subcommands::Compact { write, .. } => {
+                compact_types!(
+                    regular: [
+                        TxDeposit
+                    ], identifier: []
+                );
+
+                if write {
+                    compact::generate_vectors_with(ETH_GENERATE_VECTORS)?;
+                    compact::generate_vectors_with(GENERATE_VECTORS)?;
+                } else {
+                    compact::read_vectors_with(ETH_READ_VECTORS)?;
+                    compact::read_vectors_with(READ_VECTORS)?;
+                }
+            }
+        }
+        Ok(())
+    }
+}

crates/optimism/cli/src/lib.rs

@@ -169,6 +169,8 @@ where
                 runner.run_command_until_exit(|ctx| command.execute::<OptimismNode>(ctx))
             }
             Commands::Prune(command) => runner.run_until_ctrl_c(command.execute::<OptimismNode>()),
+            #[cfg(feature = "dev")]
+            Commands::TestVectors(command) => runner.run_until_ctrl_c(command.execute()),
         }
     }