From bce70311559e20db5b4b6ec37c0883d231dc2355 Mon Sep 17 00:00:00 2001 From: Arsenii Kulikov Date: Fri, 8 Nov 2024 15:57:01 +0400 Subject: [PATCH] feat: allow generic values in `tables!` macro (#12400) --- .../cli/commands/src/test_vectors/tables.rs | 21 +-- crates/storage/db/src/tables/mod.rs | 170 ++++++++++++++---- .../src/providers/static_file/manager.rs | 7 +- 3 files changed, 150 insertions(+), 48 deletions(-) diff --git a/crates/cli/commands/src/test_vectors/tables.rs b/crates/cli/commands/src/test_vectors/tables.rs index 29ba50c8d83e..22e54ea13368 100644 --- a/crates/cli/commands/src/test_vectors/tables.rs +++ b/crates/cli/commands/src/test_vectors/tables.rs @@ -10,6 +10,7 @@ use proptest_arbitrary_interop::arb; use reth_db::tables; use reth_db_api::table::{DupSort, Table, TableRow}; use reth_fs_util as fs; +use reth_primitives::{Header, TransactionSignedNoHash}; use std::collections::HashSet; use tracing::error; @@ -31,16 +32,16 @@ pub fn generate_vectors(mut tables: Vec) -> Result<()> { fs::create_dir_all(VECTORS_FOLDER)?; macro_rules! generate_vector { - ($table_type:ident, $per_table:expr, TABLE) => { - generate_table_vector::(&mut runner, $per_table)?; + ($table_type:ident$(<$($generic:ident),+>)?, $per_table:expr, TABLE) => { + generate_table_vector::)?>(&mut runner, $per_table)?; }; - ($table_type:ident, $per_table:expr, DUPSORT) => { - generate_dupsort_vector::(&mut runner, $per_table)?; + ($table_type:ident$(<$($generic:ident),+>)?, $per_table:expr, DUPSORT) => { + generate_dupsort_vector::)?>(&mut runner, $per_table)?; }; } macro_rules! generate { - ([$(($table_type:ident, $per_table:expr, $table_or_dup:tt)),*]) => { + ([$(($table_type:ident$(<$($generic:ident),+>)?, $per_table:expr, $table_or_dup:tt)),*]) => { let all_tables = vec![$(stringify!($table_type).to_string(),)*]; if tables.is_empty() { @@ -50,10 +51,10 @@ pub fn generate_vectors(mut tables: Vec) -> Result<()> { for table in tables { match table.as_str() { $( - stringify!($table_type) => { - println!("Generating test vectors for {} <{}>.", stringify!($table_or_dup), tables::$table_type::NAME); + stringify!($table_type$(<$($generic),+>)?) => { + println!("Generating test vectors for {} <{}>.", stringify!($table_or_dup), tables::$table_type$(::<$($generic),+>)?::NAME); - generate_vector!($table_type, $per_table, $table_or_dup); + generate_vector!($table_type$(<$($generic),+>)?, $per_table, $table_or_dup); }, )* _ => { @@ -68,11 +69,11 @@ pub fn generate_vectors(mut tables: Vec) -> Result<()> { (CanonicalHeaders, PER_TABLE, TABLE), (HeaderTerminalDifficulties, PER_TABLE, TABLE), (HeaderNumbers, PER_TABLE, TABLE), - (Headers, PER_TABLE, TABLE), + (Headers
, PER_TABLE, TABLE), (BlockBodyIndices, PER_TABLE, TABLE), (BlockOmmers, 100, TABLE), (TransactionHashNumbers, PER_TABLE, TABLE), - (Transactions, 100, TABLE), + (Transactions, 100, TABLE), (PlainStorageState, PER_TABLE, DUPSORT), (PlainAccountState, PER_TABLE, TABLE) ]); diff --git a/crates/storage/db/src/tables/mod.rs b/crates/storage/db/src/tables/mod.rs index 27f58f8a1f3d..c697c3199095 100644 --- a/crates/storage/db/src/tables/mod.rs +++ b/crates/storage/db/src/tables/mod.rs @@ -106,28 +106,39 @@ macro_rules! tables { (@view $name:ident $v:ident) => { $v.view::<$name>() }; (@view $name:ident $v:ident $_subkey:ty) => { $v.view_dupsort::<$name>() }; - ($( $(#[$attr:meta])* table $name:ident; )*) => { + (@value_doc $key:ty, $value:ty) => { + concat!("[`", stringify!($value), "`]") + }; + // Don't generate links if we have generics + (@value_doc $key:ty, $value:ty, $($generic:ident),*) => { + concat!("`", stringify!($value), "`") + }; + + ($($(#[$attr:meta])* table $name:ident$(<$($generic:ident $(= $default:ty)?),*>)? { type Key = $key:ty; type Value = $value:ty; $(type SubKey = $subkey:ty;)? } )*) => { // Table marker types. $( $(#[$attr])* /// - #[doc = concat!("Marker type representing a database table mapping [`", stringify!($key), "`] to [`", stringify!($value), "`].")] + #[doc = concat!("Marker type representing a database table mapping [`", stringify!($key), "`] to ", tables!(@value_doc $key, $value, $($($generic),*)?), ".")] $( #[doc = concat!("\n\nThis table's `DUPSORT` subkey is [`", stringify!($subkey), "`].")] )? - pub struct $name { - _private: (), + pub struct $name$(<$($generic $( = $default)?),*>)? { + _private: std::marker::PhantomData<($($($generic,)*)?)>, } // Ideally this implementation wouldn't exist, but it is necessary to derive `Debug` // when a type is generic over `T: Table`. See: https://github.com/rust-lang/rust/issues/26925 - impl fmt::Debug for $name { + impl$(<$($generic),*>)? fmt::Debug for $name$(<$($generic),*>)? { fn fmt(&self, _: &mut fmt::Formatter<'_>) -> fmt::Result { unreachable!("this type cannot be instantiated") } } - impl reth_db_api::table::Table for $name { + impl$(<$($generic),*>)? reth_db_api::table::Table for $name$(<$($generic),*>)? + where + $value: reth_db_api::table::Value + 'static + { const NAME: &'static str = table_names::$name; type Key = $key; @@ -248,7 +259,7 @@ macro_rules! tables { /// use reth_db_api::table::Table; /// /// let table = Tables::Headers; - /// let result = tables_to_generic!(table, |GenericTable| GenericTable::NAME); + /// let result = tables_to_generic!(table, |GenericTable| ::NAME); /// assert_eq!(result, table.name()); /// ``` #[macro_export] @@ -269,53 +280,96 @@ macro_rules! tables { tables! { /// Stores the header hashes belonging to the canonical chain. - table CanonicalHeaders; + table CanonicalHeaders { + type Key = BlockNumber; + type Value = HeaderHash; + } /// Stores the total difficulty from a block header. - table HeaderTerminalDifficulties; + table HeaderTerminalDifficulties { + type Key = BlockNumber; + type Value = CompactU256; + } /// Stores the block number corresponding to a header. - table HeaderNumbers; + table HeaderNumbers { + type Key = BlockHash; + type Value = BlockNumber; + } /// Stores header bodies. - table Headers; + table Headers { + type Key = BlockNumber; + type Value = H; + } /// Stores block indices that contains indexes of transaction and the count of them. /// /// More information about stored indices can be found in the [`StoredBlockBodyIndices`] struct. - table BlockBodyIndices; + table BlockBodyIndices { + type Key = BlockNumber; + type Value = StoredBlockBodyIndices; + } /// Stores the uncles/ommers of the block. - table BlockOmmers; + table BlockOmmers { + type Key = BlockNumber; + type Value = StoredBlockOmmers; + } /// Stores the block withdrawals. - table BlockWithdrawals; + table BlockWithdrawals { + type Key = BlockNumber; + type Value = StoredBlockWithdrawals; + } /// Canonical only Stores the transaction body for canonical transactions. - table Transactions; + table Transactions { + type Key = TxNumber; + type Value = T; + } /// Stores the mapping of the transaction hash to the transaction number. - table TransactionHashNumbers; + table TransactionHashNumbers { + type Key = TxHash; + type Value = TxNumber; + } /// Stores the mapping of transaction number to the blocks number. /// /// The key is the highest transaction ID in the block. - table TransactionBlocks; + table TransactionBlocks { + type Key = TxNumber; + type Value = BlockNumber; + } /// Canonical only Stores transaction receipts. - table Receipts; + table Receipts { + type Key = TxNumber; + type Value = Receipt; + } /// Stores all smart contract bytecodes. /// There will be multiple accounts that have same bytecode /// So we would need to introduce reference counter. /// This will be small optimization on state. - table Bytecodes; + table Bytecodes { + type Key = B256; + type Value = Bytecode; + } /// Stores the current state of an [`Account`]. - table PlainAccountState; + table PlainAccountState { + type Key = Address; + type Value = Account; + } /// Stores the current value of a storage key. - table PlainStorageState; + table PlainStorageState { + type Key = Address; + type Value = StorageEntry; + type SubKey = B256; + } /// Stores pointers to block changeset with changes for each account key. /// @@ -335,7 +389,10 @@ tables! { /// * If there were no shard we would get `None` entry or entry of different storage key. /// /// Code example can be found in `reth_provider::HistoricalStateProviderRef` - table AccountsHistory, Value = BlockNumberList>; + table AccountsHistory { + type Key = ShardedKey
; + type Value = BlockNumberList; + } /// Stores pointers to block number changeset with changes for each storage key. /// @@ -355,55 +412,98 @@ tables! { /// * If there were no shard we would get `None` entry or entry of different storage key. /// /// Code example can be found in `reth_provider::HistoricalStateProviderRef` - table StoragesHistory; + table StoragesHistory { + type Key = StorageShardedKey; + type Value = BlockNumberList; + } /// Stores the state of an account before a certain transaction changed it. /// Change on state can be: account is created, selfdestructed, touched while empty /// or changed balance,nonce. - table AccountChangeSets; + table AccountChangeSets { + type Key = BlockNumber; + type Value = AccountBeforeTx; + type SubKey = Address; + } /// Stores the state of a storage key before a certain transaction changed it. /// If [`StorageEntry::value`] is zero, this means storage was not existing /// and needs to be removed. - table StorageChangeSets; + table StorageChangeSets { + type Key = BlockNumberAddress; + type Value = StorageEntry; + type SubKey = B256; + } /// Stores the current state of an [`Account`] indexed with `keccak256Address` /// This table is in preparation for merklization and calculation of state root. /// We are saving whole account data as it is needed for partial update when /// part of storage is changed. Benefit for merklization is that hashed addresses are sorted. - table HashedAccounts; + table HashedAccounts { + type Key = B256; + type Value = Account; + } /// Stores the current storage values indexed with `keccak256Address` and /// hash of storage key `keccak256key`. /// This table is in preparation for merklization and calculation of state root. /// Benefit for merklization is that hashed addresses/keys are sorted. - table HashedStorages; + table HashedStorages { + type Key = B256; + type Value = StorageEntry; + type SubKey = B256; + } /// Stores the current state's Merkle Patricia Tree. - table AccountsTrie; + table AccountsTrie { + type Key = StoredNibbles; + type Value = BranchNodeCompact; + } /// From HashedAddress => NibblesSubKey => Intermediate value - table StoragesTrie; + table StoragesTrie { + type Key = B256; + type Value = StorageTrieEntry; + type SubKey = StoredNibblesSubKey; + } /// Stores the transaction sender for each canonical transaction. /// It is needed to speed up execution stage and allows fetching signer without doing /// transaction signed recovery - table TransactionSenders; + table TransactionSenders { + type Key = TxNumber; + type Value = Address; + } /// Stores the highest synced block number and stage-specific checkpoint of each stage. - table StageCheckpoints; + table StageCheckpoints { + type Key = StageId; + type Value = StageCheckpoint; + } /// Stores arbitrary data to keep track of a stage first-sync progress. - table StageCheckpointProgresses>; + table StageCheckpointProgresses { + type Key = StageId; + type Value = Vec; + } /// Stores the highest pruned block number and prune mode of each prune segment. - table PruneCheckpoints; + table PruneCheckpoints { + type Key = PruneSegment; + type Value = PruneCheckpoint; + } /// Stores the history of client versions that have accessed the database with write privileges by unix timestamp in seconds. - table VersionHistory; + table VersionHistory { + type Key = u64; + type Value = ClientVersion; + } /// Stores generic chain state info, like the last finalized block. - table ChainState; + table ChainState { + type Key = ChainStateKey; + type Value = BlockNumber; + } } /// Keys for the `ChainState` table. diff --git a/crates/storage/provider/src/providers/static_file/manager.rs b/crates/storage/provider/src/providers/static_file/manager.rs index 9ccaf0514639..66914b00abc4 100644 --- a/crates/storage/provider/src/providers/static_file/manager.rs +++ b/crates/storage/provider/src/providers/static_file/manager.rs @@ -1645,7 +1645,7 @@ impl StatsReader for StaticFileProvider { fn count_entries(&self) -> ProviderResult { match T::NAME { tables::CanonicalHeaders::NAME | - tables::Headers::NAME | + tables::Headers::
::NAME | tables::HeaderTerminalDifficulties::NAME => Ok(self .get_highest_static_file_block(StaticFileSegment::Headers) .map(|block| block + 1) @@ -1655,10 +1655,11 @@ impl StatsReader for StaticFileProvider { .get_highest_static_file_tx(StaticFileSegment::Receipts) .map(|receipts| receipts + 1) .unwrap_or_default() as usize), - tables::Transactions::NAME => Ok(self + tables::Transactions::::NAME => Ok(self .get_highest_static_file_tx(StaticFileSegment::Transactions) .map(|txs| txs + 1) - .unwrap_or_default() as usize), + .unwrap_or_default() + as usize), _ => Err(ProviderError::UnsupportedProvider), } }