From c77045f26146ab8b61ca368a5a2775c55e69ef92 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A5rten=20Blankfors?= Date: Thu, 24 Oct 2024 14:39:22 +0200 Subject: [PATCH 01/37] feat: Multi-get with boxed iterators --- crates/fuel-core/src/state/rocks_db.rs | 8 ++++++++ crates/storage/src/blueprint.rs | 16 ++++++++++++++++ crates/storage/src/kv_store.rs | 9 +++++++++ 3 files changed, 33 insertions(+) diff --git a/crates/fuel-core/src/state/rocks_db.rs b/crates/fuel-core/src/state/rocks_db.rs index c751ebe7c55..d98ea48804c 100644 --- a/crates/fuel-core/src/state/rocks_db.rs +++ b/crates/fuel-core/src/state/rocks_db.rs @@ -728,6 +728,14 @@ where Ok(value.map(Arc::new)) } + fn get_multi( + &self, + _keys: Box>, + _column: Self::Column, + ) -> Box>>> { + todo!(); // TODO + } + fn read( &self, key: &[u8], diff --git a/crates/storage/src/blueprint.rs b/crates/storage/src/blueprint.rs index c561254b11c..80cfd031a4d 100644 --- a/crates/storage/src/blueprint.rs +++ b/crates/storage/src/blueprint.rs @@ -19,6 +19,9 @@ use crate::{ }; use fuel_vm_private::prelude::MerkleRoot; +#[cfg(feature = "alloc")] +use alloc::boxed::Box; + pub mod merklized; pub mod plain; pub mod sparse; @@ -75,6 +78,18 @@ where }) .transpose() } + + /// Returns multiple values from the storage. + fn get_multi<'a>( + storage: &'a S, + keys: Box>, + column: S::Column, + ) -> Box>> + 'a> { + let keys = + Box::new(keys.map(|key| Self::KeyCodec::encode(key).as_bytes().as_ref())); + + storage.get_multi(keys, column) + } } /// It is an extension of the [`BlueprintInspect`] that allows mutating the storage. @@ -111,6 +126,7 @@ where fn delete(storage: &mut S, key: &M::Key, column: S::Column) -> StorageResult<()>; } +// TODO: Rename /// It is an extension of the blueprint that allows supporting batch operations. /// Usually, they are more performant than initializing/inserting/removing values one by one. pub trait SupportsBatching: BlueprintMutate diff --git a/crates/storage/src/kv_store.rs b/crates/storage/src/kv_store.rs index 67ab9493c7f..1677a63802f 100644 --- a/crates/storage/src/kv_store.rs +++ b/crates/storage/src/kv_store.rs @@ -62,6 +62,15 @@ pub trait KeyValueInspect { /// Returns the value from the storage. fn get(&self, key: &[u8], column: Self::Column) -> StorageResult>; + /// Returns multiple values from the storage. + fn get_multi<'a>( + &'a self, + keys: Box>, + column: Self::Column, + ) -> Box>> + 'a> { + Box::new(keys.map(move |key| self.get(key, column))) + } + /// Reads the value from the storage into the `buf` and returns the number of read bytes. fn read( &self, From e66db04e6892b94b0a7fd59a5c10a3f0ab9462f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A5rten=20Blankfors?= Date: Thu, 24 Oct 2024 22:01:08 +0200 Subject: [PATCH 02/37] =?UTF-8?q?feat:=20Multi=20get=20on=20b=C4=BAueprint?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- crates/fuel-core/src/state/rocks_db.rs | 8 ++++---- crates/storage/src/blueprint.rs | 17 +++++++++++++---- crates/storage/src/kv_store.rs | 4 ++-- 3 files changed, 19 insertions(+), 10 deletions(-) diff --git a/crates/fuel-core/src/state/rocks_db.rs b/crates/fuel-core/src/state/rocks_db.rs index d98ea48804c..47eb4a4de7a 100644 --- a/crates/fuel-core/src/state/rocks_db.rs +++ b/crates/fuel-core/src/state/rocks_db.rs @@ -728,11 +728,11 @@ where Ok(value.map(Arc::new)) } - fn get_multi( - &self, - _keys: Box>, + fn get_multi<'a>( + &'a self, + _keys: Box> + 'a>, _column: Self::Column, - ) -> Box>>> { + ) -> Box>> + 'a> { todo!(); // TODO } diff --git a/crates/storage/src/blueprint.rs b/crates/storage/src/blueprint.rs index 80cfd031a4d..b0343a39e5d 100644 --- a/crates/storage/src/blueprint.rs +++ b/crates/storage/src/blueprint.rs @@ -85,10 +85,19 @@ where keys: Box>, column: S::Column, ) -> Box>> + 'a> { - let keys = - Box::new(keys.map(|key| Self::KeyCodec::encode(key).as_bytes().as_ref())); - - storage.get_multi(keys, column) + let keys = Box::new( + keys.map(|key| Self::KeyCodec::encode(&key).as_bytes().into_owned()), + ); + + Box::new(storage.get_multi(keys, column).map(|result| { + result.and_then(|opt| { + opt.map(|value| { + Self::ValueCodec::decode_from_value(value) + .map_err(crate::Error::Codec) + }) + .transpose() + }) + })) } } diff --git a/crates/storage/src/kv_store.rs b/crates/storage/src/kv_store.rs index 1677a63802f..b69522e055c 100644 --- a/crates/storage/src/kv_store.rs +++ b/crates/storage/src/kv_store.rs @@ -65,10 +65,10 @@ pub trait KeyValueInspect { /// Returns multiple values from the storage. fn get_multi<'a>( &'a self, - keys: Box>, + keys: Box> + 'a>, column: Self::Column, ) -> Box>> + 'a> { - Box::new(keys.map(move |key| self.get(key, column))) + Box::new(keys.map(move |key| self.get(&key, column))) } /// Reads the value from the storage into the `buf` and returns the number of read bytes. From 9eace871c21953815b6c7182ababcea18d6abe59 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A5rten=20Blankfors?= Date: Thu, 24 Oct 2024 22:23:02 +0200 Subject: [PATCH 03/37] feat: get_multi on RocksDB --- crates/fuel-core/src/state/rocks_db.rs | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/crates/fuel-core/src/state/rocks_db.rs b/crates/fuel-core/src/state/rocks_db.rs index 47eb4a4de7a..4c4b5ff64ef 100644 --- a/crates/fuel-core/src/state/rocks_db.rs +++ b/crates/fuel-core/src/state/rocks_db.rs @@ -28,6 +28,7 @@ use fuel_core_storage::{ WriteOperation, }, transactional::Changes, + Error as StorageError, Result as StorageResult, }; use itertools::Itertools; @@ -730,10 +731,27 @@ where fn get_multi<'a>( &'a self, - _keys: Box> + 'a>, - _column: Self::Column, + keys: Box> + 'a>, + column: Self::Column, ) -> Box>> + 'a> { - todo!(); // TODO + // TODO: Metrics + + let column = self.cf(column); + let keys = keys.map(|key| (&column, key)); + + let values = self + .db + .multi_get_cf_opt(keys, &self.read_options) + .into_iter() + .map(|value_opt| { + value_opt + .map_err(|e| { + StorageError::Other(DatabaseError::Other(e.into()).into()) + }) + .map(|value| value.map(Arc::new)) + }); + + Box::new(values) } fn read( From 8ba0c9d9018adf5c7f826dadf0fa7b99e695613f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A5rten=20Blankfors?= Date: Thu, 24 Oct 2024 22:33:07 +0200 Subject: [PATCH 04/37] feat: Use local fuel-vm --- Cargo.lock | 18 ------------------ Cargo.toml | 2 +- 2 files changed, 1 insertion(+), 19 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 67a97f72c3e..dee020fe5d6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3166,8 +3166,6 @@ dependencies = [ [[package]] name = "fuel-asm" version = "0.58.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f325971bf9047ec70004f80a989e03456316bc19cbef3ff3a39a38b192ab56e" dependencies = [ "bitflags 2.6.0", "fuel-types 0.58.2", @@ -3178,8 +3176,6 @@ dependencies = [ [[package]] name = "fuel-compression" version = "0.58.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24e42841f56f76ed759b3f516e5188d5c42de47015bee951651660c13b6dfa6c" dependencies = [ "fuel-derive 0.58.2", "fuel-types 0.58.2", @@ -3893,8 +3889,6 @@ dependencies = [ [[package]] name = "fuel-crypto" version = "0.58.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65e318850ca64890ff123a99b6b866954ef49da94ab9bc6827cf6ee045568585" dependencies = [ "coins-bip32", "coins-bip39", @@ -3926,8 +3920,6 @@ dependencies = [ [[package]] name = "fuel-derive" version = "0.58.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab0bc46a3552964bae5169e79b383761a54bd115ea66951a1a7a229edcefa55a" dependencies = [ "proc-macro2", "quote", @@ -3963,8 +3955,6 @@ dependencies = [ [[package]] name = "fuel-merkle" version = "0.58.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c79eca6a452311c70978a5df796c0f99f27e474b69719e0db4c1d82e68800d07" dependencies = [ "derive_more 0.99.18", "digest 0.10.7", @@ -3984,8 +3974,6 @@ checksum = "4c1b711f28553ddc5f3546711bd220e144ce4c1af7d9e9a1f70b2f20d9f5b791" [[package]] name = "fuel-storage" version = "0.58.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d0c46b5d76b3e11197bd31e036cd8b1cb46c4d822cacc48836638080c6d2b76" [[package]] name = "fuel-tx" @@ -4012,8 +4000,6 @@ dependencies = [ [[package]] name = "fuel-tx" version = "0.58.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6723bb8710ba2b70516ac94d34459593225870c937670fb3afaf82e0354667ac" dependencies = [ "bitflags 2.6.0", "derivative", @@ -4046,8 +4032,6 @@ dependencies = [ [[package]] name = "fuel-types" version = "0.58.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "982265415a99b5bd6277bc24194a233bb2e18764df11c937b3dbb11a02c9e545" dependencies = [ "fuel-derive 0.58.2", "hex", @@ -4089,8 +4073,6 @@ dependencies = [ [[package]] name = "fuel-vm" version = "0.58.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54b5362d7d072c72eec20581f67fc5400090c356a7f3ae77c79880b3b177b667" dependencies = [ "anyhow", "async-trait", diff --git a/Cargo.toml b/Cargo.toml index d80bf4f5bdc..40ac40d52dd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -87,7 +87,7 @@ fuel-core-xtask = { version = "0.0.0", path = "./xtask" } fuel-gas-price-algorithm = { version = "0.40.0", path = "crates/fuel-gas-price-algorithm" } # Fuel dependencies -fuel-vm-private = { version = "0.58.2", package = "fuel-vm", default-features = false } +fuel-vm-private = { version = "0.58.2", path = "../fuel-vm/fuel-vm", package = "fuel-vm", default-features = false } # Common dependencies anyhow = "1.0" From 76b919b39f21adbd8bed5ea8f2d66f1af3b01147 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A5rten=20Blankfors?= Date: Thu, 24 Oct 2024 22:52:42 +0200 Subject: [PATCH 05/37] feat: Implementation for structured storage --- crates/storage/src/blueprint.rs | 2 +- crates/storage/src/structured_storage.rs | 21 +++++++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/crates/storage/src/blueprint.rs b/crates/storage/src/blueprint.rs index b0343a39e5d..282a7f90abd 100644 --- a/crates/storage/src/blueprint.rs +++ b/crates/storage/src/blueprint.rs @@ -82,7 +82,7 @@ where /// Returns multiple values from the storage. fn get_multi<'a>( storage: &'a S, - keys: Box>, + keys: Box + 'a>, column: S::Column, ) -> Box>> + 'a> { let keys = Box::new( diff --git a/crates/storage/src/structured_storage.rs b/crates/storage/src/structured_storage.rs index e78e9637484..c1b20523ed5 100644 --- a/crates/storage/src/structured_storage.rs +++ b/crates/storage/src/structured_storage.rs @@ -58,8 +58,12 @@ use alloc::{ fmt::Debug, }; +// TODO: Format +#[cfg(feature = "alloc")] +use alloc::boxed::Box; #[cfg(feature = "alloc")] use alloc::vec::Vec; + use fuel_vm_private::storage::{ predicate::PredicateStorageRequirements, BlobData, @@ -263,6 +267,23 @@ where .map(|value| value.map(Cow::Owned)) } + fn get_multi<'a>( + &'a self, + keys: Box::Key> + 'a>, + ) -> Box< + dyn Iterator< + Item = Result::OwnedValue>>, Self::Error>, + > + 'a, + > + where + >::Error: 'a, + { + Box::new( + ::Blueprint::get_multi(self, keys, M::column()) + .map(|result| result.map(|option| option.map(Cow::Owned))), + ) + } + fn contains_key(&self, key: &M::Key) -> Result { ::Blueprint::exists(self, key, M::column()) } From 2333dfb84551cc81befc96c58875cae23ec58354 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A5rten=20Blankfors?= Date: Fri, 25 Oct 2024 09:38:47 +0200 Subject: [PATCH 06/37] feat: Use multi-get --- crates/fuel-core/src/database/block.rs | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/crates/fuel-core/src/database/block.rs b/crates/fuel-core/src/database/block.rs index 374a76a9663..5e68f68ab07 100644 --- a/crates/fuel-core/src/database/block.rs +++ b/crates/fuel-core/src/database/block.rs @@ -69,14 +69,15 @@ impl OnChainIterableKeyValueView { // fetch all the transactions // TODO: Use multiget when it's implemented. // https://github.com/FuelLabs/fuel-core/issues/2344 - let txs = block - .transactions() - .iter() - .map(|tx_id| { - self.storage::() - .get(tx_id) - .and_then(|tx| tx.ok_or(not_found!(Transactions))) - .map(Cow::into_owned) + + let transaction_ids = Box::new(block.transactions().iter()); + let txs = self + .storage::() + .get_multi(transaction_ids) + .map(|tx| { + tx.and_then(|tx| { + tx.ok_or(not_found!(Transactions)).map(Cow::into_owned) + }) }) .try_collect()?; Ok(Some(block.into_owned().uncompress(txs))) From 49994b93042a39518d86a4beec38ac72615b07c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A5rten=20Blankfors?= Date: Fri, 25 Oct 2024 10:01:45 +0200 Subject: [PATCH 07/37] feat: Messages impl --- .../src/query/balance/asset_query.rs | 2 +- crates/fuel-core/src/query/message.rs | 25 +++++++++++++------ 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/crates/fuel-core/src/query/balance/asset_query.rs b/crates/fuel-core/src/query/balance/asset_query.rs index 13a289ec1e4..89c30b67d04 100644 --- a/crates/fuel-core/src/query/balance/asset_query.rs +++ b/crates/fuel-core/src/query/balance/asset_query.rs @@ -170,7 +170,7 @@ impl<'a> AssetsQuery<'a> { }) .try_filter_map(move |chunk| async move { let chunk = database.messages(chunk).await; - Ok::<_, StorageError>(Some(futures::stream::iter(chunk))) + Ok::<_, StorageError>(Some(chunk)) }) .try_flatten() .filter(|result| { diff --git a/crates/fuel-core/src/query/message.rs b/crates/fuel-core/src/query/message.rs index 07d2cd9ab56..5e01316f037 100644 --- a/crates/fuel-core/src/query/message.rs +++ b/crates/fuel-core/src/query/message.rs @@ -81,13 +81,24 @@ impl ReadView { pub async fn messages( &self, ids: Vec, - ) -> impl Iterator> + '_ { - // TODO: Use multiget when it's implemented. - // https://github.com/FuelLabs/fuel-core/issues/2344 - let messages = ids.into_iter().map(|id| self.message(&id)); - // Give a chance to other tasks to run. + ) -> impl Stream> { + let ids = Box::new(ids.iter()); + let messages: Vec<_> = self + .on_chain + .as_ref() + .storage::() + .get_multi(Box::new(ids)) + .map(|res| { + res.and_then(|opt| opt.ok_or(not_found!(Messages)).map(Cow::into_owned)) + }) + .collect(); + + //// TODO: Use multiget when it's implemented. + //// https://github.com/FuelLabs/fuel-core/issues/2344 + // let messages = ids.into_iter().map(|id| self.message(&id)); + //// Give a chance to other tasks to run. tokio::task::yield_now().await; - messages + futures::stream::iter(messages) } pub fn owned_messages<'a>( @@ -104,7 +115,7 @@ impl ReadView { }) .try_filter_map(move |chunk| async move { let chunk = self.messages(chunk).await; - Ok::<_, StorageError>(Some(futures::stream::iter(chunk))) + Ok::<_, StorageError>(Some(chunk)) }) .try_flatten() } From 2fb9e88487b2c3608e6e025ea7b0588c01649a2c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A5rten=20Blankfors?= Date: Fri, 25 Oct 2024 13:03:30 +0200 Subject: [PATCH 08/37] feat: Don't rely on modified StorageInspect --- Cargo.lock | 18 ++++++++++++++++++ Cargo.toml | 2 +- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index dee020fe5d6..67a97f72c3e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3166,6 +3166,8 @@ dependencies = [ [[package]] name = "fuel-asm" version = "0.58.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f325971bf9047ec70004f80a989e03456316bc19cbef3ff3a39a38b192ab56e" dependencies = [ "bitflags 2.6.0", "fuel-types 0.58.2", @@ -3176,6 +3178,8 @@ dependencies = [ [[package]] name = "fuel-compression" version = "0.58.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24e42841f56f76ed759b3f516e5188d5c42de47015bee951651660c13b6dfa6c" dependencies = [ "fuel-derive 0.58.2", "fuel-types 0.58.2", @@ -3889,6 +3893,8 @@ dependencies = [ [[package]] name = "fuel-crypto" version = "0.58.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65e318850ca64890ff123a99b6b866954ef49da94ab9bc6827cf6ee045568585" dependencies = [ "coins-bip32", "coins-bip39", @@ -3920,6 +3926,8 @@ dependencies = [ [[package]] name = "fuel-derive" version = "0.58.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab0bc46a3552964bae5169e79b383761a54bd115ea66951a1a7a229edcefa55a" dependencies = [ "proc-macro2", "quote", @@ -3955,6 +3963,8 @@ dependencies = [ [[package]] name = "fuel-merkle" version = "0.58.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c79eca6a452311c70978a5df796c0f99f27e474b69719e0db4c1d82e68800d07" dependencies = [ "derive_more 0.99.18", "digest 0.10.7", @@ -3974,6 +3984,8 @@ checksum = "4c1b711f28553ddc5f3546711bd220e144ce4c1af7d9e9a1f70b2f20d9f5b791" [[package]] name = "fuel-storage" version = "0.58.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d0c46b5d76b3e11197bd31e036cd8b1cb46c4d822cacc48836638080c6d2b76" [[package]] name = "fuel-tx" @@ -4000,6 +4012,8 @@ dependencies = [ [[package]] name = "fuel-tx" version = "0.58.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6723bb8710ba2b70516ac94d34459593225870c937670fb3afaf82e0354667ac" dependencies = [ "bitflags 2.6.0", "derivative", @@ -4032,6 +4046,8 @@ dependencies = [ [[package]] name = "fuel-types" version = "0.58.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "982265415a99b5bd6277bc24194a233bb2e18764df11c937b3dbb11a02c9e545" dependencies = [ "fuel-derive 0.58.2", "hex", @@ -4073,6 +4089,8 @@ dependencies = [ [[package]] name = "fuel-vm" version = "0.58.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54b5362d7d072c72eec20581f67fc5400090c356a7f3ae77c79880b3b177b667" dependencies = [ "anyhow", "async-trait", diff --git a/Cargo.toml b/Cargo.toml index 40ac40d52dd..d80bf4f5bdc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -87,7 +87,7 @@ fuel-core-xtask = { version = "0.0.0", path = "./xtask" } fuel-gas-price-algorithm = { version = "0.40.0", path = "crates/fuel-gas-price-algorithm" } # Fuel dependencies -fuel-vm-private = { version = "0.58.2", path = "../fuel-vm/fuel-vm", package = "fuel-vm", default-features = false } +fuel-vm-private = { version = "0.58.2", package = "fuel-vm", default-features = false } # Common dependencies anyhow = "1.0" From b2c2632ace69a396c0fd1900b032b9ce5a9aced9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A5rten=20Blankfors?= Date: Fri, 25 Oct 2024 13:41:22 +0200 Subject: [PATCH 09/37] wip: Introduce specific StorageBatchInspect trait --- crates/fuel-core/src/database/block.rs | 31 +++++++++--------- crates/fuel-core/src/graphql_api/ports.rs | 5 ++- crates/fuel-core/src/query/message.rs | 25 ++++++++------- .../fuel-core/src/state/generic_database.rs | 15 +++++++++ crates/storage/src/lib.rs | 10 ++++++ crates/storage/src/structured_storage.rs | 32 +++++++++---------- 6 files changed, 74 insertions(+), 44 deletions(-) diff --git a/crates/fuel-core/src/database/block.rs b/crates/fuel-core/src/database/block.rs index 5e68f68ab07..0612855b683 100644 --- a/crates/fuel-core/src/database/block.rs +++ b/crates/fuel-core/src/database/block.rs @@ -18,7 +18,7 @@ use fuel_core_storage::{ FuelBlockMerkleMetadata, }, FuelBlocks, - Transactions, + // Transactions, }, Error as StorageError, Result as StorageResult, @@ -36,8 +36,7 @@ use fuel_core_types::{ fuel_merkle::binary::MerkleTree, fuel_types::BlockHeight, }; -use itertools::Itertools; -use std::borrow::Cow; +// use std::borrow::Cow; impl OffChainIterableKeyValueView { pub fn get_block_height(&self, id: &BlockId) -> StorageResult> { @@ -65,22 +64,24 @@ impl OnChainIterableKeyValueView { /// Retrieve the full block and all associated transactions pub fn get_full_block(&self, height: &BlockHeight) -> StorageResult> { let db_block = self.storage::().get(height)?; - if let Some(block) = db_block { + if let Some(_block) = db_block { // fetch all the transactions // TODO: Use multiget when it's implemented. // https://github.com/FuelLabs/fuel-core/issues/2344 - let transaction_ids = Box::new(block.transactions().iter()); - let txs = self - .storage::() - .get_multi(transaction_ids) - .map(|tx| { - tx.and_then(|tx| { - tx.ok_or(not_found!(Transactions)).map(Cow::into_owned) - }) - }) - .try_collect()?; - Ok(Some(block.into_owned().uncompress(txs))) + // let transaction_ids = Box::new(block.transactions().iter()); + // let txs = self + // .storage::() + // .get_multi(transaction_ids) + // .map(|tx| { + // tx.and_then(|tx| { + // tx.ok_or(not_found!(Transactions)).map(Cow::into_owned) + // }) + // }) + // .try_collect()?; + // + // Ok(Some(block.into_owned().uncompress(txs))) + Ok(None) } else { Ok(None) } diff --git a/crates/fuel-core/src/graphql_api/ports.rs b/crates/fuel-core/src/graphql_api/ports.rs index 077a48d1637..e72a1619d52 100644 --- a/crates/fuel-core/src/graphql_api/ports.rs +++ b/crates/fuel-core/src/graphql_api/ports.rs @@ -16,6 +16,7 @@ use fuel_core_storage::{ }, Error as StorageError, Result as StorageResult, + StorageBatchInspect, StorageInspect, StorageRead, }; @@ -159,7 +160,9 @@ pub trait DatabaseDaCompressedBlocks { } /// Trait that specifies all the getters required for messages. -pub trait DatabaseMessages: StorageInspect { +pub trait DatabaseMessages: + StorageInspect + StorageBatchInspect +{ fn all_messages( &self, start_message_id: Option, diff --git a/crates/fuel-core/src/query/message.rs b/crates/fuel-core/src/query/message.rs index 5e01316f037..beae8d7c176 100644 --- a/crates/fuel-core/src/query/message.rs +++ b/crates/fuel-core/src/query/message.rs @@ -80,18 +80,21 @@ impl ReadView { pub async fn messages( &self, - ids: Vec, + _ids: Vec, ) -> impl Stream> { - let ids = Box::new(ids.iter()); - let messages: Vec<_> = self - .on_chain - .as_ref() - .storage::() - .get_multi(Box::new(ids)) - .map(|res| { - res.and_then(|opt| opt.ok_or(not_found!(Messages)).map(Cow::into_owned)) - }) - .collect(); + // let ids = Box::new(ids.iter()); + // let messages: Vec<_> = self + // .on_chain + // .as_ref() + // .storage::() + // .get_multi(Box::new(ids)) + // .map(|res| { + // res.and_then(|opt| opt.ok_or(not_found!(Messages)).map(Cow::into_owned)) + // }) + // .collect(); + + // TODO + let messages = Vec::new(); //// TODO: Use multiget when it's implemented. //// https://github.com/FuelLabs/fuel-core/issues/2344 diff --git a/crates/fuel-core/src/state/generic_database.rs b/crates/fuel-core/src/state/generic_database.rs index 8306df1b1fb..e6eeee4dd12 100644 --- a/crates/fuel-core/src/state/generic_database.rs +++ b/crates/fuel-core/src/state/generic_database.rs @@ -18,6 +18,7 @@ use fuel_core_storage::{ PredicateStorageRequirements, Result as StorageResult, StorageAsRef, + StorageBatchInspect, StorageInspect, StorageRead, StorageSize, @@ -64,6 +65,20 @@ where } } +impl StorageBatchInspect for GenericDatabase +where + M: Mappable, + StructuredStorage: StorageBatchInspect, +{ + fn get_batch<'a>( + &'a self, + keys: Box::Key> + 'a>, + ) -> Box::OwnedValue>>> + 'a> + { + self.storage.get_batch(keys) + } +} + impl StorageSize for GenericDatabase where M: Mappable, diff --git a/crates/storage/src/lib.rs b/crates/storage/src/lib.rs index c54fbf889b8..8b6fa2604f3 100644 --- a/crates/storage/src/lib.rs +++ b/crates/storage/src/lib.rs @@ -176,6 +176,16 @@ pub trait StorageBatchMutate: StorageMutate { Type::Key: 'a; } +// TODO: Document +/// TODO +pub trait StorageBatchInspect { + /// TODO + fn get_batch<'a>( + &'a self, + keys: Box + 'a>, + ) -> Box>> + 'a>; +} + /// Creates `StorageError::NotFound` error with file and line information inside. /// /// # Examples diff --git a/crates/storage/src/structured_storage.rs b/crates/storage/src/structured_storage.rs index c1b20523ed5..9fe78dbdb52 100644 --- a/crates/storage/src/structured_storage.rs +++ b/crates/storage/src/structured_storage.rs @@ -37,6 +37,7 @@ use crate::{ MerkleRoot, MerkleRootStorage, Result as StorageResult, + StorageBatchInspect, StorageBatchMutate, StorageInspect, StorageMutate, @@ -267,28 +268,25 @@ where .map(|value| value.map(Cow::Owned)) } - fn get_multi<'a>( - &'a self, - keys: Box::Key> + 'a>, - ) -> Box< - dyn Iterator< - Item = Result::OwnedValue>>, Self::Error>, - > + 'a, - > - where - >::Error: 'a, - { - Box::new( - ::Blueprint::get_multi(self, keys, M::column()) - .map(|result| result.map(|option| option.map(Cow::Owned))), - ) - } - fn contains_key(&self, key: &M::Key) -> Result { ::Blueprint::exists(self, key, M::column()) } } +impl StorageBatchInspect for StructuredStorage +where + S: KeyValueInspect, + M: TableWithBlueprint, + M::Blueprint: BlueprintInspect>, +{ + fn get_batch<'a>( + &'a self, + keys: Box + 'a>, + ) -> Box>> + 'a> { + M::Blueprint::get_multi(self, keys, M::column()) + } +} + impl StorageMutate for StructuredStorage where S: KeyValueMutate, From df63d482503037aaffdcb43844d15f3f6f13396a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A5rten=20Blankfors?= Date: Fri, 25 Oct 2024 14:01:15 +0200 Subject: [PATCH 10/37] wip: Use specific trait --- crates/fuel-core/src/graphql_api/ports.rs | 5 +++++ .../src/service/adapters/graphql_api/on_chain.rs | 11 +++++++++++ 2 files changed, 16 insertions(+) diff --git a/crates/fuel-core/src/graphql_api/ports.rs b/crates/fuel-core/src/graphql_api/ports.rs index e72a1619d52..9e62b371f7d 100644 --- a/crates/fuel-core/src/graphql_api/ports.rs +++ b/crates/fuel-core/src/graphql_api/ports.rs @@ -169,6 +169,11 @@ pub trait DatabaseMessages: direction: IterDirection, ) -> BoxedIter<'_, StorageResult>; + fn message_batch<'a>( + &'a self, + ids: BoxedIter<'a, &'a Nonce>, + ) -> BoxedIter<'a, StorageResult>; + fn message_exists(&self, nonce: &Nonce) -> StorageResult; } diff --git a/crates/fuel-core/src/service/adapters/graphql_api/on_chain.rs b/crates/fuel-core/src/service/adapters/graphql_api/on_chain.rs index b3a6d860e76..d6bbb274861 100644 --- a/crates/fuel-core/src/service/adapters/graphql_api/on_chain.rs +++ b/crates/fuel-core/src/service/adapters/graphql_api/on_chain.rs @@ -23,12 +23,14 @@ use fuel_core_storage::{ not_found, tables::{ FuelBlocks, + Messages, SealedBlockConsensus, Transactions, }, Error as StorageError, Result as StorageResult, StorageAsRef, + StorageBatchInspect, }; use fuel_core_types::{ blockchain::{ @@ -103,6 +105,15 @@ impl DatabaseMessages for OnChainIterableKeyValueView { .into_boxed() } + fn message_batch<'a>( + &'a self, + ids: BoxedIter<'a, &'a Nonce>, + ) -> BoxedIter<'a, StorageResult> { + self.get_batch(ids) + .map(|result| result.and_then(|opt| opt.ok_or(not_found!(Messages)))) + .into_boxed() + } + fn message_exists(&self, nonce: &Nonce) -> StorageResult { self.message_exists(nonce) } From 4d5ca7d5feb2bd32a967cf0f5d365016148b8792 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A5rten=20Blankfors?= Date: Fri, 25 Oct 2024 14:08:38 +0200 Subject: [PATCH 11/37] feat: Use boxed iterator --- crates/fuel-core/src/query/message.rs | 9 +++-- .../service/adapters/graphql_api/on_chain.rs | 2 +- crates/fuel-core/src/state/data_source.rs | 2 ++ .../fuel-core/src/state/generic_database.rs | 5 ++- crates/fuel-core/src/state/rocks_db.rs | 12 +++---- crates/storage/src/blueprint.rs | 36 ++++++++++--------- crates/storage/src/kv_store.rs | 14 +++++--- crates/storage/src/lib.rs | 5 +-- crates/storage/src/structured_storage.rs | 7 ++-- 9 files changed, 50 insertions(+), 42 deletions(-) diff --git a/crates/fuel-core/src/query/message.rs b/crates/fuel-core/src/query/message.rs index beae8d7c176..f66b7e99e4c 100644 --- a/crates/fuel-core/src/query/message.rs +++ b/crates/fuel-core/src/query/message.rs @@ -2,6 +2,7 @@ use crate::fuel_core_graphql_api::database::ReadView; use fuel_core_storage::{ iter::{ BoxedIter, + IntoBoxedIter, IterDirection, }, not_found, @@ -80,7 +81,7 @@ impl ReadView { pub async fn messages( &self, - _ids: Vec, + ids: Vec, ) -> impl Stream> { // let ids = Box::new(ids.iter()); // let messages: Vec<_> = self @@ -93,8 +94,10 @@ impl ReadView { // }) // .collect(); - // TODO - let messages = Vec::new(); + let messages: Vec<_> = self + .on_chain + .message_batch(ids.iter().into_boxed()) + .collect(); //// TODO: Use multiget when it's implemented. //// https://github.com/FuelLabs/fuel-core/issues/2344 diff --git a/crates/fuel-core/src/service/adapters/graphql_api/on_chain.rs b/crates/fuel-core/src/service/adapters/graphql_api/on_chain.rs index d6bbb274861..ed5fcdfac51 100644 --- a/crates/fuel-core/src/service/adapters/graphql_api/on_chain.rs +++ b/crates/fuel-core/src/service/adapters/graphql_api/on_chain.rs @@ -109,7 +109,7 @@ impl DatabaseMessages for OnChainIterableKeyValueView { &'a self, ids: BoxedIter<'a, &'a Nonce>, ) -> BoxedIter<'a, StorageResult> { - self.get_batch(ids) + >::get_batch(self, ids) .map(|result| result.and_then(|opt| opt.ok_or(not_found!(Messages)))) .into_boxed() } diff --git a/crates/fuel-core/src/state/data_source.rs b/crates/fuel-core/src/state/data_source.rs index cf107d0e1b9..10ff17b5046 100644 --- a/crates/fuel-core/src/state/data_source.rs +++ b/crates/fuel-core/src/state/data_source.rs @@ -44,6 +44,7 @@ where impl KeyValueInspect for DataSource where Description: DatabaseDescription, + Stage: Send + Sync, { type Column = Description::Column; @@ -76,6 +77,7 @@ where impl IterableStore for DataSource where Description: DatabaseDescription, + Stage: Send + Sync, { fn iter_store( &self, diff --git a/crates/fuel-core/src/state/generic_database.rs b/crates/fuel-core/src/state/generic_database.rs index e6eeee4dd12..ee82958abc9 100644 --- a/crates/fuel-core/src/state/generic_database.rs +++ b/crates/fuel-core/src/state/generic_database.rs @@ -72,9 +72,8 @@ where { fn get_batch<'a>( &'a self, - keys: Box::Key> + 'a>, - ) -> Box::OwnedValue>>> + 'a> - { + keys: BoxedIter<'a, &'a M::Key>, + ) -> BoxedIter<'a, StorageResult::OwnedValue>>> { self.storage.get_batch(keys) } } diff --git a/crates/fuel-core/src/state/rocks_db.rs b/crates/fuel-core/src/state/rocks_db.rs index 4c4b5ff64ef..c5bb6e01fb8 100644 --- a/crates/fuel-core/src/state/rocks_db.rs +++ b/crates/fuel-core/src/state/rocks_db.rs @@ -731,16 +731,15 @@ where fn get_multi<'a>( &'a self, - keys: Box> + 'a>, + keys: BoxedIter<'a, Vec>, column: Self::Column, - ) -> Box>> + 'a> { + ) -> BoxedIter<'a, StorageResult>> { // TODO: Metrics let column = self.cf(column); let keys = keys.map(|key| (&column, key)); - let values = self - .db + self.db .multi_get_cf_opt(keys, &self.read_options) .into_iter() .map(|value_opt| { @@ -749,9 +748,8 @@ where StorageError::Other(DatabaseError::Other(e.into()).into()) }) .map(|value| value.map(Arc::new)) - }); - - Box::new(values) + }) + .into_boxed() } fn read( diff --git a/crates/storage/src/blueprint.rs b/crates/storage/src/blueprint.rs index 282a7f90abd..aa6a0e57650 100644 --- a/crates/storage/src/blueprint.rs +++ b/crates/storage/src/blueprint.rs @@ -9,6 +9,10 @@ use crate::{ Encode, Encoder, }, + iter::{ + BoxedIter, + IntoBoxedIter, + }, kv_store::{ BatchOperations, KeyValueInspect, @@ -19,9 +23,6 @@ use crate::{ }; use fuel_vm_private::prelude::MerkleRoot; -#[cfg(feature = "alloc")] -use alloc::boxed::Box; - pub mod merklized; pub mod plain; pub mod sparse; @@ -82,22 +83,25 @@ where /// Returns multiple values from the storage. fn get_multi<'a>( storage: &'a S, - keys: Box + 'a>, + keys: BoxedIter<'a, &'a M::Key>, column: S::Column, - ) -> Box>> + 'a> { - let keys = Box::new( - keys.map(|key| Self::KeyCodec::encode(&key).as_bytes().into_owned()), - ); - - Box::new(storage.get_multi(keys, column).map(|result| { - result.and_then(|opt| { - opt.map(|value| { - Self::ValueCodec::decode_from_value(value) - .map_err(crate::Error::Codec) + ) -> BoxedIter<'a, StorageResult>> { + let keys = keys + .map(|key| Self::KeyCodec::encode(&key).as_bytes().into_owned()) + .into_boxed(); + + storage + .get_multi(keys, column) + .map(|result| { + result.and_then(|opt| { + opt.map(|value| { + Self::ValueCodec::decode_from_value(value) + .map_err(crate::Error::Codec) + }) + .transpose() }) - .transpose() }) - })) + .into_boxed() } } diff --git a/crates/storage/src/kv_store.rs b/crates/storage/src/kv_store.rs index b69522e055c..f54586f4377 100644 --- a/crates/storage/src/kv_store.rs +++ b/crates/storage/src/kv_store.rs @@ -1,6 +1,10 @@ //! The module provides plain abstract definition of the key-value store. use crate::{ + iter::{ + BoxedIter, + IntoBoxedIter, + }, Error as StorageError, Result as StorageResult, }; @@ -26,7 +30,7 @@ pub type KVItem = StorageResult<(Key, Value)>; pub type KeyItem = StorageResult; /// A column of the storage. -pub trait StorageColumn: Copy + core::fmt::Debug { +pub trait StorageColumn: Copy + core::fmt::Debug + Send + Sync { /// Returns the name of the column. fn name(&self) -> String; @@ -41,7 +45,7 @@ pub trait StorageColumn: Copy + core::fmt::Debug { /// The definition of the key-value inspection store. #[impl_tools::autoimpl(for &T, &mut T, Box)] -pub trait KeyValueInspect { +pub trait KeyValueInspect: Send + Sync { /// The type of the column. type Column: StorageColumn; @@ -65,10 +69,10 @@ pub trait KeyValueInspect { /// Returns multiple values from the storage. fn get_multi<'a>( &'a self, - keys: Box> + 'a>, + keys: BoxedIter<'a, Vec>, column: Self::Column, - ) -> Box>> + 'a> { - Box::new(keys.map(move |key| self.get(&key, column))) + ) -> BoxedIter<'a, StorageResult>> { + keys.map(move |key| self.get(&key, column)).into_boxed() } /// Reads the value from the storage into the `buf` and returns the number of read bytes. diff --git a/crates/storage/src/lib.rs b/crates/storage/src/lib.rs index 8b6fa2604f3..185785cd7b7 100644 --- a/crates/storage/src/lib.rs +++ b/crates/storage/src/lib.rs @@ -17,6 +17,7 @@ extern crate alloc; use anyhow::anyhow; use core::array::TryFromSliceError; use fuel_core_types::services::executor::Error as ExecutorError; +use iter::BoxedIter; #[cfg(feature = "alloc")] use alloc::{ @@ -182,8 +183,8 @@ pub trait StorageBatchInspect { /// TODO fn get_batch<'a>( &'a self, - keys: Box + 'a>, - ) -> Box>> + 'a>; + keys: BoxedIter<'a, &'a Type::Key>, + ) -> BoxedIter<'a, Result>>; } /// Creates `StorageError::NotFound` error with file and line information inside. diff --git a/crates/storage/src/structured_storage.rs b/crates/storage/src/structured_storage.rs index 9fe78dbdb52..9cc97270ca6 100644 --- a/crates/storage/src/structured_storage.rs +++ b/crates/storage/src/structured_storage.rs @@ -59,9 +59,6 @@ use alloc::{ fmt::Debug, }; -// TODO: Format -#[cfg(feature = "alloc")] -use alloc::boxed::Box; #[cfg(feature = "alloc")] use alloc::vec::Vec; @@ -281,8 +278,8 @@ where { fn get_batch<'a>( &'a self, - keys: Box + 'a>, - ) -> Box>> + 'a> { + keys: BoxedIter<'a, &'a M::Key>, + ) -> BoxedIter<'a, StorageResult>> { M::Blueprint::get_multi(self, keys, M::column()) } } From 36a50bb73fceabacd1ab6fb6c5d2e6bc9d0a9e63 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A5rten=20Blankfors?= Date: Fri, 25 Oct 2024 22:17:55 +0200 Subject: [PATCH 12/37] feat: Use multi-get when getting full block --- crates/fuel-core/src/database/block.rs | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/crates/fuel-core/src/database/block.rs b/crates/fuel-core/src/database/block.rs index 0612855b683..09148d8ee3c 100644 --- a/crates/fuel-core/src/database/block.rs +++ b/crates/fuel-core/src/database/block.rs @@ -7,6 +7,7 @@ use crate::{ }; use fuel_core_storage::{ iter::{ + IntoBoxedIter, IterDirection, IteratorOverTable, }, @@ -18,11 +19,12 @@ use fuel_core_storage::{ FuelBlockMerkleMetadata, }, FuelBlocks, - // Transactions, + Transactions, }, Error as StorageError, Result as StorageResult, StorageAsRef, + StorageBatchInspect, }; use fuel_core_types::{ blockchain::{ @@ -36,7 +38,7 @@ use fuel_core_types::{ fuel_merkle::binary::MerkleTree, fuel_types::BlockHeight, }; -// use std::borrow::Cow; +use itertools::Itertools; impl OffChainIterableKeyValueView { pub fn get_block_height(&self, id: &BlockId) -> StorageResult> { @@ -64,10 +66,17 @@ impl OnChainIterableKeyValueView { /// Retrieve the full block and all associated transactions pub fn get_full_block(&self, height: &BlockHeight) -> StorageResult> { let db_block = self.storage::().get(height)?; - if let Some(_block) = db_block { + if let Some(block) = db_block { // fetch all the transactions // TODO: Use multiget when it's implemented. // https://github.com/FuelLabs/fuel-core/issues/2344 + let transaction_ids = block.transactions().iter().into_boxed(); + let txs = >::get_batch( + self, + transaction_ids, + ) + .map(|res| res.and_then(|opt| opt.ok_or(not_found!(Transactions)))) + .try_collect()?; // let transaction_ids = Box::new(block.transactions().iter()); // let txs = self @@ -80,8 +89,7 @@ impl OnChainIterableKeyValueView { // }) // .try_collect()?; // - // Ok(Some(block.into_owned().uncompress(txs))) - Ok(None) + Ok(Some(block.into_owned().uncompress(txs))) } else { Ok(None) } From 6275b36c333c6e35415b03bad66b3bb6386fed77 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A5rten=20Blankfors?= Date: Fri, 25 Oct 2024 22:26:59 +0200 Subject: [PATCH 13/37] feat: Use multi-get when getting coins --- crates/fuel-core/src/graphql_api/ports.rs | 32 +++++++++--- crates/fuel-core/src/query/coin.rs | 33 ++++++------ .../service/adapters/graphql_api/on_chain.rs | 50 ++++++++++++++++++- 3 files changed, 94 insertions(+), 21 deletions(-) diff --git a/crates/fuel-core/src/graphql_api/ports.rs b/crates/fuel-core/src/graphql_api/ports.rs index 9e62b371f7d..d74f9cdc6f0 100644 --- a/crates/fuel-core/src/graphql_api/ports.rs +++ b/crates/fuel-core/src/graphql_api/ports.rs @@ -31,12 +31,15 @@ use fuel_core_types::{ DaBlockHeight, }, }, - entities::relayer::{ - message::{ - MerkleProof, - Message, + entities::{ + coins::coin::Coin, + relayer::{ + message::{ + MerkleProof, + Message, + }, + transaction::RelayedTransactionStatus, }, - transaction::RelayedTransactionStatus, }, fuel_tx::{ Bytes32, @@ -121,7 +124,7 @@ pub trait OnChainDatabase: + Sync + DatabaseBlocks + DatabaseMessages - + StorageInspect + + DatabaseCoins + StorageRead + StorageInspect + StorageInspect @@ -136,6 +139,12 @@ pub trait DatabaseBlocks { /// Get a transaction by its id. fn transaction(&self, tx_id: &TxId) -> StorageResult; + /// Get a batch of transactions by their ids. + fn transactions<'a>( + &'a self, + tx_ids: BoxedIter<'a, &'a TxId>, + ) -> BoxedIter<'a, StorageResult>; + /// Get a block by its height. fn block(&self, height: &BlockHeight) -> StorageResult; @@ -184,6 +193,17 @@ pub trait DatabaseRelayedTransactions { ) -> StorageResult>; } +/// Trait that specifies all the getters reqired for coins +pub trait DatabaseCoins: + StorageInspect + StorageBatchInspect +{ + fn coin(&self, utxo_id: UtxoId) -> StorageResult; + fn coins<'a>( + &'a self, + utxo_ids: BoxedIter<'a, &'a UtxoId>, + ) -> BoxedIter<'a, StorageResult>; +} + /// Trait that specifies all the getters required for contract. pub trait DatabaseContracts: StorageInspect diff --git a/crates/fuel-core/src/query/coin.rs b/crates/fuel-core/src/query/coin.rs index c487bdba23c..5c61fa52c2b 100644 --- a/crates/fuel-core/src/query/coin.rs +++ b/crates/fuel-core/src/query/coin.rs @@ -1,11 +1,14 @@ use crate::fuel_core_graphql_api::database::ReadView; use fuel_core_storage::{ - iter::IterDirection, - not_found, - tables::Coins, + iter::{ + IntoBoxedIter, + IterDirection, + }, + // not_found, + // tables::Coins, Error as StorageError, Result as StorageResult, - StorageAsRef, + // StorageAsRef, }; use fuel_core_types::{ entities::coins::coin::Coin, @@ -20,27 +23,29 @@ use futures::{ impl ReadView { pub fn coin(&self, utxo_id: UtxoId) -> StorageResult { - let coin = self - .on_chain - .as_ref() - .storage::() - .get(&utxo_id)? - .ok_or(not_found!(Coins))? - .into_owned(); + self.on_chain.coin(utxo_id) + // let coin = self + // .on_chain + // .as_ref() + // .storage::() + // .get(&utxo_id)? + // .ok_or(not_found!(Coins))? + // .into_owned(); - Ok(coin.uncompress(utxo_id)) + // Ok(coin.uncompress(utxo_id)) } pub async fn coins( &self, utxo_ids: Vec, ) -> impl Iterator> + '_ { + let coins: Vec<_> = self.on_chain.coins(utxo_ids.iter().into_boxed()).collect(); // TODO: Use multiget when it's implemented. // https://github.com/FuelLabs/fuel-core/issues/2344 - let coins = utxo_ids.into_iter().map(|id| self.coin(id)); + // let coins = utxo_ids.into_iter().map(|id| self.coin(id)); // Give a chance to other tasks to run. tokio::task::yield_now().await; - coins + coins.into_iter() } pub fn owned_coins( diff --git a/crates/fuel-core/src/service/adapters/graphql_api/on_chain.rs b/crates/fuel-core/src/service/adapters/graphql_api/on_chain.rs index ed5fcdfac51..984155fda55 100644 --- a/crates/fuel-core/src/service/adapters/graphql_api/on_chain.rs +++ b/crates/fuel-core/src/service/adapters/graphql_api/on_chain.rs @@ -11,7 +11,10 @@ use crate::{ DatabaseMessages, OnChainDatabase, }, - graphql_api::ports::worker, + graphql_api::ports::{ + worker, + DatabaseCoins, + }, }; use fuel_core_storage::{ iter::{ @@ -22,6 +25,7 @@ use fuel_core_storage::{ }, not_found, tables::{ + Coins, FuelBlocks, Messages, SealedBlockConsensus, @@ -44,6 +48,7 @@ use fuel_core_types::{ ContractId, Transaction, TxId, + UtxoId, }, fuel_types::{ BlockHeight, @@ -62,6 +67,15 @@ impl DatabaseBlocks for OnChainIterableKeyValueView { .into_owned()) } + fn transactions<'a>( + &'a self, + tx_ids: BoxedIter<'a, &'a TxId>, + ) -> BoxedIter<'a, StorageResult> { + >::get_batch(self, tx_ids) + .map(|result| result.and_then(|opt| opt.ok_or(not_found!(Transactions)))) + .into_boxed() + } + fn block(&self, height: &BlockHeight) -> StorageResult { let block = self .storage_as_ref::() @@ -119,6 +133,40 @@ impl DatabaseMessages for OnChainIterableKeyValueView { } } +impl DatabaseCoins for OnChainIterableKeyValueView { + fn coin( + &self, + utxo_id: fuel_core_types::fuel_tx::UtxoId, + ) -> StorageResult { + let coin = self + .storage::() + .get(&utxo_id)? + .ok_or(not_found!(Coins))? + .into_owned(); + + Ok(coin.uncompress(utxo_id)) + } + + fn coins<'a>( + &'a self, + utxo_ids: BoxedIter<'a, &'a UtxoId>, + ) -> BoxedIter<'a, StorageResult> { + let utxo_ids_1: Vec<_> = utxo_ids.collect(); + let utxo_ids_2 = utxo_ids_1.clone(); + + >::get_batch( + self, + utxo_ids_1.into_iter().into_boxed(), + ) + .zip(utxo_ids_2.into_iter().into_boxed()) + .map(|(res, utxo_id)| { + res.and_then(|opt| opt.ok_or(not_found!(Coins))) + .map(|coin| coin.uncompress(*utxo_id)) + }) + .into_boxed() + } +} + impl DatabaseContracts for OnChainIterableKeyValueView { fn contract_balances( &self, From e1e50faf8598cdab7597b979bf3d3d1b3b395d99 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A5rten=20Blankfors?= Date: Sat, 26 Oct 2024 22:26:06 +0200 Subject: [PATCH 14/37] feat: Use multi-get when getting transactions --- crates/fuel-core/src/graphql_api/database.rs | 44 ++++++++++++++----- crates/fuel-core/src/graphql_api/ports.rs | 7 ++- .../service/adapters/graphql_api/off_chain.rs | 18 ++++++-- 3 files changed, 54 insertions(+), 15 deletions(-) diff --git a/crates/fuel-core/src/graphql_api/database.rs b/crates/fuel-core/src/graphql_api/database.rs index bf47c8d92a7..f887d2759c9 100644 --- a/crates/fuel-core/src/graphql_api/database.rs +++ b/crates/fuel-core/src/graphql_api/database.rs @@ -12,8 +12,6 @@ use fuel_core_storage::{ IntoBoxedIter, IterDirection, }, - not_found, - tables::Transactions, transactional::AtomicView, Error as StorageError, IsNotFound, @@ -65,6 +63,7 @@ use fuel_core_types::{ use futures::Stream; use std::{ borrow::Cow, + collections::BTreeMap, sync::Arc, }; @@ -141,29 +140,52 @@ impl ReadView { pub fn transaction(&self, tx_id: &TxId) -> StorageResult { let result = self.on_chain.transaction(tx_id); if result.is_not_found() { - if let Some(tx) = self.off_chain.old_transaction(tx_id)? { - Ok(tx) - } else { - Err(not_found!(Transactions)) - } + self.off_chain.old_transaction(tx_id) } else { result } } + // TODO: Test case to ensure order is preserved pub async fn transactions( &self, tx_ids: Vec, ) -> Vec> { // TODO: Use multiget when it's implemented. // https://github.com/FuelLabs/fuel-core/issues/2344 - let result = tx_ids + let on_chain_results: BTreeMap<_, _> = tx_ids + .iter() + .enumerate() + .zip(self.on_chain.transactions(tx_ids.iter().into_boxed())) + .collect(); + + let off_chain_indexed_txids: Vec<_> = on_chain_results .iter() - .map(|tx_id| self.transaction(tx_id)) - .collect::>(); + .filter_map(|(indexed_tx_id, result)| { + result.is_not_found().then_some(*indexed_tx_id) + }) + .collect(); + + let off_chain_results = off_chain_indexed_txids.iter().copied().zip( + self.off_chain.old_transactions( + off_chain_indexed_txids + .iter() + .map(|(_, tx_id)| *tx_id) + .into_boxed(), + ), + ); + + let mut results = on_chain_results; + results.extend(off_chain_results); + + // let result = tx_ids + // .iter() + // .map(|tx_id| self.transaction(tx_id)) + // .collect::>(); // Give a chance to other tasks to run. tokio::task::yield_now().await; - result + + results.into_values().collect() } pub fn block(&self, height: &BlockHeight) -> StorageResult { diff --git a/crates/fuel-core/src/graphql_api/ports.rs b/crates/fuel-core/src/graphql_api/ports.rs index d74f9cdc6f0..77282b37908 100644 --- a/crates/fuel-core/src/graphql_api/ports.rs +++ b/crates/fuel-core/src/graphql_api/ports.rs @@ -108,7 +108,12 @@ pub trait OffChainDatabase: Send + Sync { fn old_block_consensus(&self, height: &BlockHeight) -> StorageResult; - fn old_transaction(&self, id: &TxId) -> StorageResult>; + fn old_transaction(&self, id: &TxId) -> StorageResult; + + fn old_transactions<'a>( + &'a self, + ids: BoxedIter<'a, &'a TxId>, + ) -> BoxedIter<'a, StorageResult>; fn relayed_tx_status( &self, diff --git a/crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs b/crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs index d554c7ddc45..20ed5c2d6b8 100644 --- a/crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs +++ b/crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs @@ -41,6 +41,7 @@ use fuel_core_storage::{ Error as StorageError, Result as StorageResult, StorageAsRef, + StorageBatchInspect, }; use fuel_core_types::{ blockchain::{ @@ -166,10 +167,21 @@ impl OffChainDatabase for OffChainIterableKeyValueView { .into_owned()) } - fn old_transaction(&self, id: &TxId) -> StorageResult> { + fn old_transaction(&self, id: &TxId) -> StorageResult { self.storage_as_ref::() - .get(id) - .map(|tx| tx.map(|tx| tx.into_owned())) + .get(id)? + // Note: Breaking change (previously we returned not_found!(Transactions)) + .ok_or(not_found!(OldTransactions)) + .map(|tx| tx.into_owned()) + } + + fn old_transactions<'a>( + &'a self, + ids: BoxedIter<'a, &'a TxId>, + ) -> BoxedIter<'a, StorageResult> { + >::get_batch(self, ids) + .map(|result| result.and_then(|opt| opt.ok_or(not_found!(OldTransactions)))) + .into_boxed() } fn relayed_tx_status( From f76d2100bf311b275de20df0ce888a34aa325dbb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A5rten=20Blankfors?= Date: Tue, 29 Oct 2024 09:22:44 +0100 Subject: [PATCH 15/37] refactor: Rename multi_get -> get_batch --- crates/fuel-core/src/state/rocks_db.rs | 2 +- crates/storage/src/blueprint.rs | 4 ++-- crates/storage/src/kv_store.rs | 2 +- crates/storage/src/structured_storage.rs | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/crates/fuel-core/src/state/rocks_db.rs b/crates/fuel-core/src/state/rocks_db.rs index c5bb6e01fb8..9749e087b94 100644 --- a/crates/fuel-core/src/state/rocks_db.rs +++ b/crates/fuel-core/src/state/rocks_db.rs @@ -729,7 +729,7 @@ where Ok(value.map(Arc::new)) } - fn get_multi<'a>( + fn get_batch<'a>( &'a self, keys: BoxedIter<'a, Vec>, column: Self::Column, diff --git a/crates/storage/src/blueprint.rs b/crates/storage/src/blueprint.rs index aa6a0e57650..eea1de51965 100644 --- a/crates/storage/src/blueprint.rs +++ b/crates/storage/src/blueprint.rs @@ -81,7 +81,7 @@ where } /// Returns multiple values from the storage. - fn get_multi<'a>( + fn get_batch<'a>( storage: &'a S, keys: BoxedIter<'a, &'a M::Key>, column: S::Column, @@ -91,7 +91,7 @@ where .into_boxed(); storage - .get_multi(keys, column) + .get_batch(keys, column) .map(|result| { result.and_then(|opt| { opt.map(|value| { diff --git a/crates/storage/src/kv_store.rs b/crates/storage/src/kv_store.rs index f54586f4377..462935b9611 100644 --- a/crates/storage/src/kv_store.rs +++ b/crates/storage/src/kv_store.rs @@ -67,7 +67,7 @@ pub trait KeyValueInspect: Send + Sync { fn get(&self, key: &[u8], column: Self::Column) -> StorageResult>; /// Returns multiple values from the storage. - fn get_multi<'a>( + fn get_batch<'a>( &'a self, keys: BoxedIter<'a, Vec>, column: Self::Column, diff --git a/crates/storage/src/structured_storage.rs b/crates/storage/src/structured_storage.rs index 9cc97270ca6..5fb3de982b7 100644 --- a/crates/storage/src/structured_storage.rs +++ b/crates/storage/src/structured_storage.rs @@ -280,7 +280,7 @@ where &'a self, keys: BoxedIter<'a, &'a M::Key>, ) -> BoxedIter<'a, StorageResult>> { - M::Blueprint::get_multi(self, keys, M::column()) + M::Blueprint::get_batch(self, keys, M::column()) } } From 4feadfdf1e0a4605d8e7bb278ecdbc0792f8ca09 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A5rten=20Blankfors?= Date: Tue, 29 Oct 2024 09:30:47 +0100 Subject: [PATCH 16/37] feat: Implement multi-get for more backends --- crates/fuel-core/src/state/data_source.rs | 8 ++++++++ crates/fuel-core/src/state/generic_database.rs | 8 ++++++++ crates/fuel-core/src/state/iterable_key_value_view.rs | 8 ++++++++ crates/fuel-core/src/state/key_value_view.rs | 9 +++++++++ crates/storage/src/structured_storage.rs | 8 ++++++++ 5 files changed, 41 insertions(+) diff --git a/crates/fuel-core/src/state/data_source.rs b/crates/fuel-core/src/state/data_source.rs index 10ff17b5046..dd861ea2ac4 100644 --- a/crates/fuel-core/src/state/data_source.rs +++ b/crates/fuel-core/src/state/data_source.rs @@ -64,6 +64,14 @@ where self.data.get(key, column) } + fn get_batch<'a>( + &'a self, + keys: BoxedIter<'a, Vec>, + column: Self::Column, + ) -> BoxedIter<'a, StorageResult>> { + self.data.get_batch(keys, column) + } + fn read( &self, key: &[u8], diff --git a/crates/fuel-core/src/state/generic_database.rs b/crates/fuel-core/src/state/generic_database.rs index ee82958abc9..a71870bc405 100644 --- a/crates/fuel-core/src/state/generic_database.rs +++ b/crates/fuel-core/src/state/generic_database.rs @@ -134,6 +134,14 @@ where KeyValueInspect::get(&self.storage, key, column) } + fn get_batch<'a>( + &'a self, + keys: BoxedIter<'a, Vec>, + column: Self::Column, + ) -> BoxedIter<'a, StorageResult>> { + KeyValueInspect::get_batch(&self.storage, keys, column) + } + fn read( &self, key: &[u8], diff --git a/crates/fuel-core/src/state/iterable_key_value_view.rs b/crates/fuel-core/src/state/iterable_key_value_view.rs index 40ab3483ede..4b739967735 100644 --- a/crates/fuel-core/src/state/iterable_key_value_view.rs +++ b/crates/fuel-core/src/state/iterable_key_value_view.rs @@ -63,6 +63,14 @@ where self.0.get(key, column) } + fn get_batch<'a>( + &'a self, + keys: BoxedIter<'a, Vec>, + column: Self::Column, + ) -> BoxedIter<'a, StorageResult>> { + self.0.get_batch(keys, column) + } + fn read( &self, key: &[u8], diff --git a/crates/fuel-core/src/state/key_value_view.rs b/crates/fuel-core/src/state/key_value_view.rs index 9e70037fc21..84d99f7be79 100644 --- a/crates/fuel-core/src/state/key_value_view.rs +++ b/crates/fuel-core/src/state/key_value_view.rs @@ -1,4 +1,5 @@ use fuel_core_storage::{ + iter::BoxedIter, kv_store::{ KeyValueInspect, StorageColumn, @@ -50,6 +51,14 @@ where self.0.get(key, column) } + fn get_batch<'a>( + &'a self, + keys: BoxedIter<'a, Vec>, + column: Self::Column, + ) -> BoxedIter<'a, StorageResult>> { + self.0.get_batch(keys, column) + } + fn read( &self, key: &[u8], diff --git a/crates/storage/src/structured_storage.rs b/crates/storage/src/structured_storage.rs index 5fb3de982b7..a58b51ebe42 100644 --- a/crates/storage/src/structured_storage.rs +++ b/crates/storage/src/structured_storage.rs @@ -146,6 +146,14 @@ where self.inner.get(key, column) } + fn get_batch<'a>( + &'a self, + keys: BoxedIter<'a, Vec>, + column: Self::Column, + ) -> BoxedIter<'a, StorageResult>> { + self.inner.get_batch(keys, column) + } + fn read( &self, key: &[u8], From e141f56588b463179f97d42aa05c5614c93c9f8e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A5rten=20Blankfors?= Date: Tue, 29 Oct 2024 10:23:18 +0100 Subject: [PATCH 17/37] chore: Clean up comments and add docstrings --- crates/fuel-core/src/database/block.rs | 15 +-------------- crates/fuel-core/src/graphql_api/database.rs | 8 +------- crates/fuel-core/src/query/coin.rs | 18 ++---------------- crates/fuel-core/src/query/message.rs | 17 ++--------------- crates/storage/src/lib.rs | 7 ++++--- 5 files changed, 10 insertions(+), 55 deletions(-) diff --git a/crates/fuel-core/src/database/block.rs b/crates/fuel-core/src/database/block.rs index 09148d8ee3c..7c9ab130072 100644 --- a/crates/fuel-core/src/database/block.rs +++ b/crates/fuel-core/src/database/block.rs @@ -66,10 +66,8 @@ impl OnChainIterableKeyValueView { /// Retrieve the full block and all associated transactions pub fn get_full_block(&self, height: &BlockHeight) -> StorageResult> { let db_block = self.storage::().get(height)?; + if let Some(block) = db_block { - // fetch all the transactions - // TODO: Use multiget when it's implemented. - // https://github.com/FuelLabs/fuel-core/issues/2344 let transaction_ids = block.transactions().iter().into_boxed(); let txs = >::get_batch( self, @@ -78,17 +76,6 @@ impl OnChainIterableKeyValueView { .map(|res| res.and_then(|opt| opt.ok_or(not_found!(Transactions)))) .try_collect()?; - // let transaction_ids = Box::new(block.transactions().iter()); - // let txs = self - // .storage::() - // .get_multi(transaction_ids) - // .map(|tx| { - // tx.and_then(|tx| { - // tx.ok_or(not_found!(Transactions)).map(Cow::into_owned) - // }) - // }) - // .try_collect()?; - // Ok(Some(block.into_owned().uncompress(txs))) } else { Ok(None) diff --git a/crates/fuel-core/src/graphql_api/database.rs b/crates/fuel-core/src/graphql_api/database.rs index f887d2759c9..470f8f9dd70 100644 --- a/crates/fuel-core/src/graphql_api/database.rs +++ b/crates/fuel-core/src/graphql_api/database.rs @@ -151,8 +151,6 @@ impl ReadView { &self, tx_ids: Vec, ) -> Vec> { - // TODO: Use multiget when it's implemented. - // https://github.com/FuelLabs/fuel-core/issues/2344 let on_chain_results: BTreeMap<_, _> = tx_ids .iter() .enumerate() @@ -178,11 +176,7 @@ impl ReadView { let mut results = on_chain_results; results.extend(off_chain_results); - // let result = tx_ids - // .iter() - // .map(|tx_id| self.transaction(tx_id)) - // .collect::>(); - // Give a chance to other tasks to run. + // Give a chance for other tasks to run. tokio::task::yield_now().await; results.into_values().collect() diff --git a/crates/fuel-core/src/query/coin.rs b/crates/fuel-core/src/query/coin.rs index 5c61fa52c2b..7f8ea37424e 100644 --- a/crates/fuel-core/src/query/coin.rs +++ b/crates/fuel-core/src/query/coin.rs @@ -4,11 +4,8 @@ use fuel_core_storage::{ IntoBoxedIter, IterDirection, }, - // not_found, - // tables::Coins, Error as StorageError, Result as StorageResult, - // StorageAsRef, }; use fuel_core_types::{ entities::coins::coin::Coin, @@ -24,15 +21,6 @@ use futures::{ impl ReadView { pub fn coin(&self, utxo_id: UtxoId) -> StorageResult { self.on_chain.coin(utxo_id) - // let coin = self - // .on_chain - // .as_ref() - // .storage::() - // .get(&utxo_id)? - // .ok_or(not_found!(Coins))? - // .into_owned(); - - // Ok(coin.uncompress(utxo_id)) } pub async fn coins( @@ -40,10 +28,8 @@ impl ReadView { utxo_ids: Vec, ) -> impl Iterator> + '_ { let coins: Vec<_> = self.on_chain.coins(utxo_ids.iter().into_boxed()).collect(); - // TODO: Use multiget when it's implemented. - // https://github.com/FuelLabs/fuel-core/issues/2344 - // let coins = utxo_ids.into_iter().map(|id| self.coin(id)); - // Give a chance to other tasks to run. + + // Give a chance for other tasks to run. tokio::task::yield_now().await; coins.into_iter() } diff --git a/crates/fuel-core/src/query/message.rs b/crates/fuel-core/src/query/message.rs index f66b7e99e4c..625c86e8ecb 100644 --- a/crates/fuel-core/src/query/message.rs +++ b/crates/fuel-core/src/query/message.rs @@ -83,27 +83,14 @@ impl ReadView { &self, ids: Vec, ) -> impl Stream> { - // let ids = Box::new(ids.iter()); - // let messages: Vec<_> = self - // .on_chain - // .as_ref() - // .storage::() - // .get_multi(Box::new(ids)) - // .map(|res| { - // res.and_then(|opt| opt.ok_or(not_found!(Messages)).map(Cow::into_owned)) - // }) - // .collect(); - let messages: Vec<_> = self .on_chain .message_batch(ids.iter().into_boxed()) .collect(); - //// TODO: Use multiget when it's implemented. - //// https://github.com/FuelLabs/fuel-core/issues/2344 - // let messages = ids.into_iter().map(|id| self.message(&id)); - //// Give a chance to other tasks to run. + // Give a chance for other tasks to run. tokio::task::yield_now().await; + futures::stream::iter(messages) } diff --git a/crates/storage/src/lib.rs b/crates/storage/src/lib.rs index 185785cd7b7..306b4ade192 100644 --- a/crates/storage/src/lib.rs +++ b/crates/storage/src/lib.rs @@ -177,10 +177,11 @@ pub trait StorageBatchMutate: StorageMutate { Type::Key: 'a; } -// TODO: Document -/// TODO +/// Allows getting batches of values from the storage, +/// which can be faster in certain implementatoins than +/// getting values one by one. pub trait StorageBatchInspect { - /// TODO + /// Get a batch of values associated with the provided keys. fn get_batch<'a>( &'a self, keys: BoxedIter<'a, &'a Type::Key>, From 8e3fd0f50d9f0e903bba287bd1e739176a999c3e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A5rten=20Blankfors?= Date: Tue, 29 Oct 2024 10:43:59 +0100 Subject: [PATCH 18/37] chore: Add changelog entry --- CHANGELOG.md | 4 ++++ .../fuel-core/src/service/adapters/graphql_api/off_chain.rs | 1 - 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a6ed7c7bb58..54fe2c04ea7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,11 +15,15 @@ and this project adheres to [Semantic Versioning](http://semver.org/). - [2378](https://github.com/FuelLabs/fuel-core/pull/2378): Use cached hash of the topic instead of calculating it on each publishing gossip message. +#### Breaking +- [2396](https://github.com/FuelLabs/fuel-core/pull/2396): Return `StorageResult` in `OffChainDatabase::old_transaction`. + ### Added - [2321](https://github.com/FuelLabs/fuel-core/pull/2321): New metrics for the txpool: "The size of transactions in the txpool" (`txpool_tx_size`), "The time spent by a transaction in the txpool in seconds" (`txpool_tx_time_in_txpool_seconds`), The number of transactions in the txpool (`txpool_number_of_transactions`), "The number of transactions pending verification before entering the txpool" (`txpool_number_of_transactions_pending_verification`), "The number of executable transactions in the txpool" (`txpool_number_of_executable_transactions`), "The time it took to select transactions for inclusion in a block in nanoseconds" (`txpool_select_transaction_time_nanoseconds`), The time it took to insert a transaction in the txpool in milliseconds (`txpool_insert_transaction_time_milliseconds`). - [2347](https://github.com/FuelLabs/fuel-core/pull/2364): Add activity concept in order to protect against infinitely increasing DA gas price scenarios - [2362](https://github.com/FuelLabs/fuel-core/pull/2362): Added a new request_response protocol version `/fuel/req_res/0.0.2`. In comparison with `/fuel/req/0.0.1`, which returns an empty response when a request cannot be fulfilled, this version returns more meaningful error codes. Nodes still support the version `0.0.1` of the protocol to guarantee backward compatibility with fuel-core nodes. Empty responses received from nodes using the old protocol `/fuel/req/0.0.1` are automatically converted into an error `ProtocolV1EmptyResponse` with error code 0, which is also the only error code implemented. More specific error codes will be added in the future. - [2386](https://github.com/FuelLabs/fuel-core/pull/2386): Add a flag to define the maximum number of file descriptors that RocksDB can use. By default it's half of the OS limit. +- [2396](https://github.com/FuelLabs/fuel-core/pull/2396): Add support for multi get operation for database queries. ## [Version 0.40.0] diff --git a/crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs b/crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs index 20ed5c2d6b8..1c0c79953d3 100644 --- a/crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs +++ b/crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs @@ -170,7 +170,6 @@ impl OffChainDatabase for OffChainIterableKeyValueView { fn old_transaction(&self, id: &TxId) -> StorageResult { self.storage_as_ref::() .get(id)? - // Note: Breaking change (previously we returned not_found!(Transactions)) .ok_or(not_found!(OldTransactions)) .map(|tx| tx.into_owned()) } From 39664bb5428333f1a75264a4f0a44340ff73e785 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A5rten=20Blankfors?= Date: Tue, 29 Oct 2024 13:38:34 +0100 Subject: [PATCH 19/37] fix: Take slice instead of Vec as input to function --- crates/fuel-core/src/graphql_api/database.rs | 5 +---- crates/fuel-core/src/query/tx.rs | 2 +- crates/fuel-core/src/schema/block.rs | 2 +- crates/fuel-core/src/schema/tx.rs | 2 +- 4 files changed, 4 insertions(+), 7 deletions(-) diff --git a/crates/fuel-core/src/graphql_api/database.rs b/crates/fuel-core/src/graphql_api/database.rs index 470f8f9dd70..15ddebba500 100644 --- a/crates/fuel-core/src/graphql_api/database.rs +++ b/crates/fuel-core/src/graphql_api/database.rs @@ -147,10 +147,7 @@ impl ReadView { } // TODO: Test case to ensure order is preserved - pub async fn transactions( - &self, - tx_ids: Vec, - ) -> Vec> { + pub async fn transactions(&self, tx_ids: &[TxId]) -> Vec> { let on_chain_results: BTreeMap<_, _> = tx_ids .iter() .enumerate() diff --git a/crates/fuel-core/src/query/tx.rs b/crates/fuel-core/src/query/tx.rs index 0bceeef8809..c17b184154d 100644 --- a/crates/fuel-core/src/query/tx.rs +++ b/crates/fuel-core/src/query/tx.rs @@ -50,7 +50,7 @@ impl ReadView { }) .try_filter_map(move |chunk| async move { let tx_ids = chunk.iter().map(|(_, tx_id)| *tx_id).collect::>(); - let txs = self.transactions(tx_ids).await; + let txs = self.transactions(&tx_ids).await; let txs = txs .into_iter() .zip(chunk) diff --git a/crates/fuel-core/src/schema/block.rs b/crates/fuel-core/src/schema/block.rs index 7cdfeff2d5b..737c3a6ada9 100644 --- a/crates/fuel-core/src/schema/block.rs +++ b/crates/fuel-core/src/schema/block.rs @@ -144,7 +144,7 @@ impl Block { .filter_map(move |tx_ids: Vec| { let async_query = query.as_ref().clone(); async move { - let txs = async_query.transactions(tx_ids.clone()).await; + let txs = async_query.transactions(&tx_ids).await; let txs = txs .into_iter() .zip(tx_ids.into_iter()) diff --git a/crates/fuel-core/src/schema/tx.rs b/crates/fuel-core/src/schema/tx.rs index 8ee21edf972..306d1f7ccaa 100644 --- a/crates/fuel-core/src/schema/tx.rs +++ b/crates/fuel-core/src/schema/tx.rs @@ -173,7 +173,7 @@ impl TxQuery { .iter() .map(|sorted| sorted.tx_id.0) .collect::>(); - let txs = async_query.transactions(tx_ids).await; + let txs = async_query.transactions(&tx_ids).await; let txs = txs.into_iter().zip(chunk.into_iter()).map( |(result, sorted)| { result.map(|tx| { From 52eca6436ab546087740e279033ff27d5653bbfa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A5rten=20Blankfors?= Date: Tue, 29 Oct 2024 14:16:54 +0100 Subject: [PATCH 20/37] test: Assert the returned tx is in the expected place in get_full_block_with_tx --- crates/client/src/client/schema/primitives.rs | 4 +-- crates/fuel-core/src/graphql_api/database.rs | 1 - tests/tests/blocks.rs | 28 ++++++++++++++----- 3 files changed, 23 insertions(+), 10 deletions(-) diff --git a/crates/client/src/client/schema/primitives.rs b/crates/client/src/client/schema/primitives.rs index 1559c835844..540a7b37f2b 100644 --- a/crates/client/src/client/schema/primitives.rs +++ b/crates/client/src/client/schema/primitives.rs @@ -177,7 +177,7 @@ impl LowerHex for TxPointer { } } -#[derive(cynic::Scalar, Debug, Clone)] +#[derive(cynic::Scalar, Debug, Clone, PartialEq, Eq)] pub struct HexString(pub Bytes); impl From for Vec { @@ -194,7 +194,7 @@ impl Deref for HexString { } } -#[derive(Debug, Clone)] +#[derive(Debug, Clone, PartialEq, Eq)] pub struct Bytes(pub Vec); impl FromStr for Bytes { diff --git a/crates/fuel-core/src/graphql_api/database.rs b/crates/fuel-core/src/graphql_api/database.rs index 15ddebba500..848931f3b3a 100644 --- a/crates/fuel-core/src/graphql_api/database.rs +++ b/crates/fuel-core/src/graphql_api/database.rs @@ -146,7 +146,6 @@ impl ReadView { } } - // TODO: Test case to ensure order is preserved pub async fn transactions(&self, tx_ids: &[TxId]) -> Vec> { let on_chain_results: BTreeMap<_, _> = tx_ids .iter() diff --git a/tests/tests/blocks.rs b/tests/tests/blocks.rs index 7a5ba4688d5..886532ababc 100644 --- a/tests/tests/blocks.rs +++ b/tests/tests/blocks.rs @@ -30,6 +30,7 @@ use fuel_core_storage::{ vm_storage::VmStorageRequirements, StorageAsMut, }; + use fuel_core_types::{ blockchain::{ block::CompressedBlock, @@ -43,6 +44,10 @@ use itertools::{ rev, Itertools, }; +use rand::{ + rngs::StdRng, + SeedableRng, +}; use rstest::rstest; use std::{ ops::Deref, @@ -50,11 +55,6 @@ use std::{ }; use test_helpers::send_graph_ql_query; -use rand::{ - rngs::StdRng, - SeedableRng, -}; - #[tokio::test] async fn block() { // setup test data in the node @@ -370,7 +370,10 @@ mod full_block { HeavyWorkConfig, PoolLimits, }; - use fuel_core_types::fuel_types::BlockHeight; + use fuel_core_types::fuel_types::{ + BlockHeight, + ChainId, + }; #[derive(cynic::QueryFragment, Debug)] #[cynic( @@ -428,11 +431,22 @@ mod full_block { let client = FuelClient::from(srv.bound_address); let tx = Transaction::default_test_tx(); + let tx_id = tx.id(&ChainId::default()); + client.submit_and_await_commit(&tx).await.unwrap(); let block = client.full_block_by_height(1).await.unwrap().unwrap(); assert_eq!(block.header.height.0, 1); - assert_eq!(block.transactions.len(), 2 /* mint + our tx */); + + let [returned_tx, _mint_tx] = block.transactions.as_slice() else { + panic!("expected two transactions"); + }; + + let returned_tx_id = Transaction::try_from(returned_tx.clone()) + .unwrap() + .id(&ChainId::default()); + + assert_eq!(returned_tx_id, tx_id); } #[tokio::test] From 5b0e975677ffbdd48ce978e052904d311336a9d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A5rten=20Blankfors?= Date: Tue, 29 Oct 2024 14:26:09 +0100 Subject: [PATCH 21/37] fix: Typo --- crates/storage/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/storage/src/lib.rs b/crates/storage/src/lib.rs index 306b4ade192..2def12e2d03 100644 --- a/crates/storage/src/lib.rs +++ b/crates/storage/src/lib.rs @@ -178,7 +178,7 @@ pub trait StorageBatchMutate: StorageMutate { } /// Allows getting batches of values from the storage, -/// which can be faster in certain implementatoins than +/// which can be faster in certain implementations than /// getting values one by one. pub trait StorageBatchInspect { /// Get a batch of values associated with the provided keys. From bc955fdcc327ea19125512cbabff4e11dc838db0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A5rten=20Blankfors?= Date: Tue, 29 Oct 2024 14:27:47 +0100 Subject: [PATCH 22/37] fix: Typo --- crates/fuel-core/src/graphql_api/ports.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/fuel-core/src/graphql_api/ports.rs b/crates/fuel-core/src/graphql_api/ports.rs index 77282b37908..dbc2a200808 100644 --- a/crates/fuel-core/src/graphql_api/ports.rs +++ b/crates/fuel-core/src/graphql_api/ports.rs @@ -198,7 +198,7 @@ pub trait DatabaseRelayedTransactions { ) -> StorageResult>; } -/// Trait that specifies all the getters reqired for coins +/// Trait that specifies all the getters required for coins pub trait DatabaseCoins: StorageInspect + StorageBatchInspect { From 491c452f6fad6976a5302488b4b66cabf16d506b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A5rten=20Blankfors?= Date: Tue, 29 Oct 2024 16:11:25 +0100 Subject: [PATCH 23/37] feat: Add metrics for RocksDB get_batch implementation --- crates/fuel-core/src/state/rocks_db.rs | 82 ++++++++++++++++---------- 1 file changed, 51 insertions(+), 31 deletions(-) diff --git a/crates/fuel-core/src/state/rocks_db.rs b/crates/fuel-core/src/state/rocks_db.rs index 9749e087b94..7b6acbbce28 100644 --- a/crates/fuel-core/src/state/rocks_db.rs +++ b/crates/fuel-core/src/state/rocks_db.rs @@ -640,6 +640,41 @@ where } } } + + fn register_read( + &self, + result: StorageResult>, + column_id: u32, + ) -> StorageResult> + where + T: NumberOfBytes, + { + self.metrics.read_meter.inc(); + let column_metrics = self.metrics.columns_read_statistic.get(&column_id); + column_metrics.map(|metric| metric.inc()); + + if let Ok(Some(value)) = &result { + self.metrics.bytes_read.inc_by(value.number_of_bytes()); + }; + + result + } +} + +trait NumberOfBytes { + fn number_of_bytes(&self) -> u64; +} + +impl NumberOfBytes for Vec { + fn number_of_bytes(&self) -> u64 { + self.len() as u64 + } +} + +impl NumberOfBytes for usize { + fn number_of_bytes(&self) -> u64 { + *self as u64 + } } pub(crate) struct KeyOnly; @@ -713,20 +748,13 @@ where } fn get(&self, key: &[u8], column: Self::Column) -> StorageResult> { - self.metrics.read_meter.inc(); - let column_metrics = self.metrics.columns_read_statistic.get(&column.id()); - column_metrics.map(|metric| metric.inc()); - - let value = self + let result = self .db .get_cf_opt(&self.cf(column), key, &self.read_options) - .map_err(|e| DatabaseError::Other(e.into()))?; - - if let Some(value) = &value { - self.metrics.bytes_read.inc_by(value.len() as u64); - } + .map_err(|e| StorageError::Other(DatabaseError::Other(e.into()).into())); - Ok(value.map(Arc::new)) + self.register_read(result, column.id()) + .map(|opt| opt.map(Arc::new)) } fn get_batch<'a>( @@ -734,20 +762,20 @@ where keys: BoxedIter<'a, Vec>, column: Self::Column, ) -> BoxedIter<'a, StorageResult>> { - // TODO: Metrics - - let column = self.cf(column); - let keys = keys.map(|key| (&column, key)); + let column_family = self.cf(column); + let keys = keys.map(|key| (&column_family, key)); self.db .multi_get_cf_opt(keys, &self.read_options) .into_iter() - .map(|value_opt| { - value_opt - .map_err(|e| { + .map(move |result| { + self.register_read( + result.map_err(|e| { StorageError::Other(DatabaseError::Other(e.into()).into()) - }) - .map(|value| value.map(Arc::new)) + }), + column.id(), + ) + .map(|opt| opt.map(Arc::new)) }) .into_boxed() } @@ -758,11 +786,7 @@ where column: Self::Column, mut buf: &mut [u8], ) -> StorageResult> { - self.metrics.read_meter.inc(); - let column_metrics = self.metrics.columns_read_statistic.get(&column.id()); - column_metrics.map(|metric| metric.inc()); - - let r = self + let result = self .db .get_pinned_cf_opt(&self.cf(column), key, &self.read_options) .map_err(|e| DatabaseError::Other(e.into()))? @@ -772,13 +796,9 @@ where .map_err(|e| DatabaseError::Other(anyhow::anyhow!(e)))?; StorageResult::Ok(read) }) - .transpose()?; - - if let Some(r) = &r { - self.metrics.bytes_read.inc_by(*r as u64); - } + .transpose(); - Ok(r) + self.register_read(result, column.id()) } } From 676a5d4984453254fa9849f90de2f674425660c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A5rten=20Blankfors?= Date: Tue, 29 Oct 2024 20:11:55 +0100 Subject: [PATCH 24/37] fix: Clippy --- crates/storage/src/blueprint.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/storage/src/blueprint.rs b/crates/storage/src/blueprint.rs index eea1de51965..a7904f53f08 100644 --- a/crates/storage/src/blueprint.rs +++ b/crates/storage/src/blueprint.rs @@ -87,7 +87,7 @@ where column: S::Column, ) -> BoxedIter<'a, StorageResult>> { let keys = keys - .map(|key| Self::KeyCodec::encode(&key).as_bytes().into_owned()) + .map(|key| Self::KeyCodec::encode(key).as_bytes().into_owned()) .into_boxed(); storage From 8b35460e2c60b3de75e0d32f58928aca264884c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A5rten=20Blankfors?= Date: Thu, 31 Oct 2024 09:05:22 +0100 Subject: [PATCH 25/37] fix: Remove stale TODO comment --- crates/storage/src/blueprint.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/crates/storage/src/blueprint.rs b/crates/storage/src/blueprint.rs index a7904f53f08..c0e9c04ddf9 100644 --- a/crates/storage/src/blueprint.rs +++ b/crates/storage/src/blueprint.rs @@ -139,7 +139,6 @@ where fn delete(storage: &mut S, key: &M::Key, column: S::Column) -> StorageResult<()>; } -// TODO: Rename /// It is an extension of the blueprint that allows supporting batch operations. /// Usually, they are more performant than initializing/inserting/removing values one by one. pub trait SupportsBatching: BlueprintMutate From f6977073d73ff04110a102bf5a972b4b3a71c316 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A5rten=20Blankfors?= Date: Thu, 31 Oct 2024 09:12:51 +0100 Subject: [PATCH 26/37] Revert "feat: Add metrics for RocksDB get_batch implementation" This reverts commit 491c452f6fad6976a5302488b4b66cabf16d506b. --- crates/fuel-core/src/state/rocks_db.rs | 82 ++++++++++---------------- 1 file changed, 31 insertions(+), 51 deletions(-) diff --git a/crates/fuel-core/src/state/rocks_db.rs b/crates/fuel-core/src/state/rocks_db.rs index 7b6acbbce28..9749e087b94 100644 --- a/crates/fuel-core/src/state/rocks_db.rs +++ b/crates/fuel-core/src/state/rocks_db.rs @@ -640,41 +640,6 @@ where } } } - - fn register_read( - &self, - result: StorageResult>, - column_id: u32, - ) -> StorageResult> - where - T: NumberOfBytes, - { - self.metrics.read_meter.inc(); - let column_metrics = self.metrics.columns_read_statistic.get(&column_id); - column_metrics.map(|metric| metric.inc()); - - if let Ok(Some(value)) = &result { - self.metrics.bytes_read.inc_by(value.number_of_bytes()); - }; - - result - } -} - -trait NumberOfBytes { - fn number_of_bytes(&self) -> u64; -} - -impl NumberOfBytes for Vec { - fn number_of_bytes(&self) -> u64 { - self.len() as u64 - } -} - -impl NumberOfBytes for usize { - fn number_of_bytes(&self) -> u64 { - *self as u64 - } } pub(crate) struct KeyOnly; @@ -748,13 +713,20 @@ where } fn get(&self, key: &[u8], column: Self::Column) -> StorageResult> { - let result = self + self.metrics.read_meter.inc(); + let column_metrics = self.metrics.columns_read_statistic.get(&column.id()); + column_metrics.map(|metric| metric.inc()); + + let value = self .db .get_cf_opt(&self.cf(column), key, &self.read_options) - .map_err(|e| StorageError::Other(DatabaseError::Other(e.into()).into())); + .map_err(|e| DatabaseError::Other(e.into()))?; + + if let Some(value) = &value { + self.metrics.bytes_read.inc_by(value.len() as u64); + } - self.register_read(result, column.id()) - .map(|opt| opt.map(Arc::new)) + Ok(value.map(Arc::new)) } fn get_batch<'a>( @@ -762,20 +734,20 @@ where keys: BoxedIter<'a, Vec>, column: Self::Column, ) -> BoxedIter<'a, StorageResult>> { - let column_family = self.cf(column); - let keys = keys.map(|key| (&column_family, key)); + // TODO: Metrics + + let column = self.cf(column); + let keys = keys.map(|key| (&column, key)); self.db .multi_get_cf_opt(keys, &self.read_options) .into_iter() - .map(move |result| { - self.register_read( - result.map_err(|e| { + .map(|value_opt| { + value_opt + .map_err(|e| { StorageError::Other(DatabaseError::Other(e.into()).into()) - }), - column.id(), - ) - .map(|opt| opt.map(Arc::new)) + }) + .map(|value| value.map(Arc::new)) }) .into_boxed() } @@ -786,7 +758,11 @@ where column: Self::Column, mut buf: &mut [u8], ) -> StorageResult> { - let result = self + self.metrics.read_meter.inc(); + let column_metrics = self.metrics.columns_read_statistic.get(&column.id()); + column_metrics.map(|metric| metric.inc()); + + let r = self .db .get_pinned_cf_opt(&self.cf(column), key, &self.read_options) .map_err(|e| DatabaseError::Other(e.into()))? @@ -796,9 +772,13 @@ where .map_err(|e| DatabaseError::Other(anyhow::anyhow!(e)))?; StorageResult::Ok(read) }) - .transpose(); + .transpose()?; + + if let Some(r) = &r { + self.metrics.bytes_read.inc_by(*r as u64); + } - self.register_read(result, column.id()) + Ok(r) } } From 3d0a71f59870fc787e1e85d16ce8fcb76864054e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A5rten=20Blankfors?= Date: Thu, 31 Oct 2024 09:20:59 +0100 Subject: [PATCH 27/37] refactor: Return iterator over results in old multi_get function and use it in get_batch --- benches/src/db_lookup_times_utils/utils.rs | 4 ++- .../fuel-core/src/state/historical_rocksdb.rs | 5 ++-- crates/fuel-core/src/state/rocks_db.rs | 30 ++++++------------- 3 files changed, 15 insertions(+), 24 deletions(-) diff --git a/benches/src/db_lookup_times_utils/utils.rs b/benches/src/db_lookup_times_utils/utils.rs index 4db547a5094..96bb7ef86c7 100644 --- a/benches/src/db_lookup_times_utils/utils.rs +++ b/benches/src/db_lookup_times_utils/utils.rs @@ -90,7 +90,9 @@ fn get_block_multi_get_method( .ok_or(anyhow!("empty raw block"))?; let block: CompressedBlock = postcard::from_bytes(raw_block.as_slice())?; let tx_ids = block.transactions().iter(); - let raw_txs = database.multi_get(BenchDbColumn::Transactions.id(), tx_ids)?; + let raw_txs: Vec<_> = database + .multi_get(BenchDbColumn::Transactions.id(), tx_ids) + .try_collect()?; let txs: Vec = raw_txs .iter() .flatten() diff --git a/crates/fuel-core/src/state/historical_rocksdb.rs b/crates/fuel-core/src/state/historical_rocksdb.rs index 52887fdaf2e..77c6e586434 100644 --- a/crates/fuel-core/src/state/historical_rocksdb.rs +++ b/crates/fuel-core/src/state/historical_rocksdb.rs @@ -115,9 +115,10 @@ where let mut reverse_changes = Changes::default(); for (column, column_changes) in changes { - let results = self + let results: Vec<_> = self .db - .multi_get(*column, column_changes.iter().map(|(k, _)| k))?; + .multi_get(*column, column_changes.iter().map(|(k, _)| k)) + .try_collect()?; let entry = reverse_changes .entry(*column) diff --git a/crates/fuel-core/src/state/rocks_db.rs b/crates/fuel-core/src/state/rocks_db.rs index 9749e087b94..acb2c71b705 100644 --- a/crates/fuel-core/src/state/rocks_db.rs +++ b/crates/fuel-core/src/state/rocks_db.rs @@ -31,7 +31,6 @@ use fuel_core_storage::{ Error as StorageError, Result as StorageResult, }; -use itertools::Itertools; use rocksdb::{ BlockBasedOptions, BoundColumnFamily, @@ -534,18 +533,18 @@ where &self, column: u32, iterator: I, - ) -> DatabaseResult>>> + ) -> impl Iterator>>> + '_ where I: Iterator, K: AsRef<[u8]>, { let column_metrics = self.metrics.columns_read_statistic.get(&column); let cl = self.cf_u32(column); - let results = self - .db + + self.db .multi_get_cf_opt(iterator.map(|k| (&cl, k)), &self.read_options) .into_iter() - .map(|el| { + .map(move |el| { self.metrics.read_meter.inc(); column_metrics.map(|metric| metric.inc()); el.map(|value| { @@ -556,8 +555,6 @@ where }) .map_err(|err| DatabaseError::Other(err.into())) }) - .try_collect()?; - Ok(results) } fn _iter_store( @@ -734,20 +731,11 @@ where keys: BoxedIter<'a, Vec>, column: Self::Column, ) -> BoxedIter<'a, StorageResult>> { - // TODO: Metrics - - let column = self.cf(column); - let keys = keys.map(|key| (&column, key)); - - self.db - .multi_get_cf_opt(keys, &self.read_options) - .into_iter() - .map(|value_opt| { - value_opt - .map_err(|e| { - StorageError::Other(DatabaseError::Other(e.into()).into()) - }) - .map(|value| value.map(Arc::new)) + self.multi_get(column.id(), keys) + .map(|result| { + result + .map(|opt| opt.map(Value::from)) + .map_err(|err| StorageError::Other(err.into())) }) .into_boxed() } From 5de80b77b587eed062f3dcfc5769f0d878f7dc3b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A5rten=20Blankfors?= Date: Thu, 31 Oct 2024 09:33:38 +0100 Subject: [PATCH 28/37] perf: Only fall back to fetch off chain transactions if any result is not found --- crates/fuel-core/src/graphql_api/database.rs | 22 ++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/crates/fuel-core/src/graphql_api/database.rs b/crates/fuel-core/src/graphql_api/database.rs index 848931f3b3a..a19ecca258e 100644 --- a/crates/fuel-core/src/graphql_api/database.rs +++ b/crates/fuel-core/src/graphql_api/database.rs @@ -147,12 +147,26 @@ impl ReadView { } pub async fn transactions(&self, tx_ids: &[TxId]) -> Vec> { - let on_chain_results: BTreeMap<_, _> = tx_ids - .iter() - .enumerate() - .zip(self.on_chain.transactions(tx_ids.iter().into_boxed())) + let transactions: Vec<_> = self + .on_chain + .transactions(tx_ids.iter().into_boxed()) .collect(); + // Give a chance for other tasks to run. + tokio::task::yield_now().await; + + if transactions.iter().any(|result| result.is_not_found()) { + let on_chain_results = tx_ids.iter().enumerate().zip(transactions).collect(); + + self.extend_with_off_chain_results(on_chain_results).await + } else { + transactions + } + } + pub async fn extend_with_off_chain_results( + &self, + on_chain_results: BTreeMap<(usize, &TxId), StorageResult>, + ) -> Vec> { let off_chain_indexed_txids: Vec<_> = on_chain_results .iter() .filter_map(|(indexed_tx_id, result)| { From d7a30c8c6db96f5365e6d5dae6b34b5aa44b21bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A5rten=20Blankfors?= Date: Thu, 31 Oct 2024 09:42:20 +0100 Subject: [PATCH 29/37] fix: Whitespace --- crates/fuel-core/src/graphql_api/database.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/fuel-core/src/graphql_api/database.rs b/crates/fuel-core/src/graphql_api/database.rs index a19ecca258e..bb1d54049a1 100644 --- a/crates/fuel-core/src/graphql_api/database.rs +++ b/crates/fuel-core/src/graphql_api/database.rs @@ -163,6 +163,7 @@ impl ReadView { transactions } } + pub async fn extend_with_off_chain_results( &self, on_chain_results: BTreeMap<(usize, &TxId), StorageResult>, From f27bdbbb29a930bbf7b60d0350db3572a23161b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A5rten=20Blankfors?= Date: Thu, 31 Oct 2024 09:53:14 +0100 Subject: [PATCH 30/37] feat: Take Cow as keys in `KeyValueInspect::get_batch` --- crates/fuel-core/src/state/data_source.rs | 7 +++++-- .../fuel-core/src/state/generic_database.rs | 2 +- .../src/state/iterable_key_value_view.rs | 7 +++++-- crates/fuel-core/src/state/key_value_view.rs | 7 +++++-- crates/fuel-core/src/state/rocks_db.rs | 3 ++- crates/storage/src/blueprint.rs | 7 +++++-- crates/storage/src/codec.rs | 21 ++++++++++++++----- crates/storage/src/kv_store.rs | 3 ++- crates/storage/src/structured_storage.rs | 3 ++- 9 files changed, 43 insertions(+), 17 deletions(-) diff --git a/crates/fuel-core/src/state/data_source.rs b/crates/fuel-core/src/state/data_source.rs index dd861ea2ac4..702085aa5c2 100644 --- a/crates/fuel-core/src/state/data_source.rs +++ b/crates/fuel-core/src/state/data_source.rs @@ -15,7 +15,10 @@ use fuel_core_storage::{ }, Result as StorageResult, }; -use std::sync::Arc; +use std::{ + borrow::Cow, + sync::Arc, +}; #[allow(type_alias_bounds)] pub type DataSourceType @@ -66,7 +69,7 @@ where fn get_batch<'a>( &'a self, - keys: BoxedIter<'a, Vec>, + keys: BoxedIter<'a, Cow<'a, [u8]>>, column: Self::Column, ) -> BoxedIter<'a, StorageResult>> { self.data.get_batch(keys, column) diff --git a/crates/fuel-core/src/state/generic_database.rs b/crates/fuel-core/src/state/generic_database.rs index a71870bc405..178d1066a22 100644 --- a/crates/fuel-core/src/state/generic_database.rs +++ b/crates/fuel-core/src/state/generic_database.rs @@ -136,7 +136,7 @@ where fn get_batch<'a>( &'a self, - keys: BoxedIter<'a, Vec>, + keys: BoxedIter<'a, Cow<'a, [u8]>>, column: Self::Column, ) -> BoxedIter<'a, StorageResult>> { KeyValueInspect::get_batch(&self.storage, keys, column) diff --git a/crates/fuel-core/src/state/iterable_key_value_view.rs b/crates/fuel-core/src/state/iterable_key_value_view.rs index 4b739967735..f8a8870a11a 100644 --- a/crates/fuel-core/src/state/iterable_key_value_view.rs +++ b/crates/fuel-core/src/state/iterable_key_value_view.rs @@ -13,7 +13,10 @@ use fuel_core_storage::{ }, Result as StorageResult, }; -use std::sync::Arc; +use std::{ + borrow::Cow, + sync::Arc, +}; #[derive(Clone)] pub struct IterableKeyValueViewWrapper( @@ -65,7 +68,7 @@ where fn get_batch<'a>( &'a self, - keys: BoxedIter<'a, Vec>, + keys: BoxedIter<'a, Cow<'a, [u8]>>, column: Self::Column, ) -> BoxedIter<'a, StorageResult>> { self.0.get_batch(keys, column) diff --git a/crates/fuel-core/src/state/key_value_view.rs b/crates/fuel-core/src/state/key_value_view.rs index 84d99f7be79..a62d14cf097 100644 --- a/crates/fuel-core/src/state/key_value_view.rs +++ b/crates/fuel-core/src/state/key_value_view.rs @@ -7,7 +7,10 @@ use fuel_core_storage::{ }, Result as StorageResult, }; -use std::sync::Arc; +use std::{ + borrow::Cow, + sync::Arc, +}; #[derive(Clone)] pub struct KeyValueViewWrapper( @@ -53,7 +56,7 @@ where fn get_batch<'a>( &'a self, - keys: BoxedIter<'a, Vec>, + keys: BoxedIter<'a, Cow<'a, [u8]>>, column: Self::Column, ) -> BoxedIter<'a, StorageResult>> { self.0.get_batch(keys, column) diff --git a/crates/fuel-core/src/state/rocks_db.rs b/crates/fuel-core/src/state/rocks_db.rs index acb2c71b705..ada8b458fa9 100644 --- a/crates/fuel-core/src/state/rocks_db.rs +++ b/crates/fuel-core/src/state/rocks_db.rs @@ -48,6 +48,7 @@ use rocksdb::{ WriteBatch, }; use std::{ + borrow::Cow, cmp, collections::BTreeMap, fmt, @@ -728,7 +729,7 @@ where fn get_batch<'a>( &'a self, - keys: BoxedIter<'a, Vec>, + keys: BoxedIter<'a, Cow<'a, [u8]>>, column: Self::Column, ) -> BoxedIter<'a, StorageResult>> { self.multi_get(column.id(), keys) diff --git a/crates/storage/src/blueprint.rs b/crates/storage/src/blueprint.rs index c0e9c04ddf9..a0e7d185144 100644 --- a/crates/storage/src/blueprint.rs +++ b/crates/storage/src/blueprint.rs @@ -85,9 +85,12 @@ where storage: &'a S, keys: BoxedIter<'a, &'a M::Key>, column: S::Column, - ) -> BoxedIter<'a, StorageResult>> { + ) -> BoxedIter<'a, StorageResult>> + where + Self::KeyCodec: 'a, + { let keys = keys - .map(|key| Self::KeyCodec::encode(key).as_bytes().into_owned()) + .map(|key| Self::KeyCodec::encode(key).into_bytes()) .into_boxed(); storage diff --git a/crates/storage/src/codec.rs b/crates/storage/src/codec.rs index 9154112fe55..f80dbfb6d1f 100644 --- a/crates/storage/src/codec.rs +++ b/crates/storage/src/codec.rs @@ -16,9 +16,12 @@ pub mod primitive; pub mod raw; /// The trait is usually implemented by the encoder that stores serialized objects. -pub trait Encoder { +pub trait Encoder<'a> { /// Returns the serialized object as a slice. fn as_bytes(&self) -> Cow<[u8]>; + + /// Consumes the encoder, returning the serialized object as a slice. + fn into_bytes(self) -> Cow<'a, [u8]>; } /// The trait encodes the type to the bytes and passes it to the `Encoder`, @@ -28,12 +31,12 @@ pub trait Encoder { /// it is always possible to take ownership of the serialized value. pub trait Encode { /// The encoder type that stores serialized object. - type Encoder<'a>: Encoder + type Encoder<'a>: Encoder<'a> where T: 'a; /// Encodes the object to the bytes and passes it to the `Encoder`. - fn encode(t: &T) -> Self::Encoder<'_>; + fn encode<'a>(t: &'a T) -> Self::Encoder<'a>; /// Returns the serialized object as an [`Value`]. fn encode_as_value(t: &T) -> Value { @@ -52,17 +55,25 @@ pub trait Decode { } } -impl<'a> Encoder for Cow<'a, [u8]> { +impl<'a> Encoder<'a> for Cow<'a, [u8]> { fn as_bytes(&self) -> Cow<[u8]> { match self { Cow::Borrowed(borrowed) => Cow::Borrowed(borrowed), Cow::Owned(owned) => Cow::Borrowed(owned.as_ref()), } } + + fn into_bytes(self) -> Cow<'a, [u8]> { + self + } } -impl Encoder for [u8; SIZE] { +impl<'a, const SIZE: usize> Encoder<'a> for [u8; SIZE] { fn as_bytes(&self) -> Cow<[u8]> { Cow::Borrowed(self.as_slice()) } + + fn into_bytes(self) -> Cow<'a, [u8]> { + Cow::Owned(self.to_vec()) + } } diff --git a/crates/storage/src/kv_store.rs b/crates/storage/src/kv_store.rs index 462935b9611..42fb1dda9b2 100644 --- a/crates/storage/src/kv_store.rs +++ b/crates/storage/src/kv_store.rs @@ -11,6 +11,7 @@ use crate::{ #[cfg(feature = "alloc")] use alloc::{ + borrow::Cow, boxed::Box, string::String, vec::Vec, @@ -69,7 +70,7 @@ pub trait KeyValueInspect: Send + Sync { /// Returns multiple values from the storage. fn get_batch<'a>( &'a self, - keys: BoxedIter<'a, Vec>, + keys: BoxedIter<'a, Cow<'a, [u8]>>, column: Self::Column, ) -> BoxedIter<'a, StorageResult>> { keys.map(move |key| self.get(&key, column)).into_boxed() diff --git a/crates/storage/src/structured_storage.rs b/crates/storage/src/structured_storage.rs index a58b51ebe42..b6a36028058 100644 --- a/crates/storage/src/structured_storage.rs +++ b/crates/storage/src/structured_storage.rs @@ -148,7 +148,7 @@ where fn get_batch<'a>( &'a self, - keys: BoxedIter<'a, Vec>, + keys: BoxedIter<'a, Cow<'a, [u8]>>, column: Self::Column, ) -> BoxedIter<'a, StorageResult>> { self.inner.get_batch(keys, column) @@ -283,6 +283,7 @@ where S: KeyValueInspect, M: TableWithBlueprint, M::Blueprint: BlueprintInspect>, + >>::KeyCodec: 'static, { fn get_batch<'a>( &'a self, From 2c2d2222f5b66ca8d287a23ec582488b8df251fe Mon Sep 17 00:00:00 2001 From: Green Baneling Date: Thu, 31 Oct 2024 12:00:00 +0100 Subject: [PATCH 31/37] Proposals to multi get PR (#2419) Proposals to the https://github.com/FuelLabs/fuel-core/pull/2396 --- crates/fuel-core/src/graphql_api/ports.rs | 10 ++----- .../fuel-core/src/state/generic_database.rs | 10 +++++-- .../fuel-core/src/state/historical_rocksdb.rs | 18 +++++++++++ crates/storage/src/blueprint.rs | 30 ++++++++----------- crates/storage/src/codec.rs | 2 +- crates/storage/src/kv_store.rs | 16 ++++++---- crates/storage/src/lib.rs | 10 ++++--- crates/storage/src/structured_storage.rs | 10 +++++-- crates/storage/src/transactional.rs | 18 ++++++++++- 9 files changed, 83 insertions(+), 41 deletions(-) diff --git a/crates/fuel-core/src/graphql_api/ports.rs b/crates/fuel-core/src/graphql_api/ports.rs index dbc2a200808..2ce9ecdf4b8 100644 --- a/crates/fuel-core/src/graphql_api/ports.rs +++ b/crates/fuel-core/src/graphql_api/ports.rs @@ -16,7 +16,6 @@ use fuel_core_storage::{ }, Error as StorageError, Result as StorageResult, - StorageBatchInspect, StorageInspect, StorageRead, }; @@ -174,9 +173,7 @@ pub trait DatabaseDaCompressedBlocks { } /// Trait that specifies all the getters required for messages. -pub trait DatabaseMessages: - StorageInspect + StorageBatchInspect -{ +pub trait DatabaseMessages: StorageInspect { fn all_messages( &self, start_message_id: Option, @@ -199,10 +196,9 @@ pub trait DatabaseRelayedTransactions { } /// Trait that specifies all the getters required for coins -pub trait DatabaseCoins: - StorageInspect + StorageBatchInspect -{ +pub trait DatabaseCoins: StorageInspect { fn coin(&self, utxo_id: UtxoId) -> StorageResult; + fn coins<'a>( &'a self, utxo_ids: BoxedIter<'a, &'a UtxoId>, diff --git a/crates/fuel-core/src/state/generic_database.rs b/crates/fuel-core/src/state/generic_database.rs index 178d1066a22..6d0114c4508 100644 --- a/crates/fuel-core/src/state/generic_database.rs +++ b/crates/fuel-core/src/state/generic_database.rs @@ -70,10 +70,14 @@ where M: Mappable, StructuredStorage: StorageBatchInspect, { - fn get_batch<'a>( + fn get_batch<'a, Iter>( &'a self, - keys: BoxedIter<'a, &'a M::Key>, - ) -> BoxedIter<'a, StorageResult::OwnedValue>>> { + keys: Iter, + ) -> impl Iterator>> + 'a + where + Iter: 'a + Iterator + Send, + M::Key: 'a, + { self.storage.get_batch(keys) } } diff --git a/crates/fuel-core/src/state/historical_rocksdb.rs b/crates/fuel-core/src/state/historical_rocksdb.rs index 77c6e586434..559505ba67b 100644 --- a/crates/fuel-core/src/state/historical_rocksdb.rs +++ b/crates/fuel-core/src/state/historical_rocksdb.rs @@ -29,6 +29,7 @@ use crate::{ use fuel_core_storage::{ iter::{ BoxedIter, + IntoBoxedIter, IterDirection, IterableStore, IteratorOverTable, @@ -36,6 +37,7 @@ use fuel_core_storage::{ kv_store::{ KVItem, KeyValueInspect, + StorageColumn, Value, WriteOperation, }, @@ -56,6 +58,7 @@ use serde::{ Serialize, }; use std::{ + borrow::Cow, num::NonZeroU64, path::Path, }; @@ -395,6 +398,21 @@ where self.db.get(key, Column::OriginalColumn(column)) } + fn get_batch<'a>( + &'a self, + keys: BoxedIter<'a, Cow<'a, [u8]>>, + column: Self::Column, + ) -> BoxedIter<'a, StorageResult>> { + self.db + .multi_get(Column::::OriginalColumn(column).id(), keys) + .map(|result| { + result + .map(|value| value.map(Into::into)) + .map_err(Into::into) + }) + .into_boxed() + } + fn read( &self, key: &[u8], diff --git a/crates/storage/src/blueprint.rs b/crates/storage/src/blueprint.rs index a0e7d185144..656b76f615c 100644 --- a/crates/storage/src/blueprint.rs +++ b/crates/storage/src/blueprint.rs @@ -9,10 +9,7 @@ use crate::{ Encode, Encoder, }, - iter::{ - BoxedIter, - IntoBoxedIter, - }, + iter::IntoBoxedIter, kv_store::{ BatchOperations, KeyValueInspect, @@ -81,30 +78,29 @@ where } /// Returns multiple values from the storage. - fn get_batch<'a>( + fn get_batch<'a, Iter>( storage: &'a S, - keys: BoxedIter<'a, &'a M::Key>, + keys: Iter, column: S::Column, - ) -> BoxedIter<'a, StorageResult>> + ) -> impl Iterator>> + 'a where + Iter: 'a + Iterator + Send, + M::Key: 'a, Self::KeyCodec: 'a, { let keys = keys .map(|key| Self::KeyCodec::encode(key).into_bytes()) .into_boxed(); - storage - .get_batch(keys, column) - .map(|result| { - result.and_then(|opt| { - opt.map(|value| { - Self::ValueCodec::decode_from_value(value) - .map_err(crate::Error::Codec) - }) - .transpose() + storage.get_batch(keys, column).map(|result| { + result.and_then(|opt| { + opt.map(|value| { + Self::ValueCodec::decode_from_value(value) + .map_err(crate::Error::Codec) }) + .transpose() }) - .into_boxed() + }) } } diff --git a/crates/storage/src/codec.rs b/crates/storage/src/codec.rs index f80dbfb6d1f..f77f91cc044 100644 --- a/crates/storage/src/codec.rs +++ b/crates/storage/src/codec.rs @@ -36,7 +36,7 @@ pub trait Encode { T: 'a; /// Encodes the object to the bytes and passes it to the `Encoder`. - fn encode<'a>(t: &'a T) -> Self::Encoder<'a>; + fn encode(t: &T) -> Self::Encoder<'_>; /// Returns the serialized object as an [`Value`]. fn encode_as_value(t: &T) -> Value { diff --git a/crates/storage/src/kv_store.rs b/crates/storage/src/kv_store.rs index 42fb1dda9b2..46f6b404f1c 100644 --- a/crates/storage/src/kv_store.rs +++ b/crates/storage/src/kv_store.rs @@ -1,10 +1,7 @@ //! The module provides plain abstract definition of the key-value store. use crate::{ - iter::{ - BoxedIter, - IntoBoxedIter, - }, + iter::BoxedIter, Error as StorageError, Result as StorageResult, }; @@ -17,6 +14,7 @@ use alloc::{ vec::Vec, }; +use crate::iter::IntoBoxedIter; #[cfg(feature = "std")] use core::ops::Deref; @@ -31,7 +29,7 @@ pub type KVItem = StorageResult<(Key, Value)>; pub type KeyItem = StorageResult; /// A column of the storage. -pub trait StorageColumn: Copy + core::fmt::Debug + Send + Sync { +pub trait StorageColumn: Copy + core::fmt::Debug + Send + Sync + 'static { /// Returns the name of the column. fn name(&self) -> String; @@ -121,6 +119,14 @@ where self.deref().get(key, column) } + fn get_batch<'a>( + &'a self, + keys: BoxedIter<'a, Cow<'a, [u8]>>, + column: Self::Column, + ) -> BoxedIter<'a, StorageResult>> { + self.deref().get_batch(keys, column) + } + fn read( &self, key: &[u8], diff --git a/crates/storage/src/lib.rs b/crates/storage/src/lib.rs index 2def12e2d03..1977dacd1ef 100644 --- a/crates/storage/src/lib.rs +++ b/crates/storage/src/lib.rs @@ -17,7 +17,6 @@ extern crate alloc; use anyhow::anyhow; use core::array::TryFromSliceError; use fuel_core_types::services::executor::Error as ExecutorError; -use iter::BoxedIter; #[cfg(feature = "alloc")] use alloc::{ @@ -182,10 +181,13 @@ pub trait StorageBatchMutate: StorageMutate { /// getting values one by one. pub trait StorageBatchInspect { /// Get a batch of values associated with the provided keys. - fn get_batch<'a>( + fn get_batch<'a, Iter>( &'a self, - keys: BoxedIter<'a, &'a Type::Key>, - ) -> BoxedIter<'a, Result>>; + keys: Iter, + ) -> impl Iterator>> + 'a + where + Iter: 'a + Iterator + Send, + Type::Key: 'a; } /// Creates `StorageError::NotFound` error with file and line information inside. diff --git a/crates/storage/src/structured_storage.rs b/crates/storage/src/structured_storage.rs index b6a36028058..d0396a528f2 100644 --- a/crates/storage/src/structured_storage.rs +++ b/crates/storage/src/structured_storage.rs @@ -285,10 +285,14 @@ where M::Blueprint: BlueprintInspect>, >>::KeyCodec: 'static, { - fn get_batch<'a>( + fn get_batch<'a, Iter>( &'a self, - keys: BoxedIter<'a, &'a M::Key>, - ) -> BoxedIter<'a, StorageResult>> { + keys: Iter, + ) -> impl Iterator>> + 'a + where + Iter: 'a + Iterator + Send, + M::Key: 'a, + { M::Blueprint::get_batch(self, keys, M::column()) } } diff --git a/crates/storage/src/transactional.rs b/crates/storage/src/transactional.rs index 14ec74159ed..14cb4cfe0c8 100644 --- a/crates/storage/src/transactional.rs +++ b/crates/storage/src/transactional.rs @@ -1,6 +1,10 @@ //! The primitives to work with storage in transactional mode. use crate::{ + iter::{ + BoxedIter, + IntoBoxedIter, + }, kv_store::{ BatchOperations, KeyValueInspect, @@ -14,6 +18,7 @@ use crate::{ }; use core::borrow::Borrow; +use alloc::borrow::Cow; #[cfg(feature = "alloc")] use alloc::{ boxed::Box, @@ -27,7 +32,6 @@ use alloc::{ #[cfg(feature = "test-helpers")] use crate::{ iter::{ - BoxedIter, IterDirection, IterableStore, }, @@ -384,6 +388,18 @@ where } } + fn get_batch<'a>( + &'a self, + keys: BoxedIter<'a, Cow<'a, [u8]>>, + column: Self::Column, + ) -> BoxedIter<'a, StorageResult>> { + if !self.changes.contains_key(&column.id()) { + return self.storage.get_batch(keys, column); + } else { + keys.map(move |key| self.get(&key, column)).into_boxed() + } + } + fn read( &self, key: &[u8], From df33ef2bde75eb7a445232371d1a87926756487d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A5rten=20Blankfors?= Date: Thu, 31 Oct 2024 13:03:36 +0100 Subject: [PATCH 32/37] feat: Simplify `DatabaseCoins` port --- crates/fuel-core/src/graphql_api/ports.rs | 5 +---- crates/fuel-core/src/query/coin.rs | 7 ++---- .../service/adapters/graphql_api/on_chain.rs | 22 +++++++------------ 3 files changed, 11 insertions(+), 23 deletions(-) diff --git a/crates/fuel-core/src/graphql_api/ports.rs b/crates/fuel-core/src/graphql_api/ports.rs index 2ce9ecdf4b8..1e65554853d 100644 --- a/crates/fuel-core/src/graphql_api/ports.rs +++ b/crates/fuel-core/src/graphql_api/ports.rs @@ -199,10 +199,7 @@ pub trait DatabaseRelayedTransactions { pub trait DatabaseCoins: StorageInspect { fn coin(&self, utxo_id: UtxoId) -> StorageResult; - fn coins<'a>( - &'a self, - utxo_ids: BoxedIter<'a, &'a UtxoId>, - ) -> BoxedIter<'a, StorageResult>; + fn coins<'a>(&'a self, utxo_ids: &'a [UtxoId]) -> BoxedIter<'a, StorageResult>; } /// Trait that specifies all the getters required for contract. diff --git a/crates/fuel-core/src/query/coin.rs b/crates/fuel-core/src/query/coin.rs index 7f8ea37424e..6fcec9fe106 100644 --- a/crates/fuel-core/src/query/coin.rs +++ b/crates/fuel-core/src/query/coin.rs @@ -1,9 +1,6 @@ use crate::fuel_core_graphql_api::database::ReadView; use fuel_core_storage::{ - iter::{ - IntoBoxedIter, - IterDirection, - }, + iter::IterDirection, Error as StorageError, Result as StorageResult, }; @@ -27,7 +24,7 @@ impl ReadView { &self, utxo_ids: Vec, ) -> impl Iterator> + '_ { - let coins: Vec<_> = self.on_chain.coins(utxo_ids.iter().into_boxed()).collect(); + let coins: Vec<_> = self.on_chain.coins(&utxo_ids).collect(); // Give a chance for other tasks to run. tokio::task::yield_now().await; diff --git a/crates/fuel-core/src/service/adapters/graphql_api/on_chain.rs b/crates/fuel-core/src/service/adapters/graphql_api/on_chain.rs index 984155fda55..b392482237c 100644 --- a/crates/fuel-core/src/service/adapters/graphql_api/on_chain.rs +++ b/crates/fuel-core/src/service/adapters/graphql_api/on_chain.rs @@ -42,7 +42,10 @@ use fuel_core_types::{ consensus::Consensus, primitives::DaBlockHeight, }, - entities::relayer::message::Message, + entities::{ + coins::coin::Coin, + relayer::message::Message, + }, fuel_tx::{ AssetId, ContractId, @@ -134,10 +137,7 @@ impl DatabaseMessages for OnChainIterableKeyValueView { } impl DatabaseCoins for OnChainIterableKeyValueView { - fn coin( - &self, - utxo_id: fuel_core_types::fuel_tx::UtxoId, - ) -> StorageResult { + fn coin(&self, utxo_id: fuel_core_types::fuel_tx::UtxoId) -> StorageResult { let coin = self .storage::() .get(&utxo_id)? @@ -147,18 +147,12 @@ impl DatabaseCoins for OnChainIterableKeyValueView { Ok(coin.uncompress(utxo_id)) } - fn coins<'a>( - &'a self, - utxo_ids: BoxedIter<'a, &'a UtxoId>, - ) -> BoxedIter<'a, StorageResult> { - let utxo_ids_1: Vec<_> = utxo_ids.collect(); - let utxo_ids_2 = utxo_ids_1.clone(); - + fn coins<'a>(&'a self, utxo_ids: &'a [UtxoId]) -> BoxedIter<'a, StorageResult> { >::get_batch( self, - utxo_ids_1.into_iter().into_boxed(), + utxo_ids.iter().into_boxed(), ) - .zip(utxo_ids_2.into_iter().into_boxed()) + .zip(utxo_ids.iter().into_boxed()) .map(|(res, utxo_id)| { res.and_then(|opt| opt.ok_or(not_found!(Coins))) .map(|coin| coin.uncompress(*utxo_id)) From 79e42daf12d66b50f67a07feb98e99858c32c800 Mon Sep 17 00:00:00 2001 From: Green Baneling Date: Thu, 31 Oct 2024 14:29:36 +0100 Subject: [PATCH 33/37] Another proposals to multi get PR (#2420) Since this PR add `into_bytes` for the encoder, we can optimize the batch mutate operations as well https://github.com/FuelLabs/fuel-core/pull/2396#pullrequestreview-2407673340 --- crates/storage/src/blueprint.rs | 1 - crates/storage/src/blueprint/plain.rs | 4 ++-- crates/storage/src/blueprint/sparse.rs | 10 ++++------ crates/storage/src/codec.rs | 2 +- crates/storage/src/kv_store.rs | 8 ++++++-- crates/storage/src/structured_storage.rs | 8 ++++++-- crates/storage/src/transactional.rs | 6 +++--- 7 files changed, 22 insertions(+), 17 deletions(-) diff --git a/crates/storage/src/blueprint.rs b/crates/storage/src/blueprint.rs index 656b76f615c..9845c37b967 100644 --- a/crates/storage/src/blueprint.rs +++ b/crates/storage/src/blueprint.rs @@ -86,7 +86,6 @@ where where Iter: 'a + Iterator + Send, M::Key: 'a, - Self::KeyCodec: 'a, { let keys = keys .map(|key| Self::KeyCodec::encode(key).into_bytes()) diff --git a/crates/storage/src/blueprint/plain.rs b/crates/storage/src/blueprint/plain.rs index 509fc1e9dab..9c41ac63b2f 100644 --- a/crates/storage/src/blueprint/plain.rs +++ b/crates/storage/src/blueprint/plain.rs @@ -134,7 +134,7 @@ where set.map(|(key, value)| { let key_encoder = >::KeyCodec::encode(key); - let key_bytes = key_encoder.as_bytes().to_vec(); + let key_bytes = key_encoder.into_bytes(); let value = >::ValueCodec::encode_as_value( value, @@ -158,7 +158,7 @@ where set.map(|key| { let key_encoder = >::KeyCodec::encode(key); - let key_bytes = key_encoder.as_bytes().to_vec(); + let key_bytes = key_encoder.into_bytes(); (key_bytes, WriteOperation::Remove) }), ) diff --git a/crates/storage/src/blueprint/sparse.rs b/crates/storage/src/blueprint/sparse.rs index 9d21db77d00..fdd54d266ce 100644 --- a/crates/storage/src/blueprint/sparse.rs +++ b/crates/storage/src/blueprint/sparse.rs @@ -326,7 +326,7 @@ where let encoded_set = set .map(|(key, value)| { - let key = KeyCodec::encode(key).as_bytes().into_owned(); + let key = KeyCodec::encode(key).into_bytes(); let value = ValueCodec::encode(value).as_bytes().into_owned(); (key, value) }) @@ -346,9 +346,7 @@ where )?; let nodes = nodes.iter().map(|(key, value)| { - let key = NodeKeyCodec::::encode(key) - .as_bytes() - .into_owned(); + let key = NodeKeyCodec::::encode(key).into_bytes(); let value = NodeValueCodec::::encode_as_value(value); (key, WriteOperation::Insert(value)) }); @@ -392,7 +390,7 @@ where let encoded_set = set .map(|(key, value)| { - let key = KeyCodec::encode(key).as_bytes().into_owned(); + let key = KeyCodec::encode(key).into_bytes(); let value = ValueCodec::encode(value).as_bytes().into_owned(); (key, value) }) @@ -449,7 +447,7 @@ where .map_err(|err| StorageError::Other(anyhow::anyhow!("{err:?}")))?; let encoded_set = set - .map(|key| KeyCodec::encode(key).as_bytes().into_owned()) + .map(|key| KeyCodec::encode(key).into_bytes()) .collect_vec(); for key_bytes in encoded_set.iter() { diff --git a/crates/storage/src/codec.rs b/crates/storage/src/codec.rs index f77f91cc044..373f5d298cf 100644 --- a/crates/storage/src/codec.rs +++ b/crates/storage/src/codec.rs @@ -29,7 +29,7 @@ pub trait Encoder<'a> { /// flexibility and more performant encoding, allowing the use of slices and arrays /// instead of vectors in some cases. Since the [`Encoder`] returns `Cow<[u8]>`, /// it is always possible to take ownership of the serialized value. -pub trait Encode { +pub trait Encode: 'static { /// The encoder type that stores serialized object. type Encoder<'a>: Encoder<'a> where diff --git a/crates/storage/src/kv_store.rs b/crates/storage/src/kv_store.rs index 46f6b404f1c..7d4564395d3 100644 --- a/crates/storage/src/kv_store.rs +++ b/crates/storage/src/kv_store.rs @@ -194,7 +194,11 @@ pub enum WriteOperation { #[impl_tools::autoimpl(for &mut T, Box)] pub trait BatchOperations: KeyValueMutate { /// Writes the batch of the entries into the storage. - fn batch_write(&mut self, column: Self::Column, entries: I) -> StorageResult<()> + fn batch_write<'a, I>( + &mut self, + column: Self::Column, + entries: I, + ) -> StorageResult<()> where - I: Iterator, WriteOperation)>; + I: Iterator, WriteOperation)> + 'a; } diff --git a/crates/storage/src/structured_storage.rs b/crates/storage/src/structured_storage.rs index d0396a528f2..3bdb6f2f30b 100644 --- a/crates/storage/src/structured_storage.rs +++ b/crates/storage/src/structured_storage.rs @@ -218,9 +218,13 @@ impl BatchOperations for StructuredStorage where S: BatchOperations, { - fn batch_write(&mut self, column: Self::Column, entries: I) -> StorageResult<()> + fn batch_write<'a, I>( + &mut self, + column: Self::Column, + entries: I, + ) -> StorageResult<()> where - I: Iterator, WriteOperation)>, + I: Iterator, WriteOperation)> + 'a, { self.inner.batch_write(column, entries) } diff --git a/crates/storage/src/transactional.rs b/crates/storage/src/transactional.rs index 14cb4cfe0c8..9041b56e07e 100644 --- a/crates/storage/src/transactional.rs +++ b/crates/storage/src/transactional.rs @@ -519,13 +519,13 @@ where Column: StorageColumn, S: KeyValueInspect, { - fn batch_write(&mut self, column: Column, entries: I) -> StorageResult<()> + fn batch_write<'a, I>(&mut self, column: Column, entries: I) -> StorageResult<()> where - I: Iterator, WriteOperation)>, + I: Iterator, WriteOperation)> + 'a, { let btree = self.changes.entry(column.id()).or_default(); entries.for_each(|(key, operation)| { - btree.insert(key.into(), operation); + btree.insert(key.to_vec().into(), operation); }); Ok(()) } From cbb6efc4332490122e8de70d082c7048cc97e0a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A5rten=20Blankfors?= Date: Fri, 1 Nov 2024 09:02:39 +0100 Subject: [PATCH 34/37] feat: Don't require `Send` in BoxedIter --- crates/fuel-core/src/database/block.rs | 4 +- crates/fuel-core/src/graphql_api/database.rs | 13 +-- crates/fuel-core/src/graphql_api/ports.rs | 20 ++-- crates/fuel-core/src/query/message.rs | 8 +- .../service/adapters/graphql_api/off_chain.rs | 18 ++-- .../service/adapters/graphql_api/on_chain.rs | 23 +++-- .../service/genesis/importer/import_task.rs | 6 +- crates/fuel-core/src/state/data_source.rs | 5 +- .../fuel-core/src/state/generic_database.rs | 7 +- .../fuel-core/src/state/historical_rocksdb.rs | 5 +- .../src/state/in_memory/memory_store.rs | 12 +-- .../src/state/in_memory/memory_view.rs | 12 +-- .../src/state/iterable_key_value_view.rs | 5 +- crates/fuel-core/src/state/rocks_db.rs | 25 ++--- crates/storage/src/blueprint.rs | 3 +- crates/storage/src/iter.rs | 96 ++++++++++++------- crates/storage/src/iter/changes_iterator.rs | 16 ++-- crates/storage/src/kv_store.rs | 8 +- crates/storage/src/lib.rs | 2 +- crates/storage/src/structured_storage.rs | 7 +- crates/storage/src/transactional.rs | 5 +- 21 files changed, 177 insertions(+), 123 deletions(-) diff --git a/crates/fuel-core/src/database/block.rs b/crates/fuel-core/src/database/block.rs index 7c9ab130072..00b39ab23b5 100644 --- a/crates/fuel-core/src/database/block.rs +++ b/crates/fuel-core/src/database/block.rs @@ -7,7 +7,7 @@ use crate::{ }; use fuel_core_storage::{ iter::{ - IntoBoxedIter, + IntoBoxedIterSend, IterDirection, IteratorOverTable, }, @@ -68,7 +68,7 @@ impl OnChainIterableKeyValueView { let db_block = self.storage::().get(height)?; if let Some(block) = db_block { - let transaction_ids = block.transactions().iter().into_boxed(); + let transaction_ids = block.transactions().iter().into_boxed_send(); let txs = >::get_batch( self, transaction_ids, diff --git a/crates/fuel-core/src/graphql_api/database.rs b/crates/fuel-core/src/graphql_api/database.rs index bb1d54049a1..287189c9025 100644 --- a/crates/fuel-core/src/graphql_api/database.rs +++ b/crates/fuel-core/src/graphql_api/database.rs @@ -8,7 +8,8 @@ use crate::fuel_core_graphql_api::{ use fuel_core_services::yield_stream::StreamYieldExt; use fuel_core_storage::{ iter::{ - BoxedIter, + BoxedIterSend, + IntoBoxedIterSend, IntoBoxedIter, IterDirection, }, @@ -205,7 +206,7 @@ impl ReadView { &self, height: Option, direction: IterDirection, - ) -> BoxedIter<'_, StorageResult> { + ) -> BoxedIterSend<'_, StorageResult> { // Chain together blocks from the off-chain db and the on-chain db // The blocks in off-chain db, if any, are from time before regenesis @@ -218,12 +219,12 @@ impl ReadView { .on_chain .blocks(Some(height), direction) .chain(self.off_chain.old_blocks(None, direction)) - .into_boxed(), + .into_boxed_send(), (false, IterDirection::Forward) => self .off_chain .old_blocks(Some(height), direction) .chain(self.on_chain.blocks(None, direction)) - .into_boxed(), + .into_boxed_send(), (false, IterDirection::Reverse) => { self.off_chain.old_blocks(Some(height), direction) } @@ -234,12 +235,12 @@ impl ReadView { .off_chain .old_blocks(None, direction) .chain(self.on_chain.blocks(None, direction)) - .into_boxed(), + .into_boxed_send(), IterDirection::Reverse => self .on_chain .blocks(None, direction) .chain(self.off_chain.old_blocks(None, direction)) - .into_boxed(), + .into_boxed_send(), } } } diff --git a/crates/fuel-core/src/graphql_api/ports.rs b/crates/fuel-core/src/graphql_api/ports.rs index 1e65554853d..243b0894bcc 100644 --- a/crates/fuel-core/src/graphql_api/ports.rs +++ b/crates/fuel-core/src/graphql_api/ports.rs @@ -2,6 +2,7 @@ use async_trait::async_trait; use fuel_core_services::stream::BoxStream; use fuel_core_storage::{ iter::{ + BoxedIterSend, BoxedIter, IterDirection, }, @@ -79,21 +80,21 @@ pub trait OffChainDatabase: Send + Sync { owner: &Address, start_coin: Option, direction: IterDirection, - ) -> BoxedIter<'_, StorageResult>; + ) -> BoxedIterSend<'_, StorageResult>; fn owned_message_ids( &self, owner: &Address, start_message_id: Option, direction: IterDirection, - ) -> BoxedIter<'_, StorageResult>; + ) -> BoxedIterSend<'_, StorageResult>; fn owned_transactions_ids( &self, owner: Address, start: Option, direction: IterDirection, - ) -> BoxedIter>; + ) -> BoxedIterSend>; fn contract_salt(&self, contract_id: &ContractId) -> StorageResult; @@ -103,7 +104,7 @@ pub trait OffChainDatabase: Send + Sync { &self, height: Option, direction: IterDirection, - ) -> BoxedIter<'_, StorageResult>; + ) -> BoxedIterSend<'_, StorageResult>; fn old_block_consensus(&self, height: &BlockHeight) -> StorageResult; @@ -156,7 +157,7 @@ pub trait DatabaseBlocks { &self, height: Option, direction: IterDirection, - ) -> BoxedIter<'_, StorageResult>; + ) -> BoxedIterSend<'_, StorageResult>; fn latest_height(&self) -> StorageResult; @@ -178,7 +179,7 @@ pub trait DatabaseMessages: StorageInspect { &self, start_message_id: Option, direction: IterDirection, - ) -> BoxedIter<'_, StorageResult>; + ) -> BoxedIterSend<'_, StorageResult>; fn message_batch<'a>( &'a self, @@ -199,7 +200,10 @@ pub trait DatabaseRelayedTransactions { pub trait DatabaseCoins: StorageInspect { fn coin(&self, utxo_id: UtxoId) -> StorageResult; - fn coins<'a>(&'a self, utxo_ids: &'a [UtxoId]) -> BoxedIter<'a, StorageResult>; + fn coins<'a>( + &'a self, + utxo_ids: &'a [UtxoId], + ) -> BoxedIter<'a, StorageResult>; } /// Trait that specifies all the getters required for contract. @@ -212,7 +216,7 @@ pub trait DatabaseContracts: contract: ContractId, start_asset: Option, direction: IterDirection, - ) -> BoxedIter>; + ) -> BoxedIterSend>; } /// Trait that specifies all the getters required for chain metadata. diff --git a/crates/fuel-core/src/query/message.rs b/crates/fuel-core/src/query/message.rs index 625c86e8ecb..f9a537dd0e2 100644 --- a/crates/fuel-core/src/query/message.rs +++ b/crates/fuel-core/src/query/message.rs @@ -1,7 +1,7 @@ use crate::fuel_core_graphql_api::database::ReadView; use fuel_core_storage::{ iter::{ - BoxedIter, + BoxedIterSend, IntoBoxedIter, IterDirection, }, @@ -53,20 +53,20 @@ pub trait MessageQueryData: Send + Sync { owner: &Address, start_message_id: Option, direction: IterDirection, - ) -> BoxedIter>; + ) -> BoxedIterSend>; fn owned_messages( &self, owner: &Address, start_message_id: Option, direction: IterDirection, - ) -> BoxedIter>; + ) -> BoxedIterSend>; fn all_messages( &self, start_message_id: Option, direction: IterDirection, - ) -> BoxedIter>; + ) -> BoxedIterSend>; } impl ReadView { diff --git a/crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs b/crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs index 1c0c79953d3..527ed925174 100644 --- a/crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs +++ b/crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs @@ -26,7 +26,9 @@ use fuel_core_storage::{ blueprint::BlueprintInspect, codec::Encode, iter::{ + BoxedIterSend, BoxedIter, + IntoBoxedIterSend, IntoBoxedIter, IterDirection, IteratorOverTable, @@ -97,10 +99,10 @@ impl OffChainDatabase for OffChainIterableKeyValueView { owner: &Address, start_coin: Option, direction: IterDirection, - ) -> BoxedIter<'_, StorageResult> { + ) -> BoxedIterSend<'_, StorageResult> { self.owned_coins_ids(owner, start_coin, Some(direction)) .map(|res| res.map_err(StorageError::from)) - .into_boxed() + .into_boxed_send() } fn owned_message_ids( @@ -108,10 +110,10 @@ impl OffChainDatabase for OffChainIterableKeyValueView { owner: &Address, start_message_id: Option, direction: IterDirection, - ) -> BoxedIter<'_, StorageResult> { + ) -> BoxedIterSend<'_, StorageResult> { self.owned_message_ids(owner, start_message_id, Some(direction)) .map(|result| result.map_err(StorageError::from)) - .into_boxed() + .into_boxed_send() } fn owned_transactions_ids( @@ -119,14 +121,14 @@ impl OffChainDatabase for OffChainIterableKeyValueView { owner: Address, start: Option, direction: IterDirection, - ) -> BoxedIter> { + ) -> BoxedIterSend> { let start = start.map(|tx_pointer| OwnedTransactionIndexCursor { block_height: tx_pointer.block_height(), tx_idx: tx_pointer.tx_index(), }); self.owned_transactions(owner, start, Some(direction)) .map(|result| result.map_err(StorageError::from)) - .into_boxed() + .into_boxed_send() } fn contract_salt(&self, contract_id: &ContractId) -> StorageResult { @@ -153,10 +155,10 @@ impl OffChainDatabase for OffChainIterableKeyValueView { &self, height: Option, direction: IterDirection, - ) -> BoxedIter<'_, StorageResult> { + ) -> BoxedIterSend<'_, StorageResult> { self.iter_all_by_start::(height.as_ref(), Some(direction)) .map(|r| r.map(|(_, block)| block)) - .into_boxed() + .into_boxed_send() } fn old_block_consensus(&self, height: &BlockHeight) -> StorageResult { diff --git a/crates/fuel-core/src/service/adapters/graphql_api/on_chain.rs b/crates/fuel-core/src/service/adapters/graphql_api/on_chain.rs index b392482237c..55d3d34712d 100644 --- a/crates/fuel-core/src/service/adapters/graphql_api/on_chain.rs +++ b/crates/fuel-core/src/service/adapters/graphql_api/on_chain.rs @@ -18,7 +18,9 @@ use crate::{ }; use fuel_core_storage::{ iter::{ + BoxedIterSend, BoxedIter, + IntoBoxedIterSend, IntoBoxedIter, IterDirection, IteratorOverTable, @@ -93,10 +95,10 @@ impl DatabaseBlocks for OnChainIterableKeyValueView { &self, height: Option, direction: IterDirection, - ) -> BoxedIter<'_, StorageResult> { + ) -> BoxedIterSend<'_, StorageResult> { self.iter_all_by_start::(height.as_ref(), Some(direction)) .map(|result| result.map(|(_, block)| block)) - .into_boxed() + .into_boxed_send() } fn latest_height(&self) -> StorageResult { @@ -116,10 +118,10 @@ impl DatabaseMessages for OnChainIterableKeyValueView { &self, start_message_id: Option, direction: IterDirection, - ) -> BoxedIter<'_, StorageResult> { + ) -> BoxedIterSend<'_, StorageResult> { self.all_messages(start_message_id, Some(direction)) .map(|result| result.map_err(StorageError::from)) - .into_boxed() + .into_boxed_send() } fn message_batch<'a>( @@ -147,12 +149,15 @@ impl DatabaseCoins for OnChainIterableKeyValueView { Ok(coin.uncompress(utxo_id)) } - fn coins<'a>(&'a self, utxo_ids: &'a [UtxoId]) -> BoxedIter<'a, StorageResult> { + fn coins<'a>( + &'a self, + utxo_ids: &'a [UtxoId], + ) -> BoxedIter<'a, StorageResult> { >::get_batch( self, - utxo_ids.iter().into_boxed(), + utxo_ids.iter().into_boxed_send(), ) - .zip(utxo_ids.iter().into_boxed()) + .zip(utxo_ids.iter().into_boxed_send()) .map(|(res, utxo_id)| { res.and_then(|opt| opt.ok_or(not_found!(Coins))) .map(|coin| coin.uncompress(*utxo_id)) @@ -167,7 +172,7 @@ impl DatabaseContracts for OnChainIterableKeyValueView { contract: ContractId, start_asset: Option, direction: IterDirection, - ) -> BoxedIter> { + ) -> BoxedIterSend> { self.filter_contract_balances(contract, start_asset, Some(direction)) .map_ok(|entry| ContractBalance { owner: *entry.key.contract_id(), @@ -175,7 +180,7 @@ impl DatabaseContracts for OnChainIterableKeyValueView { asset_id: *entry.key.asset_id(), }) .map(|res| res.map_err(StorageError::from)) - .into_boxed() + .into_boxed_send() } } diff --git a/crates/fuel-core/src/service/genesis/importer/import_task.rs b/crates/fuel-core/src/service/genesis/importer/import_task.rs index 5cdcf636c8c..5a6476f2730 100644 --- a/crates/fuel-core/src/service/genesis/importer/import_task.rs +++ b/crates/fuel-core/src/service/genesis/importer/import_task.rs @@ -166,7 +166,7 @@ mod tests { use fuel_core_storage::{ column::Column, iter::{ - BoxedIter, + BoxedIterSend, IterDirection, IterableStore, }, @@ -551,7 +551,7 @@ mod tests { _: Option<&[u8]>, _: Option<&[u8]>, _: IterDirection, - ) -> BoxedIter { + ) -> BoxedIterSend { unimplemented!() } @@ -561,7 +561,7 @@ mod tests { _: Option<&[u8]>, _: Option<&[u8]>, _: IterDirection, - ) -> BoxedIter { + ) -> BoxedIterSend { unimplemented!() } } diff --git a/crates/fuel-core/src/state/data_source.rs b/crates/fuel-core/src/state/data_source.rs index 702085aa5c2..2b13c754bce 100644 --- a/crates/fuel-core/src/state/data_source.rs +++ b/crates/fuel-core/src/state/data_source.rs @@ -4,6 +4,7 @@ use crate::{ }; use fuel_core_storage::{ iter::{ + BoxedIterSend, BoxedIter, IterDirection, IterableStore, @@ -96,7 +97,7 @@ where prefix: Option<&[u8]>, start: Option<&[u8]>, direction: IterDirection, - ) -> BoxedIter { + ) -> BoxedIterSend { self.data.iter_store(column, prefix, start, direction) } @@ -106,7 +107,7 @@ where prefix: Option<&[u8]>, start: Option<&[u8]>, direction: IterDirection, - ) -> BoxedIter { + ) -> BoxedIterSend { self.data.iter_store_keys(column, prefix, start, direction) } } diff --git a/crates/fuel-core/src/state/generic_database.rs b/crates/fuel-core/src/state/generic_database.rs index 6d0114c4508..c6b1d677dde 100644 --- a/crates/fuel-core/src/state/generic_database.rs +++ b/crates/fuel-core/src/state/generic_database.rs @@ -1,5 +1,6 @@ use fuel_core_storage::{ iter::{ + BoxedIterSend, BoxedIter, IterDirection, IterableStore, @@ -75,7 +76,7 @@ where keys: Iter, ) -> impl Iterator>> + 'a where - Iter: 'a + Iterator + Send, + Iter: 'a + IntoIterator, M::Key: 'a, { self.storage.get_batch(keys) @@ -166,7 +167,7 @@ where prefix: Option<&[u8]>, start: Option<&[u8]>, direction: IterDirection, - ) -> BoxedIter { + ) -> BoxedIterSend { self.storage.iter_store(column, prefix, start, direction) } @@ -176,7 +177,7 @@ where prefix: Option<&[u8]>, start: Option<&[u8]>, direction: IterDirection, - ) -> BoxedIter { + ) -> BoxedIterSend { self.storage .iter_store_keys(column, prefix, start, direction) } diff --git a/crates/fuel-core/src/state/historical_rocksdb.rs b/crates/fuel-core/src/state/historical_rocksdb.rs index 559505ba67b..2749673bd2b 100644 --- a/crates/fuel-core/src/state/historical_rocksdb.rs +++ b/crates/fuel-core/src/state/historical_rocksdb.rs @@ -29,6 +29,7 @@ use crate::{ use fuel_core_storage::{ iter::{ BoxedIter, + BoxedIterSend, IntoBoxedIter, IterDirection, IterableStore, @@ -433,7 +434,7 @@ where prefix: Option<&[u8]>, start: Option<&[u8]>, direction: IterDirection, - ) -> BoxedIter { + ) -> BoxedIterSend { self.db .iter_store(Column::OriginalColumn(column), prefix, start, direction) } @@ -444,7 +445,7 @@ where prefix: Option<&[u8]>, start: Option<&[u8]>, direction: IterDirection, - ) -> BoxedIter { + ) -> BoxedIterSend { self.db .iter_store_keys(Column::OriginalColumn(column), prefix, start, direction) } diff --git a/crates/fuel-core/src/state/in_memory/memory_store.rs b/crates/fuel-core/src/state/in_memory/memory_store.rs index c23eafe48e3..a839ab3ecff 100644 --- a/crates/fuel-core/src/state/in_memory/memory_store.rs +++ b/crates/fuel-core/src/state/in_memory/memory_store.rs @@ -16,8 +16,8 @@ use fuel_core_storage::{ iter::{ iterator, keys_iterator, - BoxedIter, - IntoBoxedIter, + BoxedIterSend, + IntoBoxedIterSend, IterableStore, }, kv_store::{ @@ -144,8 +144,8 @@ where prefix: Option<&[u8]>, start: Option<&[u8]>, direction: IterDirection, - ) -> BoxedIter { - self.iter_all(column, prefix, start, direction).into_boxed() + ) -> BoxedIterSend { + self.iter_all(column, prefix, start, direction).into_boxed_send() } fn iter_store_keys( @@ -154,9 +154,9 @@ where prefix: Option<&[u8]>, start: Option<&[u8]>, direction: IterDirection, - ) -> BoxedIter { + ) -> BoxedIterSend { self.iter_all_keys(column, prefix, start, direction) - .into_boxed() + .into_boxed_send() } } diff --git a/crates/fuel-core/src/state/in_memory/memory_view.rs b/crates/fuel-core/src/state/in_memory/memory_view.rs index 8047785033e..9f8c16c7b64 100644 --- a/crates/fuel-core/src/state/in_memory/memory_view.rs +++ b/crates/fuel-core/src/state/in_memory/memory_view.rs @@ -6,8 +6,8 @@ use fuel_core_storage::{ iter::{ iterator, keys_iterator, - BoxedIter, - IntoBoxedIter, + BoxedIterSend, + IntoBoxedIterSend, IterDirection, IterableStore, }, @@ -86,8 +86,8 @@ where prefix: Option<&[u8]>, start: Option<&[u8]>, direction: IterDirection, - ) -> BoxedIter { - self.iter_all(column, prefix, start, direction).into_boxed() + ) -> BoxedIterSend { + self.iter_all(column, prefix, start, direction).into_boxed_send() } fn iter_store_keys( @@ -96,8 +96,8 @@ where prefix: Option<&[u8]>, start: Option<&[u8]>, direction: IterDirection, - ) -> BoxedIter { + ) -> BoxedIterSend { self.iter_all_keys(column, prefix, start, direction) - .into_boxed() + .into_boxed_send() } } diff --git a/crates/fuel-core/src/state/iterable_key_value_view.rs b/crates/fuel-core/src/state/iterable_key_value_view.rs index f8a8870a11a..f6a4aa9459d 100644 --- a/crates/fuel-core/src/state/iterable_key_value_view.rs +++ b/crates/fuel-core/src/state/iterable_key_value_view.rs @@ -1,6 +1,7 @@ use fuel_core_storage::{ iter::{ BoxedIter, + BoxedIterSend, IterDirection, IterableStore, }, @@ -94,7 +95,7 @@ where prefix: Option<&[u8]>, start: Option<&[u8]>, direction: IterDirection, - ) -> BoxedIter { + ) -> BoxedIterSend { self.0.iter_store(column, prefix, start, direction) } @@ -104,7 +105,7 @@ where prefix: Option<&[u8]>, start: Option<&[u8]>, direction: IterDirection, - ) -> BoxedIter { + ) -> BoxedIterSend { self.0.iter_store_keys(column, prefix, start, direction) } } diff --git a/crates/fuel-core/src/state/rocks_db.rs b/crates/fuel-core/src/state/rocks_db.rs index ada8b458fa9..85761940d5c 100644 --- a/crates/fuel-core/src/state/rocks_db.rs +++ b/crates/fuel-core/src/state/rocks_db.rs @@ -16,7 +16,9 @@ use fuel_core_metrics::core_metrics::DatabaseMetrics; use fuel_core_storage::{ iter::{ BoxedIter, + BoxedIterSend, IntoBoxedIter, + IntoBoxedIterSend, IterableStore, }, kv_store::{ @@ -487,7 +489,7 @@ where true } }) - .into_boxed() + .into_boxed_send() } else { // No next item, so we can start backward iteration from the end. let prefix = prefix.to_vec(); @@ -499,7 +501,7 @@ where true } }) - .into_boxed() + .into_boxed_send() } } @@ -564,7 +566,7 @@ where prefix: Option<&[u8]>, start: Option<&[u8]>, direction: IterDirection, - ) -> BoxedIter> + ) -> BoxedIterSend> where T: ExtractItem, { @@ -578,11 +580,12 @@ where IterDirection::Reverse => IteratorMode::End, }; self.iterator::(column, self.read_options(), iter_mode) - .into_boxed() + .into_boxed_send() } (Some(prefix), None) => { if direction == IterDirection::Reverse { - self.reverse_prefix_iter::(prefix, column).into_boxed() + self.reverse_prefix_iter::(prefix, column) + .into_boxed_send() } else { // start iterating in a certain direction within the keyspace let iter_mode = IteratorMode::From( @@ -604,7 +607,7 @@ where true } }) - .into_boxed() + .into_boxed_send() } } (None, Some(start)) => { @@ -612,13 +615,13 @@ where let iter_mode = IteratorMode::From(start, convert_to_rocksdb_direction(direction)); self.iterator::(column, self.read_options(), iter_mode) - .into_boxed() + .into_boxed_send() } (Some(prefix), Some(start)) => { // TODO: Maybe we want to allow the `start` to be without a `prefix` in the future. // If the `start` doesn't have the same `prefix`, return nothing. if !start.starts_with(prefix) { - return iter::empty().into_boxed(); + return iter::empty().into_boxed_send(); } // start iterating in a certain direction from the start key @@ -634,7 +637,7 @@ where true } }) - .into_boxed() + .into_boxed_send() } } } @@ -781,7 +784,7 @@ where prefix: Option<&[u8]>, start: Option<&[u8]>, direction: IterDirection, - ) -> BoxedIter { + ) -> BoxedIterSend { self._iter_store::(column, prefix, start, direction) } @@ -791,7 +794,7 @@ where prefix: Option<&[u8]>, start: Option<&[u8]>, direction: IterDirection, - ) -> BoxedIter { + ) -> BoxedIterSend { self._iter_store::(column, prefix, start, direction) } } diff --git a/crates/storage/src/blueprint.rs b/crates/storage/src/blueprint.rs index 9845c37b967..8acdd7c79b1 100644 --- a/crates/storage/src/blueprint.rs +++ b/crates/storage/src/blueprint.rs @@ -84,10 +84,11 @@ where column: S::Column, ) -> impl Iterator>> + 'a where - Iter: 'a + Iterator + Send, + Iter: 'a + IntoIterator, M::Key: 'a, { let keys = keys + .into_iter() .map(|key| Self::KeyCodec::encode(key).into_bytes()) .into_boxed(); diff --git a/crates/storage/src/iter.rs b/crates/storage/src/iter.rs index 35c550851fb..8a3f7d33c67 100644 --- a/crates/storage/src/iter.rs +++ b/crates/storage/src/iter.rs @@ -26,10 +26,9 @@ use alloc::{ pub mod changes_iterator; -// TODO: BoxedIter to be used until RPITIT lands in stable rust. /// A boxed variant of the iterator that can be used as a return type of the traits. pub struct BoxedIter<'a, T> { - iter: Box + 'a + Send>, + iter: Box + 'a>, } impl<'a, T> Iterator for BoxedIter<'a, T> { @@ -48,7 +47,7 @@ pub trait IntoBoxedIter<'a, T> { impl<'a, T, I> IntoBoxedIter<'a, T> for I where - I: Iterator + 'a + Send, + I: Iterator + 'a, { fn into_boxed(self) -> BoxedIter<'a, T> { BoxedIter { @@ -57,6 +56,37 @@ where } } +/// A boxed iterator that can be sent to other threads. +/// This is useful for example when converting it to a `futures::Stream`. +pub struct BoxedIterSend<'a, T> { + iter: Box + 'a + Send>, +} + +impl<'a, T> Iterator for BoxedIterSend<'a, T> { + type Item = T; + + fn next(&mut self) -> Option { + self.iter.next() + } +} + +/// The traits simplifies conversion into `BoxedIter`. +pub trait IntoBoxedIterSend<'a, T> { + /// Converts `Self` iterator into `BoxedIter`. + fn into_boxed_send(self) -> BoxedIterSend<'a, T>; +} + +impl<'a, T, I> IntoBoxedIterSend<'a, T> for I +where + I: Iterator + 'a + Send, +{ + fn into_boxed_send(self) -> BoxedIterSend<'a, T> { + BoxedIterSend { + iter: Box::new(self), + } + } +} + /// A enum for iterating across the database #[derive(Copy, Clone, Debug, PartialOrd, Eq, PartialEq)] pub enum IterDirection { @@ -82,7 +112,7 @@ pub trait IterableStore: KeyValueInspect { prefix: Option<&[u8]>, start: Option<&[u8]>, direction: IterDirection, - ) -> BoxedIter; + ) -> BoxedIterSend; /// Returns an iterator over keys in the storage. fn iter_store_keys( @@ -91,7 +121,7 @@ pub trait IterableStore: KeyValueInspect { prefix: Option<&[u8]>, start: Option<&[u8]>, direction: IterDirection, - ) -> BoxedIter; + ) -> BoxedIterSend; } #[cfg(feature = "std")] @@ -105,7 +135,7 @@ where prefix: Option<&[u8]>, start: Option<&[u8]>, direction: IterDirection, - ) -> BoxedIter { + ) -> BoxedIterSend { use core::ops::Deref; self.deref().iter_store(column, prefix, start, direction) } @@ -116,7 +146,7 @@ where prefix: Option<&[u8]>, start: Option<&[u8]>, direction: IterDirection, - ) -> BoxedIter { + ) -> BoxedIterSend { use core::ops::Deref; self.deref() .iter_store_keys(column, prefix, start, direction) @@ -134,7 +164,7 @@ where prefix: Option

, start: Option<&M::Key>, direction: Option, - ) -> BoxedIter> + ) -> BoxedIterSend> where P: AsRef<[u8]>; @@ -144,7 +174,7 @@ where prefix: Option

, start: Option<&M::Key>, direction: Option, - ) -> BoxedIter> + ) -> BoxedIterSend> where P: AsRef<[u8]>; } @@ -160,7 +190,7 @@ where prefix: Option

, start: Option<&M::Key>, direction: Option, - ) -> BoxedIter> + ) -> BoxedIterSend> where P: AsRef<[u8]>, { @@ -186,7 +216,7 @@ where Ok(key) }) }) - .into_boxed() + .into_boxed_send() } fn iter_table

( @@ -194,7 +224,7 @@ where prefix: Option

, start: Option<&M::Key>, direction: Option, - ) -> BoxedIter> + ) -> BoxedIterSend> where P: AsRef<[u8]>, { @@ -225,7 +255,7 @@ where Ok((key, value)) }) }) - .into_boxed() + .into_boxed_send() } } @@ -235,7 +265,7 @@ pub trait IteratorOverTable { fn iter_all_keys( &self, direction: Option, - ) -> BoxedIter> + ) -> BoxedIterSend> where M: Mappable, Self: IterableTable, @@ -247,7 +277,7 @@ pub trait IteratorOverTable { fn iter_all_by_prefix_keys( &self, prefix: Option

, - ) -> BoxedIter> + ) -> BoxedIterSend> where M: Mappable, P: AsRef<[u8]>, @@ -261,7 +291,7 @@ pub trait IteratorOverTable { &self, start: Option<&M::Key>, direction: Option, - ) -> BoxedIter> + ) -> BoxedIterSend> where M: Mappable, Self: IterableTable, @@ -275,7 +305,7 @@ pub trait IteratorOverTable { prefix: Option

, start: Option<&M::Key>, direction: Option, - ) -> BoxedIter> + ) -> BoxedIterSend> where M: Mappable, P: AsRef<[u8]>, @@ -288,7 +318,7 @@ pub trait IteratorOverTable { fn iter_all( &self, direction: Option, - ) -> BoxedIter> + ) -> BoxedIterSend> where M: Mappable, Self: IterableTable, @@ -300,7 +330,7 @@ pub trait IteratorOverTable { fn iter_all_by_prefix( &self, prefix: Option

, - ) -> BoxedIter> + ) -> BoxedIterSend> where M: Mappable, P: AsRef<[u8]>, @@ -314,7 +344,7 @@ pub trait IteratorOverTable { &self, start: Option<&M::Key>, direction: Option, - ) -> BoxedIter> + ) -> BoxedIterSend> where M: Mappable, Self: IterableTable, @@ -328,7 +358,7 @@ pub trait IteratorOverTable { prefix: Option

, start: Option<&M::Key>, direction: Option, - ) -> BoxedIter> + ) -> BoxedIterSend> where M: Mappable, P: AsRef<[u8]>, @@ -353,9 +383,9 @@ where match (prefix, start) { (None, None) => { if direction == IterDirection::Forward { - tree.iter().into_boxed() + tree.iter().into_boxed_send() } else { - tree.iter().rev().into_boxed() + tree.iter().rev().into_boxed_send() } } (Some(prefix), None) => { @@ -363,23 +393,23 @@ where if direction == IterDirection::Forward { tree.range(prefix.clone()..) .take_while(move |(key, _)| key.starts_with(prefix.as_slice())) - .into_boxed() + .into_boxed_send() } else { let mut vec: Vec<_> = tree .range(prefix.clone()..) - .into_boxed() + .into_boxed_send() .take_while(|(key, _)| key.starts_with(prefix.as_slice())) .collect(); vec.reverse(); - vec.into_iter().into_boxed() + vec.into_iter().into_boxed_send() } } (None, Some(start)) => { if direction == IterDirection::Forward { - tree.range(start.to_vec()..).into_boxed() + tree.range(start.to_vec()..).into_boxed_send() } else { - tree.range(..=start.to_vec()).rev().into_boxed() + tree.range(..=start.to_vec()).rev().into_boxed_send() } } (Some(prefix), Some(start)) => { @@ -387,12 +417,12 @@ where if direction == IterDirection::Forward { tree.range(start.to_vec()..) .take_while(move |(key, _)| key.starts_with(prefix.as_slice())) - .into_boxed() + .into_boxed_send() } else { tree.range(..=start.to_vec()) .rev() .take_while(move |(key, _)| key.starts_with(prefix.as_slice())) - .into_boxed() + .into_boxed_send() } } } @@ -411,14 +441,14 @@ where match (prefix, start) { (None, None) => { if direction == IterDirection::Forward { - tree.keys().into_boxed() + tree.keys().into_boxed_send() } else { - tree.keys().rev().into_boxed() + tree.keys().rev().into_boxed_send() } } // all the other cases require using a range, so we can't use the keys() method (_, _) => iterator(tree, prefix, start, direction) .map(|(key, _)| key) - .into_boxed(), + .into_boxed_send(), } } diff --git a/crates/storage/src/iter/changes_iterator.rs b/crates/storage/src/iter/changes_iterator.rs index 9a3e566bdf3..b5260392c4f 100644 --- a/crates/storage/src/iter/changes_iterator.rs +++ b/crates/storage/src/iter/changes_iterator.rs @@ -2,8 +2,8 @@ use crate::{ iter::{ - BoxedIter, - IntoBoxedIter, + BoxedIterSend, + IntoBoxedIterSend, IterDirection, IterableStore, }, @@ -61,7 +61,7 @@ where prefix: Option<&[u8]>, start: Option<&[u8]>, direction: IterDirection, - ) -> BoxedIter { + ) -> BoxedIterSend { if let Some(tree) = self.changes.get(&column.id()) { crate::iter::iterator(tree, prefix, start, direction) .filter_map(|(key, value)| match value { @@ -71,9 +71,9 @@ where WriteOperation::Remove => None, }) .map(Ok) - .into_boxed() + .into_boxed_send() } else { - core::iter::empty().into_boxed() + core::iter::empty().into_boxed_send() } } @@ -83,7 +83,7 @@ where prefix: Option<&[u8]>, start: Option<&[u8]>, direction: IterDirection, - ) -> BoxedIter { + ) -> BoxedIterSend { // We cannot define iter_store_keys appropriately for the `ChangesIterator`, // because we have to filter out the keys that were removed, which are // marked as `WriteOperation::Remove` in the value @@ -95,9 +95,9 @@ where WriteOperation::Remove => None, }) .map(Ok) - .into_boxed() + .into_boxed_send() } else { - core::iter::empty().into_boxed() + core::iter::empty().into_boxed_send() } } } diff --git a/crates/storage/src/kv_store.rs b/crates/storage/src/kv_store.rs index 7d4564395d3..9eb3b25bf34 100644 --- a/crates/storage/src/kv_store.rs +++ b/crates/storage/src/kv_store.rs @@ -1,7 +1,10 @@ //! The module provides plain abstract definition of the key-value store. use crate::{ - iter::BoxedIter, + iter::{ + BoxedIter, + IntoBoxedIter, + }, Error as StorageError, Result as StorageResult, }; @@ -14,7 +17,6 @@ use alloc::{ vec::Vec, }; -use crate::iter::IntoBoxedIter; #[cfg(feature = "std")] use core::ops::Deref; @@ -44,7 +46,7 @@ pub trait StorageColumn: Copy + core::fmt::Debug + Send + Sync + 'static { /// The definition of the key-value inspection store. #[impl_tools::autoimpl(for &T, &mut T, Box)] -pub trait KeyValueInspect: Send + Sync { +pub trait KeyValueInspect { /// The type of the column. type Column: StorageColumn; diff --git a/crates/storage/src/lib.rs b/crates/storage/src/lib.rs index 1977dacd1ef..6ce8c2fd8cf 100644 --- a/crates/storage/src/lib.rs +++ b/crates/storage/src/lib.rs @@ -186,7 +186,7 @@ pub trait StorageBatchInspect { keys: Iter, ) -> impl Iterator>> + 'a where - Iter: 'a + Iterator + Send, + Iter: 'a + IntoIterator, Type::Key: 'a; } diff --git a/crates/storage/src/structured_storage.rs b/crates/storage/src/structured_storage.rs index 3bdb6f2f30b..882308bc8ad 100644 --- a/crates/storage/src/structured_storage.rs +++ b/crates/storage/src/structured_storage.rs @@ -15,6 +15,7 @@ use crate::{ }, iter::{ BoxedIter, + BoxedIterSend, IterDirection, IterableStore, }, @@ -240,7 +241,7 @@ where prefix: Option<&[u8]>, start: Option<&[u8]>, direction: IterDirection, - ) -> BoxedIter { + ) -> BoxedIterSend { self.inner.iter_store(column, prefix, start, direction) } @@ -250,7 +251,7 @@ where prefix: Option<&[u8]>, start: Option<&[u8]>, direction: IterDirection, - ) -> BoxedIter { + ) -> BoxedIterSend { self.inner.iter_store_keys(column, prefix, start, direction) } } @@ -294,7 +295,7 @@ where keys: Iter, ) -> impl Iterator>> + 'a where - Iter: 'a + Iterator + Send, + Iter: 'a + IntoIterator, M::Key: 'a, { M::Blueprint::get_batch(self, keys, M::column()) diff --git a/crates/storage/src/transactional.rs b/crates/storage/src/transactional.rs index 9041b56e07e..804f495d415 100644 --- a/crates/storage/src/transactional.rs +++ b/crates/storage/src/transactional.rs @@ -32,6 +32,7 @@ use alloc::{ #[cfg(feature = "test-helpers")] use crate::{ iter::{ + BoxedIterSend, IterDirection, IterableStore, }, @@ -550,7 +551,7 @@ where _: Option<&[u8]>, _: Option<&[u8]>, _: IterDirection, - ) -> BoxedIter { + ) -> BoxedIterSend { unimplemented!() } @@ -560,7 +561,7 @@ where _: Option<&[u8]>, _: Option<&[u8]>, _: IterDirection, - ) -> BoxedIter { + ) -> BoxedIterSend { unimplemented!() } } From 75fedc78cab82d4822015dac8bfd2f99d26d60a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A5rten=20Blankfors?= Date: Fri, 1 Nov 2024 11:32:04 +0100 Subject: [PATCH 35/37] fix: Cargo fmt --- crates/fuel-core/src/graphql_api/database.rs | 2 +- crates/fuel-core/src/graphql_api/ports.rs | 7 ++----- .../src/service/adapters/graphql_api/off_chain.rs | 4 ++-- .../src/service/adapters/graphql_api/on_chain.rs | 9 +++------ crates/fuel-core/src/state/data_source.rs | 2 +- crates/fuel-core/src/state/generic_database.rs | 2 +- crates/fuel-core/src/state/in_memory/memory_store.rs | 3 ++- crates/fuel-core/src/state/in_memory/memory_view.rs | 3 ++- 8 files changed, 14 insertions(+), 18 deletions(-) diff --git a/crates/fuel-core/src/graphql_api/database.rs b/crates/fuel-core/src/graphql_api/database.rs index 287189c9025..ef79c2ff8bf 100644 --- a/crates/fuel-core/src/graphql_api/database.rs +++ b/crates/fuel-core/src/graphql_api/database.rs @@ -9,8 +9,8 @@ use fuel_core_services::yield_stream::StreamYieldExt; use fuel_core_storage::{ iter::{ BoxedIterSend, - IntoBoxedIterSend, IntoBoxedIter, + IntoBoxedIterSend, IterDirection, }, transactional::AtomicView, diff --git a/crates/fuel-core/src/graphql_api/ports.rs b/crates/fuel-core/src/graphql_api/ports.rs index 243b0894bcc..b2c2eedcf55 100644 --- a/crates/fuel-core/src/graphql_api/ports.rs +++ b/crates/fuel-core/src/graphql_api/ports.rs @@ -2,8 +2,8 @@ use async_trait::async_trait; use fuel_core_services::stream::BoxStream; use fuel_core_storage::{ iter::{ - BoxedIterSend, BoxedIter, + BoxedIterSend, IterDirection, }, tables::{ @@ -200,10 +200,7 @@ pub trait DatabaseRelayedTransactions { pub trait DatabaseCoins: StorageInspect { fn coin(&self, utxo_id: UtxoId) -> StorageResult; - fn coins<'a>( - &'a self, - utxo_ids: &'a [UtxoId], - ) -> BoxedIter<'a, StorageResult>; + fn coins<'a>(&'a self, utxo_ids: &'a [UtxoId]) -> BoxedIter<'a, StorageResult>; } /// Trait that specifies all the getters required for contract. diff --git a/crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs b/crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs index 527ed925174..5669688b672 100644 --- a/crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs +++ b/crates/fuel-core/src/service/adapters/graphql_api/off_chain.rs @@ -26,10 +26,10 @@ use fuel_core_storage::{ blueprint::BlueprintInspect, codec::Encode, iter::{ - BoxedIterSend, BoxedIter, - IntoBoxedIterSend, + BoxedIterSend, IntoBoxedIter, + IntoBoxedIterSend, IterDirection, IteratorOverTable, }, diff --git a/crates/fuel-core/src/service/adapters/graphql_api/on_chain.rs b/crates/fuel-core/src/service/adapters/graphql_api/on_chain.rs index 55d3d34712d..764b08e777a 100644 --- a/crates/fuel-core/src/service/adapters/graphql_api/on_chain.rs +++ b/crates/fuel-core/src/service/adapters/graphql_api/on_chain.rs @@ -18,10 +18,10 @@ use crate::{ }; use fuel_core_storage::{ iter::{ - BoxedIterSend, BoxedIter, - IntoBoxedIterSend, + BoxedIterSend, IntoBoxedIter, + IntoBoxedIterSend, IterDirection, IteratorOverTable, }, @@ -149,10 +149,7 @@ impl DatabaseCoins for OnChainIterableKeyValueView { Ok(coin.uncompress(utxo_id)) } - fn coins<'a>( - &'a self, - utxo_ids: &'a [UtxoId], - ) -> BoxedIter<'a, StorageResult> { + fn coins<'a>(&'a self, utxo_ids: &'a [UtxoId]) -> BoxedIter<'a, StorageResult> { >::get_batch( self, utxo_ids.iter().into_boxed_send(), diff --git a/crates/fuel-core/src/state/data_source.rs b/crates/fuel-core/src/state/data_source.rs index 2b13c754bce..1647bb4d441 100644 --- a/crates/fuel-core/src/state/data_source.rs +++ b/crates/fuel-core/src/state/data_source.rs @@ -4,8 +4,8 @@ use crate::{ }; use fuel_core_storage::{ iter::{ - BoxedIterSend, BoxedIter, + BoxedIterSend, IterDirection, IterableStore, }, diff --git a/crates/fuel-core/src/state/generic_database.rs b/crates/fuel-core/src/state/generic_database.rs index c6b1d677dde..3aaebcc207d 100644 --- a/crates/fuel-core/src/state/generic_database.rs +++ b/crates/fuel-core/src/state/generic_database.rs @@ -1,7 +1,7 @@ use fuel_core_storage::{ iter::{ - BoxedIterSend, BoxedIter, + BoxedIterSend, IterDirection, IterableStore, }, diff --git a/crates/fuel-core/src/state/in_memory/memory_store.rs b/crates/fuel-core/src/state/in_memory/memory_store.rs index a839ab3ecff..ebc137dab88 100644 --- a/crates/fuel-core/src/state/in_memory/memory_store.rs +++ b/crates/fuel-core/src/state/in_memory/memory_store.rs @@ -145,7 +145,8 @@ where start: Option<&[u8]>, direction: IterDirection, ) -> BoxedIterSend { - self.iter_all(column, prefix, start, direction).into_boxed_send() + self.iter_all(column, prefix, start, direction) + .into_boxed_send() } fn iter_store_keys( diff --git a/crates/fuel-core/src/state/in_memory/memory_view.rs b/crates/fuel-core/src/state/in_memory/memory_view.rs index 9f8c16c7b64..5f40d7e8722 100644 --- a/crates/fuel-core/src/state/in_memory/memory_view.rs +++ b/crates/fuel-core/src/state/in_memory/memory_view.rs @@ -87,7 +87,8 @@ where start: Option<&[u8]>, direction: IterDirection, ) -> BoxedIterSend { - self.iter_all(column, prefix, start, direction).into_boxed_send() + self.iter_all(column, prefix, start, direction) + .into_boxed_send() } fn iter_store_keys( From 7ae04bc0b2f152a6b636fffb23046076ae07386a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A5rten=20Blankfors?= Date: Fri, 15 Nov 2024 10:25:52 +0100 Subject: [PATCH 36/37] Update crates/fuel-core/src/graphql_api/database.rs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Rafał Chabowski <88321181+rafal-ch@users.noreply.github.com> --- crates/fuel-core/src/graphql_api/database.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/fuel-core/src/graphql_api/database.rs b/crates/fuel-core/src/graphql_api/database.rs index ef79c2ff8bf..899f2772b42 100644 --- a/crates/fuel-core/src/graphql_api/database.rs +++ b/crates/fuel-core/src/graphql_api/database.rs @@ -165,7 +165,7 @@ impl ReadView { } } - pub async fn extend_with_off_chain_results( + async fn extend_with_off_chain_results( &self, on_chain_results: BTreeMap<(usize, &TxId), StorageResult>, ) -> Vec> { From c3b72642dacdbbe44fff3c6a3a95d68beb3ff6c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A5rten=20Blankfors?= Date: Fri, 15 Nov 2024 10:31:38 +0100 Subject: [PATCH 37/37] fix: Remove redundant 'static bound --- crates/storage/src/codec.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/storage/src/codec.rs b/crates/storage/src/codec.rs index 373f5d298cf..f77f91cc044 100644 --- a/crates/storage/src/codec.rs +++ b/crates/storage/src/codec.rs @@ -29,7 +29,7 @@ pub trait Encoder<'a> { /// flexibility and more performant encoding, allowing the use of slices and arrays /// instead of vectors in some cases. Since the [`Encoder`] returns `Cow<[u8]>`, /// it is always possible to take ownership of the serialized value. -pub trait Encode: 'static { +pub trait Encode { /// The encoder type that stores serialized object. type Encoder<'a>: Encoder<'a> where