From 2c2253ff6dd18e2ea847aefeebacaeacfe0036ff Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Fri, 22 Sep 2023 21:53:46 +0700 Subject: [PATCH 01/18] some refactoring --- grovedb/src/batch/mod.rs | 3 +- grovedb/src/operations/proof/generate.rs | 21 +- merk/src/lib.rs | 5 +- merk/src/merk/apply.rs | 314 +++++++ merk/src/merk/clear.rs | 32 + merk/src/merk/committer.rs | 120 +++ merk/src/merk/get.rs | 245 ++++++ merk/src/merk/mod.rs | 1013 +--------------------- merk/src/merk/open.rs | 171 ++++ merk/src/merk/prove.rs | 147 ++++ merk/src/merk/restore.rs | 3 +- merk/src/merk/source.rs | 45 + merk/src/test_utils/mod.rs | 2 + merk/src/tree/ops.rs | 105 ++- merk/src/tree/walk/mod.rs | 8 +- 15 files changed, 1190 insertions(+), 1044 deletions(-) create mode 100644 merk/src/merk/apply.rs create mode 100644 merk/src/merk/clear.rs create mode 100644 merk/src/merk/committer.rs create mode 100644 merk/src/merk/get.rs create mode 100644 merk/src/merk/open.rs create mode 100644 merk/src/merk/prove.rs create mode 100644 merk/src/merk/source.rs diff --git a/grovedb/src/batch/mod.rs b/grovedb/src/batch/mod.rs index f61202091..22f859d54 100644 --- a/grovedb/src/batch/mod.rs +++ b/grovedb/src/batch/mod.rs @@ -1150,7 +1150,8 @@ where let Element::Reference(path_reference, max_reference_hop, _) = &element else { return Err(Error::InvalidInput( "trying to refresh a an element that is not a reference", - )).wrap_with_cost(cost) + )) + .wrap_with_cost(cost); }; let merk_feature_type = if is_sum_tree { diff --git a/grovedb/src/operations/proof/generate.rs b/grovedb/src/operations/proof/generate.rs index 22bf49a95..292b8cbf1 100644 --- a/grovedb/src/operations/proof/generate.rs +++ b/grovedb/src/operations/proof/generate.rs @@ -32,40 +32,31 @@ // that supports multiple implementations for verbose and non-verbose // generation -use grovedb_costs::cost_return_on_error_default; -#[cfg(feature = "full")] use grovedb_costs::{ - cost_return_on_error, cost_return_on_error_no_add, CostResult, CostsExt, OperationCost, + cost_return_on_error, cost_return_on_error_default, cost_return_on_error_no_add, CostResult, + CostsExt, OperationCost, }; -#[cfg(feature = "full")] use grovedb_merk::{ proofs::{encode_into, Node, Op}, tree::value_hash, KVIterator, Merk, ProofWithoutEncodingResult, }; use grovedb_path::SubtreePath; -#[cfg(feature = "full")] use grovedb_storage::StorageContext; -#[cfg(feature = "full")] -use crate::element::helpers::raw_decode; -#[cfg(feature = "full")] use crate::{ + element::helpers::raw_decode, operations::proof::util::{ - reduce_limit_and_offset_by, write_to_vec, ProofTokenType, EMPTY_TREE_HASH, + reduce_limit_and_offset_by, write_slice_of_slice_to_slice, write_slice_to_vec, + write_to_vec, ProofTokenType, EMPTY_TREE_HASH, }, reference_path::path_from_reference_path_type, - Element, Error, GroveDb, PathQuery, Query, -}; -use crate::{ - operations::proof::util::{write_slice_of_slice_to_slice, write_slice_to_vec}, versioning::{prepend_version_to_bytes, PROOF_VERSION}, + Element, Error, GroveDb, PathQuery, Query, }; -#[cfg(feature = "full")] type LimitOffset = (Option, Option); -#[cfg(feature = "full")] impl GroveDb { /// Prove one or more path queries. /// If we more than one path query, we merge into a single path query before diff --git a/merk/src/lib.rs b/merk/src/lib.rs index b780b6f4e..caf3837ce 100644 --- a/merk/src/lib.rs +++ b/merk/src/lib.rs @@ -86,8 +86,9 @@ pub use tree::{CryptoHash, TreeFeatureType}; #[cfg(feature = "full")] pub use crate::merk::{ - defaults::ROOT_KEY_KEY, IsSumTree, KVIterator, Merk, MerkType, ProofConstructionResult, - ProofWithoutEncodingResult, RootHashKeyAndSum, + defaults::ROOT_KEY_KEY, + prove::{ProofConstructionResult, ProofWithoutEncodingResult}, + IsSumTree, KVIterator, Merk, MerkType, RootHashKeyAndSum, }; #[cfg(feature = "full")] pub use crate::visualize::VisualizeableMerk; diff --git a/merk/src/merk/apply.rs b/merk/src/merk/apply.rs new file mode 100644 index 000000000..22d6f55cc --- /dev/null +++ b/merk/src/merk/apply.rs @@ -0,0 +1,314 @@ +use std::cmp::Ordering; + +use grovedb_costs::{ + storage_cost::{ + removal::{StorageRemovedBytes, StorageRemovedBytes::BasicStorageRemoval}, + StorageCost, + }, + CostResult, CostsExt, +}; +use grovedb_storage::StorageContext; + +use crate::{ + tree::{ + kv::{ValueDefinedCostType, KV}, + AuxMerkBatch, Walker, + }, + Error, Merk, MerkBatch, MerkOptions, +}; + +impl<'db, S> Merk +where + S: StorageContext<'db>, +{ + /// Applies a batch of operations (puts and deletes) to the tree. + /// + /// This will fail if the keys in `batch` are not sorted and unique. This + /// check creates some overhead, so if you are sure your batch is sorted and + /// unique you can use the unsafe `apply_unchecked` for a small performance + /// gain. + /// + /// # Example + /// ``` + /// # let mut store = grovedb_merk::test_utils::TempMerk::new(); + /// # store.apply::<_, Vec<_>>(&[(vec![4,5,6], Op::Put(vec![0], BasicMerk))], &[], None) + /// .unwrap().expect(""); + /// + /// use grovedb_merk::Op; + /// use grovedb_merk::TreeFeatureType::BasicMerk; + /// + /// let batch = &[ + /// // puts value [4,5,6] to key[1,2,3] + /// (vec![1, 2, 3], Op::Put(vec![4, 5, 6], BasicMerk)), + /// // deletes key [4,5,6] + /// (vec![4, 5, 6], Op::Delete), + /// ]; + /// store.apply::<_, Vec<_>>(batch, &[], None).unwrap().expect(""); + /// ``` + pub fn apply( + &mut self, + batch: &MerkBatch, + aux: &AuxMerkBatch, + options: Option, + ) -> CostResult<(), Error> + where + KB: AsRef<[u8]>, + KA: AsRef<[u8]>, + { + let use_sum_nodes = self.is_sum_tree; + self.apply_with_costs_just_in_time_value_update( + batch, + aux, + options, + &|key, value| { + Ok(KV::layered_value_byte_cost_size_for_key_and_value_lengths( + key.len() as u32, + value.len() as u32, + use_sum_nodes, + )) + }, + &mut |_costs, _old_value, _value| Ok((false, None)), + &mut |_a, key_bytes_to_remove, value_bytes_to_remove| { + Ok(( + BasicStorageRemoval(key_bytes_to_remove), + BasicStorageRemoval(value_bytes_to_remove), + )) + }, + ) + } + + /// Applies a batch of operations (puts and deletes) to the tree. + /// + /// This will fail if the keys in `batch` are not sorted and unique. This + /// check creates some overhead, so if you are sure your batch is sorted and + /// unique you can use the unsafe `apply_unchecked` for a small performance + /// gain. + /// + /// # Example + /// ``` + /// # let mut store = grovedb_merk::test_utils::TempMerk::new(); + /// # store.apply::<_, Vec<_>>(&[(vec![4,5,6], Op::Put(vec![0], BasicMerk))], &[], None) + /// .unwrap().expect(""); + /// + /// use grovedb_merk::Op; + /// use grovedb_merk::TreeFeatureType::BasicMerk; + /// + /// let batch = &[ + /// // puts value [4,5,6] to key[1,2,3] + /// (vec![1, 2, 3], Op::Put(vec![4, 5, 6], BasicMerk)), + /// // deletes key [4,5,6] + /// (vec![4, 5, 6], Op::Delete), + /// ]; + /// store.apply::<_, Vec<_>>(batch, &[], None).unwrap().expect(""); + /// ``` + pub fn apply_with_specialized_costs( + &mut self, + batch: &MerkBatch, + aux: &AuxMerkBatch, + options: Option, + old_specialized_cost: &impl Fn(&Vec, &Vec) -> Result, + ) -> CostResult<(), Error> + where + KB: AsRef<[u8]>, + KA: AsRef<[u8]>, + { + self.apply_with_costs_just_in_time_value_update( + batch, + aux, + options, + old_specialized_cost, + &mut |_costs, _old_value, _value| Ok((false, None)), + &mut |_a, key_bytes_to_remove, value_bytes_to_remove| { + Ok(( + BasicStorageRemoval(key_bytes_to_remove), + BasicStorageRemoval(value_bytes_to_remove), + )) + }, + ) + } + + /// Applies a batch of operations (puts and deletes) to the tree with the + /// ability to update values based on costs. + /// + /// This will fail if the keys in `batch` are not sorted and unique. This + /// check creates some overhead, so if you are sure your batch is sorted and + /// unique you can use the unsafe `apply_unchecked` for a small performance + /// gain. + /// + /// # Example + /// ``` + /// # let mut store = grovedb_merk::test_utils::TempMerk::new(); + /// # store.apply_with_costs_just_in_time_value_update::<_, Vec<_>>( + /// &[(vec![4,5,6], Op::Put(vec![0], BasicMerk))], + /// &[], + /// None, + /// &|k, v| Ok(0), + /// &mut |s, v, o| Ok((false, None)), + /// &mut |s, k, v| Ok((NoStorageRemoval, NoStorageRemoval)) + /// ).unwrap().expect(""); + /// + /// use grovedb_costs::storage_cost::removal::StorageRemovedBytes::NoStorageRemoval; + /// use grovedb_merk::Op; + /// use grovedb_merk::TreeFeatureType::BasicMerk; + /// + /// let batch = &[ + /// // puts value [4,5,6] to key[1,2,3] + /// (vec![1, 2, 3], Op::Put(vec![4, 5, 6], BasicMerk)), + /// // deletes key [4,5,6] + /// (vec![4, 5, 6], Op::Delete), + /// ]; + /// + /// store.apply_with_costs_just_in_time_value_update::<_, Vec<_>>( + /// batch, + /// &[], + /// None, + /// &|k, v| Ok(0), + /// &mut |s, v, o| Ok((false, None)), + /// &mut |s, k, v| Ok((NoStorageRemoval, NoStorageRemoval)) + /// ).unwrap().expect(""); + /// ``` + pub fn apply_with_costs_just_in_time_value_update( + &mut self, + batch: &MerkBatch, + aux: &AuxMerkBatch, + options: Option, + old_specialized_cost: &impl Fn(&Vec, &Vec) -> Result, + update_tree_value_based_on_costs: &mut impl FnMut( + &StorageCost, + &Vec, + &mut Vec, + ) -> Result< + (bool, Option), + Error, + >, + section_removal_bytes: &mut impl FnMut( + &Vec, + u32, + u32, + ) -> Result< + (StorageRemovedBytes, StorageRemovedBytes), + Error, + >, + ) -> CostResult<(), Error> + where + KB: AsRef<[u8]>, + KA: AsRef<[u8]>, + { + // ensure keys in batch are sorted and unique + let mut maybe_prev_key: Option<&KB> = None; + for (key, ..) in batch.iter() { + if let Some(prev_key) = maybe_prev_key { + match prev_key.as_ref().cmp(key.as_ref()) { + Ordering::Greater => { + return Err(Error::InvalidInputError("Keys in batch must be sorted")) + .wrap_with_cost(Default::default()) + } + Ordering::Equal => { + return Err(Error::InvalidInputError("Keys in batch must be unique")) + .wrap_with_cost(Default::default()) + } + _ => (), + } + } + maybe_prev_key = Some(key); + } + + self.apply_unchecked( + batch, + aux, + options, + old_specialized_cost, + update_tree_value_based_on_costs, + section_removal_bytes, + ) + } + + /// Applies a batch of operations (puts and deletes) to the tree. + /// + /// # Safety + /// This is unsafe because the keys in `batch` must be sorted and unique - + /// if they are not, there will be undefined behavior. For a safe version of + /// this method which checks to ensure the batch is sorted and unique, see + /// `apply`. + /// + /// # Example + /// ``` + /// # let mut store = grovedb_merk::test_utils::TempMerk::new(); + /// # store.apply_with_costs_just_in_time_value_update::<_, Vec<_>>( + /// &[(vec![4,5,6], Op::Put(vec![0], BasicMerk))], + /// &[], + /// None, + /// &|k, v| Ok(0), + /// &mut |s, o, v| Ok((false, None)), + /// &mut |s, k, v| Ok((NoStorageRemoval, NoStorageRemoval)) + /// ).unwrap().expect(""); + /// + /// use grovedb_costs::storage_cost::removal::StorageRemovedBytes::NoStorageRemoval; + /// use grovedb_merk::Op; + /// use grovedb_merk::TreeFeatureType::BasicMerk; + /// + /// let batch = &[ + /// // puts value [4,5,6] to key [1,2,3] + /// (vec![1, 2, 3], Op::Put(vec![4, 5, 6], BasicMerk)), + /// // deletes key [4,5,6] + /// (vec![4, 5, 6], Op::Delete), + /// ]; + /// unsafe { store.apply_unchecked::<_, Vec<_>, _, _, _>( /// /// /// + /// batch, + /// &[], + /// None, + /// &|k, v| Ok(0), + /// &mut |s, o, v| Ok((false, None)), + /// &mut |s, k, v| Ok((NoStorageRemoval, NoStorageRemoval)) + /// ).unwrap().expect(""); + /// } + /// ``` + pub fn apply_unchecked( + &mut self, + batch: &MerkBatch, + aux: &AuxMerkBatch, + options: Option, + old_specialized_cost: &C, + update_tree_value_based_on_costs: &mut U, + section_removal_bytes: &mut R, + ) -> CostResult<(), Error> + where + KB: AsRef<[u8]>, + KA: AsRef<[u8]>, + C: Fn(&Vec, &Vec) -> Result, + U: FnMut( + &StorageCost, + &Vec, + &mut Vec, + ) -> Result<(bool, Option), Error>, + R: FnMut(&Vec, u32, u32) -> Result<(StorageRemovedBytes, StorageRemovedBytes), Error>, + { + let maybe_walker = self + .tree + .take() + .take() + .map(|tree| Walker::new(tree, self.source())); + + Walker::apply_to( + maybe_walker, + batch, + self.source(), + old_specialized_cost, + update_tree_value_based_on_costs, + section_removal_bytes, + ) + .flat_map_ok(|(maybe_tree, key_updates)| { + // we set the new root node of the merk tree + self.tree.set(maybe_tree); + // commit changes to db + self.commit( + key_updates, + aux, + options, + old_specialized_cost, + update_tree_value_based_on_costs, + section_removal_bytes, + ) + }) + } +} diff --git a/merk/src/merk/clear.rs b/merk/src/merk/clear.rs new file mode 100644 index 000000000..0de28f6a7 --- /dev/null +++ b/merk/src/merk/clear.rs @@ -0,0 +1,32 @@ +use grovedb_costs::{cost_return_on_error, CostResult, CostsExt, OperationCost}; +use grovedb_storage::{Batch, RawIterator, StorageContext}; + +use crate::{Error, Error::StorageError, Merk}; + +impl<'db, S> Merk +where + S: StorageContext<'db>, +{ + /// Deletes tree data + pub fn clear(&mut self) -> CostResult<(), Error> { + let mut cost = OperationCost::default(); + + let mut iter = self.storage.raw_iter(); + iter.seek_to_first().unwrap_add_cost(&mut cost); + + let mut to_delete = self.storage.new_batch(); + while iter.valid().unwrap_add_cost(&mut cost) { + if let Some(key) = iter.key().unwrap_add_cost(&mut cost) { + // todo: deal with cost reimbursement + to_delete.delete(key, None); + } + iter.next().unwrap_add_cost(&mut cost); + } + cost_return_on_error!( + &mut cost, + self.storage.commit_batch(to_delete).map_err(StorageError) + ); + self.tree.set(None); + Ok(()).wrap_with_cost(cost) + } +} diff --git a/merk/src/merk/committer.rs b/merk/src/merk/committer.rs new file mode 100644 index 000000000..04ef7b5d3 --- /dev/null +++ b/merk/src/merk/committer.rs @@ -0,0 +1,120 @@ +use grovedb_costs::storage_cost::{ + removal::{StorageRemovedBytes, StorageRemovedBytes::BasicStorageRemoval}, + StorageCost, +}; + +use crate::{ + merk::{defaults::MAX_UPDATE_VALUE_BASED_ON_COSTS_TIMES, BatchValue}, + tree::{kv::ValueDefinedCostType, Commit, Tree}, + Error, +}; + +pub struct MerkCommitter { + /// The batch has a key, maybe a value, with the value bytes, maybe the left + /// child size and maybe the right child size, then the + /// key_value_storage_cost + pub(in crate::merk) batch: Vec, + pub(in crate::merk) height: u8, + pub(in crate::merk) levels: u8, +} + +impl MerkCommitter { + pub(in crate::merk) fn new(height: u8, levels: u8) -> Self { + Self { + batch: Vec::with_capacity(10000), + height, + levels, + } + } +} + +impl Commit for MerkCommitter { + fn write( + &mut self, + tree: &mut Tree, + old_specialized_cost: &impl Fn(&Vec, &Vec) -> Result, + update_tree_value_based_on_costs: &mut impl FnMut( + &StorageCost, + &Vec, + &mut Vec, + ) -> Result< + (bool, Option), + Error, + >, + section_removal_bytes: &mut impl FnMut( + &Vec, + u32, + u32, + ) -> Result< + (StorageRemovedBytes, StorageRemovedBytes), + Error, + >, + ) -> Result<(), Error> { + let tree_size = tree.encoding_length(); + let (mut current_tree_plus_hook_size, mut storage_costs) = + tree.kv_with_parent_hook_size_and_storage_cost(old_specialized_cost)?; + let mut i = 0; + + if let Some(old_value) = tree.old_value.clone() { + // At this point the tree value can be updated based on client requirements + // For example to store the costs + loop { + let (flags_changed, value_defined_cost) = update_tree_value_based_on_costs( + &storage_costs.value_storage_cost, + &old_value, + tree.value_mut_ref(), + )?; + if !flags_changed { + break; + } else { + tree.inner.kv.value_defined_cost = value_defined_cost; + let after_update_tree_plus_hook_size = + tree.value_encoding_length_with_parent_to_child_reference(); + if after_update_tree_plus_hook_size == current_tree_plus_hook_size { + break; + } + let new_size_and_storage_costs = + tree.kv_with_parent_hook_size_and_storage_cost(old_specialized_cost)?; + current_tree_plus_hook_size = new_size_and_storage_costs.0; + storage_costs = new_size_and_storage_costs.1; + } + if i > MAX_UPDATE_VALUE_BASED_ON_COSTS_TIMES { + return Err(Error::CyclicError( + "updated value based on costs too many times", + )); + } + i += 1; + } + + if let BasicStorageRemoval(removed_bytes) = + storage_costs.value_storage_cost.removed_bytes + { + let (_, value_removed_bytes) = section_removal_bytes(&old_value, 0, removed_bytes)?; + storage_costs.value_storage_cost.removed_bytes = value_removed_bytes; + } + } + + // Update old tree size after generating value storage_cost cost + tree.old_size_with_parent_to_child_hook = current_tree_plus_hook_size; + tree.old_value = Some(tree.value_ref().clone()); + + let mut buf = Vec::with_capacity(tree_size); + tree.encode_into(&mut buf); + + let left_child_sizes = tree.child_ref_and_sum_size(true); + let right_child_sizes = tree.child_ref_and_sum_size(false); + self.batch.push(( + tree.key().to_vec(), + tree.feature_type().sum_length(), + Some((buf, left_child_sizes, right_child_sizes)), + Some(storage_costs), + )); + Ok(()) + } + + fn prune(&self, tree: &Tree) -> (bool, bool) { + // keep N top levels of tree + let prune = (self.height - tree.height()) >= self.levels; + (prune, prune) + } +} diff --git a/merk/src/merk/get.rs b/merk/src/merk/get.rs new file mode 100644 index 000000000..1e9090c89 --- /dev/null +++ b/merk/src/merk/get.rs @@ -0,0 +1,245 @@ +use grovedb_costs::{CostContext, CostResult, CostsExt, OperationCost}; +use grovedb_storage::StorageContext; + +use crate::{tree::Tree, CryptoHash, Error, Error::StorageError, Merk, TreeFeatureType}; + +impl<'db, S> Merk +where + S: StorageContext<'db>, +{ + /// Gets an auxiliary value. + pub fn get_aux(&self, key: &[u8]) -> CostResult>, Error> { + self.storage.get_aux(key).map_err(StorageError) + } + + /// Returns if the value at the given key exists + /// + /// Note that this is essentially the same as a normal RocksDB `get`, so + /// should be a fast operation and has almost no tree overhead. + pub fn exists(&self, key: &[u8]) -> CostResult { + self.has_node_direct(key) + } + + /// Returns if the value at the given key exists + /// + /// Note that this is essentially the same as a normal RocksDB `get`, so + /// should be a fast operation and has almost no tree overhead. + /// Contrary to a simple exists, this traverses the tree and can be faster + /// if the tree is cached, but slower if it is not + pub fn exists_by_traversing_tree(&self, key: &[u8]) -> CostResult { + self.has_node(key) + } + + /// Gets a value for the given key. If the key is not found, `None` is + /// returned. + /// + /// Note that this is essentially the same as a normal RocksDB `get`, so + /// should be a fast operation and has almost no tree overhead. + pub fn get(&self, key: &[u8], allow_cache: bool) -> CostResult>, Error> { + if allow_cache { + self.get_node_fn(key, |node| { + node.value_as_slice() + .to_vec() + .wrap_with_cost(Default::default()) + }) + } else { + self.get_node_direct_fn(key, |node| { + node.value_as_slice() + .to_vec() + .wrap_with_cost(Default::default()) + }) + } + } + + /// Returns the feature type for the node at the given key. + pub fn get_feature_type( + &self, + key: &[u8], + allow_cache: bool, + ) -> CostResult, Error> { + if allow_cache { + self.get_node_fn(key, |node| { + node.feature_type().wrap_with_cost(Default::default()) + }) + } else { + self.get_node_direct_fn(key, |node| { + node.feature_type().wrap_with_cost(Default::default()) + }) + } + } + + /// Gets a hash of a node by a given key, `None` is returned in case + /// when node not found by the key. + pub fn get_hash(&self, key: &[u8], allow_cache: bool) -> CostResult, Error> { + if allow_cache { + self.get_node_fn(key, |node| node.hash()) + } else { + self.get_node_direct_fn(key, |node| node.hash()) + } + } + + /// Gets the value hash of a node by a given key, `None` is returned in case + /// when node not found by the key. + pub fn get_value_hash( + &self, + key: &[u8], + allow_cache: bool, + ) -> CostResult, Error> { + if allow_cache { + self.get_node_fn(key, |node| { + (*node.value_hash()).wrap_with_cost(OperationCost::default()) + }) + } else { + self.get_node_direct_fn(key, |node| { + (*node.value_hash()).wrap_with_cost(OperationCost::default()) + }) + } + } + + /// Gets a hash of a node by a given key, `None` is returned in case + /// when node not found by the key. + pub fn get_kv_hash( + &self, + key: &[u8], + allow_cache: bool, + ) -> CostResult, Error> { + if allow_cache { + self.get_node_fn(key, |node| { + (*node.inner.kv.hash()).wrap_with_cost(OperationCost::default()) + }) + } else { + self.get_node_direct_fn(key, |node| { + (*node.inner.kv.hash()).wrap_with_cost(OperationCost::default()) + }) + } + } + + /// Gets the value and value hash of a node by a given key, `None` is + /// returned in case when node not found by the key. + pub fn get_value_and_value_hash( + &self, + key: &[u8], + allow_cache: bool, + ) -> CostResult, CryptoHash)>, Error> { + if allow_cache { + self.get_node_fn(key, |node| { + (node.value_as_slice().to_vec(), *node.value_hash()) + .wrap_with_cost(OperationCost::default()) + }) + } else { + self.get_node_direct_fn(key, |node| { + (node.value_as_slice().to_vec(), *node.value_hash()) + .wrap_with_cost(OperationCost::default()) + }) + } + } + + /// See if a node's field exists + fn has_node_direct(&self, key: &[u8]) -> CostResult { + Tree::get(&self.storage, key).map_ok(|x| x.is_some()) + } + + /// See if a node's field exists + fn has_node(&self, key: &[u8]) -> CostResult { + self.use_tree(move |maybe_tree| { + let mut cursor = match maybe_tree { + None => return Ok(false).wrap_with_cost(Default::default()), // empty tree + Some(tree) => tree, + }; + + loop { + if key == cursor.key() { + return Ok(true).wrap_with_cost(OperationCost::default()); + } + + let left = key < cursor.key(); + let link = match cursor.link(left) { + None => return Ok(false).wrap_with_cost(Default::default()), // not found + Some(link) => link, + }; + + let maybe_child = link.tree(); + match maybe_child { + None => { + // fetch from RocksDB + break self.has_node_direct(key); + } + Some(child) => cursor = child, // traverse to child + } + } + }) + } + + /// Generic way to get a node's field + fn get_node_direct_fn(&self, key: &[u8], f: F) -> CostResult, Error> + where + F: FnOnce(&Tree) -> CostContext, + { + Tree::get(&self.storage, key).flat_map_ok(|maybe_node| { + let mut cost = OperationCost::default(); + Ok(maybe_node.map(|node| f(&node).unwrap_add_cost(&mut cost))).wrap_with_cost(cost) + }) + } + + /// Generic way to get a node's field + fn get_node_fn(&self, key: &[u8], f: F) -> CostResult, Error> + where + F: FnOnce(&Tree) -> CostContext, + { + self.use_tree(move |maybe_tree| { + let mut cursor = match maybe_tree { + None => return Ok(None).wrap_with_cost(Default::default()), // empty tree + Some(tree) => tree, + }; + + loop { + if key == cursor.key() { + return f(cursor).map(|x| Ok(Some(x))); + } + + let left = key < cursor.key(); + let link = match cursor.link(left) { + None => return Ok(None).wrap_with_cost(Default::default()), // not found + Some(link) => link, + }; + + let maybe_child = link.tree(); + match maybe_child { + None => { + // fetch from RocksDB + break self.get_node_direct_fn(key, f); + } + Some(child) => cursor = child, // traverse to child + } + } + }) + } +} + +#[cfg(test)] +mod test { + use crate::{test_utils::TempMerk, Op, TreeFeatureType::BasicMerk}; + + #[test] + fn test_has_node_with_empty_tree() { + let mut merk = TempMerk::new(); + + let key = b"something"; + + let result = merk.has_node(key).unwrap().unwrap(); + + assert!(!result); + + let batch_entry = (key, Op::Put(vec![123; 60], BasicMerk)); + + let batch = vec![batch_entry]; + + merk.apply::<_, Vec<_>>(&batch, &[], None) + .unwrap() + .expect("should ..."); + + let result = merk.has_node(key).unwrap().unwrap(); + + assert!(result); + } +} diff --git a/merk/src/merk/mod.rs b/merk/src/merk/mod.rs index 37276b65a..5e20ade99 100644 --- a/merk/src/merk/mod.rs +++ b/merk/src/merk/mod.rs @@ -34,87 +34,44 @@ pub(crate) mod defaults; pub mod options; +pub mod apply; +pub mod clear; +pub mod committer; +pub mod get; +pub mod open; +pub mod prove; pub mod restore; +pub mod source; use std::{ cell::Cell, - cmp::Ordering, collections::{BTreeSet, LinkedList}, fmt, }; +use committer::MerkCommitter; use grovedb_costs::{ cost_return_on_error, cost_return_on_error_default, cost_return_on_error_no_add, storage_cost::{ - key_value_cost::KeyValueStorageCost, - removal::{StorageRemovedBytes, StorageRemovedBytes::BasicStorageRemoval}, - StorageCost, + key_value_cost::KeyValueStorageCost, removal::StorageRemovedBytes, StorageCost, }, ChildrenSizesWithValue, CostContext, CostResult, CostsExt, FeatureSumLength, OperationCost, }; use grovedb_storage::{self, Batch, RawIterator, StorageContext}; +use source::MerkSource; use crate::{ error::Error, - merk::{ - defaults::{MAX_UPDATE_VALUE_BASED_ON_COSTS_TIMES, ROOT_KEY_KEY}, - options::MerkOptions, - }, - proofs::{encode_into, query::query_item::QueryItem, Op as ProofOp, Query}, + merk::{defaults::ROOT_KEY_KEY, options::MerkOptions}, + proofs::{query::query_item::QueryItem, Op as ProofOp, Query}, tree::{ - kv::{ValueDefinedCostType, KV}, - AuxMerkBatch, Commit, CryptoHash, Fetch, Link, MerkBatch, Op, RefWalker, Tree, Walker, + kv::ValueDefinedCostType, AuxMerkBatch, Commit, CryptoHash, Fetch, Op, RefWalker, Tree, NULL_HASH, }, Error::{CostsError, EdError, StorageError}, MerkType::{BaseMerk, LayeredMerk, StandaloneMerk}, - TreeFeatureType, }; -type Proof = (LinkedList, Option, Option); - -/// Proof construction result -pub struct ProofConstructionResult { - /// Proof - pub proof: Vec, - /// Limit - pub limit: Option, - /// Offset - pub offset: Option, -} - -impl ProofConstructionResult { - /// New ProofConstructionResult - pub fn new(proof: Vec, limit: Option, offset: Option) -> Self { - Self { - proof, - limit, - offset, - } - } -} - -/// Proof without encoding result -pub struct ProofWithoutEncodingResult { - /// Proof - pub proof: LinkedList, - /// Limit - pub limit: Option, - /// Offset - pub offset: Option, -} - -impl ProofWithoutEncodingResult { - /// New ProofWithoutEncodingResult - pub fn new(proof: LinkedList, limit: Option, offset: Option) -> Self { - Self { - proof, - limit, - offset, - } - } -} - /// Key update types pub struct KeyUpdates { pub new_keys: BTreeSet>, @@ -303,291 +260,6 @@ impl<'db, S> Merk where S: StorageContext<'db>, { - /// Open empty tree - pub fn open_empty(storage: S, merk_type: MerkType, is_sum_tree: bool) -> Self { - Self { - tree: Cell::new(None), - root_tree_key: Cell::new(None), - storage, - merk_type, - is_sum_tree, - } - } - - /// Open standalone tree - pub fn open_standalone(storage: S, is_sum_tree: bool) -> CostResult { - let mut merk = Self { - tree: Cell::new(None), - root_tree_key: Cell::new(None), - storage, - merk_type: StandaloneMerk, - is_sum_tree, - }; - - merk.load_base_root().map_ok(|_| merk) - } - - /// Open base tree - pub fn open_base(storage: S, is_sum_tree: bool) -> CostResult { - let mut merk = Self { - tree: Cell::new(None), - root_tree_key: Cell::new(None), - storage, - merk_type: BaseMerk, - is_sum_tree, - }; - - merk.load_base_root().map_ok(|_| merk) - } - - /// Open layered tree with root key - pub fn open_layered_with_root_key( - storage: S, - root_key: Option>, - is_sum_tree: bool, - ) -> CostResult { - let mut merk = Self { - tree: Cell::new(None), - root_tree_key: Cell::new(root_key), - storage, - merk_type: LayeredMerk, - is_sum_tree, - }; - - merk.load_root().map_ok(|_| merk) - } - - /// Deletes tree data - pub fn clear(&mut self) -> CostResult<(), Error> { - let mut cost = OperationCost::default(); - - let mut iter = self.storage.raw_iter(); - iter.seek_to_first().unwrap_add_cost(&mut cost); - - let mut to_delete = self.storage.new_batch(); - while iter.valid().unwrap_add_cost(&mut cost) { - if let Some(key) = iter.key().unwrap_add_cost(&mut cost) { - // todo: deal with cost reimbursement - to_delete.delete(key, None); - } - iter.next().unwrap_add_cost(&mut cost); - } - cost_return_on_error!( - &mut cost, - self.storage.commit_batch(to_delete).map_err(StorageError) - ); - self.tree.set(None); - Ok(()).wrap_with_cost(cost) - } - - /// Gets an auxiliary value. - pub fn get_aux(&self, key: &[u8]) -> CostResult>, Error> { - self.storage.get_aux(key).map_err(StorageError) - } - - /// Returns if the value at the given key exists - /// - /// Note that this is essentially the same as a normal RocksDB `get`, so - /// should be a fast operation and has almost no tree overhead. - pub fn exists(&self, key: &[u8]) -> CostResult { - self.has_node_direct(key) - } - - /// Returns if the value at the given key exists - /// - /// Note that this is essentially the same as a normal RocksDB `get`, so - /// should be a fast operation and has almost no tree overhead. - /// Contrary to a simple exists, this traverses the tree and can be faster - /// if the tree is cached, but slower if it is not - pub fn exists_by_traversing_tree(&self, key: &[u8]) -> CostResult { - self.has_node(key) - } - - /// Gets a value for the given key. If the key is not found, `None` is - /// returned. - /// - /// Note that this is essentially the same as a normal RocksDB `get`, so - /// should be a fast operation and has almost no tree overhead. - pub fn get(&self, key: &[u8], allow_cache: bool) -> CostResult>, Error> { - if allow_cache { - self.get_node_fn(key, |node| { - node.value_as_slice() - .to_vec() - .wrap_with_cost(Default::default()) - }) - } else { - self.get_node_direct_fn(key, |node| { - node.value_as_slice() - .to_vec() - .wrap_with_cost(Default::default()) - }) - } - } - - /// Returns the feature type for the node at the given key. - pub fn get_feature_type( - &self, - key: &[u8], - allow_cache: bool, - ) -> CostResult, Error> { - if allow_cache { - self.get_node_fn(key, |node| { - node.feature_type().wrap_with_cost(Default::default()) - }) - } else { - self.get_node_direct_fn(key, |node| { - node.feature_type().wrap_with_cost(Default::default()) - }) - } - } - - /// Gets a hash of a node by a given key, `None` is returned in case - /// when node not found by the key. - pub fn get_hash(&self, key: &[u8], allow_cache: bool) -> CostResult, Error> { - if allow_cache { - self.get_node_fn(key, |node| node.hash()) - } else { - self.get_node_direct_fn(key, |node| node.hash()) - } - } - - /// Gets the value hash of a node by a given key, `None` is returned in case - /// when node not found by the key. - pub fn get_value_hash( - &self, - key: &[u8], - allow_cache: bool, - ) -> CostResult, Error> { - if allow_cache { - self.get_node_fn(key, |node| { - (*node.value_hash()).wrap_with_cost(OperationCost::default()) - }) - } else { - self.get_node_direct_fn(key, |node| { - (*node.value_hash()).wrap_with_cost(OperationCost::default()) - }) - } - } - - /// Gets a hash of a node by a given key, `None` is returned in case - /// when node not found by the key. - pub fn get_kv_hash( - &self, - key: &[u8], - allow_cache: bool, - ) -> CostResult, Error> { - if allow_cache { - self.get_node_fn(key, |node| { - (*node.inner.kv.hash()).wrap_with_cost(OperationCost::default()) - }) - } else { - self.get_node_direct_fn(key, |node| { - (*node.inner.kv.hash()).wrap_with_cost(OperationCost::default()) - }) - } - } - - /// Gets the value and value hash of a node by a given key, `None` is - /// returned in case when node not found by the key. - pub fn get_value_and_value_hash( - &self, - key: &[u8], - allow_cache: bool, - ) -> CostResult, CryptoHash)>, Error> { - if allow_cache { - self.get_node_fn(key, |node| { - (node.value_as_slice().to_vec(), *node.value_hash()) - .wrap_with_cost(OperationCost::default()) - }) - } else { - self.get_node_direct_fn(key, |node| { - (node.value_as_slice().to_vec(), *node.value_hash()) - .wrap_with_cost(OperationCost::default()) - }) - } - } - - /// See if a node's field exists - fn has_node_direct(&self, key: &[u8]) -> CostResult { - Tree::get(&self.storage, key).map_ok(|x| x.is_some()) - } - - /// See if a node's field exists - fn has_node(&self, key: &[u8]) -> CostResult { - self.use_tree(move |maybe_tree| { - let mut cursor = match maybe_tree { - None => return Ok(false).wrap_with_cost(Default::default()), // empty tree - Some(tree) => tree, - }; - - loop { - if key == cursor.key() { - return Ok(true).wrap_with_cost(OperationCost::default()); - } - - let left = key < cursor.key(); - let link = match cursor.link(left) { - None => return Ok(false).wrap_with_cost(Default::default()), // not found - Some(link) => link, - }; - - let maybe_child = link.tree(); - match maybe_child { - None => { - // fetch from RocksDB - break self.has_node_direct(key); - } - Some(child) => cursor = child, // traverse to child - } - } - }) - } - - /// Generic way to get a node's field - fn get_node_direct_fn(&self, key: &[u8], f: F) -> CostResult, Error> - where - F: FnOnce(&Tree) -> CostContext, - { - Tree::get(&self.storage, key).flat_map_ok(|maybe_node| { - let mut cost = OperationCost::default(); - Ok(maybe_node.map(|node| f(&node).unwrap_add_cost(&mut cost))).wrap_with_cost(cost) - }) - } - - /// Generic way to get a node's field - fn get_node_fn(&self, key: &[u8], f: F) -> CostResult, Error> - where - F: FnOnce(&Tree) -> CostContext, - { - self.use_tree(move |maybe_tree| { - let mut cursor = match maybe_tree { - None => return Ok(None).wrap_with_cost(Default::default()), // empty tree - Some(tree) => tree, - }; - - loop { - if key == cursor.key() { - return f(cursor).map(|x| Ok(Some(x))); - } - - let left = key < cursor.key(); - let link = match cursor.link(left) { - None => return Ok(None).wrap_with_cost(Default::default()), // not found - Some(link) => link, - }; - - let maybe_child = link.tree(); - match maybe_child { - None => { - // fetch from RocksDB - break self.get_node_direct_fn(key, f); - } - Some(child) => cursor = child, // traverse to child - } - } - }) - } - /// Returns the root hash of the tree (a digest for the entire store which /// proofs can be checked against). If the tree is empty, returns the null /// hash (zero-filled). @@ -625,384 +297,6 @@ where }) } - /// Applies a batch of operations (puts and deletes) to the tree. - /// - /// This will fail if the keys in `batch` are not sorted and unique. This - /// check creates some overhead, so if you are sure your batch is sorted and - /// unique you can use the unsafe `apply_unchecked` for a small performance - /// gain. - /// - /// # Example - /// ``` - /// # let mut store = grovedb_merk::test_utils::TempMerk::new(); - /// # store.apply::<_, Vec<_>>(&[(vec![4,5,6], Op::Put(vec![0], BasicMerk))], &[], None) - /// .unwrap().expect(""); - /// - /// use grovedb_merk::Op; - /// use grovedb_merk::TreeFeatureType::BasicMerk; - /// - /// let batch = &[ - /// // puts value [4,5,6] to key[1,2,3] - /// (vec![1, 2, 3], Op::Put(vec![4, 5, 6], BasicMerk)), - /// // deletes key [4,5,6] - /// (vec![4, 5, 6], Op::Delete), - /// ]; - /// store.apply::<_, Vec<_>>(batch, &[], None).unwrap().expect(""); - /// ``` - pub fn apply( - &mut self, - batch: &MerkBatch, - aux: &AuxMerkBatch, - options: Option, - ) -> CostResult<(), Error> - where - KB: AsRef<[u8]>, - KA: AsRef<[u8]>, - { - let use_sum_nodes = self.is_sum_tree; - self.apply_with_costs_just_in_time_value_update( - batch, - aux, - options, - &|key, value| { - Ok(KV::layered_value_byte_cost_size_for_key_and_value_lengths( - key.len() as u32, - value.len() as u32, - use_sum_nodes, - )) - }, - &mut |_costs, _old_value, _value| Ok((false, None)), - &mut |_a, key_bytes_to_remove, value_bytes_to_remove| { - Ok(( - BasicStorageRemoval(key_bytes_to_remove), - BasicStorageRemoval(value_bytes_to_remove), - )) - }, - ) - } - - /// Applies a batch of operations (puts and deletes) to the tree. - /// - /// This will fail if the keys in `batch` are not sorted and unique. This - /// check creates some overhead, so if you are sure your batch is sorted and - /// unique you can use the unsafe `apply_unchecked` for a small performance - /// gain. - /// - /// # Example - /// ``` - /// # let mut store = grovedb_merk::test_utils::TempMerk::new(); - /// # store.apply::<_, Vec<_>>(&[(vec![4,5,6], Op::Put(vec![0], BasicMerk))], &[], None) - /// .unwrap().expect(""); - /// - /// use grovedb_merk::Op; - /// use grovedb_merk::TreeFeatureType::BasicMerk; - /// - /// let batch = &[ - /// // puts value [4,5,6] to key[1,2,3] - /// (vec![1, 2, 3], Op::Put(vec![4, 5, 6], BasicMerk)), - /// // deletes key [4,5,6] - /// (vec![4, 5, 6], Op::Delete), - /// ]; - /// store.apply::<_, Vec<_>>(batch, &[], None).unwrap().expect(""); - /// ``` - pub fn apply_with_specialized_costs( - &mut self, - batch: &MerkBatch, - aux: &AuxMerkBatch, - options: Option, - old_specialized_cost: &impl Fn(&Vec, &Vec) -> Result, - ) -> CostResult<(), Error> - where - KB: AsRef<[u8]>, - KA: AsRef<[u8]>, - { - self.apply_with_costs_just_in_time_value_update( - batch, - aux, - options, - old_specialized_cost, - &mut |_costs, _old_value, _value| Ok((false, None)), - &mut |_a, key_bytes_to_remove, value_bytes_to_remove| { - Ok(( - BasicStorageRemoval(key_bytes_to_remove), - BasicStorageRemoval(value_bytes_to_remove), - )) - }, - ) - } - - /// Applies a batch of operations (puts and deletes) to the tree with the - /// ability to update values based on costs. - /// - /// This will fail if the keys in `batch` are not sorted and unique. This - /// check creates some overhead, so if you are sure your batch is sorted and - /// unique you can use the unsafe `apply_unchecked` for a small performance - /// gain. - /// - /// # Example - /// ``` - /// # let mut store = grovedb_merk::test_utils::TempMerk::new(); - /// # store.apply_with_costs_just_in_time_value_update::<_, Vec<_>>( - /// &[(vec![4,5,6], Op::Put(vec![0], BasicMerk))], - /// &[], - /// None, - /// &|k, v| Ok(0), - /// &mut |s, v, o| Ok((false, None)), - /// &mut |s, k, v| Ok((NoStorageRemoval, NoStorageRemoval)) - /// ).unwrap().expect(""); - /// - /// use grovedb_costs::storage_cost::removal::StorageRemovedBytes::NoStorageRemoval; - /// use grovedb_merk::Op; - /// use grovedb_merk::TreeFeatureType::BasicMerk; - /// - /// let batch = &[ - /// // puts value [4,5,6] to key[1,2,3] - /// (vec![1, 2, 3], Op::Put(vec![4, 5, 6], BasicMerk)), - /// // deletes key [4,5,6] - /// (vec![4, 5, 6], Op::Delete), - /// ]; - /// - /// store.apply_with_costs_just_in_time_value_update::<_, Vec<_>>( - /// batch, - /// &[], - /// None, - /// &|k, v| Ok(0), - /// &mut |s, v, o| Ok((false, None)), - /// &mut |s, k, v| Ok((NoStorageRemoval, NoStorageRemoval)) - /// ).unwrap().expect(""); - /// ``` - pub fn apply_with_costs_just_in_time_value_update( - &mut self, - batch: &MerkBatch, - aux: &AuxMerkBatch, - options: Option, - old_specialized_cost: &impl Fn(&Vec, &Vec) -> Result, - update_tree_value_based_on_costs: &mut impl FnMut( - &StorageCost, - &Vec, - &mut Vec, - ) -> Result< - (bool, Option), - Error, - >, - section_removal_bytes: &mut impl FnMut( - &Vec, - u32, - u32, - ) -> Result< - (StorageRemovedBytes, StorageRemovedBytes), - Error, - >, - ) -> CostResult<(), Error> - where - KB: AsRef<[u8]>, - KA: AsRef<[u8]>, - { - // ensure keys in batch are sorted and unique - let mut maybe_prev_key: Option<&KB> = None; - for (key, ..) in batch.iter() { - if let Some(prev_key) = maybe_prev_key { - match prev_key.as_ref().cmp(key.as_ref()) { - Ordering::Greater => { - return Err(Error::InvalidInputError("Keys in batch must be sorted")) - .wrap_with_cost(Default::default()) - } - Ordering::Equal => { - return Err(Error::InvalidInputError("Keys in batch must be unique")) - .wrap_with_cost(Default::default()) - } - _ => (), - } - } - maybe_prev_key = Some(key); - } - - self.apply_unchecked( - batch, - aux, - options, - old_specialized_cost, - update_tree_value_based_on_costs, - section_removal_bytes, - ) - } - - /// Applies a batch of operations (puts and deletes) to the tree. - /// - /// # Safety - /// This is unsafe because the keys in `batch` must be sorted and unique - - /// if they are not, there will be undefined behavior. For a safe version of - /// this method which checks to ensure the batch is sorted and unique, see - /// `apply`. - /// - /// # Example - /// ``` - /// # let mut store = grovedb_merk::test_utils::TempMerk::new(); - /// # store.apply_with_costs_just_in_time_value_update::<_, Vec<_>>( - /// &[(vec![4,5,6], Op::Put(vec![0], BasicMerk))], - /// &[], - /// None, - /// &|k, v| Ok(0), - /// &mut |s, o, v| Ok((false, None)), - /// &mut |s, k, v| Ok((NoStorageRemoval, NoStorageRemoval)) - /// ).unwrap().expect(""); - /// - /// use grovedb_costs::storage_cost::removal::StorageRemovedBytes::NoStorageRemoval; - /// use grovedb_merk::Op; - /// use grovedb_merk::TreeFeatureType::BasicMerk; - /// - /// let batch = &[ - /// // puts value [4,5,6] to key [1,2,3] - /// (vec![1, 2, 3], Op::Put(vec![4, 5, 6], BasicMerk)), - /// // deletes key [4,5,6] - /// (vec![4, 5, 6], Op::Delete), - /// ]; - /// unsafe { store.apply_unchecked::<_, Vec<_>, _, _, _>( /// /// /// - /// batch, - /// &[], - /// None, - /// &|k, v| Ok(0), - /// &mut |s, o, v| Ok((false, None)), - /// &mut |s, k, v| Ok((NoStorageRemoval, NoStorageRemoval)) - /// ).unwrap().expect(""); - /// } - /// ``` - pub fn apply_unchecked( - &mut self, - batch: &MerkBatch, - aux: &AuxMerkBatch, - options: Option, - old_specialized_cost: &C, - update_tree_value_based_on_costs: &mut U, - section_removal_bytes: &mut R, - ) -> CostResult<(), Error> - where - KB: AsRef<[u8]>, - KA: AsRef<[u8]>, - C: Fn(&Vec, &Vec) -> Result, - U: FnMut( - &StorageCost, - &Vec, - &mut Vec, - ) -> Result<(bool, Option), Error>, - R: FnMut(&Vec, u32, u32) -> Result<(StorageRemovedBytes, StorageRemovedBytes), Error>, - { - let maybe_walker = self - .tree - .take() - .take() - .map(|tree| Walker::new(tree, self.source())); - - Walker::apply_to( - maybe_walker, - batch, - self.source(), - old_specialized_cost, - section_removal_bytes, - ) - .flat_map_ok(|(maybe_tree, key_updates)| { - // we set the new root node of the merk tree - self.tree.set(maybe_tree); - // commit changes to db - self.commit( - key_updates, - aux, - options, - old_specialized_cost, - update_tree_value_based_on_costs, - section_removal_bytes, - ) - }) - } - - /// Creates a Merkle proof for the list of queried keys. For each key in the - /// query, if the key is found in the store then the value will be proven to - /// be in the tree. For each key in the query that does not exist in the - /// tree, its absence will be proven by including boundary keys. - /// - /// The proof returned is in an encoded format which can be verified with - /// `merk::verify`. - /// - /// This will fail if the keys in `query` are not sorted and unique. This - /// check adds some overhead, so if you are sure your batch is sorted and - /// unique you can use the unsafe `prove_unchecked` for a small performance - /// gain. - pub fn prove( - &self, - query: Query, - limit: Option, - offset: Option, - ) -> CostResult { - let left_to_right = query.left_to_right; - self.prove_unchecked(query, limit, offset, left_to_right) - .map_ok(|(proof, limit, offset)| { - let mut bytes = Vec::with_capacity(128); - encode_into(proof.iter(), &mut bytes); - ProofConstructionResult::new(bytes, limit, offset) - }) - } - - /// Creates a Merkle proof for the list of queried keys. For each key in the - /// query, if the key is found in the store then the value will be proven to - /// be in the tree. For each key in the query that does not exist in the - /// tree, its absence will be proven by including boundary keys. - /// - /// The proof returned is in an intermediate format to be later encoded - /// - /// This will fail if the keys in `query` are not sorted and unique. This - /// check adds some overhead, so if you are sure your batch is sorted and - /// unique you can use the unsafe `prove_unchecked` for a small performance - /// gain. - pub fn prove_without_encoding( - &self, - query: Query, - limit: Option, - offset: Option, - ) -> CostResult { - let left_to_right = query.left_to_right; - self.prove_unchecked(query, limit, offset, left_to_right) - .map_ok(|(proof, limit, offset)| ProofWithoutEncodingResult::new(proof, limit, offset)) - } - - /// Creates a Merkle proof for the list of queried keys. For each key in - /// the query, if the key is found in the store then the value will be - /// proven to be in the tree. For each key in the query that does not - /// exist in the tree, its absence will be proven by including - /// boundary keys. - /// The proof returned is in an encoded format which can be verified with - /// `merk::verify`. - /// - /// This is unsafe because the keys in `query` must be sorted and unique - - /// if they are not, there will be undefined behavior. For a safe version - /// of this method which checks to ensure the batch is sorted and - /// unique, see `prove`. - pub fn prove_unchecked( - &self, - query: I, - limit: Option, - offset: Option, - left_to_right: bool, - ) -> CostResult - where - Q: Into, - I: IntoIterator, - { - let query_vec: Vec = query.into_iter().map(Into::into).collect(); - - self.use_tree_mut(|maybe_tree| { - maybe_tree - .ok_or(Error::CorruptedCodeExecution( - "Cannot create proof for empty tree", - )) - .wrap_with_cost(Default::default()) - .flat_map_ok(|tree| { - let mut ref_walker = RefWalker::new(tree, self.source()); - ref_walker.create_proof(query_vec.as_slice(), limit, offset, left_to_right) - }) - .map_ok(|(proof, _, limit, offset, ..)| (proof, limit, offset)) - }) - } - /// Commit tree changes pub fn commit( &mut self, @@ -1186,13 +480,6 @@ where true.wrap_with_cost(cost) } - fn source(&self) -> MerkSource { - MerkSource { - storage: &self.storage, - is_sum_tree: self.is_sum_tree, - } - } - /// Use tree pub(crate) fn use_tree(&self, f: impl FnOnce(Option<&Tree>) -> T) -> T { let tree = self.tree.take(); @@ -1276,173 +563,19 @@ fn fetch_node<'db>(db: &impl StorageContext<'db>, key: &[u8]) -> Result { -// fn clone(&self) -> Self { -// let tree_clone = match self.tree.take() { -// None => None, -// Some(tree) => { -// let clone = tree.clone(); -// self.tree.set(Some(tree)); -// Some(clone) -// } -// }; -// Self { -// tree: Cell::new(tree_clone), -// storage_cost: self.storage_cost.clone(), -// } -// } -// } - // // TODO: get rid of Fetch/source and use GroveDB storage_cost abstraction -#[derive(Debug)] -pub struct MerkSource<'s, S> { - storage: &'s S, - is_sum_tree: bool, -} - -impl<'s, S> Clone for MerkSource<'s, S> { - fn clone(&self) -> Self { - MerkSource { - storage: self.storage, - is_sum_tree: self.is_sum_tree, - } - } -} - -impl<'s, 'db, S> Fetch for MerkSource<'s, S> -where - S: StorageContext<'db>, -{ - fn fetch(&self, link: &Link) -> CostResult { - Tree::get(self.storage, link.key()) - .map_ok(|x| x.ok_or(Error::KeyNotFoundError("Key not found for fetch"))) - .flatten() - } -} - -struct MerkCommitter { - /// The batch has a key, maybe a value, with the value bytes, maybe the left - /// child size and maybe the right child size, then the - /// key_value_storage_cost - batch: Vec, - height: u8, - levels: u8, -} - -impl MerkCommitter { - fn new(height: u8, levels: u8) -> Self { - Self { - batch: Vec::with_capacity(10000), - height, - levels, - } - } -} - -impl Commit for MerkCommitter { - fn write( - &mut self, - tree: &mut Tree, - old_specialized_cost: &impl Fn(&Vec, &Vec) -> Result, - update_tree_value_based_on_costs: &mut impl FnMut( - &StorageCost, - &Vec, - &mut Vec, - ) -> Result< - (bool, Option), - Error, - >, - section_removal_bytes: &mut impl FnMut( - &Vec, - u32, - u32, - ) -> Result< - (StorageRemovedBytes, StorageRemovedBytes), - Error, - >, - ) -> Result<(), Error> { - let tree_size = tree.encoding_length(); - let (mut current_tree_plus_hook_size, mut storage_costs) = - tree.kv_with_parent_hook_size_and_storage_cost(old_specialized_cost)?; - let mut i = 0; - - if let Some(old_value) = tree.old_value.clone() { - // At this point the tree value can be updated based on client requirements - // For example to store the costs - loop { - let (flags_changed, value_defined_cost) = update_tree_value_based_on_costs( - &storage_costs.value_storage_cost, - &old_value, - tree.value_mut_ref(), - )?; - if !flags_changed { - break; - } else { - tree.inner.kv.value_defined_cost = value_defined_cost; - let after_update_tree_plus_hook_size = - tree.value_encoding_length_with_parent_to_child_reference(); - if after_update_tree_plus_hook_size == current_tree_plus_hook_size { - break; - } - let new_size_and_storage_costs = - tree.kv_with_parent_hook_size_and_storage_cost(old_specialized_cost)?; - current_tree_plus_hook_size = new_size_and_storage_costs.0; - storage_costs = new_size_and_storage_costs.1; - } - if i > MAX_UPDATE_VALUE_BASED_ON_COSTS_TIMES { - return Err(Error::CyclicError( - "updated value based on costs too many times", - )); - } - i += 1; - } - - if let BasicStorageRemoval(removed_bytes) = - storage_costs.value_storage_cost.removed_bytes - { - let (_, value_removed_bytes) = section_removal_bytes(&old_value, 0, removed_bytes)?; - storage_costs.value_storage_cost.removed_bytes = value_removed_bytes; - } - } - - // Update old tree size after generating value storage_cost cost - tree.old_size_with_parent_to_child_hook = current_tree_plus_hook_size; - tree.old_value = Some(tree.value_ref().clone()); - - let mut buf = Vec::with_capacity(tree_size); - tree.encode_into(&mut buf); - - let left_child_sizes = tree.child_ref_and_sum_size(true); - let right_child_sizes = tree.child_ref_and_sum_size(false); - self.batch.push(( - tree.key().to_vec(), - tree.feature_type().sum_length(), - Some((buf, left_child_sizes, right_child_sizes)), - Some(storage_costs), - )); - Ok(()) - } - - fn prune(&self, tree: &Tree) -> (bool, bool) { - // keep N top levels of tree - let prune = (self.height - tree.height()) >= self.levels; - (prune, prune) - } -} - #[cfg(test)] mod test { - use grovedb_costs::OperationCost; use grovedb_path::SubtreePath; use grovedb_storage::{ - rocksdb_storage::{test_utils::TempStorage, PrefixedRocksDbStorageContext, RocksDbStorage}, + rocksdb_storage::{PrefixedRocksDbStorageContext, RocksDbStorage}, RawIterator, Storage, StorageBatch, StorageContext, }; use tempfile::TempDir; - use super::{Merk, MerkSource, RefWalker}; - use crate::{test_utils::*, Op, TreeFeatureType::BasicMerk}; + use super::{Merk, RefWalker}; + use crate::{merk::source::MerkSource, test_utils::*, Op, TreeFeatureType::BasicMerk}; // TODO: Close and then reopen test @@ -1453,97 +586,6 @@ mod test { }) } - #[test] - fn test_reopen_root_hash() { - let tmp_dir = TempDir::new().expect("cannot open tempdir"); - let storage = RocksDbStorage::default_rocksdb_with_path(tmp_dir.path()) - .expect("cannot open rocksdb storage"); - let test_prefix = [b"ayy"]; - - let batch = StorageBatch::new(); - let mut merk = Merk::open_base( - storage - .get_storage_context(SubtreePath::from(test_prefix.as_ref()), Some(&batch)) - .unwrap(), - false, - ) - .unwrap() - .unwrap(); - - merk.apply::<_, Vec<_>>( - &[(vec![1, 2, 3], Op::Put(vec![4, 5, 6], BasicMerk))], - &[], - None, - ) - .unwrap() - .expect("apply failed"); - - let root_hash = merk.root_hash(); - - storage - .commit_multi_context_batch(batch, None) - .unwrap() - .expect("cannot commit batch"); - - let merk = Merk::open_base( - storage - .get_storage_context(SubtreePath::from(test_prefix.as_ref()), None) - .unwrap(), - false, - ) - .unwrap() - .unwrap(); - assert_eq!(merk.root_hash(), root_hash); - } - - #[test] - fn test_open_fee() { - let storage = TempStorage::new(); - let batch = StorageBatch::new(); - - let merk_fee_context = Merk::open_base( - storage - .get_storage_context(SubtreePath::empty(), Some(&batch)) - .unwrap(), - false, - ); - - // Opening not existing merk should cost only root key seek (except context - // creation) - assert!(matches!( - merk_fee_context.cost(), - OperationCost { seek_count: 1, .. } - )); - - let mut merk = merk_fee_context.unwrap().unwrap(); - merk.apply::<_, Vec<_>>( - &[(vec![1, 2, 3], Op::Put(vec![4, 5, 6], BasicMerk))], - &[], - None, - ) - .unwrap() - .expect("apply failed"); - - storage - .commit_multi_context_batch(batch, None) - .unwrap() - .expect("cannot commit batch"); - - let merk_fee_context = Merk::open_base( - storage - .get_storage_context(SubtreePath::empty(), None) - .unwrap(), - false, - ); - - // Opening existing merk should cost two seeks. (except context creation) - assert!(matches!( - merk_fee_context.cost(), - OperationCost { seek_count: 2, .. } - )); - assert!(merk_fee_context.cost().storage_loaded_bytes > 0); - } - #[test] fn simple_insert_apply() { let batch_size = 20; @@ -1581,29 +623,6 @@ mod test { assert_invariants(&merk); } - #[test] - fn test_has_node_with_empty_tree() { - let mut merk = TempMerk::new(); - - let key = b"something"; - - let result = merk.has_node(key).unwrap().unwrap(); - - assert!(!result); - - let batch_entry = (key, Op::Put(vec![123; 60], BasicMerk)); - - let batch = vec![batch_entry]; - - merk.apply::<_, Vec<_>>(&batch, &[], None) - .unwrap() - .expect("should ..."); - - let result = merk.has_node(key).unwrap().unwrap(); - - assert!(result); - } - #[test] fn insert_two() { let tree_size = 2; diff --git a/merk/src/merk/open.rs b/merk/src/merk/open.rs new file mode 100644 index 000000000..66a0b2120 --- /dev/null +++ b/merk/src/merk/open.rs @@ -0,0 +1,171 @@ +use std::cell::Cell; + +use grovedb_costs::CostResult; +use grovedb_storage::StorageContext; + +use crate::{ + Error, Merk, MerkType, + MerkType::{BaseMerk, LayeredMerk, StandaloneMerk}, +}; + +impl<'db, S> Merk +where + S: StorageContext<'db>, +{ + /// Open empty tree + pub fn open_empty(storage: S, merk_type: MerkType, is_sum_tree: bool) -> Self { + Self { + tree: Cell::new(None), + root_tree_key: Cell::new(None), + storage, + merk_type, + is_sum_tree, + } + } + + /// Open standalone tree + pub fn open_standalone(storage: S, is_sum_tree: bool) -> CostResult { + let mut merk = Self { + tree: Cell::new(None), + root_tree_key: Cell::new(None), + storage, + merk_type: StandaloneMerk, + is_sum_tree, + }; + + merk.load_base_root().map_ok(|_| merk) + } + + /// Open base tree + pub fn open_base(storage: S, is_sum_tree: bool) -> CostResult { + let mut merk = Self { + tree: Cell::new(None), + root_tree_key: Cell::new(None), + storage, + merk_type: BaseMerk, + is_sum_tree, + }; + + merk.load_base_root().map_ok(|_| merk) + } + + /// Open layered tree with root key + pub fn open_layered_with_root_key( + storage: S, + root_key: Option>, + is_sum_tree: bool, + ) -> CostResult { + let mut merk = Self { + tree: Cell::new(None), + root_tree_key: Cell::new(root_key), + storage, + merk_type: LayeredMerk, + is_sum_tree, + }; + + merk.load_root().map_ok(|_| merk) + } +} + +#[cfg(test)] +mod test { + use grovedb_path::SubtreePath; + use grovedb_storage::{ + rocksdb_storage::{test_utils::TempStorage, RocksDbStorage}, + Storage, StorageBatch, + }; + use tempfile::TempDir; + + use crate::{Merk, Op, TreeFeatureType::BasicMerk}; + + #[test] + fn test_reopen_root_hash() { + let tmp_dir = TempDir::new().expect("cannot open tempdir"); + let storage = RocksDbStorage::default_rocksdb_with_path(tmp_dir.path()) + .expect("cannot open rocksdb storage"); + let test_prefix = [b"ayy"]; + + let batch = StorageBatch::new(); + let mut merk = Merk::open_base( + storage + .get_storage_context(SubtreePath::from(test_prefix.as_ref()), Some(&batch)) + .unwrap(), + false, + ) + .unwrap() + .unwrap(); + + merk.apply::<_, Vec<_>>( + &[(vec![1, 2, 3], Op::Put(vec![4, 5, 6], BasicMerk))], + &[], + None, + ) + .unwrap() + .expect("apply failed"); + + let root_hash = merk.root_hash(); + + storage + .commit_multi_context_batch(batch, None) + .unwrap() + .expect("cannot commit batch"); + + let merk = Merk::open_base( + storage + .get_storage_context(SubtreePath::from(test_prefix.as_ref()), None) + .unwrap(), + false, + ) + .unwrap() + .unwrap(); + assert_eq!(merk.root_hash(), root_hash); + } + + #[test] + fn test_open_fee() { + let storage = TempStorage::new(); + let batch = StorageBatch::new(); + + let merk_fee_context = Merk::open_base( + storage + .get_storage_context(SubtreePath::empty(), Some(&batch)) + .unwrap(), + false, + ); + + // Opening not existing merk should cost only root key seek (except context + // creation) + assert!(matches!( + merk_fee_context.cost(), + OperationCost { seek_count: 1, .. } + )); + + let mut merk = merk_fee_context.unwrap().unwrap(); + merk.apply::<_, Vec<_>>( + &[(vec![1, 2, 3], Op::Put(vec![4, 5, 6], BasicMerk))], + &[], + None, + ) + .unwrap() + .expect("apply failed"); + + storage + .commit_multi_context_batch(batch, None) + .unwrap() + .expect("cannot commit batch"); + + let merk_fee_context = Merk::open_base( + storage + .get_storage_context(SubtreePath::empty(), None) + .unwrap(), + false, + ); + + // Opening existing merk should cost two seeks. (except context creation) + assert!(matches!( + merk_fee_context.cost(), + OperationCost { seek_count: 2, .. } + )); + assert!(merk_fee_context.cost().storage_loaded_bytes > 0); + } +} diff --git a/merk/src/merk/prove.rs b/merk/src/merk/prove.rs new file mode 100644 index 000000000..7f2955349 --- /dev/null +++ b/merk/src/merk/prove.rs @@ -0,0 +1,147 @@ +use std::collections::LinkedList; + +use grovedb_costs::{CostResult, CostsExt}; +use grovedb_storage::StorageContext; + +use crate::{ + proofs::{encode_into, query::QueryItem, Op as ProofOp, Query}, + tree::RefWalker, + Error, Merk, +}; + +impl<'db, S> Merk +where + S: StorageContext<'db>, +{ + /// Creates a Merkle proof for the list of queried keys. For each key in the + /// query, if the key is found in the store then the value will be proven to + /// be in the tree. For each key in the query that does not exist in the + /// tree, its absence will be proven by including boundary keys. + /// + /// The proof returned is in an encoded format which can be verified with + /// `merk::verify`. + /// + /// This will fail if the keys in `query` are not sorted and unique. This + /// check adds some overhead, so if you are sure your batch is sorted and + /// unique you can use the unsafe `prove_unchecked` for a small performance + /// gain. + pub fn prove( + &self, + query: Query, + limit: Option, + offset: Option, + ) -> CostResult { + let left_to_right = query.left_to_right; + self.prove_unchecked(query, limit, offset, left_to_right) + .map_ok(|(proof, limit, offset)| { + let mut bytes = Vec::with_capacity(128); + encode_into(proof.iter(), &mut bytes); + ProofConstructionResult::new(bytes, limit, offset) + }) + } + + /// Creates a Merkle proof for the list of queried keys. For each key in the + /// query, if the key is found in the store then the value will be proven to + /// be in the tree. For each key in the query that does not exist in the + /// tree, its absence will be proven by including boundary keys. + /// + /// The proof returned is in an intermediate format to be later encoded + /// + /// This will fail if the keys in `query` are not sorted and unique. This + /// check adds some overhead, so if you are sure your batch is sorted and + /// unique you can use the unsafe `prove_unchecked` for a small performance + /// gain. + pub fn prove_without_encoding( + &self, + query: Query, + limit: Option, + offset: Option, + ) -> CostResult { + let left_to_right = query.left_to_right; + self.prove_unchecked(query, limit, offset, left_to_right) + .map_ok(|(proof, limit, offset)| ProofWithoutEncodingResult::new(proof, limit, offset)) + } + + /// Creates a Merkle proof for the list of queried keys. For each key in + /// the query, if the key is found in the store then the value will be + /// proven to be in the tree. For each key in the query that does not + /// exist in the tree, its absence will be proven by including + /// boundary keys. + /// The proof returned is in an encoded format which can be verified with + /// `merk::verify`. + /// + /// This is unsafe because the keys in `query` must be sorted and unique - + /// if they are not, there will be undefined behavior. For a safe version + /// of this method which checks to ensure the batch is sorted and + /// unique, see `prove`. + pub fn prove_unchecked( + &self, + query: I, + limit: Option, + offset: Option, + left_to_right: bool, + ) -> CostResult + where + Q: Into, + I: IntoIterator, + { + let query_vec: Vec = query.into_iter().map(Into::into).collect(); + + self.use_tree_mut(|maybe_tree| { + maybe_tree + .ok_or(Error::CorruptedCodeExecution( + "Cannot create proof for empty tree", + )) + .wrap_with_cost(Default::default()) + .flat_map_ok(|tree| { + let mut ref_walker = RefWalker::new(tree, self.source()); + ref_walker.create_proof(query_vec.as_slice(), limit, offset, left_to_right) + }) + .map_ok(|(proof, _, limit, offset, ..)| (proof, limit, offset)) + }) + } +} + +type Proof = (LinkedList, Option, Option); + +/// Proof construction result +pub struct ProofConstructionResult { + /// Proof + pub proof: Vec, + /// Limit + pub limit: Option, + /// Offset + pub offset: Option, +} + +impl ProofConstructionResult { + /// New ProofConstructionResult + pub fn new(proof: Vec, limit: Option, offset: Option) -> Self { + Self { + proof, + limit, + offset, + } + } +} + +/// Proof without encoding result +pub struct ProofWithoutEncodingResult { + /// Proof + pub proof: LinkedList, + /// Limit + pub limit: Option, + /// Offset + pub offset: Option, +} + +impl ProofWithoutEncodingResult { + /// New ProofWithoutEncodingResult + pub fn new(proof: LinkedList, limit: Option, offset: Option) -> Self { + Self { + proof, + limit, + offset, + } + } +} diff --git a/merk/src/merk/restore.rs b/merk/src/merk/restore.rs index 23cef7038..3d9862fb4 100644 --- a/merk/src/merk/restore.rs +++ b/merk/src/merk/restore.rs @@ -38,9 +38,10 @@ use grovedb_storage::{Batch, StorageContext}; #[cfg(feature = "full")] use super::Merk; #[cfg(feature = "full")] +use crate::merk::source::MerkSource; +#[cfg(feature = "full")] use crate::{ error::Error, - merk::MerkSource, proofs::{ chunk::{verify_leaf, verify_trunk, MIN_TRUNK_HEIGHT}, tree::{Child, Tree as ProofTree}, diff --git a/merk/src/merk/source.rs b/merk/src/merk/source.rs new file mode 100644 index 000000000..07be638ee --- /dev/null +++ b/merk/src/merk/source.rs @@ -0,0 +1,45 @@ +use grovedb_costs::CostResult; +use grovedb_storage::StorageContext; + +use crate::{ + tree::{Fetch, Tree}, + Error, Link, Merk, +}; + +impl<'db, S> Merk +where + S: StorageContext<'db>, +{ + pub(in crate::merk) fn source(&self) -> MerkSource { + MerkSource { + storage: &self.storage, + is_sum_tree: self.is_sum_tree, + } + } +} + +#[derive(Debug)] +pub struct MerkSource<'s, S> { + storage: &'s S, + is_sum_tree: bool, +} + +impl<'s, S> Clone for MerkSource<'s, S> { + fn clone(&self) -> Self { + MerkSource { + storage: self.storage, + is_sum_tree: self.is_sum_tree, + } + } +} + +impl<'s, 'db, S> Fetch for MerkSource<'s, S> +where + S: StorageContext<'db>, +{ + fn fetch(&self, link: &Link) -> CostResult { + Tree::get(self.storage, link.key()) + .map_ok(|x| x.ok_or(Error::KeyNotFoundError("Key not found for fetch"))) + .flatten() + } +} diff --git a/merk/src/test_utils/mod.rs b/merk/src/test_utils/mod.rs index b75181589..90cf1d339 100644 --- a/merk/src/test_utils/mod.rs +++ b/merk/src/test_utils/mod.rs @@ -85,6 +85,7 @@ pub fn apply_memonly_unchecked(tree: Tree, batch: &MerkBatch>) -> Tree { is_sum_node, )) }, + &mut |_, _, _| Ok((false, None)), &mut |_flags, key_bytes_to_remove, value_bytes_to_remove| { Ok(( BasicStorageRemoval(key_bytes_to_remove), @@ -146,6 +147,7 @@ pub fn apply_to_memonly( is_sum_tree, )) }, + &mut |_, _, _| Ok((false, None)), &mut |_flags, key_bytes_to_remove, value_bytes_to_remove| { Ok(( BasicStorageRemoval(key_bytes_to_remove), diff --git a/merk/src/tree/ops.rs b/merk/src/tree/ops.rs index 1b161cdd0..4c69fc891 100644 --- a/merk/src/tree/ops.rs +++ b/merk/src/tree/ops.rs @@ -53,7 +53,10 @@ use Op::*; use super::{Fetch, Link, Tree, Walker}; #[cfg(feature = "full")] use crate::{error::Error, tree::tree_feature_type::TreeFeatureType, CryptoHash, HASH_LENGTH_U32}; -use crate::{merk::KeyUpdates, tree::kv::ValueDefinedCostType::SpecializedValueDefinedCost}; +use crate::{ + merk::KeyUpdates, + tree::kv::{ValueDefinedCostType, ValueDefinedCostType::SpecializedValueDefinedCost}, +}; #[cfg(feature = "full")] /// An operation to be applied to a key in the store. @@ -162,15 +165,21 @@ where /// not require a non-empty tree. /// /// Keys in batch must be sorted and unique. - pub fn apply_to, C, R>( + pub fn apply_to, C, U, R>( maybe_tree: Option, batch: &MerkBatch, source: S, old_tree_cost: &C, + update_tree_value_based_on_costs: &mut U, section_removal_bytes: &mut R, ) -> CostContext, KeyUpdates), Error>> where C: Fn(&Vec, &Vec) -> Result, + U: FnMut( + &StorageCost, + &Vec, + &mut Vec, + ) -> Result<(bool, Option), Error>, R: FnMut(&Vec, u32, u32) -> Result<(StorageRemovedBytes, StorageRemovedBytes), Error>, { let mut cost = OperationCost::default(); @@ -188,28 +197,38 @@ where } else { match maybe_tree { None => { - return Self::build(batch, source, old_tree_cost, section_removal_bytes).map_ok( - |tree| { - let new_keys: BTreeSet> = batch - .iter() - .map(|batch_entry| batch_entry.0.as_ref().to_vec()) - .collect(); - ( - tree, - KeyUpdates::new( - new_keys, - BTreeSet::default(), - LinkedList::default(), - None, - ), - ) - }, + return Self::build( + batch, + source, + old_tree_cost, + update_tree_value_based_on_costs, + section_removal_bytes, ) + .map_ok(|tree| { + let new_keys: BTreeSet> = batch + .iter() + .map(|batch_entry| batch_entry.0.as_ref().to_vec()) + .collect(); + ( + tree, + KeyUpdates::new( + new_keys, + BTreeSet::default(), + LinkedList::default(), + None, + ), + ) + }) } Some(tree) => { cost_return_on_error!( &mut cost, - tree.apply_sorted(batch, old_tree_cost, section_removal_bytes) + tree.apply_sorted( + batch, + old_tree_cost, + update_tree_value_based_on_costs, + section_removal_bytes + ) ) } } @@ -222,14 +241,20 @@ where /// Builds a `Tree` from a batch of operations. /// /// Keys in batch must be sorted and unique. - fn build, C, R>( + fn build, C, U, R>( batch: &MerkBatch, source: S, old_tree_cost: &C, + update_tree_value_based_on_costs: &mut U, section_removal_bytes: &mut R, ) -> CostResult, Error> where C: Fn(&Vec, &Vec) -> Result, + U: FnMut( + &StorageCost, + &Vec, + &mut Vec, + ) -> Result<(bool, Option), Error>, R: FnMut(&Vec, u32, u32) -> Result<(StorageRemovedBytes, StorageRemovedBytes), Error>, { let mut cost = OperationCost::default(); @@ -251,6 +276,7 @@ where left_batch, source.clone(), old_tree_cost, + update_tree_value_based_on_costs, section_removal_bytes ) ) @@ -259,7 +285,12 @@ where Some(tree) => { cost_return_on_error!( &mut cost, - tree.apply_sorted(right_batch, old_tree_cost, section_removal_bytes) + tree.apply_sorted( + right_batch, + old_tree_cost, + update_tree_value_based_on_costs, + section_removal_bytes + ) ) .0 } @@ -269,6 +300,7 @@ where right_batch, source.clone(), old_tree_cost, + update_tree_value_based_on_costs, section_removal_bytes ) ) @@ -338,6 +370,7 @@ where None ), old_tree_cost, + update_tree_value_based_on_costs, section_removal_bytes, ) ) @@ -354,6 +387,7 @@ where self.apply_sorted( batch, &|_, _| Ok(0), + &mut |_, _, _| Ok((false, None)), &mut |_flags, key_bytes_to_remove, value_bytes_to_remove| { Ok(( BasicStorageRemoval(key_bytes_to_remove), @@ -367,14 +401,20 @@ where /// `Walker::apply`_to, but requires a populated tree. /// /// Keys in batch must be sorted and unique. - fn apply_sorted, C, R>( + fn apply_sorted, C, U, R>( self, batch: &MerkBatch, old_specialized_cost: &C, + update_tree_value_based_on_costs: &mut U, section_removal_bytes: &mut R, ) -> CostResult<(Option, KeyUpdates), Error> where C: Fn(&Vec, &Vec) -> Result, + U: FnMut( + &StorageCost, + &Vec, + &mut Vec, + ) -> Result<(bool, Option), Error>, R: FnMut(&Vec, u32, u32) -> Result<(StorageRemovedBytes, StorageRemovedBytes), Error>, { let mut cost = OperationCost::default(); @@ -467,6 +507,7 @@ where &batch[..index], source.clone(), old_specialized_cost, + update_tree_value_based_on_costs, section_removal_bytes ) ); @@ -479,6 +520,7 @@ where &batch[index + 1..], source.clone(), old_specialized_cost, + update_tree_value_based_on_costs, section_removal_bytes ) ); @@ -520,6 +562,7 @@ where exclusive, KeyUpdates::new(new_keys, updated_keys, LinkedList::default(), None), old_specialized_cost, + update_tree_value_based_on_costs, section_removal_bytes, ) .add_cost(cost) @@ -530,17 +573,23 @@ where /// /// This recursion executes serially in the same thread, but in the future /// will be dispatched to workers in other threads. - fn recurse, C, R>( + fn recurse, C, U, R>( self, batch: &MerkBatch, mid: usize, exclusive: bool, mut key_updates: KeyUpdates, old_tree_cost: &C, + update_tree_value_based_on_costs: &mut U, section_removal_bytes: &mut R, ) -> CostResult<(Option, KeyUpdates), Error> where C: Fn(&Vec, &Vec) -> Result, + U: FnMut( + &StorageCost, + &Vec, + &mut Vec, + ) -> Result<(bool, Option), Error>, R: FnMut(&Vec, u32, u32) -> Result<(StorageRemovedBytes, StorageRemovedBytes), Error>, { let mut cost = OperationCost::default(); @@ -564,6 +613,7 @@ where left_batch, source, old_tree_cost, + update_tree_value_based_on_costs, section_removal_bytes, ) .map_ok(|(maybe_left, mut key_updates_left)| { @@ -592,6 +642,7 @@ where right_batch, source, old_tree_cost, + update_tree_value_based_on_costs, section_removal_bytes, ) .map_ok(|(maybe_right, mut key_updates_right)| { @@ -895,11 +946,12 @@ mod test { #[test] fn apply_empty_none() { - let (maybe_tree, key_updates) = Walker::::apply_to::, _, _>( + let (maybe_tree, key_updates) = Walker::::apply_to::, _, _, _>( None, &[], PanicSource {}, &|_, _| Ok(0), + &mut |_, _, _| Ok((false, None)), &mut |_flags, key_bytes_to_remove, value_bytes_to_remove| { Ok(( BasicStorageRemoval(key_bytes_to_remove), @@ -922,6 +974,7 @@ mod test { &batch, PanicSource {}, &|_, _| Ok(0), + &mut |_, _, _| Ok((false, None)), &mut |_flags, key_bytes_to_remove, value_bytes_to_remove| { Ok(( BasicStorageRemoval(key_bytes_to_remove), @@ -947,6 +1000,7 @@ mod test { &batch, PanicSource {}, &|_, _| Ok(0), + &mut |_, _, _| Ok((false, None)), &mut |_flags, key_bytes_to_remove, value_bytes_to_remove| { Ok(( BasicStorageRemoval(key_bytes_to_remove), @@ -969,6 +1023,7 @@ mod test { &batch, PanicSource {}, &|_, _| Ok(0), + &mut |_, _, _| Ok((false, None)), &mut |_flags, key_bytes_to_remove, value_bytes_to_remove| { Ok(( BasicStorageRemoval(key_bytes_to_remove), @@ -997,6 +1052,7 @@ mod test { &batch, PanicSource {}, &|_, _| Ok(0), + &mut |_, _, _| Ok((false, None)), &mut |_flags, key_bytes_to_remove, value_bytes_to_remove| { Ok(( BasicStorageRemoval(key_bytes_to_remove), @@ -1020,6 +1076,7 @@ mod test { &batch, PanicSource {}, &|_, _| Ok(0), + &mut |_, _, _| Ok((false, None)), &mut |_flags, key_bytes_to_remove, value_bytes_to_remove| { Ok(( BasicStorageRemoval(key_bytes_to_remove), diff --git a/merk/src/tree/walk/mod.rs b/merk/src/tree/walk/mod.rs index 9cc5bb163..77e4f595c 100644 --- a/merk/src/tree/walk/mod.rs +++ b/merk/src/tree/walk/mod.rs @@ -182,7 +182,7 @@ where self } - /// Similar to `Tree#with_value`. + /// Similar to `Tree#put_value`. pub fn put_value(mut self, value: Vec, feature_type: TreeFeatureType) -> CostContext { let mut cost = OperationCost::default(); self.tree @@ -190,7 +190,7 @@ where self.wrap_with_cost(cost) } - /// Similar to `Tree#with_value`. + /// Similar to `Tree#put_value_with_fixed_cost`. pub fn put_value_with_fixed_cost( mut self, value: Vec, @@ -205,7 +205,7 @@ where self.wrap_with_cost(cost) } - /// Similar to `Tree#with_value_and_value_hash`. + /// Similar to `Tree#put_value_and_reference_value_hash`. pub fn put_value_and_reference_value_hash( mut self, value: Vec, @@ -220,7 +220,7 @@ where self.wrap_with_cost(cost) } - /// Similar to `Tree#with_value_and_value_hash`. + /// Similar to `Tree#put_value_with_reference_value_hash_and_value_cost`. pub fn put_value_with_reference_value_hash_and_value_cost( mut self, value: Vec, From db7208f191d22dd00f9be6783675f93204a86d66 Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Sat, 23 Sep 2023 00:41:57 +0700 Subject: [PATCH 02/18] tree to tree node --- grovedb/src/element/get.rs | 4 +- grovedb/src/element/helpers.rs | 4 +- .../src/estimated_costs/average_case_costs.rs | 8 +-- .../src/estimated_costs/worst_case_costs.rs | 8 +-- .../src/estimated_costs/average_case_costs.rs | 6 +- merk/src/estimated_costs/worst_case_costs.rs | 6 +- merk/src/merk/committer.rs | 6 +- merk/src/merk/get.rs | 10 +-- merk/src/merk/mod.rs | 16 ++--- merk/src/merk/open.rs | 2 +- merk/src/merk/restore.rs | 10 +-- merk/src/merk/source.rs | 6 +- merk/src/proofs/chunk.rs | 8 +-- merk/src/proofs/query/mod.rs | 50 ++++++------- merk/src/test_utils/mod.rs | 20 +++--- merk/src/tree/commit.rs | 10 +-- merk/src/tree/debug.rs | 6 +- merk/src/tree/encoding.rs | 40 +++++------ merk/src/tree/fuzz_tests.rs | 4 +- merk/src/tree/iter.rs | 10 +-- merk/src/tree/link.rs | 28 ++++---- merk/src/tree/mod.rs | 72 +++++++++---------- merk/src/tree/ops.rs | 38 +++++----- merk/src/tree/walk/fetch.rs | 4 +- merk/src/tree/walk/mod.rs | 46 ++++++------ merk/src/tree/walk/ref_walker.rs | 8 +-- merk/src/visualize.rs | 6 +- 27 files changed, 218 insertions(+), 218 deletions(-) diff --git a/grovedb/src/element/get.rs b/grovedb/src/element/get.rs index eae7baa0e..57cd40121 100644 --- a/grovedb/src/element/get.rs +++ b/grovedb/src/element/get.rs @@ -37,7 +37,7 @@ use grovedb_merk::tree::kv::KV; #[cfg(feature = "full")] use grovedb_merk::Merk; #[cfg(feature = "full")] -use grovedb_merk::{ed::Decode, tree::TreeInner}; +use grovedb_merk::{ed::Decode, tree::TreeNodeInner}; #[cfg(feature = "full")] use grovedb_storage::StorageContext; use integer_encoding::VarInt; @@ -129,7 +129,7 @@ impl Element { .get(key_ref) .map_err(|e| Error::CorruptedData(e.to_string())) ); - let maybe_tree_inner: Option = cost_return_on_error_no_add!( + let maybe_tree_inner: Option = cost_return_on_error_no_add!( &cost, node_value_opt .map(|node_value| { diff --git a/grovedb/src/element/helpers.rs b/grovedb/src/element/helpers.rs index c049e7baa..60b529317 100644 --- a/grovedb/src/element/helpers.rs +++ b/grovedb/src/element/helpers.rs @@ -31,7 +31,7 @@ #[cfg(feature = "full")] use grovedb_merk::{ - tree::{kv::KV, Tree}, + tree::{kv::KV, TreeNode}, TreeFeatureType, TreeFeatureType::{BasicMerk, SummedMerk}, }; @@ -312,7 +312,7 @@ impl Element { #[cfg(feature = "full")] /// Decode from bytes pub fn raw_decode(bytes: &[u8]) -> Result { - let tree = Tree::decode_raw(bytes, vec![]).map_err(|e| Error::CorruptedData(e.to_string()))?; + let tree = TreeNode::decode_raw(bytes, vec![]).map_err(|e| Error::CorruptedData(e.to_string()))?; let element: Element = Element::deserialize(tree.value_as_slice())?; Ok(element) } diff --git a/grovedb/src/estimated_costs/average_case_costs.rs b/grovedb/src/estimated_costs/average_case_costs.rs index 919d4d5a0..e93e1b8fd 100644 --- a/grovedb/src/estimated_costs/average_case_costs.rs +++ b/grovedb/src/estimated_costs/average_case_costs.rs @@ -40,7 +40,7 @@ use grovedb_merk::{ add_average_case_merk_replace_layered, EstimatedLayerInformation, }, }, - tree::Tree, + tree::TreeNode, HASH_LENGTH, }; use grovedb_storage::{worst_case_costs::WorstKeyLength, Storage}; @@ -68,7 +68,7 @@ impl GroveDb { match path.last() { None => {} Some(key) => { - cost.storage_loaded_bytes += Tree::average_case_encoded_tree_size( + cost.storage_loaded_bytes += TreeNode::average_case_encoded_tree_size( key.max_length() as u32, HASH_LENGTH as u32, is_sum_tree, @@ -359,7 +359,7 @@ impl GroveDb { estimated_element_size: u32, in_parent_tree_using_sums: bool, ) { - let value_size = Tree::average_case_encoded_tree_size( + let value_size = TreeNode::average_case_encoded_tree_size( key.max_length() as u32, estimated_element_size, in_parent_tree_using_sums, @@ -443,7 +443,7 @@ impl GroveDb { estimated_references_sizes: Vec, ) { // todo: verify - let value_size: u32 = Tree::average_case_encoded_tree_size( + let value_size: u32 = TreeNode::average_case_encoded_tree_size( key.max_length() as u32, estimated_element_size, in_parent_tree_using_sums, diff --git a/grovedb/src/estimated_costs/worst_case_costs.rs b/grovedb/src/estimated_costs/worst_case_costs.rs index d84b3df22..d72b86e8d 100644 --- a/grovedb/src/estimated_costs/worst_case_costs.rs +++ b/grovedb/src/estimated_costs/worst_case_costs.rs @@ -42,7 +42,7 @@ use grovedb_merk::{ MERK_BIGGEST_VALUE_SIZE, }, }, - tree::Tree, + tree::TreeNode, HASH_LENGTH, }; use grovedb_storage::{worst_case_costs::WorstKeyLength, Storage}; @@ -67,7 +67,7 @@ impl GroveDb { match path.last() { None => {} Some(key) => { - cost.storage_loaded_bytes += Tree::worst_case_encoded_tree_size( + cost.storage_loaded_bytes += TreeNode::worst_case_encoded_tree_size( key.max_length() as u32, HASH_LENGTH as u32, is_sum_tree, @@ -333,7 +333,7 @@ impl GroveDb { max_element_size: u32, in_parent_tree_using_sums: bool, ) { - let value_size = Tree::worst_case_encoded_tree_size( + let value_size = TreeNode::worst_case_encoded_tree_size( key.max_length() as u32, max_element_size, in_parent_tree_using_sums, @@ -392,7 +392,7 @@ impl GroveDb { max_references_sizes: Vec, ) { // todo: verify - let value_size: u32 = Tree::worst_case_encoded_tree_size( + let value_size: u32 = TreeNode::worst_case_encoded_tree_size( key.max_length() as u32, max_element_size, in_parent_tree_using_sums, diff --git a/merk/src/estimated_costs/average_case_costs.rs b/merk/src/estimated_costs/average_case_costs.rs index 23501844b..53d6cc9b5 100644 --- a/merk/src/estimated_costs/average_case_costs.rs +++ b/merk/src/estimated_costs/average_case_costs.rs @@ -37,7 +37,7 @@ use integer_encoding::VarInt; use crate::{ error::Error, estimated_costs::LAYER_COST_SIZE, - tree::{kv::KV, Link, Tree}, + tree::{kv::KV, Link, TreeNode}, HASH_BLOCK_SIZE, HASH_BLOCK_SIZE_U32, HASH_LENGTH, HASH_LENGTH_U32, }; @@ -318,7 +318,7 @@ impl EstimatedLayerCount { } #[cfg(feature = "full")] -impl Tree { +impl TreeNode { /// Return estimate of average encoded tree size pub fn average_case_encoded_tree_size( not_prefixed_key_len: u32, @@ -347,7 +347,7 @@ pub fn add_average_case_get_merk_node( // To write a node to disk, the left link, right link and kv nodes are encoded. // worst case, the node has both the left and right link present. - cost.storage_loaded_bytes += Tree::average_case_encoded_tree_size( + cost.storage_loaded_bytes += TreeNode::average_case_encoded_tree_size( not_prefixed_key_len, approximate_element_size, is_sum_tree, diff --git a/merk/src/estimated_costs/worst_case_costs.rs b/merk/src/estimated_costs/worst_case_costs.rs index 42911b37a..407e2c706 100644 --- a/merk/src/estimated_costs/worst_case_costs.rs +++ b/merk/src/estimated_costs/worst_case_costs.rs @@ -37,7 +37,7 @@ use grovedb_costs::{CostResult, CostsExt, OperationCost}; use crate::{ error::Error, merk::defaults::MAX_PREFIXED_KEY_SIZE, - tree::{kv::KV, Link, Tree}, + tree::{kv::KV, Link, TreeNode}, HASH_BLOCK_SIZE, HASH_BLOCK_SIZE_U32, HASH_LENGTH, }; @@ -52,7 +52,7 @@ pub enum WorstCaseLayerInformation { } #[cfg(feature = "full")] -impl Tree { +impl TreeNode { /// Return worst case size of encoded tree pub fn worst_case_encoded_tree_size( not_prefixed_key_len: u32, @@ -82,7 +82,7 @@ pub fn add_worst_case_get_merk_node( // To write a node to disk, the left link, right link and kv nodes are encoded. // worst case, the node has both the left and right link present. cost.storage_loaded_bytes += - Tree::worst_case_encoded_tree_size(not_prefixed_key_len, max_element_size, is_sum_node); + TreeNode::worst_case_encoded_tree_size(not_prefixed_key_len, max_element_size, is_sum_node); } #[cfg(feature = "full")] diff --git a/merk/src/merk/committer.rs b/merk/src/merk/committer.rs index 04ef7b5d3..f959cfbf3 100644 --- a/merk/src/merk/committer.rs +++ b/merk/src/merk/committer.rs @@ -5,7 +5,7 @@ use grovedb_costs::storage_cost::{ use crate::{ merk::{defaults::MAX_UPDATE_VALUE_BASED_ON_COSTS_TIMES, BatchValue}, - tree::{kv::ValueDefinedCostType, Commit, Tree}, + tree::{kv::ValueDefinedCostType, Commit, TreeNode}, Error, }; @@ -31,7 +31,7 @@ impl MerkCommitter { impl Commit for MerkCommitter { fn write( &mut self, - tree: &mut Tree, + tree: &mut TreeNode, old_specialized_cost: &impl Fn(&Vec, &Vec) -> Result, update_tree_value_based_on_costs: &mut impl FnMut( &StorageCost, @@ -112,7 +112,7 @@ impl Commit for MerkCommitter { Ok(()) } - fn prune(&self, tree: &Tree) -> (bool, bool) { + fn prune(&self, tree: &TreeNode) -> (bool, bool) { // keep N top levels of tree let prune = (self.height - tree.height()) >= self.levels; (prune, prune) diff --git a/merk/src/merk/get.rs b/merk/src/merk/get.rs index 1e9090c89..573b895a4 100644 --- a/merk/src/merk/get.rs +++ b/merk/src/merk/get.rs @@ -1,7 +1,7 @@ use grovedb_costs::{CostContext, CostResult, CostsExt, OperationCost}; use grovedb_storage::StorageContext; -use crate::{tree::Tree, CryptoHash, Error, Error::StorageError, Merk, TreeFeatureType}; +use crate::{tree::TreeNode, CryptoHash, Error, Error::StorageError, Merk, TreeFeatureType}; impl<'db, S> Merk where @@ -136,7 +136,7 @@ where /// See if a node's field exists fn has_node_direct(&self, key: &[u8]) -> CostResult { - Tree::get(&self.storage, key).map_ok(|x| x.is_some()) + TreeNode::get(&self.storage, key).map_ok(|x| x.is_some()) } /// See if a node's field exists @@ -173,9 +173,9 @@ where /// Generic way to get a node's field fn get_node_direct_fn(&self, key: &[u8], f: F) -> CostResult, Error> where - F: FnOnce(&Tree) -> CostContext, + F: FnOnce(&TreeNode) -> CostContext, { - Tree::get(&self.storage, key).flat_map_ok(|maybe_node| { + TreeNode::get(&self.storage, key).flat_map_ok(|maybe_node| { let mut cost = OperationCost::default(); Ok(maybe_node.map(|node| f(&node).unwrap_add_cost(&mut cost))).wrap_with_cost(cost) }) @@ -184,7 +184,7 @@ where /// Generic way to get a node's field fn get_node_fn(&self, key: &[u8], f: F) -> CostResult, Error> where - F: FnOnce(&Tree) -> CostContext, + F: FnOnce(&TreeNode) -> CostContext, { self.use_tree(move |maybe_tree| { let mut cursor = match maybe_tree { diff --git a/merk/src/merk/mod.rs b/merk/src/merk/mod.rs index 5e20ade99..c9304be0f 100644 --- a/merk/src/merk/mod.rs +++ b/merk/src/merk/mod.rs @@ -65,7 +65,7 @@ use crate::{ merk::{defaults::ROOT_KEY_KEY, options::MerkOptions}, proofs::{query::query_item::QueryItem, Op as ProofOp, Query}, tree::{ - kv::ValueDefinedCostType, AuxMerkBatch, Commit, CryptoHash, Fetch, Op, RefWalker, Tree, + kv::ValueDefinedCostType, AuxMerkBatch, Commit, CryptoHash, Fetch, Op, RefWalker, TreeNode, NULL_HASH, }, Error::{CostsError, EdError, StorageError}, @@ -229,7 +229,7 @@ impl MerkType { /// A handle to a Merkle key/value store backed by RocksDB. pub struct Merk { - pub(crate) tree: Cell>, + pub(crate) tree: Cell>, pub(crate) root_tree_key: Cell>>, /// Storage pub storage: S, @@ -481,14 +481,14 @@ where } /// Use tree - pub(crate) fn use_tree(&self, f: impl FnOnce(Option<&Tree>) -> T) -> T { + pub(crate) fn use_tree(&self, f: impl FnOnce(Option<&TreeNode>) -> T) -> T { let tree = self.tree.take(); let res = f(tree.as_ref()); self.tree.set(tree); res } - fn use_tree_mut(&self, mut f: impl FnMut(Option<&mut Tree>) -> T) -> T { + fn use_tree_mut(&self, mut f: impl FnMut(Option<&mut TreeNode>) -> T) -> T { let mut tree = self.tree.take(); let res = f(tree.as_mut()); self.tree.set(tree); @@ -524,7 +524,7 @@ where if let Some(tree_root_key) = tree_root_key_opt { // Trying to build a tree out of it, costs will be accumulated because // `Tree::get` returns `CostContext` and this call happens inside `flat_map_ok`. - Tree::get(&self.storage, tree_root_key).map_ok(|tree| { + TreeNode::get(&self.storage, tree_root_key).map_ok(|tree| { if let Some(t) = tree.as_ref() { self.root_tree_key = Cell::new(Some(t.key().to_vec())); } @@ -544,7 +544,7 @@ where if let Some(tree_root_key) = self.root_tree_key.get_mut() { // Trying to build a tree out of it, costs will be accumulated because // `Tree::get` returns `CostContext` and this call happens inside `flat_map_ok`. - Tree::get(&self.storage, tree_root_key).map_ok(|tree| { + TreeNode::get(&self.storage, tree_root_key).map_ok(|tree| { self.tree = Cell::new(tree); }) } else { @@ -554,10 +554,10 @@ where } } -fn fetch_node<'db>(db: &impl StorageContext<'db>, key: &[u8]) -> Result, Error> { +fn fetch_node<'db>(db: &impl StorageContext<'db>, key: &[u8]) -> Result, Error> { let bytes = db.get(key).unwrap().map_err(StorageError)?; // TODO: get_pinned ? if let Some(bytes) = bytes { - Ok(Some(Tree::decode(key.to_vec(), &bytes).map_err(EdError)?)) + Ok(Some(TreeNode::decode(key.to_vec(), &bytes).map_err(EdError)?)) } else { Ok(None) } diff --git a/merk/src/merk/open.rs b/merk/src/merk/open.rs index 66a0b2120..63eb37093 100644 --- a/merk/src/merk/open.rs +++ b/merk/src/merk/open.rs @@ -75,6 +75,7 @@ mod test { Storage, StorageBatch, }; use tempfile::TempDir; + use grovedb_costs::OperationCost; use crate::{Merk, Op, TreeFeatureType::BasicMerk}; @@ -132,7 +133,6 @@ mod test { .unwrap(), false, ); - // Opening not existing merk should cost only root key seek (except context // creation) assert!(matches!( diff --git a/merk/src/merk/restore.rs b/merk/src/merk/restore.rs index 3d9862fb4..b4e4c39f0 100644 --- a/merk/src/merk/restore.rs +++ b/merk/src/merk/restore.rs @@ -47,7 +47,7 @@ use crate::{ tree::{Child, Tree as ProofTree}, Node, Op, }, - tree::{combine_hash, value_hash, Link, RefWalker, Tree}, + tree::{combine_hash, value_hash, Link, RefWalker, TreeNode}, CryptoHash, Error::{CostsError, EdError, StorageError}, TreeFeatureType::BasicMerk, @@ -136,16 +136,16 @@ impl<'db, S: StorageContext<'db>> Restorer { tree.visit_refs(&mut |proof_node| { if let Some((mut node, key)) = match &proof_node.node { Node::KV(key, value) => Some(( - Tree::new(key.clone(), value.clone(), None, BasicMerk).unwrap(), + TreeNode::new(key.clone(), value.clone(), None, BasicMerk).unwrap(), key, )), Node::KVValueHash(key, value, value_hash) => Some(( - Tree::new_with_value_hash(key.clone(), value.clone(), *value_hash, BasicMerk) + TreeNode::new_with_value_hash(key.clone(), value.clone(), *value_hash, BasicMerk) .unwrap(), key, )), Node::KVValueHashFeatureType(key, value, value_hash, feature_type) => Some(( - Tree::new_with_value_hash( + TreeNode::new_with_value_hash( key.clone(), value.clone(), *value_hash, @@ -301,7 +301,7 @@ impl<'db, S: StorageContext<'db>> Restorer { } let mut cloned_node = - Tree::decode(node.tree().key().to_vec(), node.tree().encode().as_slice()) + TreeNode::decode(node.tree().key().to_vec(), node.tree().encode().as_slice()) .map_err(EdError)?; let left_child = node.walk(true).unwrap()?.unwrap(); diff --git a/merk/src/merk/source.rs b/merk/src/merk/source.rs index 07be638ee..709063a69 100644 --- a/merk/src/merk/source.rs +++ b/merk/src/merk/source.rs @@ -2,7 +2,7 @@ use grovedb_costs::CostResult; use grovedb_storage::StorageContext; use crate::{ - tree::{Fetch, Tree}, + tree::{Fetch, TreeNode}, Error, Link, Merk, }; @@ -37,8 +37,8 @@ impl<'s, 'db, S> Fetch for MerkSource<'s, S> where S: StorageContext<'db>, { - fn fetch(&self, link: &Link) -> CostResult { - Tree::get(self.storage, link.key()) + fn fetch(&self, link: &Link) -> CostResult { + TreeNode::get(self.storage, link.key()) .map_ok(|x| x.ok_or(Error::KeyNotFoundError("Key not found for fetch"))) .flatten() } diff --git a/merk/src/proofs/chunk.rs b/merk/src/proofs/chunk.rs index 48afe8f30..bec53af49 100644 --- a/merk/src/proofs/chunk.rs +++ b/merk/src/proofs/chunk.rs @@ -38,7 +38,7 @@ use grovedb_storage::RawIterator; use { super::tree::{execute, Tree as ProofTree}, crate::tree::CryptoHash, - crate::tree::Tree, + crate::tree::TreeNode, }; #[cfg(feature = "full")] @@ -199,7 +199,7 @@ pub(crate) fn get_next_chunk( let mut chunk = Vec::with_capacity(512); let mut stack = Vec::with_capacity(32); - let mut node = Tree::new(vec![], vec![], None, BasicMerk).unwrap_add_cost(&mut cost); + let mut node = TreeNode::new(vec![], vec![], None, BasicMerk).unwrap_add_cost(&mut cost); while iter.valid().unwrap_add_cost(&mut cost) { let key = iter.key().unwrap_add_cost(&mut cost).unwrap(); @@ -213,7 +213,7 @@ pub(crate) fn get_next_chunk( let encoded_node = iter.value().unwrap_add_cost(&mut cost).unwrap(); cost_return_on_error_no_add!( &cost, - Tree::decode_into(&mut node, vec![], encoded_node).map_err(EdError) + TreeNode::decode_into(&mut node, vec![], encoded_node).map_err(EdError) ); // TODO: Only use the KVValueHash if needed, saves 32 bytes @@ -386,7 +386,7 @@ mod tests { use super::{super::tree::Tree, *}; use crate::{ test_utils::*, - tree::{NoopCommit, PanicSource, Tree as BaseTree}, + tree::{NoopCommit, PanicSource, TreeNode as BaseTree}, }; #[derive(Default)] diff --git a/merk/src/proofs/query/mod.rs b/merk/src/proofs/query/mod.rs index d63d2d478..a68bd159d 100644 --- a/merk/src/proofs/query/mod.rs +++ b/merk/src/proofs/query/mod.rs @@ -792,7 +792,7 @@ mod test { verify::{verify_query, ProvedKeyValue}, }, test_utils::make_tree_seq, - tree::{NoopCommit, PanicSource, RefWalker, Tree}, + tree::{NoopCommit, PanicSource, RefWalker, TreeNode}, TreeFeatureType::BasicMerk, }; @@ -807,16 +807,16 @@ mod test { } } - fn make_3_node_tree() -> Tree { - let mut tree = Tree::new(vec![5], vec![5], None, BasicMerk) + fn make_3_node_tree() -> TreeNode { + let mut tree = TreeNode::new(vec![5], vec![5], None, BasicMerk) .unwrap() .attach( true, - Some(Tree::new(vec![3], vec![3], None, BasicMerk).unwrap()), + Some(TreeNode::new(vec![3], vec![3], None, BasicMerk).unwrap()), ) .attach( false, - Some(Tree::new(vec![7], vec![7], None, BasicMerk).unwrap()), + Some(TreeNode::new(vec![7], vec![7], None, BasicMerk).unwrap()), ); tree.commit( &mut NoopCommit {}, @@ -829,10 +829,10 @@ mod test { tree } - fn make_6_node_tree() -> Tree { - let two_tree = Tree::new(vec![2], vec![2], None, BasicMerk).unwrap(); - let four_tree = Tree::new(vec![4], vec![4], None, BasicMerk).unwrap(); - let mut three_tree = Tree::new(vec![3], vec![3], None, BasicMerk) + fn make_6_node_tree() -> TreeNode { + let two_tree = TreeNode::new(vec![2], vec![2], None, BasicMerk).unwrap(); + let four_tree = TreeNode::new(vec![4], vec![4], None, BasicMerk).unwrap(); + let mut three_tree = TreeNode::new(vec![3], vec![3], None, BasicMerk) .unwrap() .attach(true, Some(two_tree)) .attach(false, Some(four_tree)); @@ -846,8 +846,8 @@ mod test { .unwrap() .expect("commit failed"); - let seven_tree = Tree::new(vec![7], vec![7], None, BasicMerk).unwrap(); - let mut eight_tree = Tree::new(vec![8], vec![8], None, BasicMerk) + let seven_tree = TreeNode::new(vec![7], vec![7], None, BasicMerk).unwrap(); + let mut eight_tree = TreeNode::new(vec![8], vec![8], None, BasicMerk) .unwrap() .attach(true, Some(seven_tree)); eight_tree @@ -860,7 +860,7 @@ mod test { .unwrap() .expect("commit failed"); - let mut root_tree = Tree::new(vec![5], vec![5], None, BasicMerk) + let mut root_tree = TreeNode::new(vec![5], vec![5], None, BasicMerk) .unwrap() .attach(true, Some(three_tree)) .attach(false, Some(eight_tree)); @@ -1553,25 +1553,25 @@ mod test { #[test] fn doc_proof() { - let mut tree = Tree::new(vec![5], vec![5], None, BasicMerk) + let mut tree = TreeNode::new(vec![5], vec![5], None, BasicMerk) .unwrap() .attach( true, Some( - Tree::new(vec![2], vec![2], None, BasicMerk) + TreeNode::new(vec![2], vec![2], None, BasicMerk) .unwrap() .attach( true, - Some(Tree::new(vec![1], vec![1], None, BasicMerk).unwrap()), + Some(TreeNode::new(vec![1], vec![1], None, BasicMerk).unwrap()), ) .attach( false, Some( - Tree::new(vec![4], vec![4], None, BasicMerk) + TreeNode::new(vec![4], vec![4], None, BasicMerk) .unwrap() .attach( true, - Some(Tree::new(vec![3], vec![3], None, BasicMerk).unwrap()), + Some(TreeNode::new(vec![3], vec![3], None, BasicMerk).unwrap()), ), ), ), @@ -1580,32 +1580,32 @@ mod test { .attach( false, Some( - Tree::new(vec![9], vec![9], None, BasicMerk) + TreeNode::new(vec![9], vec![9], None, BasicMerk) .unwrap() .attach( true, Some( - Tree::new(vec![7], vec![7], None, BasicMerk) + TreeNode::new(vec![7], vec![7], None, BasicMerk) .unwrap() .attach( true, - Some(Tree::new(vec![6], vec![6], None, BasicMerk).unwrap()), + Some(TreeNode::new(vec![6], vec![6], None, BasicMerk).unwrap()), ) .attach( false, - Some(Tree::new(vec![8], vec![8], None, BasicMerk).unwrap()), + Some(TreeNode::new(vec![8], vec![8], None, BasicMerk).unwrap()), ), ), ) .attach( false, Some( - Tree::new(vec![11], vec![11], None, BasicMerk) + TreeNode::new(vec![11], vec![11], None, BasicMerk) .unwrap() .attach( true, Some( - Tree::new(vec![10], vec![10], None, BasicMerk).unwrap(), + TreeNode::new(vec![10], vec![10], None, BasicMerk).unwrap(), ), ), ), @@ -5755,7 +5755,7 @@ mod test { #[test] fn verify_ops() { - let mut tree = Tree::new(vec![5], vec![5], None, BasicMerk).unwrap(); + let mut tree = TreeNode::new(vec![5], vec![5], None, BasicMerk).unwrap(); tree.commit( &mut NoopCommit {}, &|_, _| Ok(0), @@ -5786,7 +5786,7 @@ mod test { #[test] #[should_panic(expected = "verify failed")] fn verify_ops_mismatched_hash() { - let mut tree = Tree::new(vec![5], vec![5], None, BasicMerk).unwrap(); + let mut tree = TreeNode::new(vec![5], vec![5], None, BasicMerk).unwrap(); tree.commit( &mut NoopCommit {}, &|_, _| Ok(0), diff --git a/merk/src/test_utils/mod.rs b/merk/src/test_utils/mod.rs index 90cf1d339..4d257bde0 100644 --- a/merk/src/test_utils/mod.rs +++ b/merk/src/test_utils/mod.rs @@ -39,13 +39,13 @@ use rand::prelude::*; pub use temp_merk::TempMerk; use crate::{ - tree::{kv::KV, BatchEntry, MerkBatch, NoopCommit, Op, PanicSource, Tree, Walker}, + tree::{kv::KV, BatchEntry, MerkBatch, NoopCommit, Op, PanicSource, TreeNode, Walker}, Merk, TreeFeatureType::{BasicMerk, SummedMerk}, }; /// Assert tree invariants -pub fn assert_tree_invariants(tree: &Tree) { +pub fn assert_tree_invariants(tree: &TreeNode) { assert!(tree.balance_factor().abs() < 2); let maybe_left = tree.link(true); @@ -71,7 +71,7 @@ pub fn assert_tree_invariants(tree: &Tree) { /// Apply given batch to given tree and commit using memory only. /// Used by `apply_memonly` which also performs checks using /// `assert_tree_invariants`. Return Tree. -pub fn apply_memonly_unchecked(tree: Tree, batch: &MerkBatch>) -> Tree { +pub fn apply_memonly_unchecked(tree: TreeNode, batch: &MerkBatch>) -> TreeNode { let is_sum_node = tree.is_sum_node(); let walker = Walker::::new(tree, PanicSource {}); let mut tree = Walker::::apply_to( @@ -122,7 +122,7 @@ pub fn apply_memonly_unchecked(tree: Tree, batch: &MerkBatch>) -> Tree { /// Apply given batch to given tree and commit using memory only. /// Perform checks using `assert_tree_invariants`. Return Tree. -pub fn apply_memonly(tree: Tree, batch: &MerkBatch>) -> Tree { +pub fn apply_memonly(tree: TreeNode, batch: &MerkBatch>) -> TreeNode { let tree = apply_memonly_unchecked(tree, batch); assert_tree_invariants(&tree); tree @@ -131,10 +131,10 @@ pub fn apply_memonly(tree: Tree, batch: &MerkBatch>) -> Tree { /// Applies given batch to given tree or creates a new tree to apply to and /// commits to memory only. pub fn apply_to_memonly( - maybe_tree: Option, + maybe_tree: Option, batch: &MerkBatch>, is_sum_tree: bool, -) -> Option { +) -> Option { let maybe_walker = maybe_tree.map(|tree| Walker::::new(tree, PanicSource {})); Walker::::apply_to( maybe_walker, @@ -250,7 +250,7 @@ pub fn make_tree_rand( batch_size: u64, initial_seed: u64, is_sum_tree: bool, -) -> Tree { +) -> TreeNode { assert!(node_count >= batch_size); assert_eq!((node_count % batch_size), 0); @@ -260,7 +260,7 @@ pub fn make_tree_rand( } else { BasicMerk }; - let mut tree = Tree::new(vec![0; 20], value, None, feature_type).unwrap(); + let mut tree = TreeNode::new(vec![0; 20], value, None, feature_type).unwrap(); let mut seed = initial_seed; @@ -276,7 +276,7 @@ pub fn make_tree_rand( /// Create tree with initial fixed values and apply `node count` Put ops using /// sequential keys using memory only -pub fn make_tree_seq(node_count: u64) -> Tree { +pub fn make_tree_seq(node_count: u64) -> TreeNode { let batch_size = if node_count >= 10_000 { assert_eq!(node_count % 10_000, 0); 10_000 @@ -285,7 +285,7 @@ pub fn make_tree_seq(node_count: u64) -> Tree { }; let value = vec![123; 60]; - let mut tree = Tree::new(vec![0; 20], value, None, BasicMerk).unwrap(); + let mut tree = TreeNode::new(vec![0; 20], value, None, BasicMerk).unwrap(); let batch_count = node_count / batch_size; for i in 0..batch_count { diff --git a/merk/src/tree/commit.rs b/merk/src/tree/commit.rs index ccecfb48f..c50daced7 100644 --- a/merk/src/tree/commit.rs +++ b/merk/src/tree/commit.rs @@ -32,7 +32,7 @@ use grovedb_costs::storage_cost::{removal::StorageRemovedBytes, StorageCost}; #[cfg(feature = "full")] -use super::Tree; +use super::TreeNode; #[cfg(feature = "full")] use crate::error::Error; use crate::tree::kv::ValueDefinedCostType; @@ -45,7 +45,7 @@ pub trait Commit { /// backing store or cache. fn write( &mut self, - tree: &mut Tree, + tree: &mut TreeNode, old_specialized_cost: &impl Fn(&Vec, &Vec) -> Result, update_tree_value_based_on_costs: &mut impl FnMut( &StorageCost, @@ -69,7 +69,7 @@ pub trait Commit { /// tuple specifies whether or not to prune the left and right child nodes, /// respectively. For example, returning `(true, true)` will prune both /// nodes, removing them from memory. - fn prune(&self, _tree: &Tree) -> (bool, bool) { + fn prune(&self, _tree: &TreeNode) -> (bool, bool) { (true, true) } } @@ -83,7 +83,7 @@ pub struct NoopCommit {} impl Commit for NoopCommit { fn write( &mut self, - _tree: &mut Tree, + _tree: &mut TreeNode, _old_specialized_cost: &impl Fn(&Vec, &Vec) -> Result, _update_tree_value_based_on_costs: &mut impl FnMut( &StorageCost, @@ -105,7 +105,7 @@ impl Commit for NoopCommit { Ok(()) } - fn prune(&self, _tree: &Tree) -> (bool, bool) { + fn prune(&self, _tree: &TreeNode) -> (bool, bool) { (false, false) } } diff --git a/merk/src/tree/debug.rs b/merk/src/tree/debug.rs index 33889ebf0..3e88c60b9 100644 --- a/merk/src/tree/debug.rs +++ b/merk/src/tree/debug.rs @@ -32,15 +32,15 @@ use std::fmt::{Debug, Formatter, Result}; use colored::Colorize; -use super::{Link, Tree}; +use super::{Link, TreeNode}; #[cfg(feature = "full")] -impl Debug for Tree { +impl Debug for TreeNode { // TODO: unwraps should be results that bubble up fn fmt(&self, f: &mut Formatter) -> Result { fn traverse( f: &mut Formatter, - cursor: &Tree, + cursor: &TreeNode, stack: &mut Vec<(Vec, Vec)>, left: bool, ) { diff --git a/merk/src/tree/encoding.rs b/merk/src/tree/encoding.rs index 98f6a7c60..836b770ab 100644 --- a/merk/src/tree/encoding.rs +++ b/merk/src/tree/encoding.rs @@ -38,20 +38,20 @@ use grovedb_costs::{ use grovedb_storage::StorageContext; #[cfg(feature = "full")] -use super::Tree; +use super::TreeNode; #[cfg(feature = "full")] use crate::{ error::{Error, Error::EdError}, - tree::TreeInner, + tree::TreeNodeInner, Error::StorageError, }; #[cfg(feature = "full")] -impl Tree { +impl TreeNode { /// Decode given bytes and set as Tree fields. Set key to value of given /// key. pub fn decode_raw(bytes: &[u8], key: Vec) -> Result { - Tree::decode(key, bytes).map_err(EdError) + TreeNode::decode(key, bytes).map_err(EdError) } /// Get value from storage given key. @@ -66,7 +66,7 @@ impl Tree { let tree_opt = cost_return_on_error_no_add!( &cost, tree_bytes - .map(|x| Tree::decode_raw(&x, key.as_ref().to_vec())) + .map(|x| TreeNode::decode_raw(&x, key.as_ref().to_vec())) .transpose() ); @@ -75,7 +75,7 @@ impl Tree { } #[cfg(feature = "full")] -impl Tree { +impl TreeNode { #[inline] /// Encode pub fn encode(&self) -> Vec { @@ -112,7 +112,7 @@ impl Tree { #[inline] /// Decode bytes from reader, set as Tree fields and set key to given key pub fn decode_into(&mut self, key: Vec, input: &[u8]) -> ed::Result<()> { - let mut tree_inner: TreeInner = Decode::decode(input)?; + let mut tree_inner: TreeNodeInner = Decode::decode(input)?; tree_inner.kv.key = key; self.inner = Box::new(tree_inner); Ok(()) @@ -121,9 +121,9 @@ impl Tree { #[inline] /// Decode input and set as Tree fields. Set the key as the given key. pub fn decode(key: Vec, input: &[u8]) -> ed::Result { - let mut tree_inner: TreeInner = Decode::decode(input)?; + let mut tree_inner: TreeNodeInner = Decode::decode(input)?; tree_inner.kv.key = key; - Ok(Tree::new_with_tree_inner(tree_inner)) + Ok(TreeNode::new_with_tree_inner(tree_inner)) } } @@ -135,7 +135,7 @@ mod tests { #[test] fn encode_leaf_tree() { - let tree = Tree::from_fields(vec![0], vec![1], [55; 32], None, None, BasicMerk).unwrap(); + let tree = TreeNode::from_fields(vec![0], vec![1], [55; 32], None, None, BasicMerk).unwrap(); assert_eq!(tree.encoding_length(), 68); assert_eq!( tree.value_encoding_length_with_parent_to_child_reference(), @@ -155,14 +155,14 @@ mod tests { #[test] #[should_panic] fn encode_modified_tree() { - let tree = Tree::from_fields( + let tree = TreeNode::from_fields( vec![0], vec![1], [55; 32], Some(Link::Modified { pending_writes: 1, child_heights: (123, 124), - tree: Tree::new(vec![2], vec![3], None, BasicMerk).unwrap(), + tree: TreeNode::new(vec![2], vec![3], None, BasicMerk).unwrap(), }), None, BasicMerk, @@ -173,7 +173,7 @@ mod tests { #[test] fn encode_loaded_tree() { - let tree = Tree::from_fields( + let tree = TreeNode::from_fields( vec![0], vec![1], [55; 32], @@ -181,7 +181,7 @@ mod tests { hash: [66; 32], sum: None, child_heights: (123, 124), - tree: Tree::new(vec![2], vec![3], None, BasicMerk).unwrap(), + tree: TreeNode::new(vec![2], vec![3], None, BasicMerk).unwrap(), }), None, BasicMerk, @@ -202,7 +202,7 @@ mod tests { #[test] fn encode_uncommitted_tree() { - let tree = Tree::from_fields( + let tree = TreeNode::from_fields( vec![0], vec![1], [55; 32], @@ -210,7 +210,7 @@ mod tests { hash: [66; 32], sum: Some(10), child_heights: (123, 124), - tree: Tree::new(vec![2], vec![3], None, BasicMerk).unwrap(), + tree: TreeNode::new(vec![2], vec![3], None, BasicMerk).unwrap(), }), None, SummedMerk(5), @@ -231,7 +231,7 @@ mod tests { #[test] fn encode_reference_tree() { - let tree = Tree::from_fields( + let tree = TreeNode::from_fields( vec![0], vec![1], [55; 32], @@ -275,7 +275,7 @@ mod tests { 131, 208, 25, 73, 98, 245, 209, 227, 170, 26, 72, 212, 134, 166, 126, 39, 98, 166, 199, 149, 144, 21, 1, ]; - let tree = Tree::decode(vec![0], bytes.as_slice()).expect("should decode correctly"); + let tree = TreeNode::decode(vec![0], bytes.as_slice()).expect("should decode correctly"); assert_eq!(tree.key(), &[0]); assert_eq!(tree.value_as_slice(), &[1]); assert_eq!(tree.inner.kv.feature_type, BasicMerk); @@ -290,7 +290,7 @@ mod tests { 55, 55, 55, 55, 55, 55, 32, 34, 236, 157, 87, 27, 167, 116, 207, 158, 131, 208, 25, 73, 98, 245, 209, 227, 170, 26, 72, 212, 134, 166, 126, 39, 98, 166, 199, 149, 144, 21, 1, ]; - let tree = Tree::decode(vec![0], bytes.as_slice()).expect("should decode correctly"); + let tree = TreeNode::decode(vec![0], bytes.as_slice()).expect("should decode correctly"); assert_eq!(tree.key(), &[0]); assert_eq!(tree.value_as_slice(), &[1]); if let Some(Link::Reference { @@ -311,7 +311,7 @@ mod tests { #[test] fn decode_invalid_bytes_as_tree() { let bytes = vec![2, 3, 4, 5]; - let tree = Tree::decode(vec![0], bytes.as_slice()); + let tree = TreeNode::decode(vec![0], bytes.as_slice()); assert!(matches!(tree, Err(_))); } } diff --git a/merk/src/tree/fuzz_tests.rs b/merk/src/tree/fuzz_tests.rs index cd9d22d50..631918ff8 100644 --- a/merk/src/tree/fuzz_tests.rs +++ b/merk/src/tree/fuzz_tests.rs @@ -95,7 +95,7 @@ fn fuzz_case(seed: u64, using_sum_trees: bool) { } #[cfg(feature = "full")] -fn make_batch(maybe_tree: Option<&Tree>, size: u64, seed: u64) -> Vec { +fn make_batch(maybe_tree: Option<&TreeNode>, size: u64, seed: u64) -> Vec { let rng: RefCell = RefCell::new(SeedableRng::seed_from_u64(seed)); let mut batch = Vec::with_capacity(size as usize); @@ -170,7 +170,7 @@ fn apply_to_map(map: &mut Map, batch: &Batch) { } #[cfg(feature = "full")] -fn assert_map(maybe_tree: Option<&Tree>, map: &Map) { +fn assert_map(maybe_tree: Option<&TreeNode>, map: &Map) { if map.is_empty() { assert!(maybe_tree.is_none(), "expected tree to be None"); return; diff --git a/merk/src/tree/iter.rs b/merk/src/tree/iter.rs index 2daa5a027..6ca58df72 100644 --- a/merk/src/tree/iter.rs +++ b/merk/src/tree/iter.rs @@ -29,7 +29,7 @@ //! Merk tree iterator #[cfg(feature = "full")] -use super::Tree; +use super::TreeNode; #[cfg(feature = "full")] /// An entry stored on an `Iter`'s stack, containing a reference to a `Tree`, @@ -38,7 +38,7 @@ use super::Tree; /// The `traversed` field represents whether or not the left child, self, and /// right child have been visited, respectively (`(left, self, right)`). struct StackItem<'a> { - tree: &'a Tree, + tree: &'a TreeNode, traversed: (bool, bool, bool), } @@ -47,7 +47,7 @@ impl<'a> StackItem<'a> { /// Creates a new `StackItem` for the given tree. The `traversed` state will /// be `false` since the children and self have not been visited yet, but /// will default to `true` for sides that do not have a child. - const fn new(tree: &'a Tree) -> Self { + const fn new(tree: &'a TreeNode) -> Self { StackItem { tree, traversed: ( @@ -77,14 +77,14 @@ pub struct Iter<'a> { #[cfg(feature = "full")] impl<'a> Iter<'a> { /// Creates a new iterator for the given tree. - pub fn new(tree: &'a Tree) -> Self { + pub fn new(tree: &'a TreeNode) -> Self { let stack = vec![StackItem::new(tree)]; Iter { stack } } } #[cfg(feature = "full")] -impl<'a> Tree { +impl<'a> TreeNode { /// Creates an iterator which yields `(key, value)` tuples for all of the /// tree's nodes which are retained in memory (skipping pruned subtrees). pub fn iter(&'a self) -> Iter<'a> { diff --git a/merk/src/tree/link.rs b/merk/src/tree/link.rs index 56d9f1b0f..8cdb948b4 100644 --- a/merk/src/tree/link.rs +++ b/merk/src/tree/link.rs @@ -37,7 +37,7 @@ use ed::{Decode, Encode, Result, Terminated}; use integer_encoding::{VarInt, VarIntReader, VarIntWriter}; #[cfg(feature = "full")] -use super::{hash::CryptoHash, Tree}; +use super::{hash::CryptoHash, TreeNode}; #[cfg(feature = "full")] use crate::HASH_LENGTH_U32; @@ -72,7 +72,7 @@ pub enum Link { /// Child heights child_heights: (u8, u8), /// Tree - tree: Tree + tree: TreeNode }, /// Represents a tree node which has been modified since the `Tree`'s last @@ -84,7 +84,7 @@ pub enum Link { /// Child heights child_heights: (u8, u8), /// Tree - tree: Tree, + tree: TreeNode, /// Sum sum: Option, }, @@ -97,7 +97,7 @@ pub enum Link { /// Child heights child_heights: (u8, u8), /// Tree - tree: Tree, + tree: TreeNode, /// Sum sum: Option, }, @@ -107,7 +107,7 @@ pub enum Link { impl Link { /// Creates a `Link::Modified` from the given `Tree`. #[inline] - pub const fn from_modified_tree(tree: Tree) -> Self { + pub const fn from_modified_tree(tree: TreeNode) -> Self { let pending_writes = 1 + tree.child_pending_writes(true) + tree.child_pending_writes(false); Self::Modified { @@ -119,7 +119,7 @@ impl Link { /// Creates a `Link::Modified` from the given tree, if any. If `None`, /// returns `None`. - pub fn maybe_from_modified_tree(maybe_tree: Option) -> Option { + pub fn maybe_from_modified_tree(maybe_tree: Option) -> Option { maybe_tree.map(Self::from_modified_tree) } @@ -161,7 +161,7 @@ impl Link { /// Returns the `Tree` instance of the tree referenced by the link. If the /// link is of variant `Link::Reference`, the returned value will be `None`. #[inline] - pub const fn tree(&self) -> Option<&Tree> { + pub const fn tree(&self) -> Option<&TreeNode> { match self { // TODO: panic for Reference, don't return Option? Link::Reference { .. } => None, @@ -483,14 +483,14 @@ fn read_u8(mut input: R) -> Result { #[cfg(test)] mod test { use super::{ - super::{hash::NULL_HASH, Tree}, + super::{hash::NULL_HASH, TreeNode}, *, }; use crate::TreeFeatureType::BasicMerk; #[test] fn from_modified_tree() { - let tree = Tree::new(vec![0], vec![1], None, BasicMerk).unwrap(); + let tree = TreeNode::new(vec![0], vec![1], None, BasicMerk).unwrap(); let link = Link::from_modified_tree(tree); assert!(link.is_modified()); assert_eq!(link.height(), 1); @@ -507,7 +507,7 @@ mod test { let link = Link::maybe_from_modified_tree(None); assert!(link.is_none()); - let tree = Tree::new(vec![0], vec![1], None, BasicMerk).unwrap(); + let tree = TreeNode::new(vec![0], vec![1], None, BasicMerk).unwrap(); let link = Link::maybe_from_modified_tree(Some(tree)); assert!(link.expect("expected link").is_modified()); } @@ -519,7 +519,7 @@ mod test { let child_heights = (0, 0); let pending_writes = 1; let key = vec![0]; - let tree = || Tree::new(vec![0], vec![1], None, BasicMerk).unwrap(); + let tree = || TreeNode::new(vec![0], vec![1], None, BasicMerk).unwrap(); let reference = Link::Reference { hash, @@ -585,7 +585,7 @@ mod test { Link::Modified { pending_writes: 1, child_heights: (1, 1), - tree: Tree::new(vec![0], vec![1], None, BasicMerk).unwrap(), + tree: TreeNode::new(vec![0], vec![1], None, BasicMerk).unwrap(), } .hash(); } @@ -596,7 +596,7 @@ mod test { Link::Modified { pending_writes: 1, child_heights: (1, 1), - tree: Tree::new(vec![0], vec![1], None, BasicMerk).unwrap(), + tree: TreeNode::new(vec![0], vec![1], None, BasicMerk).unwrap(), } .into_reference(); } @@ -608,7 +608,7 @@ mod test { hash: [1; 32], sum: None, child_heights: (1, 1), - tree: Tree::new(vec![0], vec![1], None, BasicMerk).unwrap(), + tree: TreeNode::new(vec![0], vec![1], None, BasicMerk).unwrap(), } .into_reference(); } diff --git a/merk/src/tree/mod.rs b/merk/src/tree/mod.rs index c497b8939..f060da60c 100644 --- a/merk/src/tree/mod.rs +++ b/merk/src/tree/mod.rs @@ -99,14 +99,14 @@ use crate::{error::Error, Error::Overflow}; #[cfg(feature = "full")] /// The fields of the `Tree` type, stored on the heap. #[derive(Clone, Encode, Decode, Debug)] -pub struct TreeInner { +pub struct TreeNodeInner { pub(crate) left: Option, pub(crate) right: Option, pub(crate) kv: KV, } #[cfg(feature = "full")] -impl TreeInner { +impl TreeNodeInner { /// Get the value as owned of the key value struct pub fn value_as_owned(self) -> Vec { self.kv.value @@ -129,7 +129,7 @@ impl TreeInner { } #[cfg(feature = "full")] -impl Terminated for Box {} +impl Terminated for Box {} #[cfg(feature = "full")] /// A binary AVL tree data structure, with Merkle hashes. @@ -138,14 +138,14 @@ impl Terminated for Box {} /// link to each other, and so we can detach nodes from their parents, then /// reattach without allocating or freeing heap memory. #[derive(Clone)] -pub struct Tree { - pub(crate) inner: Box, +pub struct TreeNode { + pub(crate) inner: Box, pub(crate) old_size_with_parent_to_child_hook: u32, pub(crate) old_value: Option>, } #[cfg(feature = "full")] -impl Tree { +impl TreeNode { /// Creates a new `Tree` with the given key and value, and no children. /// /// Hashes the key/value pair and initializes the `kv_hash` field. @@ -156,7 +156,7 @@ impl Tree { feature_type: TreeFeatureType, ) -> CostContext { KV::new(key, value, value_defined_cost, feature_type).map(|kv| Self { - inner: Box::new(TreeInner { + inner: Box::new(TreeNodeInner { kv, left: None, right: None, @@ -167,7 +167,7 @@ impl Tree { } /// Creates a new `Tree` given an inner tree - pub fn new_with_tree_inner(inner_tree: TreeInner) -> Self { + pub fn new_with_tree_inner(inner_tree: TreeNodeInner) -> Self { let decode_size = inner_tree.kv.value_byte_cost_size(); let old_value = inner_tree.kv.value.clone(); Self { @@ -257,7 +257,7 @@ impl Tree { feature_type: TreeFeatureType, ) -> CostContext { KV::new_with_value_hash(key, value, value_hash, feature_type).map(|kv| Self { - inner: Box::new(TreeInner { + inner: Box::new(TreeNodeInner { kv, left: None, right: None, @@ -277,7 +277,7 @@ impl Tree { feature_type: TreeFeatureType, ) -> CostContext { KV::new_with_combined_value_hash(key, value, value_hash, feature_type).map(|kv| Self { - inner: Box::new(TreeInner { + inner: Box::new(TreeNodeInner { kv, left: None, right: None, @@ -299,7 +299,7 @@ impl Tree { ) -> CostContext { KV::new_with_layered_value_hash(key, value, value_cost, value_hash, feature_type).map( |kv| Self { - inner: Box::new(TreeInner { + inner: Box::new(TreeNodeInner { kv, left: None, right: None, @@ -321,7 +321,7 @@ impl Tree { feature_type: TreeFeatureType, ) -> CostContext { value_hash(value.as_slice()).map(|vh| Self { - inner: Box::new(TreeInner { + inner: Box::new(TreeNodeInner { kv: KV::from_fields(key, value, kv_hash, vh, feature_type), left, right, @@ -888,12 +888,12 @@ pub const fn side_to_str(left: bool) -> &'static str { mod test { use grovedb_costs::storage_cost::removal::StorageRemovedBytes::NoStorageRemoval; - use super::{commit::NoopCommit, hash::NULL_HASH, Tree}; + use super::{commit::NoopCommit, hash::NULL_HASH, TreeNode}; use crate::tree::{tree_feature_type::TreeFeatureType::SummedMerk, TreeFeatureType::BasicMerk}; #[test] fn build_tree() { - let tree = Tree::new(vec![1], vec![101], None, BasicMerk).unwrap(); + let tree = TreeNode::new(vec![1], vec![101], None, BasicMerk).unwrap(); assert_eq!(tree.key(), &[1]); assert_eq!(tree.value_as_slice(), &[101]); assert!(tree.child(true).is_none()); @@ -905,13 +905,13 @@ mod test { let tree = tree.attach( true, - Some(Tree::new(vec![2], vec![102], None, BasicMerk).unwrap()), + Some(TreeNode::new(vec![2], vec![102], None, BasicMerk).unwrap()), ); assert_eq!(tree.key(), &[1]); assert_eq!(tree.child(true).unwrap().key(), &[2]); assert!(tree.child(false).is_none()); - let tree = Tree::new(vec![3], vec![103], None, BasicMerk) + let tree = TreeNode::new(vec![3], vec![103], None, BasicMerk) .unwrap() .attach(false, Some(tree)); assert_eq!(tree.key(), &[3]); @@ -922,29 +922,29 @@ mod test { #[should_panic] #[test] fn attach_existing() { - Tree::new(vec![0], vec![1], None, BasicMerk) + TreeNode::new(vec![0], vec![1], None, BasicMerk) .unwrap() .attach( true, - Some(Tree::new(vec![2], vec![3], None, BasicMerk).unwrap()), + Some(TreeNode::new(vec![2], vec![3], None, BasicMerk).unwrap()), ) .attach( true, - Some(Tree::new(vec![4], vec![5], None, BasicMerk).unwrap()), + Some(TreeNode::new(vec![4], vec![5], None, BasicMerk).unwrap()), ); } #[test] fn modify() { - let tree = Tree::new(vec![0], vec![1], None, BasicMerk) + let tree = TreeNode::new(vec![0], vec![1], None, BasicMerk) .unwrap() .attach( true, - Some(Tree::new(vec![2], vec![3], None, BasicMerk).unwrap()), + Some(TreeNode::new(vec![2], vec![3], None, BasicMerk).unwrap()), ) .attach( false, - Some(Tree::new(vec![4], vec![5], None, BasicMerk).unwrap()), + Some(TreeNode::new(vec![4], vec![5], None, BasicMerk).unwrap()), ); let tree = tree.walk(true, |left_opt| { @@ -956,7 +956,7 @@ mod test { let tree = tree.walk(true, |left_opt| { assert!(left_opt.is_none()); - Some(Tree::new(vec![2], vec![3], None, BasicMerk).unwrap()) + Some(TreeNode::new(vec![2], vec![3], None, BasicMerk).unwrap()) }); assert_eq!(tree.link(true).unwrap().key(), &[2]); @@ -970,11 +970,11 @@ mod test { #[test] fn child_and_link() { - let mut tree = Tree::new(vec![0], vec![1], None, BasicMerk) + let mut tree = TreeNode::new(vec![0], vec![1], None, BasicMerk) .unwrap() .attach( true, - Some(Tree::new(vec![2], vec![3], None, BasicMerk).unwrap()), + Some(TreeNode::new(vec![2], vec![3], None, BasicMerk).unwrap()), ); assert!(tree.link(true).expect("expected link").is_modified()); assert!(tree.child(true).is_some()); @@ -1003,11 +1003,11 @@ mod test { #[test] fn child_hash() { - let mut tree = Tree::new(vec![0], vec![1], None, BasicMerk) + let mut tree = TreeNode::new(vec![0], vec![1], None, BasicMerk) .unwrap() .attach( true, - Some(Tree::new(vec![2], vec![3], None, BasicMerk).unwrap()), + Some(TreeNode::new(vec![2], vec![3], None, BasicMerk).unwrap()), ); tree.commit( &mut NoopCommit {}, @@ -1029,7 +1029,7 @@ mod test { #[test] fn hash() { - let tree = Tree::new(vec![0], vec![1], None, BasicMerk).unwrap(); + let tree = TreeNode::new(vec![0], vec![1], None, BasicMerk).unwrap(); assert_eq!( tree.hash().unwrap(), [ @@ -1041,13 +1041,13 @@ mod test { #[test] fn child_pending_writes() { - let tree = Tree::new(vec![0], vec![1], None, BasicMerk).unwrap(); + let tree = TreeNode::new(vec![0], vec![1], None, BasicMerk).unwrap(); assert_eq!(tree.child_pending_writes(true), 0); assert_eq!(tree.child_pending_writes(false), 0); let tree = tree.attach( true, - Some(Tree::new(vec![2], vec![3], None, BasicMerk).unwrap()), + Some(TreeNode::new(vec![2], vec![3], None, BasicMerk).unwrap()), ); assert_eq!(tree.child_pending_writes(true), 1); assert_eq!(tree.child_pending_writes(false), 0); @@ -1055,7 +1055,7 @@ mod test { #[test] fn height_and_balance() { - let tree = Tree::new(vec![0], vec![1], None, BasicMerk).unwrap(); + let tree = TreeNode::new(vec![0], vec![1], None, BasicMerk).unwrap(); assert_eq!(tree.height(), 1); assert_eq!(tree.child_height(true), 0); assert_eq!(tree.child_height(false), 0); @@ -1063,7 +1063,7 @@ mod test { let tree = tree.attach( true, - Some(Tree::new(vec![2], vec![3], None, BasicMerk).unwrap()), + Some(TreeNode::new(vec![2], vec![3], None, BasicMerk).unwrap()), ); assert_eq!(tree.height(), 2); assert_eq!(tree.child_height(true), 1); @@ -1080,11 +1080,11 @@ mod test { #[test] fn commit() { - let mut tree = Tree::new(vec![0], vec![1], None, BasicMerk) + let mut tree = TreeNode::new(vec![0], vec![1], None, BasicMerk) .unwrap() .attach( false, - Some(Tree::new(vec![2], vec![3], None, BasicMerk).unwrap()), + Some(TreeNode::new(vec![2], vec![3], None, BasicMerk).unwrap()), ); tree.commit( &mut NoopCommit {}, @@ -1100,11 +1100,11 @@ mod test { #[test] fn sum_tree() { - let mut tree = Tree::new(vec![0], vec![1], None, SummedMerk(3)) + let mut tree = TreeNode::new(vec![0], vec![1], None, SummedMerk(3)) .unwrap() .attach( false, - Some(Tree::new(vec![2], vec![3], None, SummedMerk(5)).unwrap()), + Some(TreeNode::new(vec![2], vec![3], None, SummedMerk(5)).unwrap()), ); tree.commit( &mut NoopCommit {}, diff --git a/merk/src/tree/ops.rs b/merk/src/tree/ops.rs index 4c69fc891..b7fa4dd53 100644 --- a/merk/src/tree/ops.rs +++ b/merk/src/tree/ops.rs @@ -50,7 +50,7 @@ use integer_encoding::VarInt; use Op::*; #[cfg(feature = "full")] -use super::{Fetch, Link, Tree, Walker}; +use super::{Fetch, Link, TreeNode, Walker}; #[cfg(feature = "full")] use crate::{error::Error, tree::tree_feature_type::TreeFeatureType, CryptoHash, HASH_LENGTH_U32}; use crate::{ @@ -150,7 +150,7 @@ pub struct PanicSource {} #[cfg(feature = "full")] impl Fetch for PanicSource { - fn fetch(&self, _link: &Link) -> CostResult { + fn fetch(&self, _link: &Link) -> CostResult { unreachable!("'fetch' should not have been called") } } @@ -172,7 +172,7 @@ where old_tree_cost: &C, update_tree_value_based_on_costs: &mut U, section_removal_bytes: &mut R, - ) -> CostContext, KeyUpdates), Error>> + ) -> CostContext, KeyUpdates), Error>> where C: Fn(&Vec, &Vec) -> Result, U: FnMut( @@ -247,7 +247,7 @@ where old_tree_cost: &C, update_tree_value_based_on_costs: &mut U, section_removal_bytes: &mut R, - ) -> CostResult, Error> + ) -> CostResult, Error> where C: Fn(&Vec, &Vec) -> Result, U: FnMut( @@ -318,21 +318,21 @@ where // TODO: take from batch so we don't have to clone let mid_tree = match mid_op { - Put(..) => Tree::new( + Put(..) => TreeNode::new( mid_key.as_ref().to_vec(), mid_value.to_vec(), None, mid_feature_type.to_owned(), ) .unwrap_add_cost(&mut cost), - PutWithSpecializedCost(_, value_cost, _) => Tree::new( + PutWithSpecializedCost(_, value_cost, _) => TreeNode::new( mid_key.as_ref().to_vec(), mid_value.to_vec(), Some(SpecializedValueDefinedCost(*value_cost)), mid_feature_type.to_owned(), ) .unwrap_add_cost(&mut cost), - PutCombinedReference(_, referenced_value, _) => Tree::new_with_combined_value_hash( + PutCombinedReference(_, referenced_value, _) => TreeNode::new_with_combined_value_hash( mid_key.as_ref().to_vec(), mid_value, referenced_value.to_owned(), @@ -341,7 +341,7 @@ where .unwrap_add_cost(&mut cost), PutLayeredReference(_, value_cost, referenced_value, _) | ReplaceLayeredReference(_, value_cost, referenced_value, _) => { - Tree::new_with_layered_value_hash( + TreeNode::new_with_layered_value_hash( mid_key.as_ref().to_vec(), mid_value, *value_cost, @@ -456,7 +456,7 @@ where Delete | DeleteLayered | DeleteLayeredMaybeSpecialized | DeleteMaybeSpecialized => { // TODO: we shouldn't have to do this as 2 different calls to apply let source = self.clone_source(); - let wrap = |maybe_tree: Option| { + let wrap = |maybe_tree: Option| { maybe_tree.map(|tree| Self::new(tree, source.clone())) }; let key = self.tree().key().to_vec(); @@ -799,7 +799,7 @@ mod test { #[test] fn simple_insert() { let batch = [(b"foo2".to_vec(), Op::Put(b"bar2".to_vec(), BasicMerk))]; - let tree = Tree::new(b"foo".to_vec(), b"bar".to_vec(), None, BasicMerk).unwrap(); + let tree = TreeNode::new(b"foo".to_vec(), b"bar".to_vec(), None, BasicMerk).unwrap(); let (maybe_walker, key_updates) = Walker::new(tree, PanicSource {}) .apply_sorted_without_costs(&batch) .unwrap() @@ -815,7 +815,7 @@ mod test { #[test] fn simple_update() { let batch = [(b"foo".to_vec(), Op::Put(b"bar2".to_vec(), BasicMerk))]; - let tree = Tree::new(b"foo".to_vec(), b"bar".to_vec(), None, BasicMerk).unwrap(); + let tree = TreeNode::new(b"foo".to_vec(), b"bar".to_vec(), None, BasicMerk).unwrap(); let (maybe_walker, key_updates) = Walker::new(tree, PanicSource {}) .apply_sorted_without_costs(&batch) .unwrap() @@ -832,7 +832,7 @@ mod test { #[test] fn simple_delete() { let batch = [(b"foo2".to_vec(), Op::Delete)]; - let tree = Tree::from_fields( + let tree = TreeNode::from_fields( b"foo".to_vec(), b"bar".to_vec(), [123; 32], @@ -841,7 +841,7 @@ mod test { hash: [123; 32], sum: None, child_heights: (0, 0), - tree: Tree::new(b"foo2".to_vec(), b"bar2".to_vec(), None, BasicMerk).unwrap(), + tree: TreeNode::new(b"foo2".to_vec(), b"bar2".to_vec(), None, BasicMerk).unwrap(), }), BasicMerk, ) @@ -866,7 +866,7 @@ mod test { #[test] fn delete_non_existent() { let batch = [(b"foo2".to_vec(), Op::Delete)]; - let tree = Tree::new(b"foo".to_vec(), b"bar".to_vec(), None, BasicMerk).unwrap(); + let tree = TreeNode::new(b"foo".to_vec(), b"bar".to_vec(), None, BasicMerk).unwrap(); Walker::new(tree, PanicSource {}) .apply_sorted_without_costs(&batch) .unwrap() @@ -876,7 +876,7 @@ mod test { #[test] fn delete_only_node() { let batch = [(b"foo".to_vec(), Op::Delete)]; - let tree = Tree::new(b"foo".to_vec(), b"bar".to_vec(), None, BasicMerk).unwrap(); + let tree = TreeNode::new(b"foo".to_vec(), b"bar".to_vec(), None, BasicMerk).unwrap(); let (maybe_walker, key_updates) = Walker::new(tree, PanicSource {}) .apply_sorted_without_costs(&batch) .unwrap() @@ -1096,7 +1096,7 @@ mod test { #[test] fn insert_root_single() { - let tree = Tree::new(vec![5], vec![123], None, BasicMerk).unwrap(); + let tree = TreeNode::new(vec![5], vec![123], None, BasicMerk).unwrap(); let batch = vec![(vec![6], Op::Put(vec![123], BasicMerk))]; let tree = apply_memonly(tree, &batch); assert_eq!(tree.key(), &[5]); @@ -1106,7 +1106,7 @@ mod test { #[test] fn insert_root_double() { - let tree = Tree::new(vec![5], vec![123], None, BasicMerk).unwrap(); + let tree = TreeNode::new(vec![5], vec![123], None, BasicMerk).unwrap(); let batch = vec![ (vec![4], Op::Put(vec![123], BasicMerk)), (vec![6], Op::Put(vec![123], BasicMerk)), @@ -1119,7 +1119,7 @@ mod test { #[test] fn insert_rebalance() { - let tree = Tree::new(vec![5], vec![123], None, BasicMerk).unwrap(); + let tree = TreeNode::new(vec![5], vec![123], None, BasicMerk).unwrap(); let batch = vec![(vec![6], Op::Put(vec![123], BasicMerk))]; let tree = apply_memonly(tree, &batch); @@ -1134,7 +1134,7 @@ mod test { #[test] fn insert_100_sequential() { - let mut tree = Tree::new(vec![0], vec![123], None, BasicMerk).unwrap(); + let mut tree = TreeNode::new(vec![0], vec![123], None, BasicMerk).unwrap(); for i in 0..100 { let batch = vec![(vec![i + 1], Op::Put(vec![123], BasicMerk))]; diff --git a/merk/src/tree/walk/fetch.rs b/merk/src/tree/walk/fetch.rs index 94a083afc..a13be66cf 100644 --- a/merk/src/tree/walk/fetch.rs +++ b/merk/src/tree/walk/fetch.rs @@ -32,7 +32,7 @@ use grovedb_costs::CostResult; #[cfg(feature = "full")] -use super::super::{Link, Tree}; +use super::super::{Link, TreeNode}; #[cfg(feature = "full")] use crate::error::Error; @@ -43,5 +43,5 @@ use crate::error::Error; pub trait Fetch { /// Called when the tree needs to fetch a node with the given `Link`. The /// `link` value will always be a `Link::Reference` variant. - fn fetch(&self, link: &Link) -> CostResult; + fn fetch(&self, link: &Link) -> CostResult; } diff --git a/merk/src/tree/walk/mod.rs b/merk/src/tree/walk/mod.rs index 77e4f595c..061df5209 100644 --- a/merk/src/tree/walk/mod.rs +++ b/merk/src/tree/walk/mod.rs @@ -41,7 +41,7 @@ use grovedb_costs::{cost_return_on_error, CostContext, CostResult, CostsExt, Ope pub use ref_walker::RefWalker; #[cfg(feature = "full")] -use super::{Link, Tree}; +use super::{Link, TreeNode}; #[cfg(feature = "full")] use crate::{owner::Owner, tree::tree_feature_type::TreeFeatureType, CryptoHash, Error}; @@ -52,7 +52,7 @@ pub struct Walker where S: Fetch + Sized + Clone, { - tree: Owner, + tree: Owner, source: S, } @@ -62,7 +62,7 @@ where S: Fetch + Sized + Clone, { /// Creates a `Walker` with the given tree and source. - pub fn new(tree: Tree, source: S) -> Self { + pub fn new(tree: TreeNode, source: S) -> Self { Self { tree: Owner::new(tree), source, @@ -119,7 +119,7 @@ where pub fn walk(self, left: bool, f: F) -> CostResult where F: FnOnce(Option) -> CostResult, Error>, - T: Into, + T: Into, { let mut cost = OperationCost::default(); @@ -137,7 +137,7 @@ where pub fn walk_expect(self, left: bool, f: F) -> CostResult where F: FnOnce(Self) -> CostResult, Error>, - T: Into, + T: Into, { let mut cost = OperationCost::default(); @@ -151,18 +151,18 @@ where } /// Returns an immutable reference to the `Tree` wrapped by this walker. - pub fn tree(&self) -> &Tree { + pub fn tree(&self) -> &TreeNode { &self.tree } /// Consumes the `Walker` and returns the `Tree` it wraps. - pub fn into_inner(self) -> Tree { + pub fn into_inner(self) -> TreeNode { self.tree.into_inner() } /// Takes a `Tree` and returns a `Walker` which fetches from the same source /// as `self`. - fn wrap(&self, tree: Tree) -> Self { + fn wrap(&self, tree: TreeNode) -> Self { Self::new(tree, self.source.clone()) } @@ -175,7 +175,7 @@ where /// implements `Into`. pub fn attach(mut self, left: bool, maybe_child: Option) -> Self where - T: Into, + T: Into, { self.tree .own(|t| t.attach(left, maybe_child.map(|t| t.into()))); @@ -243,7 +243,7 @@ where } #[cfg(feature = "full")] -impl From> for Tree +impl From> for TreeNode where S: Fetch + Sized + Clone, { @@ -258,31 +258,31 @@ mod test { use grovedb_costs::{storage_cost::removal::StorageRemovedBytes::NoStorageRemoval, CostsExt}; use super::{super::NoopCommit, *}; - use crate::tree::{Tree, TreeFeatureType::BasicMerk}; + use crate::tree::{TreeNode, TreeFeatureType::BasicMerk}; #[derive(Clone)] struct MockSource {} impl Fetch for MockSource { - fn fetch(&self, link: &Link) -> CostResult { - Tree::new(link.key().to_vec(), b"foo".to_vec(), None, BasicMerk).map(Ok) + fn fetch(&self, link: &Link) -> CostResult { + TreeNode::new(link.key().to_vec(), b"foo".to_vec(), None, BasicMerk).map(Ok) } } #[test] fn walk_modified() { - let tree = Tree::new(b"test".to_vec(), b"abc".to_vec(), None, BasicMerk) + let tree = TreeNode::new(b"test".to_vec(), b"abc".to_vec(), None, BasicMerk) .unwrap() .attach( true, - Some(Tree::new(b"foo".to_vec(), b"bar".to_vec(), None, BasicMerk).unwrap()), + Some(TreeNode::new(b"foo".to_vec(), b"bar".to_vec(), None, BasicMerk).unwrap()), ); let source = MockSource {}; let walker = Walker::new(tree, source); let walker = walker - .walk(true, |child| -> CostResult, Error> { + .walk(true, |child| -> CostResult, Error> { assert_eq!(child.expect("should have child").tree().key(), b"foo"); Ok(None).wrap_with_cost(Default::default()) }) @@ -293,11 +293,11 @@ mod test { #[test] fn walk_stored() { - let mut tree = Tree::new(b"test".to_vec(), b"abc".to_vec(), None, BasicMerk) + let mut tree = TreeNode::new(b"test".to_vec(), b"abc".to_vec(), None, BasicMerk) .unwrap() .attach( true, - Some(Tree::new(b"foo".to_vec(), b"bar".to_vec(), None, BasicMerk).unwrap()), + Some(TreeNode::new(b"foo".to_vec(), b"bar".to_vec(), None, BasicMerk).unwrap()), ); tree.commit( &mut NoopCommit {}, @@ -312,7 +312,7 @@ mod test { let walker = Walker::new(tree, source); let walker = walker - .walk(true, |child| -> CostResult, Error> { + .walk(true, |child| -> CostResult, Error> { assert_eq!(child.expect("should have child").tree().key(), b"foo"); Ok(None).wrap_with_cost(Default::default()) }) @@ -323,7 +323,7 @@ mod test { #[test] fn walk_pruned() { - let tree = Tree::from_fields( + let tree = TreeNode::from_fields( b"test".to_vec(), b"abc".to_vec(), Default::default(), @@ -342,7 +342,7 @@ mod test { let walker = Walker::new(tree, source); let walker = walker - .walk_expect(true, |child| -> CostResult, Error> { + .walk_expect(true, |child| -> CostResult, Error> { assert_eq!(child.tree().key(), b"foo"); Ok(None).wrap_with_cost(Default::default()) }) @@ -353,13 +353,13 @@ mod test { #[test] fn walk_none() { - let tree = Tree::new(b"test".to_vec(), b"abc".to_vec(), None, BasicMerk).unwrap(); + let tree = TreeNode::new(b"test".to_vec(), b"abc".to_vec(), None, BasicMerk).unwrap(); let source = MockSource {}; let walker = Walker::new(tree, source); walker - .walk(true, |child| -> CostResult, Error> { + .walk(true, |child| -> CostResult, Error> { assert!(child.is_none()); Ok(None).wrap_with_cost(Default::default()) }) diff --git a/merk/src/tree/walk/ref_walker.rs b/merk/src/tree/walk/ref_walker.rs index a6d7e4f0b..26c23b0a6 100644 --- a/merk/src/tree/walk/ref_walker.rs +++ b/merk/src/tree/walk/ref_walker.rs @@ -33,7 +33,7 @@ use grovedb_costs::{CostResult, CostsExt, OperationCost}; #[cfg(feature = "full")] use super::{ - super::{Link, Tree}, + super::{Link, TreeNode}, Fetch, }; #[cfg(feature = "full")] @@ -50,7 +50,7 @@ pub struct RefWalker<'a, S> where S: Fetch + Sized + Clone, { - tree: &'a mut Tree, + tree: &'a mut TreeNode, source: S, } @@ -60,13 +60,13 @@ where S: Fetch + Sized + Clone, { /// Creates a `RefWalker` with the given tree and source. - pub fn new(tree: &'a mut Tree, source: S) -> Self { + pub fn new(tree: &'a mut TreeNode, source: S) -> Self { // TODO: check if tree has modified links, panic if so RefWalker { tree, source } } /// Gets an immutable reference to the `Tree` wrapped by this `RefWalker`. - pub fn tree(&self) -> &Tree { + pub fn tree(&self) -> &TreeNode { self.tree } diff --git a/merk/src/visualize.rs b/merk/src/visualize.rs index da0bec444..4b3b2fb75 100644 --- a/merk/src/visualize.rs +++ b/merk/src/visualize.rs @@ -33,7 +33,7 @@ use std::io::{Result, Write}; use grovedb_storage::StorageContext; use grovedb_visualize::{Drawer, Visualize}; -use crate::{tree::Tree, Merk}; +use crate::{tree::TreeNode, Merk}; /// Visualizeable Merk pub struct VisualizeableMerk<'a, S, F> { @@ -52,12 +52,12 @@ impl<'a, S, F> VisualizeableMerk<'a, S, F> { } struct VisualizableTree<'a, F> { - tree: &'a Tree, + tree: &'a TreeNode, deserialize_fn: F, } impl<'a, F> VisualizableTree<'a, F> { - fn new(tree: &'a Tree, deserialize_fn: F) -> Self { + fn new(tree: &'a TreeNode, deserialize_fn: F) -> Self { Self { tree, deserialize_fn, From fc4640cc79b75ad3f539efeaf3a3abbd52695fa5 Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Sat, 23 Sep 2023 01:30:08 +0700 Subject: [PATCH 03/18] removed just in time updates from commit time --- grovedb/src/element/helpers.rs | 3 +- grovedb/src/versioning.rs | 1 - merk/src/merk/committer.rs | 71 +---------------- merk/src/merk/mod.rs | 11 +-- merk/src/merk/open.rs | 2 +- merk/src/merk/restore.rs | 9 ++- merk/src/proofs/chunk.rs | 44 +++-------- merk/src/proofs/query/mod.rs | 83 +++++++------------ merk/src/test_utils/mod.rs | 49 ++++-------- merk/src/tree/commit.rs | 32 -------- merk/src/tree/encoding.rs | 3 +- merk/src/tree/just_in_time_value_update.rs | 82 +++++++++++++++++++ merk/src/tree/mod.rs | 92 ++++------------------ merk/src/tree/walk/mod.rs | 13 +-- 14 files changed, 177 insertions(+), 318 deletions(-) create mode 100644 merk/src/tree/just_in_time_value_update.rs diff --git a/grovedb/src/element/helpers.rs b/grovedb/src/element/helpers.rs index 60b529317..394ec3dae 100644 --- a/grovedb/src/element/helpers.rs +++ b/grovedb/src/element/helpers.rs @@ -312,7 +312,8 @@ impl Element { #[cfg(feature = "full")] /// Decode from bytes pub fn raw_decode(bytes: &[u8]) -> Result { - let tree = TreeNode::decode_raw(bytes, vec![]).map_err(|e| Error::CorruptedData(e.to_string()))?; + let tree = + TreeNode::decode_raw(bytes, vec![]).map_err(|e| Error::CorruptedData(e.to_string()))?; let element: Element = Element::deserialize(tree.value_as_slice())?; Ok(element) } diff --git a/grovedb/src/versioning.rs b/grovedb/src/versioning.rs index a241b1e46..ca9a29247 100644 --- a/grovedb/src/versioning.rs +++ b/grovedb/src/versioning.rs @@ -37,7 +37,6 @@ pub(crate) fn prepend_version_to_bytes(mut bytes: Vec, version: u32) -> Resu #[cfg(test)] mod tests { - use integer_encoding::VarIntWriter; use crate::versioning::{ prepend_version_to_bytes, read_and_consume_proof_version, read_proof_version, diff --git a/merk/src/merk/committer.rs b/merk/src/merk/committer.rs index f959cfbf3..a87b1c3b4 100644 --- a/merk/src/merk/committer.rs +++ b/merk/src/merk/committer.rs @@ -1,11 +1,6 @@ -use grovedb_costs::storage_cost::{ - removal::{StorageRemovedBytes, StorageRemovedBytes::BasicStorageRemoval}, - StorageCost, -}; - use crate::{ - merk::{defaults::MAX_UPDATE_VALUE_BASED_ON_COSTS_TIMES, BatchValue}, - tree::{kv::ValueDefinedCostType, Commit, TreeNode}, + merk::BatchValue, + tree::{Commit, TreeNode}, Error, }; @@ -33,70 +28,10 @@ impl Commit for MerkCommitter { &mut self, tree: &mut TreeNode, old_specialized_cost: &impl Fn(&Vec, &Vec) -> Result, - update_tree_value_based_on_costs: &mut impl FnMut( - &StorageCost, - &Vec, - &mut Vec, - ) -> Result< - (bool, Option), - Error, - >, - section_removal_bytes: &mut impl FnMut( - &Vec, - u32, - u32, - ) -> Result< - (StorageRemovedBytes, StorageRemovedBytes), - Error, - >, ) -> Result<(), Error> { let tree_size = tree.encoding_length(); - let (mut current_tree_plus_hook_size, mut storage_costs) = + let (_, storage_costs) = tree.kv_with_parent_hook_size_and_storage_cost(old_specialized_cost)?; - let mut i = 0; - - if let Some(old_value) = tree.old_value.clone() { - // At this point the tree value can be updated based on client requirements - // For example to store the costs - loop { - let (flags_changed, value_defined_cost) = update_tree_value_based_on_costs( - &storage_costs.value_storage_cost, - &old_value, - tree.value_mut_ref(), - )?; - if !flags_changed { - break; - } else { - tree.inner.kv.value_defined_cost = value_defined_cost; - let after_update_tree_plus_hook_size = - tree.value_encoding_length_with_parent_to_child_reference(); - if after_update_tree_plus_hook_size == current_tree_plus_hook_size { - break; - } - let new_size_and_storage_costs = - tree.kv_with_parent_hook_size_and_storage_cost(old_specialized_cost)?; - current_tree_plus_hook_size = new_size_and_storage_costs.0; - storage_costs = new_size_and_storage_costs.1; - } - if i > MAX_UPDATE_VALUE_BASED_ON_COSTS_TIMES { - return Err(Error::CyclicError( - "updated value based on costs too many times", - )); - } - i += 1; - } - - if let BasicStorageRemoval(removed_bytes) = - storage_costs.value_storage_cost.removed_bytes - { - let (_, value_removed_bytes) = section_removal_bytes(&old_value, 0, removed_bytes)?; - storage_costs.value_storage_cost.removed_bytes = value_removed_bytes; - } - } - - // Update old tree size after generating value storage_cost cost - tree.old_size_with_parent_to_child_hook = current_tree_plus_hook_size; - tree.old_value = Some(tree.value_ref().clone()); let mut buf = Vec::with_capacity(tree_size); tree.encode_into(&mut buf); diff --git a/merk/src/merk/mod.rs b/merk/src/merk/mod.rs index c9304be0f..dc6b785b1 100644 --- a/merk/src/merk/mod.rs +++ b/merk/src/merk/mod.rs @@ -336,12 +336,7 @@ where let mut committer = MerkCommitter::new(tree.height(), 100); cost_return_on_error!( &mut inner_cost, - tree.commit( - &mut committer, - old_specialized_cost, - update_tree_value_based_on_costs, - section_removal_bytes - ) + tree.commit(&mut committer, old_specialized_cost,) ); let tree_key = tree.key(); @@ -557,7 +552,9 @@ where fn fetch_node<'db>(db: &impl StorageContext<'db>, key: &[u8]) -> Result, Error> { let bytes = db.get(key).unwrap().map_err(StorageError)?; // TODO: get_pinned ? if let Some(bytes) = bytes { - Ok(Some(TreeNode::decode(key.to_vec(), &bytes).map_err(EdError)?)) + Ok(Some( + TreeNode::decode(key.to_vec(), &bytes).map_err(EdError)?, + )) } else { Ok(None) } diff --git a/merk/src/merk/open.rs b/merk/src/merk/open.rs index 63eb37093..115926abd 100644 --- a/merk/src/merk/open.rs +++ b/merk/src/merk/open.rs @@ -69,13 +69,13 @@ where #[cfg(test)] mod test { + use grovedb_costs::OperationCost; use grovedb_path::SubtreePath; use grovedb_storage::{ rocksdb_storage::{test_utils::TempStorage, RocksDbStorage}, Storage, StorageBatch, }; use tempfile::TempDir; - use grovedb_costs::OperationCost; use crate::{Merk, Op, TreeFeatureType::BasicMerk}; diff --git a/merk/src/merk/restore.rs b/merk/src/merk/restore.rs index b4e4c39f0..eb4cea611 100644 --- a/merk/src/merk/restore.rs +++ b/merk/src/merk/restore.rs @@ -140,8 +140,13 @@ impl<'db, S: StorageContext<'db>> Restorer { key, )), Node::KVValueHash(key, value, value_hash) => Some(( - TreeNode::new_with_value_hash(key.clone(), value.clone(), *value_hash, BasicMerk) - .unwrap(), + TreeNode::new_with_value_hash( + key.clone(), + value.clone(), + *value_hash, + BasicMerk, + ) + .unwrap(), key, )), Node::KVValueHashFeatureType(key, value, value_hash, feature_type) => Some(( diff --git a/merk/src/proofs/chunk.rs b/merk/src/proofs/chunk.rs index bec53af49..077683cdc 100644 --- a/merk/src/proofs/chunk.rs +++ b/merk/src/proofs/chunk.rs @@ -460,14 +460,9 @@ mod tests { #[test] fn one_node_tree_trunk_roundtrip() { let mut tree = BaseTree::new(vec![0], vec![], None, BasicMerk).unwrap(); - tree.commit( - &mut NoopCommit {}, - &|_, _| Ok(0), - &mut |_, _, _| Ok((false, None)), - &mut |_, _, _| Ok((NoStorageRemoval, NoStorageRemoval)), - ) - .unwrap() - .unwrap(); + tree.commit(&mut NoopCommit {}, &|_, _| Ok(0)) + .unwrap() + .unwrap(); let mut walker = RefWalker::new(&mut tree, PanicSource {}); let (proof, has_more) = walker.create_trunk_proof().unwrap().unwrap(); @@ -491,14 +486,9 @@ mod tests { false, Some(BaseTree::new(vec![1], vec![], None, BasicMerk).unwrap()), ); - tree.commit( - &mut NoopCommit {}, - &|_, _| Ok(0), - &mut |_, _, _| Ok((false, None)), - &mut |_, _, _| Ok((NoStorageRemoval, NoStorageRemoval)), - ) - .unwrap() - .unwrap(); + tree.commit(&mut NoopCommit {}, &|_, _| Ok(0)) + .unwrap() + .unwrap(); let mut walker = RefWalker::new(&mut tree, PanicSource {}); let (proof, has_more) = walker.create_trunk_proof().unwrap().unwrap(); assert!(!has_more); @@ -521,14 +511,9 @@ mod tests { true, Some(BaseTree::new(vec![0], vec![], None, BasicMerk).unwrap()), ); - tree.commit( - &mut NoopCommit {}, - &|_, _| Ok(0), - &mut |_, _, _| Ok((false, None)), - &mut |_, _, _| Ok((NoStorageRemoval, NoStorageRemoval)), - ) - .unwrap() - .unwrap(); + tree.commit(&mut NoopCommit {}, &|_, _| Ok(0)) + .unwrap() + .unwrap(); let mut walker = RefWalker::new(&mut tree, PanicSource {}); let (proof, has_more) = walker.create_trunk_proof().unwrap().unwrap(); assert!(!has_more); @@ -555,14 +540,9 @@ mod tests { false, Some(BaseTree::new(vec![2], vec![], None, BasicMerk).unwrap()), ); - tree.commit( - &mut NoopCommit {}, - &|_, _| Ok(0), - &mut |_, _, _| Ok((false, None)), - &mut |_, _, _| Ok((NoStorageRemoval, NoStorageRemoval)), - ) - .unwrap() - .unwrap(); + tree.commit(&mut NoopCommit {}, &|_, _| Ok(0)) + .unwrap() + .unwrap(); let mut walker = RefWalker::new(&mut tree, PanicSource {}); let (proof, has_more) = walker.create_trunk_proof().unwrap().unwrap(); diff --git a/merk/src/proofs/query/mod.rs b/merk/src/proofs/query/mod.rs index a68bd159d..b9ba161e6 100644 --- a/merk/src/proofs/query/mod.rs +++ b/merk/src/proofs/query/mod.rs @@ -818,14 +818,9 @@ mod test { false, Some(TreeNode::new(vec![7], vec![7], None, BasicMerk).unwrap()), ); - tree.commit( - &mut NoopCommit {}, - &|_, _| Ok(0), - &mut |_, _, _| Ok((false, None)), - &mut |_, _, _| Ok((NoStorageRemoval, NoStorageRemoval)), - ) - .unwrap() - .expect("commit failed"); + tree.commit(&mut NoopCommit {}, &|_, _| Ok(0)) + .unwrap() + .expect("commit failed"); tree } @@ -837,12 +832,7 @@ mod test { .attach(true, Some(two_tree)) .attach(false, Some(four_tree)); three_tree - .commit( - &mut NoopCommit {}, - &|_, _| Ok(0), - &mut |_, _, _| Ok((false, None)), - &mut |_, _, _| Ok((NoStorageRemoval, NoStorageRemoval)), - ) + .commit(&mut NoopCommit {}, &|_, _| Ok(0)) .unwrap() .expect("commit failed"); @@ -851,12 +841,7 @@ mod test { .unwrap() .attach(true, Some(seven_tree)); eight_tree - .commit( - &mut NoopCommit {}, - &|_, _| Ok(0), - &mut |_, _, _| Ok((false, None)), - &mut |_, _, _| Ok((NoStorageRemoval, NoStorageRemoval)), - ) + .commit(&mut NoopCommit {}, &|_, _| Ok(0)) .unwrap() .expect("commit failed"); @@ -865,12 +850,7 @@ mod test { .attach(true, Some(three_tree)) .attach(false, Some(eight_tree)); root_tree - .commit( - &mut NoopCommit {}, - &|_, _| Ok(0), - &mut |_, _, _| Ok((false, None)), - &mut |_, _, _| Ok((NoStorageRemoval, NoStorageRemoval)), - ) + .commit(&mut NoopCommit {}, &|_, _| Ok(0)) .unwrap() .expect("commit failed"); @@ -1571,7 +1551,10 @@ mod test { .unwrap() .attach( true, - Some(TreeNode::new(vec![3], vec![3], None, BasicMerk).unwrap()), + Some( + TreeNode::new(vec![3], vec![3], None, BasicMerk) + .unwrap(), + ), ), ), ), @@ -1589,11 +1572,17 @@ mod test { .unwrap() .attach( true, - Some(TreeNode::new(vec![6], vec![6], None, BasicMerk).unwrap()), + Some( + TreeNode::new(vec![6], vec![6], None, BasicMerk) + .unwrap(), + ), ) .attach( false, - Some(TreeNode::new(vec![8], vec![8], None, BasicMerk).unwrap()), + Some( + TreeNode::new(vec![8], vec![8], None, BasicMerk) + .unwrap(), + ), ), ), ) @@ -1605,21 +1594,17 @@ mod test { .attach( true, Some( - TreeNode::new(vec![10], vec![10], None, BasicMerk).unwrap(), + TreeNode::new(vec![10], vec![10], None, BasicMerk) + .unwrap(), ), ), ), ), ), ); - tree.commit( - &mut NoopCommit {}, - &|_, _| Ok(0), - &mut |_, _, _| Ok((false, None)), - &mut |_, _, _| Ok((NoStorageRemoval, NoStorageRemoval)), - ) - .unwrap() - .unwrap(); + tree.commit(&mut NoopCommit {}, &|_, _| Ok(0)) + .unwrap() + .unwrap(); let mut walker = RefWalker::new(&mut tree, PanicSource {}); @@ -5756,14 +5741,9 @@ mod test { #[test] fn verify_ops() { let mut tree = TreeNode::new(vec![5], vec![5], None, BasicMerk).unwrap(); - tree.commit( - &mut NoopCommit {}, - &|_, _| Ok(0), - &mut |_, _, _| Ok((false, None)), - &mut |_, _, _| Ok((NoStorageRemoval, NoStorageRemoval)), - ) - .unwrap() - .expect("commit failed"); + tree.commit(&mut NoopCommit {}, &|_, _| Ok(0)) + .unwrap() + .expect("commit failed"); let root_hash = tree.hash().unwrap(); let mut walker = RefWalker::new(&mut tree, PanicSource {}); @@ -5787,14 +5767,9 @@ mod test { #[should_panic(expected = "verify failed")] fn verify_ops_mismatched_hash() { let mut tree = TreeNode::new(vec![5], vec![5], None, BasicMerk).unwrap(); - tree.commit( - &mut NoopCommit {}, - &|_, _| Ok(0), - &mut |_, _, _| Ok((false, None)), - &mut |_, _, _| Ok((NoStorageRemoval, NoStorageRemoval)), - ) - .unwrap() - .expect("commit failed"); + tree.commit(&mut NoopCommit {}, &|_, _| Ok(0)) + .unwrap() + .expect("commit failed"); let mut walker = RefWalker::new(&mut tree, PanicSource {}); diff --git a/merk/src/test_utils/mod.rs b/merk/src/test_utils/mod.rs index 4d257bde0..a9a087b56 100644 --- a/merk/src/test_utils/mod.rs +++ b/merk/src/test_utils/mod.rs @@ -98,23 +98,13 @@ pub fn apply_memonly_unchecked(tree: TreeNode, batch: &MerkBatch>) -> Tr .0 .expect("expected tree"); let is_sum_node = tree.is_sum_node(); - tree.commit( - &mut NoopCommit {}, - &|key, value| { - Ok(KV::layered_value_byte_cost_size_for_key_and_value_lengths( - key.len() as u32, - value.len() as u32, - is_sum_node, - )) - }, - &mut |_, _, _| Ok((false, None)), - &mut |_, key_bytes_to_remove, value_bytes_to_remove| { - Ok(( - BasicStorageRemoval(key_bytes_to_remove), - BasicStorageRemoval(value_bytes_to_remove), - )) - }, - ) + tree.commit(&mut NoopCommit {}, &|key, value| { + Ok(KV::layered_value_byte_cost_size_for_key_and_value_lengths( + key.len() as u32, + value.len() as u32, + is_sum_node, + )) + }) .unwrap() .expect("commit failed"); tree @@ -160,26 +150,15 @@ pub fn apply_to_memonly( .0 .map(|mut tree| { let is_sum_node = tree.is_sum_node(); - tree.commit( - &mut NoopCommit {}, - &|key, value| { - Ok(KV::layered_value_byte_cost_size_for_key_and_value_lengths( - key.len() as u32, - value.len() as u32, - is_sum_node, - )) - }, - &mut |_, _, _| Ok((false, None)), - &mut |_, key_bytes_to_remove, value_bytes_to_remove| { - Ok(( - BasicStorageRemoval(key_bytes_to_remove), - BasicStorageRemoval(value_bytes_to_remove), - )) - }, - ) + tree.commit(&mut NoopCommit {}, &|key, value| { + Ok(KV::layered_value_byte_cost_size_for_key_and_value_lengths( + key.len() as u32, + value.len() as u32, + is_sum_node, + )) + }) .unwrap() .expect("commit failed"); - println!("{:?}", &tree); assert_tree_invariants(&tree); tree }) diff --git a/merk/src/tree/commit.rs b/merk/src/tree/commit.rs index c50daced7..e7bcc66a6 100644 --- a/merk/src/tree/commit.rs +++ b/merk/src/tree/commit.rs @@ -47,22 +47,6 @@ pub trait Commit { &mut self, tree: &mut TreeNode, old_specialized_cost: &impl Fn(&Vec, &Vec) -> Result, - update_tree_value_based_on_costs: &mut impl FnMut( - &StorageCost, - &Vec, - &mut Vec, - ) -> Result< - (bool, Option), - Error, - >, - section_removal_bytes: &mut impl FnMut( - &Vec, - u32, - u32, - ) -> Result< - (StorageRemovedBytes, StorageRemovedBytes), - Error, - >, ) -> Result<(), Error>; /// Called once per node after writing a node and its children. The returned @@ -85,22 +69,6 @@ impl Commit for NoopCommit { &mut self, _tree: &mut TreeNode, _old_specialized_cost: &impl Fn(&Vec, &Vec) -> Result, - _update_tree_value_based_on_costs: &mut impl FnMut( - &StorageCost, - &Vec, - &mut Vec, - ) -> Result< - (bool, Option), - Error, - >, - _section_removal_bytes: &mut impl FnMut( - &Vec, - u32, - u32, - ) -> Result< - (StorageRemovedBytes, StorageRemovedBytes), - Error, - >, ) -> Result<(), Error> { Ok(()) } diff --git a/merk/src/tree/encoding.rs b/merk/src/tree/encoding.rs index 836b770ab..d7a56ce1d 100644 --- a/merk/src/tree/encoding.rs +++ b/merk/src/tree/encoding.rs @@ -135,7 +135,8 @@ mod tests { #[test] fn encode_leaf_tree() { - let tree = TreeNode::from_fields(vec![0], vec![1], [55; 32], None, None, BasicMerk).unwrap(); + let tree = + TreeNode::from_fields(vec![0], vec![1], [55; 32], None, None, BasicMerk).unwrap(); assert_eq!(tree.encoding_length(), 68); assert_eq!( tree.value_encoding_length_with_parent_to_child_reference(), diff --git a/merk/src/tree/just_in_time_value_update.rs b/merk/src/tree/just_in_time_value_update.rs new file mode 100644 index 000000000..36459cbd0 --- /dev/null +++ b/merk/src/tree/just_in_time_value_update.rs @@ -0,0 +1,82 @@ +use grovedb_costs::storage_cost::{ + removal::{StorageRemovedBytes, StorageRemovedBytes::BasicStorageRemoval}, + StorageCost, +}; + +use crate::{ + merk::defaults::MAX_UPDATE_VALUE_BASED_ON_COSTS_TIMES, + tree::{kv::ValueDefinedCostType, TreeNode}, + Error, +}; + +impl TreeNode { + pub(in crate::tree) fn just_in_time_tree_value_update( + &mut self, + old_specialized_cost: &impl Fn(&Vec, &Vec) -> Result, + update_tree_value_based_on_costs: &mut impl FnMut( + &StorageCost, + &Vec, + &mut Vec, + ) -> Result< + (bool, Option), + Error, + >, + section_removal_bytes: &mut impl FnMut( + &Vec, + u32, + u32, + ) -> Result< + (StorageRemovedBytes, StorageRemovedBytes), + Error, + >, + ) -> Result<(), Error> { + let (mut current_tree_plus_hook_size, mut storage_costs) = + self.kv_with_parent_hook_size_and_storage_cost(old_specialized_cost)?; + let mut i = 0; + + if let Some(old_value) = self.old_value.clone() { + // At this point the tree value can be updated based on client requirements + // For example to store the costs + loop { + let (flags_changed, value_defined_cost) = update_tree_value_based_on_costs( + &storage_costs.value_storage_cost, + &old_value, + self.value_mut_ref(), + )?; + if !flags_changed { + break; + } else { + self.inner.kv.value_defined_cost = value_defined_cost; + let after_update_tree_plus_hook_size = + self.value_encoding_length_with_parent_to_child_reference(); + if after_update_tree_plus_hook_size == current_tree_plus_hook_size { + break; + } + let new_size_and_storage_costs = + self.kv_with_parent_hook_size_and_storage_cost(old_specialized_cost)?; + current_tree_plus_hook_size = new_size_and_storage_costs.0; + storage_costs = new_size_and_storage_costs.1; + } + if i > MAX_UPDATE_VALUE_BASED_ON_COSTS_TIMES { + return Err(Error::CyclicError( + "updated value based on costs too many times", + )); + } + i += 1; + } + + if let BasicStorageRemoval(removed_bytes) = + storage_costs.value_storage_cost.removed_bytes + { + let (_, value_removed_bytes) = section_removal_bytes(&old_value, 0, removed_bytes)?; + storage_costs.value_storage_cost.removed_bytes = value_removed_bytes; + } + } + + // Update old tree size after generating value storage_cost cost + self.old_size_with_parent_to_child_hook = current_tree_plus_hook_size; + self.old_value = Some(self.value_ref().clone()); + + Ok(()) + } +} diff --git a/merk/src/tree/mod.rs b/merk/src/tree/mod.rs index f060da60c..d701d4f3c 100644 --- a/merk/src/tree/mod.rs +++ b/merk/src/tree/mod.rs @@ -41,6 +41,8 @@ mod hash; #[cfg(feature = "full")] mod iter; #[cfg(feature = "full")] +mod just_in_time_value_update; +#[cfg(feature = "full")] pub mod kv; #[cfg(feature = "full")] mod link; @@ -737,22 +739,6 @@ impl TreeNode { &mut self, c: &mut C, old_specialized_cost: &impl Fn(&Vec, &Vec) -> Result, - update_tree_value_based_on_costs: &mut impl FnMut( - &StorageCost, - &Vec, - &mut Vec, - ) -> Result< - (bool, Option), - Error, - >, - section_removal_bytes: &mut impl FnMut( - &Vec, - u32, - u32, - ) -> Result< - (StorageRemovedBytes, StorageRemovedBytes), - Error, - >, ) -> CostResult<(), Error> { // TODO: make this method less ugly // TODO: call write in-order for better performance in writing batch to db? @@ -769,15 +755,7 @@ impl TreeNode { }) = self.inner.left.take() { // println!("key is {}", std::str::from_utf8(tree.key()).unwrap()); - cost_return_on_error!( - &mut cost, - tree.commit( - c, - old_specialized_cost, - update_tree_value_based_on_costs, - section_removal_bytes - ) - ); + cost_return_on_error!(&mut cost, tree.commit(c, old_specialized_cost,)); let sum = cost_return_on_error_default!(tree.sum()); self.inner.left = Some(Link::Loaded { @@ -800,15 +778,7 @@ impl TreeNode { }) = self.inner.right.take() { // println!("key is {}", std::str::from_utf8(tree.key()).unwrap()); - cost_return_on_error!( - &mut cost, - tree.commit( - c, - old_specialized_cost, - update_tree_value_based_on_costs, - section_removal_bytes - ) - ); + cost_return_on_error!(&mut cost, tree.commit(c, old_specialized_cost,)); let sum = cost_return_on_error_default!(tree.sum()); self.inner.right = Some(Link::Loaded { hash: tree.hash().unwrap_add_cost(&mut cost), @@ -821,15 +791,7 @@ impl TreeNode { } } - cost_return_on_error_no_add!( - &cost, - c.write( - self, - old_specialized_cost, - update_tree_value_based_on_costs, - section_removal_bytes - ) - ); + cost_return_on_error_no_add!(&cost, c.write(self, old_specialized_cost,)); // println!("done committing {}", std::str::from_utf8(self.key()).unwrap()); @@ -981,14 +943,9 @@ mod test { assert!(tree.link(false).is_none()); assert!(tree.child(false).is_none()); - tree.commit( - &mut NoopCommit {}, - &|_, _| Ok(0), - &mut |_, _, _| Ok((false, None)), - &mut |_, _, _| Ok((NoStorageRemoval, NoStorageRemoval)), - ) - .unwrap() - .expect("commit failed"); + tree.commit(&mut NoopCommit {}, &|_, _| Ok(0)) + .unwrap() + .expect("commit failed"); assert!(tree.link(true).expect("expected link").is_stored()); assert!(tree.child(true).is_some()); @@ -1009,14 +966,9 @@ mod test { true, Some(TreeNode::new(vec![2], vec![3], None, BasicMerk).unwrap()), ); - tree.commit( - &mut NoopCommit {}, - &|_, _| Ok(0), - &mut |_, _, _| Ok((false, None)), - &mut |_, _, _| Ok((NoStorageRemoval, NoStorageRemoval)), - ) - .unwrap() - .expect("commit failed"); + tree.commit(&mut NoopCommit {}, &|_, _| Ok(0)) + .unwrap() + .expect("commit failed"); assert_eq!( tree.child_hash(true), &[ @@ -1086,14 +1038,9 @@ mod test { false, Some(TreeNode::new(vec![2], vec![3], None, BasicMerk).unwrap()), ); - tree.commit( - &mut NoopCommit {}, - &|_, _| Ok(0), - &mut |_, _, _| Ok((false, None)), - &mut |_, _, _| Ok((NoStorageRemoval, NoStorageRemoval)), - ) - .unwrap() - .expect("commit failed"); + tree.commit(&mut NoopCommit {}, &|_, _| Ok(0)) + .unwrap() + .expect("commit failed"); assert!(tree.link(false).expect("expected link").is_stored()); } @@ -1106,14 +1053,9 @@ mod test { false, Some(TreeNode::new(vec![2], vec![3], None, SummedMerk(5)).unwrap()), ); - tree.commit( - &mut NoopCommit {}, - &|_, _| Ok(0), - &mut |_, _, _| Ok((false, None)), - &mut |_, _, _| Ok((NoStorageRemoval, NoStorageRemoval)), - ) - .unwrap() - .expect("commit failed"); + tree.commit(&mut NoopCommit {}, &|_, _| Ok(0)) + .unwrap() + .expect("commit failed"); assert_eq!(Some(8), tree.sum().expect("expected to get sum from tree")); } diff --git a/merk/src/tree/walk/mod.rs b/merk/src/tree/walk/mod.rs index 061df5209..3c757e6c8 100644 --- a/merk/src/tree/walk/mod.rs +++ b/merk/src/tree/walk/mod.rs @@ -258,7 +258,7 @@ mod test { use grovedb_costs::{storage_cost::removal::StorageRemovedBytes::NoStorageRemoval, CostsExt}; use super::{super::NoopCommit, *}; - use crate::tree::{TreeNode, TreeFeatureType::BasicMerk}; + use crate::tree::{TreeFeatureType::BasicMerk, TreeNode}; #[derive(Clone)] struct MockSource {} @@ -299,14 +299,9 @@ mod test { true, Some(TreeNode::new(b"foo".to_vec(), b"bar".to_vec(), None, BasicMerk).unwrap()), ); - tree.commit( - &mut NoopCommit {}, - &|_, _| Ok(0), - &mut |_, _, _| Ok((false, None)), - &mut |_, _, _| Ok((NoStorageRemoval, NoStorageRemoval)), - ) - .unwrap() - .expect("commit failed"); + tree.commit(&mut NoopCommit {}, &|_, _| Ok(0)) + .unwrap() + .expect("commit failed"); let source = MockSource {}; let walker = Walker::new(tree, source); From 2e74c718aa7c6a93043a5485b588d2c59746a001 Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Mon, 25 Sep 2023 11:20:55 +0700 Subject: [PATCH 04/18] more work --- merk/src/merk/apply.rs | 9 +- merk/src/merk/committer.rs | 12 +- merk/src/merk/mod.rs | 30 +--- merk/src/owner.rs | 21 +++ merk/src/tree/just_in_time_value_update.rs | 3 +- merk/src/tree/kv.rs | 45 ++++- merk/src/tree/mod.rs | 184 ++++++++++++++++++--- merk/src/tree/ops.rs | 71 +++++--- merk/src/tree/walk/mod.rs | 164 +++++++++++++++--- 9 files changed, 434 insertions(+), 105 deletions(-) diff --git a/merk/src/merk/apply.rs b/merk/src/merk/apply.rs index 22d6f55cc..52b966a24 100644 --- a/merk/src/merk/apply.rs +++ b/merk/src/merk/apply.rs @@ -301,14 +301,7 @@ where // we set the new root node of the merk tree self.tree.set(maybe_tree); // commit changes to db - self.commit( - key_updates, - aux, - options, - old_specialized_cost, - update_tree_value_based_on_costs, - section_removal_bytes, - ) + self.commit(key_updates, aux, options, old_specialized_cost) }) } } diff --git a/merk/src/merk/committer.rs b/merk/src/merk/committer.rs index a87b1c3b4..af460bd80 100644 --- a/merk/src/merk/committer.rs +++ b/merk/src/merk/committer.rs @@ -1,3 +1,5 @@ +use grovedb_costs::storage_cost::key_value_cost::KeyValueStorageCost; + use crate::{ merk::BatchValue, tree::{Commit, TreeNode}, @@ -30,8 +32,12 @@ impl Commit for MerkCommitter { old_specialized_cost: &impl Fn(&Vec, &Vec) -> Result, ) -> Result<(), Error> { let tree_size = tree.encoding_length(); - let (_, storage_costs) = - tree.kv_with_parent_hook_size_and_storage_cost(old_specialized_cost)?; + let storage_costs = if let Some(storage_costs) = tree.known_storage_cost.take() { + storage_costs + } else { + tree.kv_with_parent_hook_size_and_storage_cost(old_specialized_cost)? + .1 + }; let mut buf = Vec::with_capacity(tree_size); tree.encode_into(&mut buf); @@ -42,7 +48,7 @@ impl Commit for MerkCommitter { tree.key().to_vec(), tree.feature_type().sum_length(), Some((buf, left_child_sizes, right_child_sizes)), - Some(storage_costs), + storage_costs, )); Ok(()) } diff --git a/merk/src/merk/mod.rs b/merk/src/merk/mod.rs index dc6b785b1..4da94fb26 100644 --- a/merk/src/merk/mod.rs +++ b/merk/src/merk/mod.rs @@ -76,7 +76,7 @@ use crate::{ pub struct KeyUpdates { pub new_keys: BTreeSet>, pub updated_keys: BTreeSet>, - pub deleted_keys: LinkedList<(Vec, Option)>, + pub deleted_keys: LinkedList<(Vec, KeyValueStorageCost)>, pub updated_root_key_from: Option>, } @@ -85,7 +85,7 @@ impl KeyUpdates { pub fn new( new_keys: BTreeSet>, updated_keys: BTreeSet>, - deleted_keys: LinkedList<(Vec, Option)>, + deleted_keys: LinkedList<(Vec, KeyValueStorageCost)>, updated_root_key_from: Option>, ) -> Self { Self { @@ -102,7 +102,7 @@ pub type BatchValue = ( Vec, Option, ChildrenSizesWithValue, - Option, + KeyValueStorageCost, ); /// A bool type @@ -251,7 +251,7 @@ pub type UseTreeMutResult = CostResult< Vec, Option, ChildrenSizesWithValue, - Option, + KeyValueStorageCost, )>, Error, >; @@ -304,22 +304,6 @@ where aux: &AuxMerkBatch, options: Option, old_specialized_cost: &impl Fn(&Vec, &Vec) -> Result, - update_tree_value_based_on_costs: &mut impl FnMut( - &StorageCost, - &Vec, - &mut Vec, - ) -> Result< - (bool, Option), - Error, - >, - section_removal_bytes: &mut impl FnMut( - &Vec, - u32, - u32, - ) -> Result< - (StorageRemovedBytes, StorageRemovedBytes), - Error, - >, ) -> CostResult<(), Error> where K: AsRef<[u8]>, @@ -398,7 +382,7 @@ where to_batch.push((key, None, None, maybe_cost)); } to_batch.sort_by(|a, b| a.0.cmp(&b.0)); - for (key, maybe_sum_tree_cost, maybe_value, maybe_cost) in to_batch { + for (key, maybe_sum_tree_cost, maybe_value, storage_cost) in to_batch { if let Some((value, left_size, right_size)) = maybe_value { cost_return_on_error_no_add!( &cost, @@ -407,12 +391,12 @@ where &key, &value, Some((maybe_sum_tree_cost, left_size, right_size)), - maybe_cost + Some(storage_cost) ) .map_err(CostsError) ); } else { - batch.delete(&key, maybe_cost); + batch.delete(&key, Some(storage_cost)); } } diff --git a/merk/src/owner.rs b/merk/src/owner.rs index d84917b75..18efb8f2a 100644 --- a/merk/src/owner.rs +++ b/merk/src/owner.rs @@ -91,6 +91,27 @@ impl Owner { return_value } + /// Takes temporary ownership of the contained value by passing it to `f`. + /// The function must return a result of the same type (the same value, or a + /// new value to take its place). + /// + /// Like `own`, but uses a tuple return type which allows specifying a value + /// to return from the call to `own_result` for convenience. + pub fn own_result(&mut self, f: F) -> Result<(), E> + where + F: FnOnce(T) -> Result, + { + let old_value = unwrap(self.inner.take()); + let new_value_result = f(old_value); + match new_value_result { + Ok(new_value) => { + self.inner = Some(new_value); + Ok(()) + } + Err(e) => Err(e), + } + } + /// Sheds the `Owner` container and returns the value it contained. pub fn into_inner(mut self) -> T { unwrap(self.inner.take()) diff --git a/merk/src/tree/just_in_time_value_update.rs b/merk/src/tree/just_in_time_value_update.rs index 36459cbd0..52af4138a 100644 --- a/merk/src/tree/just_in_time_value_update.rs +++ b/merk/src/tree/just_in_time_value_update.rs @@ -10,7 +10,7 @@ use crate::{ }; impl TreeNode { - pub(in crate::tree) fn just_in_time_tree_value_update( + pub(in crate::tree) fn just_in_time_tree_node_value_update( &mut self, old_specialized_cost: &impl Fn(&Vec, &Vec) -> Result, update_tree_value_based_on_costs: &mut impl FnMut( @@ -76,6 +76,7 @@ impl TreeNode { // Update old tree size after generating value storage_cost cost self.old_size_with_parent_to_child_hook = current_tree_plus_hook_size; self.old_value = Some(self.value_ref().clone()); + self.known_storage_cost = Some(storage_costs); Ok(()) } diff --git a/merk/src/tree/kv.rs b/merk/src/tree/kv.rs index 064b18a94..2c92c9a0c 100644 --- a/merk/src/tree/kv.rs +++ b/merk/src/tree/kv.rs @@ -197,18 +197,46 @@ impl KV { } } + /// Replaces the `KV`'s value with the given value, does not update the hash + /// or value hash. + #[inline] + pub fn put_value_no_update_of_hashes(mut self, value: Vec) -> Self { + self.value = value; + self + } + /// Replaces the `KV`'s value with the given value, updates the hash, /// value hash and returns the modified `KV`. #[inline] pub fn put_value_then_update(mut self, value: Vec) -> CostContext { - let mut cost = OperationCost::default(); - // TODO: length check? self.value = value; + self.update_hashes() + } + + /// Updates the hash, value hash and returns the modified `KV`. + #[inline] + pub fn update_hashes(mut self) -> CostContext { + let mut cost = OperationCost::default(); self.value_hash = value_hash(self.value_as_slice()).unwrap_add_cost(&mut cost); self.hash = kv_digest_to_kv_hash(self.key(), self.value_hash()).unwrap_add_cost(&mut cost); self.wrap_with_cost(cost) } + /// Updates the hashes and returns the modified `KV`. + #[inline] + pub fn update_hashes_using_reference_value_hash( + mut self, + reference_value_hash: CryptoHash, + ) -> CostContext { + let mut cost = OperationCost::default(); + let actual_value_hash = value_hash(self.value_as_slice()).unwrap_add_cost(&mut cost); + let combined_value_hash = + combine_hash(&actual_value_hash, &reference_value_hash).unwrap_add_cost(&mut cost); + self.value_hash = combined_value_hash; + self.hash = kv_digest_to_kv_hash(self.key(), self.value_hash()).unwrap_add_cost(&mut cost); + self.wrap_with_cost(cost) + } + /// Replaces the `KV`'s value with the given value, updates the hash, /// value hash and returns the modified `KV`. /// This is used when we want a fixed cost, for example in sum trees @@ -222,6 +250,19 @@ impl KV { self.put_value_then_update(value) } + /// Replaces the `KV`'s value with the given value, does not update the + /// hashes, value hash and returns the modified `KV`. + /// This is used when we want a fixed cost, for example in sum trees + #[inline] + pub fn put_value_with_fixed_cost_no_update_of_hashes( + mut self, + value: Vec, + value_cost: u32, + ) -> Self { + self.value_defined_cost = Some(SpecializedValueDefinedCost(value_cost)); + self.put_value_no_update_of_hashes(value) + } + /// Replaces the `KV`'s value with the given value and value hash, /// updates the hash and returns the modified `KV`. #[inline] diff --git a/merk/src/tree/mod.rs b/merk/src/tree/mod.rs index d701d4f3c..73bf7c492 100644 --- a/merk/src/tree/mod.rs +++ b/merk/src/tree/mod.rs @@ -144,6 +144,7 @@ pub struct TreeNode { pub(crate) inner: Box, pub(crate) old_size_with_parent_to_child_hook: u32, pub(crate) old_value: Option>, + pub(crate) known_storage_cost: Option, } #[cfg(feature = "full")] @@ -165,6 +166,7 @@ impl TreeNode { }), old_size_with_parent_to_child_hook: 0, old_value: None, + known_storage_cost: None, }) } @@ -176,6 +178,7 @@ impl TreeNode { inner: Box::new(inner_tree), old_size_with_parent_to_child_hook: decode_size, old_value: Some(old_value), + known_storage_cost: None, } } @@ -266,6 +269,7 @@ impl TreeNode { }), old_size_with_parent_to_child_hook: 0, old_value: None, + known_storage_cost: None, }) } @@ -286,6 +290,7 @@ impl TreeNode { }), old_size_with_parent_to_child_hook: 0, old_value: None, + known_storage_cost: None, }) } @@ -308,6 +313,7 @@ impl TreeNode { }), old_size_with_parent_to_child_hook: 0, old_value: None, + known_storage_cost: None, }, ) } @@ -330,6 +336,7 @@ impl TreeNode { }), old_size_with_parent_to_child_hook: 0, old_value: None, + known_storage_cost: None, }) } @@ -655,15 +662,49 @@ impl TreeNode { /// Replaces the root node's value with the given value and returns the /// modified `Tree`. #[inline] - pub fn put_value(mut self, value: Vec, feature_type: TreeFeatureType) -> CostContext { + pub fn put_value( + mut self, + value: Vec, + feature_type: TreeFeatureType, + old_specialized_cost: &impl Fn(&Vec, &Vec) -> Result, + update_tree_value_based_on_costs: &mut impl FnMut( + &StorageCost, + &Vec, + &mut Vec, + ) -> Result< + (bool, Option), + Error, + >, + section_removal_bytes: &mut impl FnMut( + &Vec, + u32, + u32, + ) -> Result< + (StorageRemovedBytes, StorageRemovedBytes), + Error, + >, + ) -> CostResult { let mut cost = OperationCost::default(); - self.inner.kv = self - .inner - .kv - .put_value_then_update(value) - .unwrap_add_cost(&mut cost); + + self.inner.kv = self.inner.kv.put_value_no_update_of_hashes(value); self.inner.kv.feature_type = feature_type; - self.wrap_with_cost(cost) + + if self.old_value.is_some() { + // we are replacing a value + // in this case there is a possibility that the client would want to update the + // element flags based on the change of values + cost_return_on_error_no_add!( + &cost, + self.just_in_time_tree_node_value_update( + old_specialized_cost, + update_tree_value_based_on_costs, + section_removal_bytes + ) + ); + } + + self.inner.kv = self.inner.kv.update_hashes().unwrap_add_cost(&mut cost); + Ok(self).wrap_with_cost(cost) } /// Replaces the root node's value with the given value and returns the @@ -674,15 +715,47 @@ impl TreeNode { value: Vec, value_fixed_cost: u32, feature_type: TreeFeatureType, - ) -> CostContext { + old_specialized_cost: &impl Fn(&Vec, &Vec) -> Result, + update_tree_value_based_on_costs: &mut impl FnMut( + &StorageCost, + &Vec, + &mut Vec, + ) -> Result< + (bool, Option), + Error, + >, + section_removal_bytes: &mut impl FnMut( + &Vec, + u32, + u32, + ) -> Result< + (StorageRemovedBytes, StorageRemovedBytes), + Error, + >, + ) -> CostResult { let mut cost = OperationCost::default(); self.inner.kv = self .inner .kv - .put_value_with_fixed_cost_then_update(value, value_fixed_cost) - .unwrap_add_cost(&mut cost); + .put_value_with_fixed_cost_no_update_of_hashes(value, value_fixed_cost); self.inner.kv.feature_type = feature_type; - self.wrap_with_cost(cost) + + if self.old_value.is_some() { + // we are replacing a value + // in this case there is a possibility that the client would want to update the + // element flags based on the change of values + cost_return_on_error_no_add!( + &cost, + self.just_in_time_tree_node_value_update( + old_specialized_cost, + update_tree_value_based_on_costs, + section_removal_bytes + ) + ); + } + + self.inner.kv = self.inner.kv.update_hashes().unwrap_add_cost(&mut cost); + Ok(self).wrap_with_cost(cost) } /// Replaces the root node's value with the given value and value hash @@ -693,15 +766,49 @@ impl TreeNode { value: Vec, value_hash: CryptoHash, feature_type: TreeFeatureType, - ) -> CostContext { + old_specialized_cost: &impl Fn(&Vec, &Vec) -> Result, + update_tree_value_based_on_costs: &mut impl FnMut( + &StorageCost, + &Vec, + &mut Vec, + ) -> Result< + (bool, Option), + Error, + >, + section_removal_bytes: &mut impl FnMut( + &Vec, + u32, + u32, + ) -> Result< + (StorageRemovedBytes, StorageRemovedBytes), + Error, + >, + ) -> CostResult { let mut cost = OperationCost::default(); + + self.inner.kv = self.inner.kv.put_value_no_update_of_hashes(value); + self.inner.kv.feature_type = feature_type; + + if self.old_value.is_some() { + // we are replacing a value + // in this case there is a possibility that the client would want to update the + // element flags based on the change of values + cost_return_on_error_no_add!( + &cost, + self.just_in_time_tree_node_value_update( + old_specialized_cost, + update_tree_value_based_on_costs, + section_removal_bytes + ) + ); + } + self.inner.kv = self .inner .kv - .put_value_and_reference_value_hash_then_update(value, value_hash) + .update_hashes_using_reference_value_hash(value_hash) .unwrap_add_cost(&mut cost); - self.inner.kv.feature_type = feature_type; - self.wrap_with_cost(cost) + Ok(self).wrap_with_cost(cost) } /// Replaces the root node's value with the given value and value hash @@ -713,17 +820,52 @@ impl TreeNode { value_hash: CryptoHash, value_cost: u32, feature_type: TreeFeatureType, - ) -> CostContext { + old_specialized_cost: &impl Fn(&Vec, &Vec) -> Result, + update_tree_value_based_on_costs: &mut impl FnMut( + &StorageCost, + &Vec, + &mut Vec, + ) -> Result< + (bool, Option), + Error, + >, + section_removal_bytes: &mut impl FnMut( + &Vec, + u32, + u32, + ) -> Result< + (StorageRemovedBytes, StorageRemovedBytes), + Error, + >, + ) -> CostResult { let mut cost = OperationCost::default(); + self.inner.kv = self .inner .kv - .put_value_with_reference_value_hash_and_value_cost_then_update( - value, value_hash, value_cost, - ) - .unwrap_add_cost(&mut cost); + .put_value_with_fixed_cost_no_update_of_hashes(value, value_cost); self.inner.kv.feature_type = feature_type; - self.wrap_with_cost(cost) + + if self.old_value.is_some() { + // we are replacing a value + // in this case there is a possibility that the client would want to update the + // element flags based on the change of values + cost_return_on_error_no_add!( + &cost, + self.just_in_time_tree_node_value_update( + old_specialized_cost, + update_tree_value_based_on_costs, + section_removal_bytes + ) + ); + } + + self.inner.kv = self + .inner + .kv + .update_hashes_using_reference_value_hash(value_hash) + .unwrap_add_cost(&mut cost); + Ok(self).wrap_with_cost(cost) } // TODO: add compute_hashes method diff --git a/merk/src/tree/ops.rs b/merk/src/tree/ops.rs index b7fa4dd53..82055b111 100644 --- a/merk/src/tree/ops.rs +++ b/merk/src/tree/ops.rs @@ -430,28 +430,59 @@ where // a key matches this node's key, apply op to this node match op { // TODO: take vec from batch so we don't need to clone - Put(value, feature_type) => self - .put_value(value.to_vec(), feature_type.to_owned()) - .unwrap_add_cost(&mut cost), - PutWithSpecializedCost(value, value_cost, feature_type) => self - .put_value_with_fixed_cost(value.to_vec(), *value_cost, feature_type.to_owned()) - .unwrap_add_cost(&mut cost), - PutCombinedReference(value, referenced_value, feature_type) => self - .put_value_and_reference_value_hash( - value.to_vec(), - referenced_value.to_owned(), - feature_type.to_owned(), + Put(value, feature_type) => { + cost_return_on_error!( + &mut cost, + self.put_value( + value.to_vec(), + feature_type.to_owned(), + old_specialized_cost, + update_tree_value_based_on_costs, + section_removal_bytes + ) + ) + } + + PutWithSpecializedCost(value, value_cost, feature_type) => { + cost_return_on_error!( + &mut cost, + self.put_value_with_fixed_cost( + value.to_vec(), + *value_cost, + feature_type.to_owned(), + old_specialized_cost, + update_tree_value_based_on_costs, + section_removal_bytes + ) ) - .unwrap_add_cost(&mut cost), + } + PutCombinedReference(value, referenced_value, feature_type) => { + cost_return_on_error!( + &mut cost, + self.put_value_and_reference_value_hash( + value.to_vec(), + referenced_value.to_owned(), + feature_type.to_owned(), + old_specialized_cost, + update_tree_value_based_on_costs, + section_removal_bytes, + ) + ) + } PutLayeredReference(value, value_cost, referenced_value, feature_type) | ReplaceLayeredReference(value, value_cost, referenced_value, feature_type) => { - self.put_value_with_reference_value_hash_and_value_cost( - value.to_vec(), - referenced_value.to_owned(), - *value_cost, - feature_type.to_owned(), + cost_return_on_error!( + &mut cost, + self.put_value_with_reference_value_hash_and_value_cost( + value.to_vec(), + referenced_value.to_owned(), + *value_cost, + feature_type.to_owned(), + old_specialized_cost, + update_tree_value_based_on_costs, + section_removal_bytes, + ) ) - .unwrap_add_cost(&mut cost) } Delete | DeleteLayered | DeleteLayeredMaybeSpecialized | DeleteMaybeSpecialized => { // TODO: we shouldn't have to do this as 2 different calls to apply @@ -481,7 +512,7 @@ where &cost, section_removal_bytes(value, total_key_len, old_cost) ); - let deletion_cost = Some(KeyValueStorageCost { + let deletion_cost = KeyValueStorageCost { key_storage_cost: StorageCost { added_bytes: 0, replaced_bytes: 0, @@ -494,7 +525,7 @@ where }, new_node: false, needs_value_verification: false, - }); + }; let maybe_tree = cost_return_on_error!(&mut cost, self.remove()); diff --git a/merk/src/tree/walk/mod.rs b/merk/src/tree/walk/mod.rs index 3c757e6c8..d4a6a5377 100644 --- a/merk/src/tree/walk/mod.rs +++ b/merk/src/tree/walk/mod.rs @@ -37,11 +37,16 @@ mod ref_walker; pub use fetch::Fetch; #[cfg(feature = "full")] use grovedb_costs::{cost_return_on_error, CostContext, CostResult, CostsExt, OperationCost}; +use grovedb_costs::{ + cost_return_on_error_no_add, + storage_cost::{removal::StorageRemovedBytes, StorageCost}, +}; #[cfg(feature = "full")] pub use ref_walker::RefWalker; #[cfg(feature = "full")] use super::{Link, TreeNode}; +use crate::tree::kv::ValueDefinedCostType; #[cfg(feature = "full")] use crate::{owner::Owner, tree::tree_feature_type::TreeFeatureType, CryptoHash, Error}; @@ -183,11 +188,42 @@ where } /// Similar to `Tree#put_value`. - pub fn put_value(mut self, value: Vec, feature_type: TreeFeatureType) -> CostContext { + pub fn put_value( + mut self, + value: Vec, + feature_type: TreeFeatureType, + old_specialized_cost: &impl Fn(&Vec, &Vec) -> Result, + update_tree_value_based_on_costs: &mut impl FnMut( + &StorageCost, + &Vec, + &mut Vec, + ) -> Result< + (bool, Option), + Error, + >, + section_removal_bytes: &mut impl FnMut( + &Vec, + u32, + u32, + ) -> Result< + (StorageRemovedBytes, StorageRemovedBytes), + Error, + >, + ) -> CostResult { let mut cost = OperationCost::default(); - self.tree - .own(|t| t.put_value(value, feature_type).unwrap_add_cost(&mut cost)); - self.wrap_with_cost(cost) + cost_return_on_error_no_add!( + &cost, + self.tree.own_result(|t| t + .put_value( + value, + feature_type, + old_specialized_cost, + update_tree_value_based_on_costs, + section_removal_bytes + ) + .unwrap_add_cost(&mut cost)) + ); + Ok(self).wrap_with_cost(cost) } /// Similar to `Tree#put_value_with_fixed_cost`. @@ -196,13 +232,39 @@ where value: Vec, value_fixed_cost: u32, feature_type: TreeFeatureType, - ) -> CostContext { + old_specialized_cost: &impl Fn(&Vec, &Vec) -> Result, + update_tree_value_based_on_costs: &mut impl FnMut( + &StorageCost, + &Vec, + &mut Vec, + ) -> Result< + (bool, Option), + Error, + >, + section_removal_bytes: &mut impl FnMut( + &Vec, + u32, + u32, + ) -> Result< + (StorageRemovedBytes, StorageRemovedBytes), + Error, + >, + ) -> CostResult { let mut cost = OperationCost::default(); - self.tree.own(|t| { - t.put_value_with_fixed_cost(value, value_fixed_cost, feature_type) - .unwrap_add_cost(&mut cost) - }); - self.wrap_with_cost(cost) + cost_return_on_error_no_add!( + &cost, + self.tree.own_result(|t| t + .put_value_with_fixed_cost( + value, + value_fixed_cost, + feature_type, + old_specialized_cost, + update_tree_value_based_on_costs, + section_removal_bytes + ) + .unwrap_add_cost(&mut cost)) + ); + Ok(self).wrap_with_cost(cost) } /// Similar to `Tree#put_value_and_reference_value_hash`. @@ -211,13 +273,39 @@ where value: Vec, value_hash: CryptoHash, feature_type: TreeFeatureType, - ) -> CostContext { + old_specialized_cost: &impl Fn(&Vec, &Vec) -> Result, + update_tree_value_based_on_costs: &mut impl FnMut( + &StorageCost, + &Vec, + &mut Vec, + ) -> Result< + (bool, Option), + Error, + >, + section_removal_bytes: &mut impl FnMut( + &Vec, + u32, + u32, + ) -> Result< + (StorageRemovedBytes, StorageRemovedBytes), + Error, + >, + ) -> CostResult { let mut cost = OperationCost::default(); - self.tree.own(|t| { - t.put_value_and_reference_value_hash(value, value_hash, feature_type) - .unwrap_add_cost(&mut cost) - }); - self.wrap_with_cost(cost) + cost_return_on_error_no_add!( + &cost, + self.tree.own_result(|t| t + .put_value_and_reference_value_hash( + value, + value_hash, + feature_type, + old_specialized_cost, + update_tree_value_based_on_costs, + section_removal_bytes + ) + .unwrap_add_cost(&mut cost)) + ); + Ok(self).wrap_with_cost(cost) } /// Similar to `Tree#put_value_with_reference_value_hash_and_value_cost`. @@ -227,18 +315,40 @@ where value_hash: CryptoHash, value_fixed_cost: u32, feature_type: TreeFeatureType, - ) -> CostContext { + old_specialized_cost: &impl Fn(&Vec, &Vec) -> Result, + update_tree_value_based_on_costs: &mut impl FnMut( + &StorageCost, + &Vec, + &mut Vec, + ) -> Result< + (bool, Option), + Error, + >, + section_removal_bytes: &mut impl FnMut( + &Vec, + u32, + u32, + ) -> Result< + (StorageRemovedBytes, StorageRemovedBytes), + Error, + >, + ) -> CostResult { let mut cost = OperationCost::default(); - self.tree.own(|t| { - t.put_value_with_reference_value_hash_and_value_cost( - value, - value_hash, - value_fixed_cost, - feature_type, - ) - .unwrap_add_cost(&mut cost) - }); - self.wrap_with_cost(cost) + cost_return_on_error_no_add!( + &cost, + self.tree.own_result(|t| t + .put_value_with_reference_value_hash_and_value_cost( + value, + value_hash, + value_fixed_cost, + feature_type, + old_specialized_cost, + update_tree_value_based_on_costs, + section_removal_bytes + ) + .unwrap_add_cost(&mut cost)) + ); + Ok(self).wrap_with_cost(cost) } } From 9aed6f1ce65b9861b5e12c111c7c7d72f12ce00a Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Tue, 26 Sep 2023 03:31:07 +0700 Subject: [PATCH 05/18] more work --- merk/src/tree/just_in_time_value_update.rs | 1 - merk/src/tree/kv.rs | 4 +-- merk/src/tree/mod.rs | 33 +++++++++------------- 3 files changed, 15 insertions(+), 23 deletions(-) diff --git a/merk/src/tree/just_in_time_value_update.rs b/merk/src/tree/just_in_time_value_update.rs index 52af4138a..20861ec4d 100644 --- a/merk/src/tree/just_in_time_value_update.rs +++ b/merk/src/tree/just_in_time_value_update.rs @@ -74,7 +74,6 @@ impl TreeNode { } // Update old tree size after generating value storage_cost cost - self.old_size_with_parent_to_child_hook = current_tree_plus_hook_size; self.old_value = Some(self.value_ref().clone()); self.known_storage_cost = Some(storage_costs); diff --git a/merk/src/tree/kv.rs b/merk/src/tree/kv.rs index 2c92c9a0c..8855855ae 100644 --- a/merk/src/tree/kv.rs +++ b/merk/src/tree/kv.rs @@ -257,9 +257,9 @@ impl KV { pub fn put_value_with_fixed_cost_no_update_of_hashes( mut self, value: Vec, - value_cost: u32, + value_cost: ValueDefinedCostType, ) -> Self { - self.value_defined_cost = Some(SpecializedValueDefinedCost(value_cost)); + self.value_defined_cost = Some(value_cost); self.put_value_no_update_of_hashes(value) } diff --git a/merk/src/tree/mod.rs b/merk/src/tree/mod.rs index 73bf7c492..5f15ed945 100644 --- a/merk/src/tree/mod.rs +++ b/merk/src/tree/mod.rs @@ -92,6 +92,7 @@ pub use walk::{Fetch, RefWalker, Walker}; #[cfg(feature = "full")] use crate::tree::kv::ValueDefinedCostType; +use crate::tree::kv::ValueDefinedCostType::{LayeredValueDefinedCost, SpecializedValueDefinedCost}; #[cfg(feature = "full")] use crate::{error::Error, Error::Overflow}; @@ -142,7 +143,6 @@ impl Terminated for Box {} #[derive(Clone)] pub struct TreeNode { pub(crate) inner: Box, - pub(crate) old_size_with_parent_to_child_hook: u32, pub(crate) old_value: Option>, pub(crate) known_storage_cost: Option, } @@ -164,7 +164,6 @@ impl TreeNode { left: None, right: None, }), - old_size_with_parent_to_child_hook: 0, old_value: None, known_storage_cost: None, }) @@ -172,11 +171,9 @@ impl TreeNode { /// Creates a new `Tree` given an inner tree pub fn new_with_tree_inner(inner_tree: TreeNodeInner) -> Self { - let decode_size = inner_tree.kv.value_byte_cost_size(); let old_value = inner_tree.kv.value.clone(); Self { inner: Box::new(inner_tree), - old_size_with_parent_to_child_hook: decode_size, old_value: Some(old_value), known_storage_cost: None, } @@ -223,7 +220,7 @@ impl TreeNode { let key_value_storage_cost = KeyValueStorageCost { key_storage_cost, // the key storage cost is added later value_storage_cost, - new_node: self.old_size_with_parent_to_child_hook == 0, + new_node: self.old_value.is_none(), needs_value_verification: self.inner.kv.value_defined_cost.is_none(), }; @@ -239,10 +236,10 @@ impl TreeNode { ) -> Result<(u32, KeyValueStorageCost), Error> { let current_value_byte_cost = self.value_encoding_length_with_parent_to_child_reference(); - let old_cost = if self.inner.kv.value_defined_cost.is_some() && self.old_value.is_some() { - old_tree_cost(self.key_as_ref(), self.old_value.as_ref().unwrap()) + let old_cost = if let Some(old_value) = self.old_value.as_ref() { + old_tree_cost(self.key_as_ref(), old_value) } else { - Ok(self.old_size_with_parent_to_child_hook) + Ok(0) // there was no old value, hence old cost would be 0 }?; self.kv_with_parent_hook_size_and_storage_cost_from_old_cost( @@ -267,7 +264,6 @@ impl TreeNode { left: None, right: None, }), - old_size_with_parent_to_child_hook: 0, old_value: None, known_storage_cost: None, }) @@ -288,7 +284,6 @@ impl TreeNode { left: None, right: None, }), - old_size_with_parent_to_child_hook: 0, old_value: None, known_storage_cost: None, }) @@ -311,7 +306,6 @@ impl TreeNode { left: None, right: None, }), - old_size_with_parent_to_child_hook: 0, old_value: None, known_storage_cost: None, }, @@ -334,7 +328,6 @@ impl TreeNode { left, right, }), - old_size_with_parent_to_child_hook: 0, old_value: None, known_storage_cost: None, }) @@ -734,10 +727,10 @@ impl TreeNode { >, ) -> CostResult { let mut cost = OperationCost::default(); - self.inner.kv = self - .inner - .kv - .put_value_with_fixed_cost_no_update_of_hashes(value, value_fixed_cost); + self.inner.kv = self.inner.kv.put_value_with_fixed_cost_no_update_of_hashes( + value, + SpecializedValueDefinedCost(value_fixed_cost), + ); self.inner.kv.feature_type = feature_type; if self.old_value.is_some() { @@ -840,10 +833,10 @@ impl TreeNode { ) -> CostResult { let mut cost = OperationCost::default(); - self.inner.kv = self - .inner - .kv - .put_value_with_fixed_cost_no_update_of_hashes(value, value_cost); + self.inner.kv = self.inner.kv.put_value_with_fixed_cost_no_update_of_hashes( + value, + LayeredValueDefinedCost(value_cost), + ); self.inner.kv.feature_type = feature_type; if self.old_value.is_some() { From 986df1b7e89f6083f7f8e0a81e55fc4a98f38232 Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Tue, 26 Sep 2023 14:13:47 +0700 Subject: [PATCH 06/18] removed a comma --- merk/src/merk/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/merk/src/merk/mod.rs b/merk/src/merk/mod.rs index 4da94fb26..89982ef0d 100644 --- a/merk/src/merk/mod.rs +++ b/merk/src/merk/mod.rs @@ -320,7 +320,7 @@ where let mut committer = MerkCommitter::new(tree.height(), 100); cost_return_on_error!( &mut inner_cost, - tree.commit(&mut committer, old_specialized_cost,) + tree.commit(&mut committer, old_specialized_cost) ); let tree_key = tree.key(); From 7d0899c3a405d52729cbf7b311394fff9269af7f Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Wed, 27 Sep 2023 10:24:10 +0700 Subject: [PATCH 07/18] added a callback method to be able to get the value defined costs --- grovedb/src/batch/mod.rs | 97 +++--- grovedb/src/element/delete.rs | 15 +- grovedb/src/element/exists.rs | 7 +- grovedb/src/element/get.rs | 32 +- grovedb/src/element/helpers.rs | 45 ++- grovedb/src/element/insert.rs | 3 + .../src/estimated_costs/average_case_costs.rs | 10 +- .../src/estimated_costs/worst_case_costs.rs | 7 +- grovedb/src/lib.rs | 88 ++++-- grovedb/src/operations/delete/mod.rs | 3 +- grovedb/src/operations/insert/mod.rs | 17 +- grovedb/src/replication.rs | 1 + grovedb/src/tests/mod.rs | 44 ++- grovedb/src/tests/sum_tree_tests.rs | 151 ++++++--- grovedb/src/tests/tree_hashes_tests.rs | 76 ++++- grovedb/src/util.rs | 20 +- merk/src/merk/apply.rs | 35 ++- merk/src/merk/chunks.rs | 4 + merk/src/merk/get.rs | 195 ++++++++---- merk/src/merk/mod.rs | 133 ++++++-- merk/src/merk/open.rs | 30 +- merk/src/merk/restore.rs | 59 ++-- merk/src/merk/source.rs | 10 +- merk/src/proofs/chunk.rs | 45 ++- merk/src/proofs/encoding.rs | 18 +- merk/src/proofs/query/mod.rs | 64 ++-- merk/src/test_utils/mod.rs | 19 +- merk/src/test_utils/temp_merk.rs | 17 +- merk/src/tree/encoding.rs | 76 +++-- merk/src/tree/kv.rs | 58 +--- merk/src/tree/link.rs | 14 +- merk/src/tree/mod.rs | 66 ++-- merk/src/tree/ops.rs | 287 ++++++++++++------ merk/src/tree/tree_feature_type.rs | 32 +- merk/src/tree/walk/fetch.rs | 8 +- merk/src/tree/walk/mod.rs | 136 ++++++--- merk/src/tree/walk/ref_walker.rs | 12 +- 37 files changed, 1317 insertions(+), 617 deletions(-) diff --git a/grovedb/src/batch/mod.rs b/grovedb/src/batch/mod.rs index 22f859d54..5d463f91a 100644 --- a/grovedb/src/batch/mod.rs +++ b/grovedb/src/batch/mod.rs @@ -80,7 +80,7 @@ use grovedb_merk::{ value_hash, NULL_HASH, }, CryptoHash, Error as MerkError, Merk, MerkType, RootHashKeyAndSum, - TreeFeatureType::{BasicMerk, SummedMerk}, + TreeFeatureType::{BasicMerkNode, SummedMerkNode}, }; use grovedb_path::SubtreePath; use grovedb_storage::{ @@ -766,8 +766,12 @@ where if recursions_allowed == 1 { let referenced_element_value_hash_opt = cost_return_on_error!( &mut cost, - merk.get_value_hash(key.as_ref(), true) - .map_err(|e| Error::CorruptedData(e.to_string())) + merk.get_value_hash( + key.as_ref(), + true, + Some(Element::value_defined_cost_for_serialized_value) + ) + .map_err(|e| Error::CorruptedData(e.to_string())) ); let referenced_element_value_hash = cost_return_on_error!( @@ -806,8 +810,12 @@ where // change in the batch. let referenced_element = cost_return_on_error!( &mut cost, - merk.get(key.as_ref(), true) - .map_err(|e| Error::CorruptedData(e.to_string())) + merk.get( + key.as_ref(), + true, + Some(Element::value_defined_cost_for_serialized_value) + ) + .map_err(|e| Error::CorruptedData(e.to_string())) ); let referenced_element = cost_return_on_error_no_add!( @@ -1130,14 +1138,18 @@ where } else { let value = cost_return_on_error!( &mut cost, - merk.get(key_info.as_slice(), true) - .map(|result_value| result_value - .map_err(Error::MerkError) - .and_then(|maybe_value| maybe_value.ok_or( - Error::InvalidInput( - "trying to refresh a non existing reference", - ) - ))) + merk.get( + key_info.as_slice(), + true, + Some(Element::value_defined_cost_for_serialized_value) + ) + .map( + |result_value| result_value.map_err(Error::MerkError).and_then( + |maybe_value| maybe_value.ok_or(Error::InvalidInput( + "trying to refresh a non existing reference", + )) + ) + ) ); cost_return_on_error_no_add!( &cost, @@ -1155,9 +1167,9 @@ where }; let merk_feature_type = if is_sum_tree { - SummedMerk(0) + SummedMerkNode(0) } else { - BasicMerk + BasicMerkNode }; let path_reference = cost_return_on_error!( @@ -1276,7 +1288,7 @@ where } cost_return_on_error!( &mut cost, - merk.apply_unchecked::<_, Vec, _, _, _>( + merk.apply_unchecked::<_, Vec, _, _, _, _>( &batch_operations, &[], Some(batch_apply_options.as_merk_options()), @@ -1284,6 +1296,7 @@ where Element::specialized_costs_for_key_value(key, value, is_sum_tree) .map_err(|e| MerkError::ClientCorruptionError(e.to_string())) }, + Some(&Element::value_defined_cost_for_serialized_value), &mut |storage_costs, old_value, new_value| { // todo: change the flags without full deserialization let old_element = Element::deserialize(old_value.as_slice()) @@ -1779,13 +1792,16 @@ impl GroveDb { ); let is_sum_tree = element.is_sum_tree(); if let Element::Tree(root_key, _) | Element::SumTree(root_key, ..) = element { - Merk::open_layered_with_root_key(storage, root_key, is_sum_tree) - .map_err(|_| { - Error::CorruptedData( - "cannot open a subtree with given root key".to_owned(), - ) - }) - .add_cost(cost) + Merk::open_layered_with_root_key( + storage, + root_key, + is_sum_tree, + Some(&Element::value_defined_cost_for_serialized_value), + ) + .map_err(|_| { + Error::CorruptedData("cannot open a subtree with given root key".to_owned()) + }) + .add_cost(cost) } else { Err(Error::CorruptedPath( "cannot open a subtree as parent exists but is not a tree", @@ -1797,9 +1813,13 @@ impl GroveDb { if new_merk { Ok(Merk::open_empty(storage, MerkType::BaseMerk, false)).wrap_with_cost(cost) } else { - Merk::open_base(storage, false) - .map_err(|_| Error::CorruptedData("cannot open a the root subtree".to_owned())) - .add_cost(cost) + Merk::open_base( + storage, + false, + Some(&Element::value_defined_cost_for_serialized_value), + ) + .map_err(|_| Error::CorruptedData("cannot open a the root subtree".to_owned())) + .add_cost(cost) } } } @@ -1835,11 +1855,16 @@ impl GroveDb { ); let is_sum_tree = element.is_sum_tree(); if let Element::Tree(root_key, _) | Element::SumTree(root_key, ..) = element { - Merk::open_layered_with_root_key(storage, root_key, is_sum_tree) - .map_err(|_| { - Error::CorruptedData("cannot open a subtree with given root key".to_owned()) - }) - .add_cost(local_cost) + Merk::open_layered_with_root_key( + storage, + root_key, + is_sum_tree, + Some(&Element::value_defined_cost_for_serialized_value), + ) + .map_err(|_| { + Error::CorruptedData("cannot open a subtree with given root key".to_owned()) + }) + .add_cost(local_cost) } else { Err(Error::CorruptedData( "cannot open a subtree as parent exists but is not a tree".to_owned(), @@ -1847,9 +1872,13 @@ impl GroveDb { .wrap_with_cost(local_cost) } } else { - Merk::open_base(storage, false) - .map_err(|_| Error::CorruptedData("cannot open a subtree".to_owned())) - .add_cost(local_cost) + Merk::open_base( + storage, + false, + Some(&Element::value_defined_cost_for_serialized_value), + ) + .map_err(|_| Error::CorruptedData("cannot open a subtree".to_owned())) + .add_cost(local_cost) } } diff --git a/grovedb/src/element/delete.rs b/grovedb/src/element/delete.rs index 9766c5464..92087bc48 100644 --- a/grovedb/src/element/delete.rs +++ b/grovedb/src/element/delete.rs @@ -57,10 +57,16 @@ impl Element { }; let batch = [(key, op)]; let uses_sum_nodes = merk.is_sum_tree; - merk.apply_with_specialized_costs::<_, Vec>(&batch, &[], merk_options, &|key, value| { - Self::specialized_costs_for_key_value(key, value, uses_sum_nodes) - .map_err(|e| MerkError::ClientCorruptionError(e.to_string())) - }) + merk.apply_with_specialized_costs::<_, Vec>( + &batch, + &[], + merk_options, + &|key, value| { + Self::specialized_costs_for_key_value(key, value, uses_sum_nodes) + .map_err(|e| MerkError::ClientCorruptionError(e.to_string())) + }, + Some(&Element::value_defined_cost_for_serialized_value), + ) .map_err(|e| Error::CorruptedData(e.to_string())) } @@ -97,6 +103,7 @@ impl Element { Self::specialized_costs_for_key_value(key, value, uses_sum_nodes) .map_err(|e| MerkError::ClientCorruptionError(e.to_string())) }, + Some(&Element::value_defined_cost_for_serialized_value), &mut |_costs, _old_value, _value| Ok((false, None)), sectioned_removal, ) diff --git a/grovedb/src/element/exists.rs b/grovedb/src/element/exists.rs index d0bf5ecb0..c3bf61fa4 100644 --- a/grovedb/src/element/exists.rs +++ b/grovedb/src/element/exists.rs @@ -48,7 +48,10 @@ impl Element { merk: &mut Merk, key: K, ) -> CostResult { - merk.exists(key.as_ref()) - .map_err(|e| Error::CorruptedData(e.to_string())) + merk.exists( + key.as_ref(), + Some(&Element::value_defined_cost_for_serialized_value), + ) + .map_err(|e| Error::CorruptedData(e.to_string())) } } diff --git a/grovedb/src/element/get.rs b/grovedb/src/element/get.rs index 57cd40121..b6f75b101 100644 --- a/grovedb/src/element/get.rs +++ b/grovedb/src/element/get.rs @@ -78,8 +78,12 @@ impl Element { let value_opt = cost_return_on_error!( &mut cost, - merk.get(key.as_ref(), allow_cache) - .map_err(|e| Error::CorruptedData(e.to_string())) + merk.get( + key.as_ref(), + allow_cache, + Some(&Element::value_defined_cost_for_serialized_value) + ) + .map_err(|e| Error::CorruptedData(e.to_string())) ); let element = cost_return_on_error_no_add!( &cost, @@ -230,8 +234,12 @@ impl Element { let value_hash = cost_return_on_error!( &mut cost, - merk.get_value_hash(key.as_ref(), allow_cache) - .map_err(|e| Error::CorruptedData(e.to_string())) + merk.get_value_hash( + key.as_ref(), + allow_cache, + Some(&Element::value_defined_cost_for_serialized_value) + ) + .map_err(|e| Error::CorruptedData(e.to_string())) ); Ok(value_hash).wrap_with_cost(cost) @@ -253,7 +261,13 @@ mod tests { let ctx = storage .get_storage_context(SubtreePath::empty(), Some(&batch)) .unwrap(); - let mut merk = Merk::open_base(ctx, false).unwrap().unwrap(); + let mut merk = Merk::open_base( + ctx, + false, + Some(&Element::value_defined_cost_for_serialized_value), + ) + .unwrap() + .unwrap(); Element::empty_tree() .insert(&mut merk, b"mykey", None) .unwrap() @@ -271,7 +285,13 @@ mod tests { let ctx = storage .get_storage_context(SubtreePath::empty(), None) .unwrap(); - let mut merk = Merk::open_base(ctx, false).unwrap().unwrap(); + let mut merk = Merk::open_base( + ctx, + false, + Some(&Element::value_defined_cost_for_serialized_value), + ) + .unwrap() + .unwrap(); assert_eq!( Element::get(&merk, b"another-key", true) diff --git a/grovedb/src/element/helpers.rs b/grovedb/src/element/helpers.rs index 394ec3dae..85ce8f4e4 100644 --- a/grovedb/src/element/helpers.rs +++ b/grovedb/src/element/helpers.rs @@ -29,11 +29,15 @@ //! Helpers //! Implements helper functions in Element +use grovedb_merk::tree::kv::{ + ValueDefinedCostType, + ValueDefinedCostType::{LayeredValueDefinedCost, SpecializedValueDefinedCost}, +}; #[cfg(feature = "full")] use grovedb_merk::{ tree::{kv::KV, TreeNode}, TreeFeatureType, - TreeFeatureType::{BasicMerk, SummedMerk}, + TreeFeatureType::{BasicMerkNode, SummedMerkNode}, }; #[cfg(feature = "full")] use integer_encoding::VarInt; @@ -114,8 +118,8 @@ impl Element { /// Get the tree feature type pub fn get_feature_type(&self, parent_is_sum_tree: bool) -> Result { match parent_is_sum_tree { - true => Ok(SummedMerk(self.sum_value_or_default())), - false => Ok(BasicMerk), + true => Ok(SummedMerkNode(self.sum_value_or_default())), + false => Ok(BasicMerkNode), } } @@ -307,13 +311,44 @@ impl Element { )), } } + + #[cfg(feature = "full")] + /// Get the value defined cost for a serialized value + pub fn value_defined_cost(&self) -> Option { + let Some(value_cost) = self.get_specialized_cost().ok() else { + return None; + }; + + let cost = value_cost + + self.get_flags().as_ref().map_or(0, |flags| { + let flags_len = flags.len() as u32; + flags_len + flags_len.required_space() as u32 + }); + match self { + Element::Tree(..) => Some(LayeredValueDefinedCost(cost)), + Element::SumTree(..) => Some(LayeredValueDefinedCost(cost)), + Element::SumItem(..) => Some(SpecializedValueDefinedCost(cost)), + _ => None, + } + } + + #[cfg(feature = "full")] + /// Get the value defined cost for a serialized value + pub fn value_defined_cost_for_serialized_value(value: &[u8]) -> Option { + let element = Element::deserialize(value).ok()?; + element.value_defined_cost() + } } #[cfg(feature = "full")] /// Decode from bytes pub fn raw_decode(bytes: &[u8]) -> Result { - let tree = - TreeNode::decode_raw(bytes, vec![]).map_err(|e| Error::CorruptedData(e.to_string()))?; + let tree = TreeNode::decode_raw( + bytes, + vec![], + Some(Element::value_defined_cost_for_serialized_value), + ) + .map_err(|e| Error::CorruptedData(e.to_string()))?; let element: Element = Element::deserialize(tree.value_as_slice())?; Ok(element) } diff --git a/grovedb/src/element/insert.rs b/grovedb/src/element/insert.rs index e500b41af..2ba7f92db 100644 --- a/grovedb/src/element/insert.rs +++ b/grovedb/src/element/insert.rs @@ -92,6 +92,7 @@ impl Element { Self::specialized_costs_for_key_value(key, value, uses_sum_nodes) .map_err(|e| MerkError::ClientCorruptionError(e.to_string())) }, + Some(&Element::value_defined_cost_for_serialized_value), ) .map_err(|e| Error::CorruptedData(e.to_string())) } @@ -287,6 +288,7 @@ impl Element { Self::specialized_costs_for_key_value(key, value, uses_sum_nodes) .map_err(|e| MerkError::ClientCorruptionError(e.to_string())) }, + Some(&Element::value_defined_cost_for_serialized_value), ) .map_err(|e| Error::CorruptedData(e.to_string())) } @@ -356,6 +358,7 @@ impl Element { Self::specialized_costs_for_key_value(key, value, uses_sum_nodes) .map_err(|e| MerkError::ClientCorruptionError(e.to_string())) }, + Some(&Element::value_defined_cost_for_serialized_value), ) .map_err(|e| Error::CorruptedData(e.to_string())) } diff --git a/grovedb/src/estimated_costs/average_case_costs.rs b/grovedb/src/estimated_costs/average_case_costs.rs index e93e1b8fd..d93b6451b 100644 --- a/grovedb/src/estimated_costs/average_case_costs.rs +++ b/grovedb/src/estimated_costs/average_case_costs.rs @@ -461,7 +461,7 @@ mod test { use grovedb_costs::OperationCost; use grovedb_merk::{ estimated_costs::average_case_costs::add_average_case_get_merk_node, - test_utils::make_batch_seq, Merk, + test_utils::make_batch_seq, tree::kv::ValueDefinedCostType, Merk, }; use grovedb_storage::{ rocksdb_storage::RocksDbStorage, worst_case_costs::WorstKeyLength, Storage, StorageBatch, @@ -487,6 +487,7 @@ mod test { .get_storage_context(EMPTY_PATH, Some(&batch)) .unwrap(), false, + None::<&fn(&[u8]) -> Option>, ) .unwrap() .expect("cannot open merk"); @@ -505,6 +506,7 @@ mod test { let merk = Merk::open_base( storage.get_storage_context(EMPTY_PATH, None).unwrap(), false, + None::<&fn(&[u8]) -> Option>, ) .unwrap() .expect("cannot open merk"); @@ -514,7 +516,11 @@ mod test { // 2. Left link exists // 3. Right link exists // Based on merk's avl rotation algorithm node is key 8 satisfies this - let node_result = merk.get(&8_u64.to_be_bytes(), true); + let node_result = merk.get( + &8_u64.to_be_bytes(), + true, + None::<&fn(&[u8]) -> Option>, + ); // By tweaking the max element size, we can adapt the average case function to // this scenario. make_batch_seq creates values that are 60 bytes in size diff --git a/grovedb/src/estimated_costs/worst_case_costs.rs b/grovedb/src/estimated_costs/worst_case_costs.rs index d72b86e8d..5a72a4056 100644 --- a/grovedb/src/estimated_costs/worst_case_costs.rs +++ b/grovedb/src/estimated_costs/worst_case_costs.rs @@ -411,6 +411,7 @@ mod test { use grovedb_merk::{ estimated_costs::worst_case_costs::add_worst_case_get_merk_node, test_utils::{empty_path_merk, empty_path_merk_read_only, make_batch_seq}, + tree::kv::ValueDefinedCostType, }; use grovedb_storage::{ rocksdb_storage::{test_utils::TempStorage, RocksDbStorage}, @@ -451,7 +452,11 @@ mod test { // 2. Left link exists // 3. Right link exists // Based on merk's avl rotation algorithm node is key 8 satisfies this - let node_result = merk.get(&8_u64.to_be_bytes(), true); + let node_result = merk.get( + &8_u64.to_be_bytes(), + true, + None::<&fn(&[u8]) -> Option>, + ); // By tweaking the max element size, we can adapt the worst case function to // this scenario. make_batch_seq creates values that are 60 bytes in size diff --git a/grovedb/src/lib.rs b/grovedb/src/lib.rs index 87ca24436..30ca2e916 100644 --- a/grovedb/src/lib.rs +++ b/grovedb/src/lib.rs @@ -193,6 +193,7 @@ pub use grovedb_merk::estimated_costs::{ pub use grovedb_merk::proofs::query::query_item::QueryItem; #[cfg(any(feature = "full", feature = "verify"))] pub use grovedb_merk::proofs::Query; +use grovedb_merk::tree::kv::ValueDefinedCostType; #[cfg(feature = "full")] use grovedb_merk::{ self, @@ -289,11 +290,16 @@ impl GroveDb { ); let is_sum_tree = element.is_sum_tree(); if let Element::Tree(root_key, _) | Element::SumTree(root_key, ..) = element { - Merk::open_layered_with_root_key(storage, root_key, is_sum_tree) - .map_err(|_| { - Error::CorruptedData("cannot open a subtree with given root key".to_owned()) - }) - .add_cost(cost) + Merk::open_layered_with_root_key( + storage, + root_key, + is_sum_tree, + Some(&Element::value_defined_cost_for_serialized_value), + ) + .map_err(|_| { + Error::CorruptedData("cannot open a subtree with given root key".to_owned()) + }) + .add_cost(cost) } else { Err(Error::CorruptedPath( "cannot open a subtree as parent exists but is not a tree", @@ -301,9 +307,13 @@ impl GroveDb { .wrap_with_cost(cost) } } else { - Merk::open_base(storage, false) - .map_err(|_| Error::CorruptedData("cannot open a the root subtree".to_owned())) - .add_cost(cost) + Merk::open_base( + storage, + false, + Some(&Element::value_defined_cost_for_serialized_value), + ) + .map_err(|_| Error::CorruptedData("cannot open a the root subtree".to_owned())) + .add_cost(cost) } } @@ -340,20 +350,29 @@ impl GroveDb { .unwrap()?; let is_sum_tree = element.is_sum_tree(); if let Element::Tree(root_key, _) | Element::SumTree(root_key, ..) = element { - Merk::open_layered_with_root_key(storage, root_key, is_sum_tree) - .map_err(|_| { - Error::CorruptedData("cannot open a subtree with given root key".to_owned()) - }) - .unwrap() + Merk::open_layered_with_root_key( + storage, + root_key, + is_sum_tree, + Some(&Element::value_defined_cost_for_serialized_value), + ) + .map_err(|_| { + Error::CorruptedData("cannot open a subtree with given root key".to_owned()) + }) + .unwrap() } else { Err(Error::CorruptedPath( "cannot open a subtree as parent exists but is not a tree", )) } } else { - Merk::open_base(storage, false) - .map_err(|_| Error::CorruptedData("cannot open a the root subtree".to_owned())) - .unwrap() + Merk::open_base( + storage, + false, + None::<&fn(&[u8]) -> Option>, + ) + .map_err(|_| Error::CorruptedData("cannot open a the root subtree".to_owned())) + .unwrap() } } @@ -391,11 +410,16 @@ impl GroveDb { ); let is_sum_tree = element.is_sum_tree(); if let Element::Tree(root_key, _) | Element::SumTree(root_key, ..) = element { - Merk::open_layered_with_root_key(storage, root_key, is_sum_tree) - .map_err(|_| { - Error::CorruptedData("cannot open a subtree with given root key".to_owned()) - }) - .add_cost(cost) + Merk::open_layered_with_root_key( + storage, + root_key, + is_sum_tree, + Some(&Element::value_defined_cost_for_serialized_value), + ) + .map_err(|_| { + Error::CorruptedData("cannot open a subtree with given root key".to_owned()) + }) + .add_cost(cost) } else { Err(Error::CorruptedPath( "cannot open a subtree as parent exists but is not a tree", @@ -403,9 +427,13 @@ impl GroveDb { .wrap_with_cost(cost) } } else { - Merk::open_base(storage, false) - .map_err(|_| Error::CorruptedData("cannot open a the root subtree".to_owned())) - .add_cost(cost) + Merk::open_base( + storage, + false, + Some(&Element::value_defined_cost_for_serialized_value), + ) + .map_err(|_| Error::CorruptedData("cannot open a the root subtree".to_owned())) + .add_cost(cost) } } @@ -675,7 +703,11 @@ impl GroveDb { key: K, ) -> CostResult { subtree - .get(key.as_ref(), true) + .get( + key.as_ref(), + true, + Some(&Element::value_defined_cost_for_serialized_value), + ) .map_err(|_| { Error::InvalidPath("can't find subtree in parent during propagation".to_owned()) }) @@ -831,7 +863,11 @@ impl GroveDb { let element = raw_decode(&element_value).unwrap(); if element.is_tree() { let (kv_value, element_value_hash) = merk - .get_value_and_value_hash(&key, true) + .get_value_and_value_hash( + &key, + true, + None::<&fn(&[u8]) -> Option>, + ) .unwrap() .unwrap() .unwrap(); diff --git a/grovedb/src/operations/delete/mod.rs b/grovedb/src/operations/delete/mod.rs index fb2ce5ce7..590926979 100644 --- a/grovedb/src/operations/delete/mod.rs +++ b/grovedb/src/operations/delete/mod.rs @@ -495,7 +495,8 @@ impl GroveDb { Merk::open_layered_with_root_key( storage, subtree_to_delete_from.root_key(), - element.is_sum_tree() + element.is_sum_tree(), + Some(&Element::value_defined_cost_for_serialized_value), ) .map_err(|_| { Error::CorruptedData("cannot open a subtree with given root key".to_owned()) diff --git a/grovedb/src/operations/insert/mod.rs b/grovedb/src/operations/insert/mod.rs index 9b83ff87f..d691ea346 100644 --- a/grovedb/src/operations/insert/mod.rs +++ b/grovedb/src/operations/insert/mod.rs @@ -218,7 +218,11 @@ impl GroveDb { let maybe_element_bytes = cost_return_on_error!( &mut cost, subtree_to_insert_into - .get(key, true) + .get( + key, + true, + Some(&Element::value_defined_cost_for_serialized_value) + ) .map_err(|e| Error::CorruptedData(e.to_string())) ); if let Some(element_bytes) = maybe_element_bytes { @@ -353,7 +357,11 @@ impl GroveDb { let maybe_element_bytes = cost_return_on_error!( &mut cost, subtree_to_insert_into - .get(key, true) + .get( + key, + true, + Some(&Element::value_defined_cost_for_serialized_value) + ) .map_err(|e| Error::CorruptedData(e.to_string())) ); if let Some(element_bytes) = maybe_element_bytes { @@ -867,6 +875,11 @@ mod tests { // Child Heights 2 // Total 37 + 85 + 48 = 170 + + // replaced bytes + // 133 for key1 (higher node/same merk level) + // ? + assert_eq!( cost, OperationCost { diff --git a/grovedb/src/replication.rs b/grovedb/src/replication.rs index 86c1c3f04..898f5ff1f 100644 --- a/grovedb/src/replication.rs +++ b/grovedb/src/replication.rs @@ -181,6 +181,7 @@ impl<'db> Restorer<'db> { .get_immediate_storage_context(SubtreePath::empty(), tx) .unwrap(), false, + Some(&Element::value_defined_cost_for_serialized_value), ) .unwrap() .map_err(|e| RestorerError(e.to_string()))?, diff --git a/grovedb/src/tests/mod.rs b/grovedb/src/tests/mod.rs index e078237db..922f5e9b0 100644 --- a/grovedb/src/tests/mod.rs +++ b/grovedb/src/tests/mod.rs @@ -2365,9 +2365,13 @@ fn test_find_subtrees() { fn test_root_subtree_has_root_key() { let db = make_test_grovedb(); let storage = db.db.get_storage_context(EMPTY_PATH, None).unwrap(); - let root_merk = Merk::open_base(storage, false) - .unwrap() - .expect("expected to get root merk"); + let root_merk = Merk::open_base( + storage, + false, + Some(&Element::value_defined_cost_for_serialized_value), + ) + .unwrap() + .expect("expected to get root merk"); let (_, root_key, _) = root_merk .root_hash_key_and_sum() .unwrap() @@ -2457,10 +2461,14 @@ fn test_get_subtree() { .db .get_storage_context([TEST_LEAF, b"key1", b"key2"].as_ref().into(), None) .unwrap(); - let subtree = - Merk::open_layered_with_root_key(subtree_storage, Some(b"key3".to_vec()), false) - .unwrap() - .expect("cannot open merk"); + let subtree = Merk::open_layered_with_root_key( + subtree_storage, + Some(b"key3".to_vec()), + false, + Some(&Element::value_defined_cost_for_serialized_value), + ) + .unwrap() + .expect("cannot open merk"); let result_element = Element::get(&subtree, b"key3", true).unwrap().unwrap(); assert_eq!(result_element, Element::new_item(b"ayy".to_vec())); } @@ -2497,9 +2505,14 @@ fn test_get_subtree() { &transaction, ) .unwrap(); - let subtree = Merk::open_layered_with_root_key(subtree_storage, Some(b"key4".to_vec()), false) - .unwrap() - .expect("cannot open merk"); + let subtree = Merk::open_layered_with_root_key( + subtree_storage, + Some(b"key4".to_vec()), + false, + Some(&Element::value_defined_cost_for_serialized_value), + ) + .unwrap() + .expect("cannot open merk"); let result_element = Element::get(&subtree, b"key4", true).unwrap().unwrap(); assert_eq!(result_element, Element::new_item(b"ayy".to_vec())); @@ -2509,9 +2522,14 @@ fn test_get_subtree() { .db .get_storage_context([TEST_LEAF, b"key1", b"key2"].as_ref().into(), None) .unwrap(); - let subtree = Merk::open_layered_with_root_key(subtree_storage, Some(b"key3".to_vec()), false) - .unwrap() - .expect("cannot open merk"); + let subtree = Merk::open_layered_with_root_key( + subtree_storage, + Some(b"key3".to_vec()), + false, + Some(&Element::value_defined_cost_for_serialized_value), + ) + .unwrap() + .expect("cannot open merk"); let result_element = Element::get(&subtree, b"key3", true).unwrap().unwrap(); assert_eq!(result_element, Element::new_item(b"ayy".to_vec())); } diff --git a/grovedb/src/tests/sum_tree_tests.rs b/grovedb/src/tests/sum_tree_tests.rs index 3bc6896e8..6c4a75895 100644 --- a/grovedb/src/tests/sum_tree_tests.rs +++ b/grovedb/src/tests/sum_tree_tests.rs @@ -30,7 +30,8 @@ use grovedb_merk::{ proofs::Query, - TreeFeatureType::{BasicMerk, SummedMerk}, + tree::kv::ValueDefinedCostType, + TreeFeatureType::{BasicMerkNode, SummedMerkNode}, }; use grovedb_storage::StorageBatch; @@ -266,28 +267,44 @@ fn test_homogenous_node_type_in_sum_trees_and_regular_trees() { .unwrap() .expect("should open tree"); assert!(matches!( - merk.get_feature_type(b"item1", true) - .unwrap() - .expect("node should exist"), - Some(SummedMerk(30)) + merk.get_feature_type( + b"item1", + true, + None::<&fn(&[u8]) -> Option> + ) + .unwrap() + .expect("node should exist"), + Some(SummedMerkNode(30)) )); assert!(matches!( - merk.get_feature_type(b"item2", true) - .unwrap() - .expect("node should exist"), - Some(SummedMerk(10)) + merk.get_feature_type( + b"item2", + true, + None::<&fn(&[u8]) -> Option> + ) + .unwrap() + .expect("node should exist"), + Some(SummedMerkNode(10)) )); assert!(matches!( - merk.get_feature_type(b"item3", true) - .unwrap() - .expect("node should exist"), - Some(SummedMerk(0)) + merk.get_feature_type( + b"item3", + true, + None::<&fn(&[u8]) -> Option> + ) + .unwrap() + .expect("node should exist"), + Some(SummedMerkNode(0)) )); assert!(matches!( - merk.get_feature_type(b"item4", true) - .unwrap() - .expect("node should exist"), - Some(SummedMerk(0)) + merk.get_feature_type( + b"item4", + true, + None::<&fn(&[u8]) -> Option> + ) + .unwrap() + .expect("node should exist"), + Some(SummedMerkNode(0)) )); assert_eq!(merk.sum().expect("expected to get sum"), Some(40)); @@ -326,16 +343,24 @@ fn test_homogenous_node_type_in_sum_trees_and_regular_trees() { .unwrap() .expect("should open tree"); assert!(matches!( - merk.get_feature_type(b"item1", true) - .unwrap() - .expect("node should exist"), - Some(BasicMerk) + merk.get_feature_type( + b"item1", + true, + Some(&Element::value_defined_cost_for_serialized_value) + ) + .unwrap() + .expect("node should exist"), + Some(BasicMerkNode) )); assert!(matches!( - merk.get_feature_type(b"item2", true) - .unwrap() - .expect("node should exist"), - Some(BasicMerk) + merk.get_feature_type( + b"item2", + true, + Some(&Element::value_defined_cost_for_serialized_value) + ) + .unwrap() + .expect("node should exist"), + Some(BasicMerkNode) )); assert_eq!(merk.sum().expect("expected to get sum"), None); } @@ -582,10 +607,14 @@ fn test_sum_tree_propagation() { .expect("should open tree"); assert!(matches!( test_leaf_merk - .get_feature_type(b"key", true) + .get_feature_type( + b"key", + true, + Some(&Element::value_defined_cost_for_serialized_value) + ) .unwrap() .expect("node should exist"), - Some(BasicMerk) + Some(BasicMerkNode) )); let parent_sum_tree = db @@ -594,12 +623,16 @@ fn test_sum_tree_propagation() { .expect("should open tree"); assert!(matches!( parent_sum_tree - .get_feature_type(b"tree2", true) + .get_feature_type( + b"tree2", + true, + Some(&Element::value_defined_cost_for_serialized_value) + ) .unwrap() .expect("node should exist"), - Some(SummedMerk(15)) /* 15 because the child sum tree has one sum item of - * value 5 and - * another of value 10 */ + Some(SummedMerkNode(15)) /* 15 because the child sum tree has one sum item of + * value 5 and + * another of value 10 */ )); let child_sum_tree = db @@ -611,33 +644,49 @@ fn test_sum_tree_propagation() { .expect("should open tree"); assert!(matches!( child_sum_tree - .get_feature_type(b"item1", true) + .get_feature_type( + b"item1", + true, + None::<&fn(&[u8]) -> Option> + ) .unwrap() .expect("node should exist"), - Some(SummedMerk(0)) + Some(SummedMerkNode(0)) )); assert!(matches!( child_sum_tree - .get_feature_type(b"sumitem1", true) + .get_feature_type( + b"sumitem1", + true, + None::<&fn(&[u8]) -> Option> + ) .unwrap() .expect("node should exist"), - Some(SummedMerk(5)) + Some(SummedMerkNode(5)) )); assert!(matches!( child_sum_tree - .get_feature_type(b"sumitem2", true) + .get_feature_type( + b"sumitem2", + true, + None::<&fn(&[u8]) -> Option> + ) .unwrap() .expect("node should exist"), - Some(SummedMerk(10)) + Some(SummedMerkNode(10)) )); // TODO: should references take the sum of the referenced element?? assert!(matches!( child_sum_tree - .get_feature_type(b"item2", true) + .get_feature_type( + b"item2", + true, + None::<&fn(&[u8]) -> Option> + ) .unwrap() .expect("node should exist"), - Some(SummedMerk(0)) + Some(SummedMerkNode(0)) )); } @@ -673,17 +722,25 @@ fn test_sum_tree_with_batches() { assert!(matches!( sum_tree - .get_feature_type(b"a", true) + .get_feature_type( + b"a", + true, + Some(&Element::value_defined_cost_for_serialized_value) + ) .unwrap() .expect("node should exist"), - Some(SummedMerk(0)) + Some(SummedMerkNode(0)) )); assert!(matches!( sum_tree - .get_feature_type(b"b", true) + .get_feature_type( + b"b", + true, + Some(&Element::value_defined_cost_for_serialized_value) + ) .unwrap() .expect("node should exist"), - Some(SummedMerk(10)) + Some(SummedMerkNode(10)) )); // Create new batch to use existing tree @@ -703,10 +760,14 @@ fn test_sum_tree_with_batches() { .expect("should open tree"); assert!(matches!( sum_tree - .get_feature_type(b"c", true) + .get_feature_type( + b"c", + true, + None::<&fn(&[u8]) -> Option> + ) .unwrap() .expect("node should exist"), - Some(SummedMerk(10)) + Some(SummedMerkNode(10)) )); assert_eq!(sum_tree.sum().expect("expected to get sum"), Some(20)); diff --git a/grovedb/src/tests/tree_hashes_tests.rs b/grovedb/src/tests/tree_hashes_tests.rs index b67d1cf95..d24181322 100644 --- a/grovedb/src/tests/tree_hashes_tests.rs +++ b/grovedb/src/tests/tree_hashes_tests.rs @@ -28,7 +28,9 @@ //! Tree hashes tests -use grovedb_merk::tree::{combine_hash, kv_digest_to_kv_hash, node_hash, value_hash, NULL_HASH}; +use grovedb_merk::tree::{ + combine_hash, kv::ValueDefinedCostType, kv_digest_to_kv_hash, node_hash, value_hash, NULL_HASH, +}; use grovedb_storage::StorageBatch; use crate::{ @@ -58,19 +60,31 @@ fn test_node_hashes_when_inserting_item() { .expect("should open merk"); let (elem_value, elem_value_hash) = test_leaf_merk - .get_value_and_value_hash(b"key1", true) + .get_value_and_value_hash( + b"key1", + true, + None::<&fn(&[u8]) -> Option>, + ) .unwrap() .expect("should get value hash") .expect("value hash should be some"); let elem_kv_hash = test_leaf_merk - .get_kv_hash(b"key1", true) + .get_kv_hash( + b"key1", + true, + None::<&fn(&[u8]) -> Option>, + ) .unwrap() .expect("should get value hash") .expect("value hash should be some"); let elem_node_hash = test_leaf_merk - .get_hash(b"key1", true) + .get_hash( + b"key1", + true, + None::<&fn(&[u8]) -> Option>, + ) .unwrap() .expect("should get value hash") .expect("value hash should be some"); @@ -110,19 +124,31 @@ fn test_tree_hashes_when_inserting_empty_tree() { .expect("should open merk"); let (elem_value, elem_value_hash) = test_leaf_merk - .get_value_and_value_hash(b"key1", true) + .get_value_and_value_hash( + b"key1", + true, + None::<&fn(&[u8]) -> Option>, + ) .unwrap() .expect("should get value hash") .expect("value hash should be some"); let elem_kv_hash = test_leaf_merk - .get_kv_hash(b"key1", true) + .get_kv_hash( + b"key1", + true, + None::<&fn(&[u8]) -> Option>, + ) .unwrap() .expect("should get value hash") .expect("value hash should be some"); let elem_node_hash = test_leaf_merk - .get_hash(b"key1", true) + .get_hash( + b"key1", + true, + None::<&fn(&[u8]) -> Option>, + ) .unwrap() .expect("should get value hash") .expect("value hash should be some"); @@ -187,7 +213,11 @@ fn test_tree_hashes_when_inserting_empty_trees_twice_under_each_other() { // Let's first verify that the lowest nodes hashes are as we expect let (elem_value, elem_value_hash) = middle_merk_key1 - .get_value_and_value_hash(b"key2", true) + .get_value_and_value_hash( + b"key2", + true, + None::<&fn(&[u8]) -> Option>, + ) .unwrap() .expect("should get value hash") .expect("value hash should be some"); @@ -210,7 +240,11 @@ fn test_tree_hashes_when_inserting_empty_trees_twice_under_each_other() { assert_eq!(elem_value_hash, combined_value_hash_key2); let elem_kv_hash = middle_merk_key1 - .get_kv_hash(b"key2", true) + .get_kv_hash( + b"key2", + true, + None::<&fn(&[u8]) -> Option>, + ) .unwrap() .expect("should get kv hash") .expect("value hash should be some"); @@ -220,7 +254,11 @@ fn test_tree_hashes_when_inserting_empty_trees_twice_under_each_other() { assert_eq!(elem_kv_hash, kv_hash_key2); let elem_node_hash = middle_merk_key1 - .get_hash(b"key2", true) + .get_hash( + b"key2", + true, + None::<&fn(&[u8]) -> Option>, + ) .unwrap() .expect("should get kv hash") .expect("value hash should be some"); @@ -238,7 +276,11 @@ fn test_tree_hashes_when_inserting_empty_trees_twice_under_each_other() { assert_eq!(root_hash, node_hash_key2); let (middle_elem_value_key1, middle_elem_value_hash_key1) = under_top_merk - .get_value_and_value_hash(b"key1", true) + .get_value_and_value_hash( + b"key1", + true, + None::<&fn(&[u8]) -> Option>, + ) .unwrap() .expect("should get value hash") .expect("value hash should be some"); @@ -276,7 +318,11 @@ fn test_tree_hashes_when_inserting_empty_trees_twice_under_each_other() { ); let middle_elem_kv_hash_key1 = under_top_merk - .get_kv_hash(b"key1", true) + .get_kv_hash( + b"key1", + true, + None::<&fn(&[u8]) -> Option>, + ) .unwrap() .expect("should get value hash") .expect("value hash should be some"); @@ -290,7 +336,11 @@ fn test_tree_hashes_when_inserting_empty_trees_twice_under_each_other() { ); let middle_elem_node_hash_key1 = under_top_merk - .get_hash(b"key1", true) + .get_hash( + b"key1", + true, + None::<&fn(&[u8]) -> Option>, + ) .unwrap() .expect("should get value hash") .expect("value hash should be some"); diff --git a/grovedb/src/util.rs b/grovedb/src/util.rs index da478128f..cbedc9af3 100644 --- a/grovedb/src/util.rs +++ b/grovedb/src/util.rs @@ -198,8 +198,11 @@ macro_rules! merk_optional_tx { { let $subtree = cost_return_on_error!( &mut $cost, - ::grovedb_merk::Merk::open_base(storage.unwrap_add_cost(&mut $cost), false) - .map(|merk_res| + ::grovedb_merk::Merk::open_base( + storage.unwrap_add_cost(&mut $cost), + false, + Some(&Element::value_defined_cost_for_serialized_value) + ).map(|merk_res| merk_res .map_err(|_| crate::Error::CorruptedData( "cannot open a subtree".to_owned() @@ -226,7 +229,8 @@ macro_rules! merk_optional_tx { ::grovedb_merk::Merk::open_layered_with_root_key( storage, root_key, - is_sum_tree + is_sum_tree, + Some(&Element::value_defined_cost_for_serialized_value), ).map(|merk_res| merk_res .map_err(|_| crate::Error::CorruptedData( @@ -271,7 +275,8 @@ macro_rules! merk_optional_tx_path_not_empty { ::grovedb_merk::Merk::open_layered_with_root_key( storage, root_key, - is_sum_tree + is_sum_tree, + Some(&Element::value_defined_cost_for_serialized_value), ).map(|merk_res| merk_res .map_err(|_| crate::Error::CorruptedData( @@ -308,8 +313,11 @@ macro_rules! root_merk_optional_tx { { let $subtree = cost_return_on_error!( &mut $cost, - ::grovedb_merk::Merk::open_base(storage.unwrap_add_cost(&mut $cost), false) - .map(|merk_res| + ::grovedb_merk::Merk::open_base( + storage.unwrap_add_cost(&mut $cost), + false, + Some(&Element::value_defined_cost_for_serialized_value) + ).map(|merk_res| merk_res .map_err(|_| crate::Error::CorruptedData( "cannot open a subtree".to_owned() diff --git a/merk/src/merk/apply.rs b/merk/src/merk/apply.rs index 52b966a24..4418dfc11 100644 --- a/merk/src/merk/apply.rs +++ b/merk/src/merk/apply.rs @@ -31,15 +31,15 @@ where /// # Example /// ``` /// # let mut store = grovedb_merk::test_utils::TempMerk::new(); - /// # store.apply::<_, Vec<_>>(&[(vec![4,5,6], Op::Put(vec![0], BasicMerk))], &[], None) + /// # store.apply::<_, Vec<_>>(&[(vec![4,5,6], Op::Put(vec![0], BasicMerkNode))], &[], None) /// .unwrap().expect(""); /// /// use grovedb_merk::Op; - /// use grovedb_merk::TreeFeatureType::BasicMerk; + /// use grovedb_merk::TreeFeatureType::BasicMerkNode; /// /// let batch = &[ /// // puts value [4,5,6] to key[1,2,3] - /// (vec![1, 2, 3], Op::Put(vec![4, 5, 6], BasicMerk)), + /// (vec![1, 2, 3], Op::Put(vec![4, 5, 6], BasicMerkNode)), /// // deletes key [4,5,6] /// (vec![4, 5, 6], Op::Delete), /// ]; @@ -67,6 +67,7 @@ where use_sum_nodes, )) }, + None::<&fn(&[u8]) -> Option>, &mut |_costs, _old_value, _value| Ok((false, None)), &mut |_a, key_bytes_to_remove, value_bytes_to_remove| { Ok(( @@ -87,15 +88,15 @@ where /// # Example /// ``` /// # let mut store = grovedb_merk::test_utils::TempMerk::new(); - /// # store.apply::<_, Vec<_>>(&[(vec![4,5,6], Op::Put(vec![0], BasicMerk))], &[], None) + /// # store.apply::<_, Vec<_>>(&[(vec![4,5,6], Op::Put(vec![0], BasicMerkNode))], &[], None) /// .unwrap().expect(""); /// /// use grovedb_merk::Op; - /// use grovedb_merk::TreeFeatureType::BasicMerk; + /// use grovedb_merk::TreeFeatureType::BasicMerkNode; /// /// let batch = &[ /// // puts value [4,5,6] to key[1,2,3] - /// (vec![1, 2, 3], Op::Put(vec![4, 5, 6], BasicMerk)), + /// (vec![1, 2, 3], Op::Put(vec![4, 5, 6], BasicMerkNode)), /// // deletes key [4,5,6] /// (vec![4, 5, 6], Op::Delete), /// ]; @@ -107,6 +108,7 @@ where aux: &AuxMerkBatch, options: Option, old_specialized_cost: &impl Fn(&Vec, &Vec) -> Result, + value_defined_cost_fn: Option<&impl Fn(&[u8]) -> Option>, ) -> CostResult<(), Error> where KB: AsRef<[u8]>, @@ -117,6 +119,7 @@ where aux, options, old_specialized_cost, + value_defined_cost_fn, &mut |_costs, _old_value, _value| Ok((false, None)), &mut |_a, key_bytes_to_remove, value_bytes_to_remove| { Ok(( @@ -139,7 +142,7 @@ where /// ``` /// # let mut store = grovedb_merk::test_utils::TempMerk::new(); /// # store.apply_with_costs_just_in_time_value_update::<_, Vec<_>>( - /// &[(vec![4,5,6], Op::Put(vec![0], BasicMerk))], + /// &[(vec![4,5,6], Op::Put(vec![0], BasicMerkNode))], /// &[], /// None, /// &|k, v| Ok(0), @@ -149,11 +152,11 @@ where /// /// use grovedb_costs::storage_cost::removal::StorageRemovedBytes::NoStorageRemoval; /// use grovedb_merk::Op; - /// use grovedb_merk::TreeFeatureType::BasicMerk; + /// use grovedb_merk::TreeFeatureType::BasicMerkNode; /// /// let batch = &[ /// // puts value [4,5,6] to key[1,2,3] - /// (vec![1, 2, 3], Op::Put(vec![4, 5, 6], BasicMerk)), + /// (vec![1, 2, 3], Op::Put(vec![4, 5, 6], BasicMerkNode)), /// // deletes key [4,5,6] /// (vec![4, 5, 6], Op::Delete), /// ]; @@ -173,6 +176,7 @@ where aux: &AuxMerkBatch, options: Option, old_specialized_cost: &impl Fn(&Vec, &Vec) -> Result, + value_defined_cost_fn: Option<&impl Fn(&[u8]) -> Option>, update_tree_value_based_on_costs: &mut impl FnMut( &StorageCost, &Vec, @@ -218,6 +222,7 @@ where aux, options, old_specialized_cost, + value_defined_cost_fn, update_tree_value_based_on_costs, section_removal_bytes, ) @@ -235,7 +240,7 @@ where /// ``` /// # let mut store = grovedb_merk::test_utils::TempMerk::new(); /// # store.apply_with_costs_just_in_time_value_update::<_, Vec<_>>( - /// &[(vec![4,5,6], Op::Put(vec![0], BasicMerk))], + /// &[(vec![4,5,6], Op::Put(vec![0], BasicMerkNode))], /// &[], /// None, /// &|k, v| Ok(0), @@ -245,11 +250,11 @@ where /// /// use grovedb_costs::storage_cost::removal::StorageRemovedBytes::NoStorageRemoval; /// use grovedb_merk::Op; - /// use grovedb_merk::TreeFeatureType::BasicMerk; + /// use grovedb_merk::TreeFeatureType::BasicMerkNode; /// /// let batch = &[ /// // puts value [4,5,6] to key [1,2,3] - /// (vec![1, 2, 3], Op::Put(vec![4, 5, 6], BasicMerk)), + /// (vec![1, 2, 3], Op::Put(vec![4, 5, 6], BasicMerkNode)), /// // deletes key [4,5,6] /// (vec![4, 5, 6], Op::Delete), /// ]; @@ -258,17 +263,19 @@ where /// &[], /// None, /// &|k, v| Ok(0), + /// &|v| None, /// &mut |s, o, v| Ok((false, None)), /// &mut |s, k, v| Ok((NoStorageRemoval, NoStorageRemoval)) /// ).unwrap().expect(""); /// } /// ``` - pub fn apply_unchecked( + pub fn apply_unchecked( &mut self, batch: &MerkBatch, aux: &AuxMerkBatch, options: Option, old_specialized_cost: &C, + value_defined_cost_fn: Option<&V>, update_tree_value_based_on_costs: &mut U, section_removal_bytes: &mut R, ) -> CostResult<(), Error> @@ -276,6 +283,7 @@ where KB: AsRef<[u8]>, KA: AsRef<[u8]>, C: Fn(&Vec, &Vec) -> Result, + V: Fn(&[u8]) -> Option, U: FnMut( &StorageCost, &Vec, @@ -294,6 +302,7 @@ where batch, self.source(), old_specialized_cost, + value_defined_cost_fn, update_tree_value_based_on_costs, section_removal_bytes, ) diff --git a/merk/src/merk/chunks.rs b/merk/src/merk/chunks.rs index 7e8c588e8..4f6564efe 100644 --- a/merk/src/merk/chunks.rs +++ b/merk/src/merk/chunks.rs @@ -214,6 +214,7 @@ mod tests { use crate::{ proofs::chunk::{verify_leaf, verify_trunk}, test_utils::*, + tree::kv::ValueDefinedCostType, }; #[test] @@ -275,6 +276,7 @@ mod tests { .get_storage_context(SubtreePath::empty(), Some(&batch)) .unwrap(), false, + None::<&fn(&[u8]) -> Option>, ) .unwrap() .unwrap(); @@ -293,6 +295,7 @@ mod tests { .get_storage_context(SubtreePath::empty(), None) .unwrap(), false, + None::<&fn(&[u8]) -> Option>, ) .unwrap() .unwrap(); @@ -311,6 +314,7 @@ mod tests { .get_storage_context(SubtreePath::empty(), None) .unwrap(), false, + None::<&fn(&[u8]) -> Option>, ) .unwrap() .unwrap(); diff --git a/merk/src/merk/get.rs b/merk/src/merk/get.rs index 573b895a4..f0c25b424 100644 --- a/merk/src/merk/get.rs +++ b/merk/src/merk/get.rs @@ -1,7 +1,12 @@ use grovedb_costs::{CostContext, CostResult, CostsExt, OperationCost}; use grovedb_storage::StorageContext; -use crate::{tree::TreeNode, CryptoHash, Error, Error::StorageError, Merk, TreeFeatureType}; +use crate::{ + tree::{kv::ValueDefinedCostType, TreeNode}, + CryptoHash, Error, + Error::StorageError, + Merk, TreeFeatureType, +}; impl<'db, S> Merk where @@ -16,8 +21,12 @@ where /// /// Note that this is essentially the same as a normal RocksDB `get`, so /// should be a fast operation and has almost no tree overhead. - pub fn exists(&self, key: &[u8]) -> CostResult { - self.has_node_direct(key) + pub fn exists( + &self, + key: &[u8], + value_defined_cost_fn: Option Option>, + ) -> CostResult { + self.has_node_direct(key, value_defined_cost_fn) } /// Returns if the value at the given key exists @@ -26,8 +35,12 @@ where /// should be a fast operation and has almost no tree overhead. /// Contrary to a simple exists, this traverses the tree and can be faster /// if the tree is cached, but slower if it is not - pub fn exists_by_traversing_tree(&self, key: &[u8]) -> CostResult { - self.has_node(key) + pub fn exists_by_traversing_tree( + &self, + key: &[u8], + value_defined_cost_fn: Option Option>, + ) -> CostResult { + self.has_node(key, value_defined_cost_fn) } /// Gets a value for the given key. If the key is not found, `None` is @@ -35,19 +48,32 @@ where /// /// Note that this is essentially the same as a normal RocksDB `get`, so /// should be a fast operation and has almost no tree overhead. - pub fn get(&self, key: &[u8], allow_cache: bool) -> CostResult>, Error> { + pub fn get( + &self, + key: &[u8], + allow_cache: bool, + value_defined_cost_fn: Option Option>, + ) -> CostResult>, Error> { if allow_cache { - self.get_node_fn(key, |node| { - node.value_as_slice() - .to_vec() - .wrap_with_cost(Default::default()) - }) + self.get_node_fn( + key, + |node| { + node.value_as_slice() + .to_vec() + .wrap_with_cost(Default::default()) + }, + value_defined_cost_fn, + ) } else { - self.get_node_direct_fn(key, |node| { - node.value_as_slice() - .to_vec() - .wrap_with_cost(Default::default()) - }) + self.get_node_direct_fn( + key, + |node| { + node.value_as_slice() + .to_vec() + .wrap_with_cost(Default::default()) + }, + value_defined_cost_fn, + ) } } @@ -56,25 +82,35 @@ where &self, key: &[u8], allow_cache: bool, + value_defined_cost_fn: Option Option>, ) -> CostResult, Error> { if allow_cache { - self.get_node_fn(key, |node| { - node.feature_type().wrap_with_cost(Default::default()) - }) + self.get_node_fn( + key, + |node| node.feature_type().wrap_with_cost(Default::default()), + value_defined_cost_fn, + ) } else { - self.get_node_direct_fn(key, |node| { - node.feature_type().wrap_with_cost(Default::default()) - }) + self.get_node_direct_fn( + key, + |node| node.feature_type().wrap_with_cost(Default::default()), + value_defined_cost_fn, + ) } } /// Gets a hash of a node by a given key, `None` is returned in case /// when node not found by the key. - pub fn get_hash(&self, key: &[u8], allow_cache: bool) -> CostResult, Error> { + pub fn get_hash( + &self, + key: &[u8], + allow_cache: bool, + value_defined_cost_fn: Option Option>, + ) -> CostResult, Error> { if allow_cache { - self.get_node_fn(key, |node| node.hash()) + self.get_node_fn(key, |node| node.hash(), value_defined_cost_fn) } else { - self.get_node_direct_fn(key, |node| node.hash()) + self.get_node_direct_fn(key, |node| node.hash(), value_defined_cost_fn) } } @@ -84,15 +120,20 @@ where &self, key: &[u8], allow_cache: bool, + value_defined_cost_fn: Option Option>, ) -> CostResult, Error> { if allow_cache { - self.get_node_fn(key, |node| { - (*node.value_hash()).wrap_with_cost(OperationCost::default()) - }) + self.get_node_fn( + key, + |node| (*node.value_hash()).wrap_with_cost(OperationCost::default()), + value_defined_cost_fn, + ) } else { - self.get_node_direct_fn(key, |node| { - (*node.value_hash()).wrap_with_cost(OperationCost::default()) - }) + self.get_node_direct_fn( + key, + |node| (*node.value_hash()).wrap_with_cost(OperationCost::default()), + value_defined_cost_fn, + ) } } @@ -102,15 +143,20 @@ where &self, key: &[u8], allow_cache: bool, + value_defined_cost_fn: Option Option>, ) -> CostResult, Error> { if allow_cache { - self.get_node_fn(key, |node| { - (*node.inner.kv.hash()).wrap_with_cost(OperationCost::default()) - }) + self.get_node_fn( + key, + |node| (*node.inner.kv.hash()).wrap_with_cost(OperationCost::default()), + value_defined_cost_fn, + ) } else { - self.get_node_direct_fn(key, |node| { - (*node.inner.kv.hash()).wrap_with_cost(OperationCost::default()) - }) + self.get_node_direct_fn( + key, + |node| (*node.inner.kv.hash()).wrap_with_cost(OperationCost::default()), + value_defined_cost_fn, + ) } } @@ -120,27 +166,44 @@ where &self, key: &[u8], allow_cache: bool, + value_defined_cost_fn: Option Option>, ) -> CostResult, CryptoHash)>, Error> { if allow_cache { - self.get_node_fn(key, |node| { - (node.value_as_slice().to_vec(), *node.value_hash()) - .wrap_with_cost(OperationCost::default()) - }) + self.get_node_fn( + key, + |node| { + (node.value_as_slice().to_vec(), *node.value_hash()) + .wrap_with_cost(OperationCost::default()) + }, + value_defined_cost_fn, + ) } else { - self.get_node_direct_fn(key, |node| { - (node.value_as_slice().to_vec(), *node.value_hash()) - .wrap_with_cost(OperationCost::default()) - }) + self.get_node_direct_fn( + key, + |node| { + (node.value_as_slice().to_vec(), *node.value_hash()) + .wrap_with_cost(OperationCost::default()) + }, + value_defined_cost_fn, + ) } } /// See if a node's field exists - fn has_node_direct(&self, key: &[u8]) -> CostResult { - TreeNode::get(&self.storage, key).map_ok(|x| x.is_some()) + fn has_node_direct( + &self, + key: &[u8], + value_defined_cost_fn: Option Option>, + ) -> CostResult { + TreeNode::get(&self.storage, key, value_defined_cost_fn).map_ok(|x| x.is_some()) } /// See if a node's field exists - fn has_node(&self, key: &[u8]) -> CostResult { + fn has_node( + &self, + key: &[u8], + value_defined_cost_fn: Option Option>, + ) -> CostResult { self.use_tree(move |maybe_tree| { let mut cursor = match maybe_tree { None => return Ok(false).wrap_with_cost(Default::default()), // empty tree @@ -162,7 +225,7 @@ where match maybe_child { None => { // fetch from RocksDB - break self.has_node_direct(key); + break self.has_node_direct(key, value_defined_cost_fn); } Some(child) => cursor = child, // traverse to child } @@ -171,18 +234,28 @@ where } /// Generic way to get a node's field - fn get_node_direct_fn(&self, key: &[u8], f: F) -> CostResult, Error> + fn get_node_direct_fn( + &self, + key: &[u8], + f: F, + value_defined_cost_fn: Option Option>, + ) -> CostResult, Error> where F: FnOnce(&TreeNode) -> CostContext, { - TreeNode::get(&self.storage, key).flat_map_ok(|maybe_node| { + TreeNode::get(&self.storage, key, value_defined_cost_fn).flat_map_ok(|maybe_node| { let mut cost = OperationCost::default(); Ok(maybe_node.map(|node| f(&node).unwrap_add_cost(&mut cost))).wrap_with_cost(cost) }) } /// Generic way to get a node's field - fn get_node_fn(&self, key: &[u8], f: F) -> CostResult, Error> + fn get_node_fn( + &self, + key: &[u8], + f: F, + value_defined_cost_fn: Option Option>, + ) -> CostResult, Error> where F: FnOnce(&TreeNode) -> CostContext, { @@ -207,7 +280,7 @@ where match maybe_child { None => { // fetch from RocksDB - break self.get_node_direct_fn(key, f); + break self.get_node_direct_fn(key, f, value_defined_cost_fn); } Some(child) => cursor = child, // traverse to child } @@ -218,7 +291,9 @@ where #[cfg(test)] mod test { - use crate::{test_utils::TempMerk, Op, TreeFeatureType::BasicMerk}; + use crate::{ + test_utils::TempMerk, tree::kv::ValueDefinedCostType, Op, TreeFeatureType::BasicMerkNode, + }; #[test] fn test_has_node_with_empty_tree() { @@ -226,11 +301,14 @@ mod test { let key = b"something"; - let result = merk.has_node(key).unwrap().unwrap(); + let result = merk + .has_node(key, None::<&fn(&[u8]) -> Option>) + .unwrap() + .unwrap(); assert!(!result); - let batch_entry = (key, Op::Put(vec![123; 60], BasicMerk)); + let batch_entry = (key, Op::Put(vec![123; 60], BasicMerkNode)); let batch = vec![batch_entry]; @@ -238,7 +316,10 @@ mod test { .unwrap() .expect("should ..."); - let result = merk.has_node(key).unwrap().unwrap(); + let result = merk + .has_node(key, None::<&fn(&[u8]) -> Option>) + .unwrap() + .unwrap(); assert!(result); } diff --git a/merk/src/merk/mod.rs b/merk/src/merk/mod.rs index 89982ef0d..9c025dcb9 100644 --- a/merk/src/merk/mod.rs +++ b/merk/src/merk/mod.rs @@ -494,7 +494,10 @@ where /// Loads the Merk from the base root key /// The base root key should only be used if the Merk tree is independent /// Meaning that it doesn't have a parent Merk - pub(crate) fn load_base_root(&mut self) -> CostResult<(), Error> { + pub(crate) fn load_base_root( + &mut self, + value_defined_cost_fn: Option Option>, + ) -> CostResult<(), Error> { self.storage .get_root(ROOT_KEY_KEY) .map(|root_result| root_result.map_err(Error::StorageError)) @@ -503,12 +506,14 @@ where if let Some(tree_root_key) = tree_root_key_opt { // Trying to build a tree out of it, costs will be accumulated because // `Tree::get` returns `CostContext` and this call happens inside `flat_map_ok`. - TreeNode::get(&self.storage, tree_root_key).map_ok(|tree| { - if let Some(t) = tree.as_ref() { - self.root_tree_key = Cell::new(Some(t.key().to_vec())); - } - self.tree = Cell::new(tree); - }) + TreeNode::get(&self.storage, tree_root_key, value_defined_cost_fn).map_ok( + |tree| { + if let Some(t) = tree.as_ref() { + self.root_tree_key = Cell::new(Some(t.key().to_vec())); + } + self.tree = Cell::new(tree); + }, + ) } else { Ok(()).wrap_with_cost(Default::default()) } @@ -518,12 +523,15 @@ where /// Loads the Merk from it's parent root key /// The base root key should only be used if the Merk tree is independent /// Meaning that it doesn't have a parent Merk - pub(crate) fn load_root(&mut self) -> CostResult<(), Error> { + pub(crate) fn load_root( + &mut self, + value_defined_cost_fn: Option Option>, + ) -> CostResult<(), Error> { // In case of successful seek for root key check if it exists if let Some(tree_root_key) = self.root_tree_key.get_mut() { // Trying to build a tree out of it, costs will be accumulated because // `Tree::get` returns `CostContext` and this call happens inside `flat_map_ok`. - TreeNode::get(&self.storage, tree_root_key).map_ok(|tree| { + TreeNode::get(&self.storage, tree_root_key, value_defined_cost_fn).map_ok(|tree| { self.tree = Cell::new(tree); }) } else { @@ -533,11 +541,15 @@ where } } -fn fetch_node<'db>(db: &impl StorageContext<'db>, key: &[u8]) -> Result, Error> { +fn fetch_node<'db>( + db: &impl StorageContext<'db>, + key: &[u8], + value_defined_cost_fn: Option Option>, +) -> Result, Error> { let bytes = db.get(key).unwrap().map_err(StorageError)?; // TODO: get_pinned ? if let Some(bytes) = bytes { Ok(Some( - TreeNode::decode(key.to_vec(), &bytes).map_err(EdError)?, + TreeNode::decode(key.to_vec(), &bytes, value_defined_cost_fn).map_err(EdError)?, )) } else { Ok(None) @@ -556,7 +568,10 @@ mod test { use tempfile::TempDir; use super::{Merk, RefWalker}; - use crate::{merk::source::MerkSource, test_utils::*, Op, TreeFeatureType::BasicMerk}; + use crate::{ + merk::source::MerkSource, test_utils::*, tree::kv::ValueDefinedCostType, Op, + TreeFeatureType::BasicMerkNode, + }; // TODO: Close and then reopen test @@ -656,7 +671,7 @@ mod test { let mut merk = TempMerk::new(); merk.apply::, _>( &[], - &[(vec![1, 2, 3], Op::Put(vec![4, 5, 6], BasicMerk), None)], + &[(vec![1, 2, 3], Op::Put(vec![4, 5, 6], BasicMerkNode), None)], None, ) .unwrap() @@ -672,27 +687,55 @@ mod test { let mut merk = TempMerk::new(); // no root - assert!(merk.get(&[1, 2, 3], true).unwrap().unwrap().is_none()); + assert!(merk + .get( + &[1, 2, 3], + true, + None::<&fn(&[u8]) -> Option> + ) + .unwrap() + .unwrap() + .is_none()); // cached - merk.apply::<_, Vec<_>>(&[(vec![5, 5, 5], Op::Put(vec![], BasicMerk))], &[], None) + merk.apply::<_, Vec<_>>( + &[(vec![5, 5, 5], Op::Put(vec![], BasicMerkNode))], + &[], + None, + ) + .unwrap() + .unwrap(); + assert!(merk + .get( + &[1, 2, 3], + true, + None::<&fn(&[u8]) -> Option> + ) .unwrap() - .unwrap(); - assert!(merk.get(&[1, 2, 3], true).unwrap().unwrap().is_none()); + .unwrap() + .is_none()); // uncached merk.apply::<_, Vec<_>>( &[ - (vec![0, 0, 0], Op::Put(vec![], BasicMerk)), - (vec![1, 1, 1], Op::Put(vec![], BasicMerk)), - (vec![2, 2, 2], Op::Put(vec![], BasicMerk)), + (vec![0, 0, 0], Op::Put(vec![], BasicMerkNode)), + (vec![1, 1, 1], Op::Put(vec![], BasicMerkNode)), + (vec![2, 2, 2], Op::Put(vec![], BasicMerkNode)), ], &[], None, ) .unwrap() .unwrap(); - assert!(merk.get(&[3, 3, 3], true).unwrap().unwrap().is_none()); + assert!(merk + .get( + &[3, 3, 3], + true, + None::<&fn(&[u8]) -> Option> + ) + .unwrap() + .unwrap() + .is_none()); } // TODO: what this test should do? @@ -706,6 +749,7 @@ mod test { .get_storage_context(SubtreePath::empty(), None) .unwrap(), false, + None::<&fn(&[u8]) -> Option>, ) .unwrap() .expect("cannot open merk"); @@ -729,6 +773,7 @@ mod test { .get_storage_context(SubtreePath::empty(), None) .unwrap(), false, + None::<&fn(&[u8]) -> Option>, ) .unwrap() .expect("cannot open merk"); @@ -746,10 +791,18 @@ mod test { nodes: &mut Vec>, ) { nodes.push(node.tree().encode()); - if let Some(c) = node.walk(true).unwrap().unwrap() { + if let Some(c) = node + .walk(true, None::<&fn(&[u8]) -> Option>) + .unwrap() + .unwrap() + { collect(c, nodes); } - if let Some(c) = node.walk(false).unwrap().unwrap() { + if let Some(c) = node + .walk(false, None::<&fn(&[u8]) -> Option>) + .unwrap() + .unwrap() + { collect(c, nodes); } } @@ -765,6 +818,7 @@ mod test { .get_storage_context(SubtreePath::empty(), Some(&batch)) .unwrap(), false, + None::<&fn(&[u8]) -> Option>, ) .unwrap() .expect("cannot open merk"); @@ -782,6 +836,7 @@ mod test { .get_storage_context(SubtreePath::empty(), None) .unwrap(), false, + None::<&fn(&[u8]) -> Option>, ) .unwrap() .expect("cannot open merk"); @@ -801,6 +856,7 @@ mod test { .get_storage_context(SubtreePath::empty(), None) .unwrap(), false, + None::<&fn(&[u8]) -> Option>, ) .unwrap() .expect("cannot open merk"); @@ -841,6 +897,7 @@ mod test { .get_storage_context(SubtreePath::empty(), Some(&batch)) .unwrap(), false, + None::<&fn(&[u8]) -> Option>, ) .unwrap() .expect("cannot open merk"); @@ -860,6 +917,7 @@ mod test { .get_storage_context(SubtreePath::empty(), None) .unwrap(), false, + None::<&fn(&[u8]) -> Option>, ) .unwrap() .expect("cannot open merk"); @@ -873,6 +931,7 @@ mod test { .get_storage_context(SubtreePath::empty(), None) .unwrap(), false, + None::<&fn(&[u8]) -> Option>, ) .unwrap() .expect("cannot open merk"); @@ -894,19 +953,20 @@ mod test { .get_storage_context(SubtreePath::empty(), Some(&batch)) .unwrap(), false, + None::<&fn(&[u8]) -> Option>, ) .unwrap() .expect("cannot open merk"); merk.apply::<_, Vec<_>>( - &[(b"9".to_vec(), Op::Put(b"a".to_vec(), BasicMerk))], + &[(b"9".to_vec(), Op::Put(b"a".to_vec(), BasicMerkNode))], &[], None, ) .unwrap() .expect("should insert successfully"); merk.apply::<_, Vec<_>>( - &[(b"10".to_vec(), Op::Put(b"a".to_vec(), BasicMerk))], + &[(b"10".to_vec(), Op::Put(b"a".to_vec(), BasicMerkNode))], &[], None, ) @@ -914,21 +974,29 @@ mod test { .expect("should insert successfully"); let result = merk - .get(b"10".as_slice(), true) + .get( + b"10".as_slice(), + true, + None::<&fn(&[u8]) -> Option>, + ) .unwrap() .expect("should get successfully"); assert_eq!(result, Some(b"a".to_vec())); // Update the node merk.apply::<_, Vec<_>>( - &[(b"10".to_vec(), Op::Put(b"b".to_vec(), BasicMerk))], + &[(b"10".to_vec(), Op::Put(b"b".to_vec(), BasicMerkNode))], &[], None, ) .unwrap() .expect("should insert successfully"); let result = merk - .get(b"10".as_slice(), true) + .get( + b"10".as_slice(), + true, + None::<&fn(&[u8]) -> Option>, + ) .unwrap() .expect("should get successfully"); assert_eq!(result, Some(b"b".to_vec())); @@ -943,20 +1011,25 @@ mod test { .get_storage_context(SubtreePath::empty(), None) .unwrap(), false, + None::<&fn(&[u8]) -> Option>, ) .unwrap() .expect("cannot open merk"); // Update the node after dropping merk merk.apply::<_, Vec<_>>( - &[(b"10".to_vec(), Op::Put(b"c".to_vec(), BasicMerk))], + &[(b"10".to_vec(), Op::Put(b"c".to_vec(), BasicMerkNode))], &[], None, ) .unwrap() .expect("should insert successfully"); let result = merk - .get(b"10".as_slice(), true) + .get( + b"10".as_slice(), + true, + None::<&fn(&[u8]) -> Option>, + ) .unwrap() .expect("should get successfully"); assert_eq!(result, Some(b"c".to_vec())); diff --git a/merk/src/merk/open.rs b/merk/src/merk/open.rs index 115926abd..af15d5969 100644 --- a/merk/src/merk/open.rs +++ b/merk/src/merk/open.rs @@ -4,6 +4,7 @@ use grovedb_costs::CostResult; use grovedb_storage::StorageContext; use crate::{ + tree::kv::ValueDefinedCostType, Error, Merk, MerkType, MerkType::{BaseMerk, LayeredMerk, StandaloneMerk}, }; @@ -24,7 +25,11 @@ where } /// Open standalone tree - pub fn open_standalone(storage: S, is_sum_tree: bool) -> CostResult { + pub fn open_standalone( + storage: S, + is_sum_tree: bool, + value_defined_cost_fn: Option Option>, + ) -> CostResult { let mut merk = Self { tree: Cell::new(None), root_tree_key: Cell::new(None), @@ -33,11 +38,15 @@ where is_sum_tree, }; - merk.load_base_root().map_ok(|_| merk) + merk.load_base_root(value_defined_cost_fn).map_ok(|_| merk) } /// Open base tree - pub fn open_base(storage: S, is_sum_tree: bool) -> CostResult { + pub fn open_base( + storage: S, + is_sum_tree: bool, + value_defined_cost_fn: Option Option>, + ) -> CostResult { let mut merk = Self { tree: Cell::new(None), root_tree_key: Cell::new(None), @@ -46,7 +55,7 @@ where is_sum_tree, }; - merk.load_base_root().map_ok(|_| merk) + merk.load_base_root(value_defined_cost_fn).map_ok(|_| merk) } /// Open layered tree with root key @@ -54,6 +63,7 @@ where storage: S, root_key: Option>, is_sum_tree: bool, + value_defined_cost_fn: Option Option>, ) -> CostResult { let mut merk = Self { tree: Cell::new(None), @@ -63,7 +73,7 @@ where is_sum_tree, }; - merk.load_root().map_ok(|_| merk) + merk.load_root(value_defined_cost_fn).map_ok(|_| merk) } } @@ -77,7 +87,7 @@ mod test { }; use tempfile::TempDir; - use crate::{Merk, Op, TreeFeatureType::BasicMerk}; + use crate::{tree::kv::ValueDefinedCostType, Merk, Op, TreeFeatureType::BasicMerkNode}; #[test] fn test_reopen_root_hash() { @@ -92,12 +102,13 @@ mod test { .get_storage_context(SubtreePath::from(test_prefix.as_ref()), Some(&batch)) .unwrap(), false, + None::<&fn(&[u8]) -> Option>, ) .unwrap() .unwrap(); merk.apply::<_, Vec<_>>( - &[(vec![1, 2, 3], Op::Put(vec![4, 5, 6], BasicMerk))], + &[(vec![1, 2, 3], Op::Put(vec![4, 5, 6], BasicMerkNode))], &[], None, ) @@ -116,6 +127,7 @@ mod test { .get_storage_context(SubtreePath::from(test_prefix.as_ref()), None) .unwrap(), false, + None::<&fn(&[u8]) -> Option>, ) .unwrap() .unwrap(); @@ -132,6 +144,7 @@ mod test { .get_storage_context(SubtreePath::empty(), Some(&batch)) .unwrap(), false, + None::<&fn(&[u8]) -> Option>, ); // Opening not existing merk should cost only root key seek (except context // creation) @@ -142,7 +155,7 @@ mod test { let mut merk = merk_fee_context.unwrap().unwrap(); merk.apply::<_, Vec<_>>( - &[(vec![1, 2, 3], Op::Put(vec![4, 5, 6], BasicMerk))], + &[(vec![1, 2, 3], Op::Put(vec![4, 5, 6], BasicMerkNode))], &[], None, ) @@ -159,6 +172,7 @@ mod test { .get_storage_context(SubtreePath::empty(), None) .unwrap(), false, + None::<&fn(&[u8]) -> Option>, ); // Opening existing merk should cost two seeks. (except context creation) diff --git a/merk/src/merk/restore.rs b/merk/src/merk/restore.rs index eb4cea611..e6ac22e22 100644 --- a/merk/src/merk/restore.rs +++ b/merk/src/merk/restore.rs @@ -39,6 +39,7 @@ use grovedb_storage::{Batch, StorageContext}; use super::Merk; #[cfg(feature = "full")] use crate::merk::source::MerkSource; +use crate::tree::kv::ValueDefinedCostType; #[cfg(feature = "full")] use crate::{ error::Error, @@ -50,7 +51,7 @@ use crate::{ tree::{combine_hash, value_hash, Link, RefWalker, TreeNode}, CryptoHash, Error::{CostsError, EdError, StorageError}, - TreeFeatureType::BasicMerk, + TreeFeatureType::BasicMerkNode, }; #[cfg(feature = "full")] @@ -116,7 +117,9 @@ impl<'db, S: StorageContext<'db>> Restorer { self.rewrite_trunk_child_heights()?; } - self.merk.load_base_root().unwrap()?; + self.merk + .load_base_root(None:: Option>) + .unwrap()?; Ok(self.merk) } @@ -136,7 +139,7 @@ impl<'db, S: StorageContext<'db>> Restorer { tree.visit_refs(&mut |proof_node| { if let Some((mut node, key)) = match &proof_node.node { Node::KV(key, value) => Some(( - TreeNode::new(key.clone(), value.clone(), None, BasicMerk).unwrap(), + TreeNode::new(key.clone(), value.clone(), None, BasicMerkNode).unwrap(), key, )), Node::KVValueHash(key, value, value_hash) => Some(( @@ -144,7 +147,7 @@ impl<'db, S: StorageContext<'db>> Restorer { key.clone(), value.clone(), *value_hash, - BasicMerk, + BasicMerkNode, ) .unwrap(), key, @@ -270,8 +273,12 @@ impl<'db, S: StorageContext<'db>> Restorer { fn rewrite_parent_link(&mut self, leaf: &ProofTree) -> Result<(), Error> { let parent_keys = self.parent_keys.as_mut().unwrap(); let parent_key = parent_keys.peek().unwrap().clone(); - let mut parent = crate::merk::fetch_node(&self.merk.storage, parent_key.as_slice())? - .expect("Could not find parent of leaf chunk"); + let mut parent = crate::merk::fetch_node( + &self.merk.storage, + parent_key.as_slice(), + None:: Option>, + )? + .expect("Could not find parent of leaf chunk"); let is_left_child = self.remaining_chunks_unchecked() % 2 == 0; if let Some(Link::Reference { ref mut key, .. }) = parent.link_mut(is_left_child) { @@ -305,16 +312,25 @@ impl<'db, S: StorageContext<'db>> Restorer { return Ok(node.tree().child_heights()); } - let mut cloned_node = - TreeNode::decode(node.tree().key().to_vec(), node.tree().encode().as_slice()) - .map_err(EdError)?; + let mut cloned_node = TreeNode::decode( + node.tree().key().to_vec(), + node.tree().encode().as_slice(), + None:: Option>, + ) + .map_err(EdError)?; - let left_child = node.walk(true).unwrap()?.unwrap(); + let left_child = node + .walk(true, None::<&fn(&[u8]) -> Option>) + .unwrap()? + .unwrap(); let left_child_heights = recurse(left_child, remaining_depth - 1, batch)?; let left_height = left_child_heights.0.max(left_child_heights.1) + 1; *cloned_node.link_mut(true).unwrap().child_heights_mut() = left_child_heights; - let right_child = node.walk(false).unwrap()?.unwrap(); + let right_child = node + .walk(false, None::<&fn(&[u8]) -> Option>) + .unwrap()? + .unwrap(); let right_child_heights = recurse(right_child, remaining_depth - 1, batch)?; let right_height = right_child_heights.0.max(right_child_heights.1) + 1; *cloned_node.link_mut(false).unwrap().child_heights_mut() = right_child_heights; @@ -327,7 +343,9 @@ impl<'db, S: StorageContext<'db>> Restorer { Ok((left_height, right_height)) } - self.merk.load_base_root().unwrap()?; + self.merk + .load_base_root(None:: Option>) + .unwrap()?; let mut batch = self.merk.storage.new_batch(); @@ -415,6 +433,7 @@ mod tests { .get_immediate_storage_context(SubtreePath::empty(), &tx) .unwrap(), false, + None::<&fn(&[u8]) -> Option>, ) .unwrap() .unwrap(); @@ -432,7 +451,13 @@ mod tests { let ctx = storage .get_immediate_storage_context(SubtreePath::empty(), &tx) .unwrap(); - let merk = Merk::open_base(ctx, false).unwrap().unwrap(); + let merk = Merk::open_base( + ctx, + false, + None::<&fn(&[u8]) -> Option>, + ) + .unwrap() + .unwrap(); let mut restorer = Merk::restore(merk, original.root_hash().unwrap()); assert_eq!(restorer.remaining_chunks(), None); @@ -466,8 +491,8 @@ mod tests { fn restore_2_left_heavy() { restore_test( &[ - &[(vec![0], Op::Put(vec![], BasicMerk))], - &[(vec![1], Op::Put(vec![], BasicMerk))], + &[(vec![0], Op::Put(vec![], BasicMerkNode))], + &[(vec![1], Op::Put(vec![], BasicMerkNode))], ], 2, ); @@ -477,8 +502,8 @@ mod tests { fn restore_2_right_heavy() { restore_test( &[ - &[(vec![1], Op::Put(vec![], BasicMerk))], - &[(vec![0], Op::Put(vec![], BasicMerk))], + &[(vec![1], Op::Put(vec![], BasicMerkNode))], + &[(vec![0], Op::Put(vec![], BasicMerkNode))], ], 2, ); diff --git a/merk/src/merk/source.rs b/merk/src/merk/source.rs index 709063a69..46782bdc8 100644 --- a/merk/src/merk/source.rs +++ b/merk/src/merk/source.rs @@ -2,7 +2,7 @@ use grovedb_costs::CostResult; use grovedb_storage::StorageContext; use crate::{ - tree::{Fetch, TreeNode}, + tree::{kv::ValueDefinedCostType, Fetch, TreeNode}, Error, Link, Merk, }; @@ -37,8 +37,12 @@ impl<'s, 'db, S> Fetch for MerkSource<'s, S> where S: StorageContext<'db>, { - fn fetch(&self, link: &Link) -> CostResult { - TreeNode::get(self.storage, link.key()) + fn fetch( + &self, + link: &Link, + value_defined_cost_fn: Option<&impl Fn(&[u8]) -> Option>, + ) -> CostResult { + TreeNode::get(self.storage, link.key(), value_defined_cost_fn) .map_ok(|x| x.ok_or(Error::KeyNotFoundError("Key not found for fetch"))) .flatten() } diff --git a/merk/src/proofs/chunk.rs b/merk/src/proofs/chunk.rs index 077683cdc..5e852ec13 100644 --- a/merk/src/proofs/chunk.rs +++ b/merk/src/proofs/chunk.rs @@ -43,12 +43,13 @@ use { #[cfg(feature = "full")] use super::{Node, Op}; +use crate::tree::kv::ValueDefinedCostType; #[cfg(feature = "full")] use crate::{ error::Error, tree::{Fetch, RefWalker}, Error::EdError, - TreeFeatureType::BasicMerk, + TreeFeatureType::BasicMerkNode, }; /// The minimum number of layers the trunk will be guaranteed to have before @@ -95,7 +96,10 @@ where depth: usize, ) -> CostResult { let mut cost = OperationCost::default(); - let maybe_left = match self.walk(true).unwrap_add_cost(&mut cost) { + let maybe_left = match self + .walk(true, None::<&fn(&[u8]) -> Option>) + .unwrap_add_cost(&mut cost) + { Ok(maybe_left) => maybe_left, Err(e) => { return Err(e).wrap_with_cost(cost); @@ -159,7 +163,11 @@ where // traverse left let has_left_child = self.tree().link(true).is_some(); if has_left_child { - let mut left = cost_return_on_error!(&mut cost, self.walk(true)).unwrap(); + let mut left = cost_return_on_error!( + &mut cost, + self.walk(true, None::<&fn(&[u8]) -> Option>) + ) + .unwrap(); cost_return_on_error!( &mut cost, left.traverse_for_trunk(proof, remaining_depth - 1, is_leftmost) @@ -174,7 +182,10 @@ where } // traverse right - if let Some(mut right) = cost_return_on_error!(&mut cost, self.walk(false)) { + if let Some(mut right) = cost_return_on_error!( + &mut cost, + self.walk(false, None::<&fn(&[u8]) -> Option>) + ) { cost_return_on_error!( &mut cost, right.traverse_for_trunk(proof, remaining_depth - 1, false) @@ -199,7 +210,7 @@ pub(crate) fn get_next_chunk( let mut chunk = Vec::with_capacity(512); let mut stack = Vec::with_capacity(32); - let mut node = TreeNode::new(vec![], vec![], None, BasicMerk).unwrap_add_cost(&mut cost); + let mut node = TreeNode::new(vec![], vec![], None, BasicMerkNode).unwrap_add_cost(&mut cost); while iter.valid().unwrap_add_cost(&mut cost) { let key = iter.key().unwrap_add_cost(&mut cost).unwrap(); @@ -213,7 +224,13 @@ pub(crate) fn get_next_chunk( let encoded_node = iter.value().unwrap_add_cost(&mut cost).unwrap(); cost_return_on_error_no_add!( &cost, - TreeNode::decode_into(&mut node, vec![], encoded_node).map_err(EdError) + TreeNode::decode_into( + &mut node, + vec![], + encoded_node, + None:: Option> + ) + .map_err(EdError) ); // TODO: Only use the KVValueHash if needed, saves 32 bytes @@ -459,7 +476,7 @@ mod tests { #[test] fn one_node_tree_trunk_roundtrip() { - let mut tree = BaseTree::new(vec![0], vec![], None, BasicMerk).unwrap(); + let mut tree = BaseTree::new(vec![0], vec![], None, BasicMerkNode).unwrap(); tree.commit(&mut NoopCommit {}, &|_, _| Ok(0)) .unwrap() .unwrap(); @@ -480,11 +497,11 @@ mod tests { // 0 // \ // 1 - let mut tree = BaseTree::new(vec![0], vec![], None, BasicMerk) + let mut tree = BaseTree::new(vec![0], vec![], None, BasicMerkNode) .unwrap() .attach( false, - Some(BaseTree::new(vec![1], vec![], None, BasicMerk).unwrap()), + Some(BaseTree::new(vec![1], vec![], None, BasicMerkNode).unwrap()), ); tree.commit(&mut NoopCommit {}, &|_, _| Ok(0)) .unwrap() @@ -505,11 +522,11 @@ mod tests { // 1 // / // 0 - let mut tree = BaseTree::new(vec![1], vec![], None, BasicMerk) + let mut tree = BaseTree::new(vec![1], vec![], None, BasicMerkNode) .unwrap() .attach( true, - Some(BaseTree::new(vec![0], vec![], None, BasicMerk).unwrap()), + Some(BaseTree::new(vec![0], vec![], None, BasicMerkNode).unwrap()), ); tree.commit(&mut NoopCommit {}, &|_, _| Ok(0)) .unwrap() @@ -530,15 +547,15 @@ mod tests { // 1 // / \ // 0 2 - let mut tree = BaseTree::new(vec![1], vec![], None, BasicMerk) + let mut tree = BaseTree::new(vec![1], vec![], None, BasicMerkNode) .unwrap() .attach( true, - Some(BaseTree::new(vec![0], vec![], None, BasicMerk).unwrap()), + Some(BaseTree::new(vec![0], vec![], None, BasicMerkNode).unwrap()), ) .attach( false, - Some(BaseTree::new(vec![2], vec![], None, BasicMerk).unwrap()), + Some(BaseTree::new(vec![2], vec![], None, BasicMerkNode).unwrap()), ); tree.commit(&mut NoopCommit {}, &|_, _| Ok(0)) .unwrap() diff --git a/merk/src/proofs/encoding.rs b/merk/src/proofs/encoding.rs index b0e828338..6b5f95b0e 100644 --- a/merk/src/proofs/encoding.rs +++ b/merk/src/proofs/encoding.rs @@ -465,7 +465,7 @@ mod test { use super::super::{Node, Op}; use crate::{ tree::HASH_LENGTH, - TreeFeatureType::{BasicMerk, SummedMerk}, + TreeFeatureType::{BasicMerkNode, SummedMerkNode}, }; #[test] @@ -567,7 +567,7 @@ mod test { vec![1, 2, 3], vec![4, 5, 6], [0; 32], - BasicMerk, + BasicMerkNode, )); assert_eq!(op.encoding_length(), 43); @@ -585,7 +585,7 @@ mod test { vec![1, 2, 3], vec![4, 5, 6], [0; 32], - SummedMerk(6), + SummedMerkNode(6), )); assert_eq!(op.encoding_length(), 44); @@ -683,7 +683,7 @@ mod test { vec![1, 2, 3], vec![4, 5, 6], [0; 32], - BasicMerk, + BasicMerkNode, )); assert_eq!(op.encoding_length(), 43); @@ -701,7 +701,7 @@ mod test { vec![1, 2, 3], vec![4, 5, 6], [0; 32], - SummedMerk(5), + SummedMerkNode(5), )); assert_eq!(op.encoding_length(), 44); @@ -860,7 +860,7 @@ mod test { vec![1, 2, 3], vec![4, 5, 6], [0; 32], - BasicMerk + BasicMerkNode )) ); @@ -875,7 +875,7 @@ mod test { vec![1, 2, 3], vec![4, 5, 6], [0; 32], - SummedMerk(6) + SummedMerkNode(6) )) ); } @@ -960,7 +960,7 @@ mod test { vec![1, 2, 3], vec![4, 5, 6], [0; 32], - BasicMerk + BasicMerkNode )) ); @@ -975,7 +975,7 @@ mod test { vec![1, 2, 3], vec![4, 5, 6], [0; 32], - SummedMerk(6) + SummedMerkNode(6) )) ); } diff --git a/merk/src/proofs/query/mod.rs b/merk/src/proofs/query/mod.rs index b9ba161e6..40aea9059 100644 --- a/merk/src/proofs/query/mod.rs +++ b/merk/src/proofs/query/mod.rs @@ -67,6 +67,7 @@ use {super::Op, std::collections::LinkedList}; use super::Node; #[cfg(any(feature = "full", feature = "verify"))] use crate::error::Error; +use crate::tree::kv::ValueDefinedCostType; #[cfg(feature = "full")] use crate::tree::{Fetch, Link, RefWalker}; @@ -752,14 +753,15 @@ where left_to_right: bool, ) -> CostResult { if !query.is_empty() { - self.walk(left).flat_map_ok(|child_opt| { - if let Some(mut child) = child_opt { - child.create_proof(query, limit, offset, left_to_right) - } else { - Ok((LinkedList::new(), (true, true), limit, offset)) - .wrap_with_cost(Default::default()) - } - }) + self.walk(left, None::<&fn(&[u8]) -> Option>) + .flat_map_ok(|child_opt| { + if let Some(mut child) = child_opt { + child.create_proof(query, limit, offset, left_to_right) + } else { + Ok((LinkedList::new(), (true, true), limit, offset)) + .wrap_with_cost(Default::default()) + } + }) } else if let Some(link) = self.tree().link(left) { let mut proof = LinkedList::new(); proof.push_back(if left_to_right { @@ -793,7 +795,7 @@ mod test { }, test_utils::make_tree_seq, tree::{NoopCommit, PanicSource, RefWalker, TreeNode}, - TreeFeatureType::BasicMerk, + TreeFeatureType::BasicMerkNode, }; fn compare_result_tuples( @@ -808,15 +810,15 @@ mod test { } fn make_3_node_tree() -> TreeNode { - let mut tree = TreeNode::new(vec![5], vec![5], None, BasicMerk) + let mut tree = TreeNode::new(vec![5], vec![5], None, BasicMerkNode) .unwrap() .attach( true, - Some(TreeNode::new(vec![3], vec![3], None, BasicMerk).unwrap()), + Some(TreeNode::new(vec![3], vec![3], None, BasicMerkNode).unwrap()), ) .attach( false, - Some(TreeNode::new(vec![7], vec![7], None, BasicMerk).unwrap()), + Some(TreeNode::new(vec![7], vec![7], None, BasicMerkNode).unwrap()), ); tree.commit(&mut NoopCommit {}, &|_, _| Ok(0)) .unwrap() @@ -825,9 +827,9 @@ mod test { } fn make_6_node_tree() -> TreeNode { - let two_tree = TreeNode::new(vec![2], vec![2], None, BasicMerk).unwrap(); - let four_tree = TreeNode::new(vec![4], vec![4], None, BasicMerk).unwrap(); - let mut three_tree = TreeNode::new(vec![3], vec![3], None, BasicMerk) + let two_tree = TreeNode::new(vec![2], vec![2], None, BasicMerkNode).unwrap(); + let four_tree = TreeNode::new(vec![4], vec![4], None, BasicMerkNode).unwrap(); + let mut three_tree = TreeNode::new(vec![3], vec![3], None, BasicMerkNode) .unwrap() .attach(true, Some(two_tree)) .attach(false, Some(four_tree)); @@ -836,8 +838,8 @@ mod test { .unwrap() .expect("commit failed"); - let seven_tree = TreeNode::new(vec![7], vec![7], None, BasicMerk).unwrap(); - let mut eight_tree = TreeNode::new(vec![8], vec![8], None, BasicMerk) + let seven_tree = TreeNode::new(vec![7], vec![7], None, BasicMerkNode).unwrap(); + let mut eight_tree = TreeNode::new(vec![8], vec![8], None, BasicMerkNode) .unwrap() .attach(true, Some(seven_tree)); eight_tree @@ -845,7 +847,7 @@ mod test { .unwrap() .expect("commit failed"); - let mut root_tree = TreeNode::new(vec![5], vec![5], None, BasicMerk) + let mut root_tree = TreeNode::new(vec![5], vec![5], None, BasicMerkNode) .unwrap() .attach(true, Some(three_tree)) .attach(false, Some(eight_tree)); @@ -1533,26 +1535,26 @@ mod test { #[test] fn doc_proof() { - let mut tree = TreeNode::new(vec![5], vec![5], None, BasicMerk) + let mut tree = TreeNode::new(vec![5], vec![5], None, BasicMerkNode) .unwrap() .attach( true, Some( - TreeNode::new(vec![2], vec![2], None, BasicMerk) + TreeNode::new(vec![2], vec![2], None, BasicMerkNode) .unwrap() .attach( true, - Some(TreeNode::new(vec![1], vec![1], None, BasicMerk).unwrap()), + Some(TreeNode::new(vec![1], vec![1], None, BasicMerkNode).unwrap()), ) .attach( false, Some( - TreeNode::new(vec![4], vec![4], None, BasicMerk) + TreeNode::new(vec![4], vec![4], None, BasicMerkNode) .unwrap() .attach( true, Some( - TreeNode::new(vec![3], vec![3], None, BasicMerk) + TreeNode::new(vec![3], vec![3], None, BasicMerkNode) .unwrap(), ), ), @@ -1563,24 +1565,24 @@ mod test { .attach( false, Some( - TreeNode::new(vec![9], vec![9], None, BasicMerk) + TreeNode::new(vec![9], vec![9], None, BasicMerkNode) .unwrap() .attach( true, Some( - TreeNode::new(vec![7], vec![7], None, BasicMerk) + TreeNode::new(vec![7], vec![7], None, BasicMerkNode) .unwrap() .attach( true, Some( - TreeNode::new(vec![6], vec![6], None, BasicMerk) + TreeNode::new(vec![6], vec![6], None, BasicMerkNode) .unwrap(), ), ) .attach( false, Some( - TreeNode::new(vec![8], vec![8], None, BasicMerk) + TreeNode::new(vec![8], vec![8], None, BasicMerkNode) .unwrap(), ), ), @@ -1589,12 +1591,12 @@ mod test { .attach( false, Some( - TreeNode::new(vec![11], vec![11], None, BasicMerk) + TreeNode::new(vec![11], vec![11], None, BasicMerkNode) .unwrap() .attach( true, Some( - TreeNode::new(vec![10], vec![10], None, BasicMerk) + TreeNode::new(vec![10], vec![10], None, BasicMerkNode) .unwrap(), ), ), @@ -5740,7 +5742,7 @@ mod test { #[test] fn verify_ops() { - let mut tree = TreeNode::new(vec![5], vec![5], None, BasicMerk).unwrap(); + let mut tree = TreeNode::new(vec![5], vec![5], None, BasicMerkNode).unwrap(); tree.commit(&mut NoopCommit {}, &|_, _| Ok(0)) .unwrap() .expect("commit failed"); @@ -5766,7 +5768,7 @@ mod test { #[test] #[should_panic(expected = "verify failed")] fn verify_ops_mismatched_hash() { - let mut tree = TreeNode::new(vec![5], vec![5], None, BasicMerk).unwrap(); + let mut tree = TreeNode::new(vec![5], vec![5], None, BasicMerkNode).unwrap(); tree.commit(&mut NoopCommit {}, &|_, _| Ok(0)) .unwrap() .expect("commit failed"); diff --git a/merk/src/test_utils/mod.rs b/merk/src/test_utils/mod.rs index a9a087b56..6abe167ec 100644 --- a/merk/src/test_utils/mod.rs +++ b/merk/src/test_utils/mod.rs @@ -39,9 +39,12 @@ use rand::prelude::*; pub use temp_merk::TempMerk; use crate::{ - tree::{kv::KV, BatchEntry, MerkBatch, NoopCommit, Op, PanicSource, TreeNode, Walker}, + tree::{ + kv::{ValueDefinedCostType, KV}, + BatchEntry, MerkBatch, NoopCommit, Op, PanicSource, TreeNode, Walker, + }, Merk, - TreeFeatureType::{BasicMerk, SummedMerk}, + TreeFeatureType::{BasicMerkNode, SummedMerkNode}, }; /// Assert tree invariants @@ -85,6 +88,7 @@ pub fn apply_memonly_unchecked(tree: TreeNode, batch: &MerkBatch>) -> Tr is_sum_node, )) }, + None::<&fn(&[u8]) -> Option>, &mut |_, _, _| Ok((false, None)), &mut |_flags, key_bytes_to_remove, value_bytes_to_remove| { Ok(( @@ -137,6 +141,7 @@ pub fn apply_to_memonly( is_sum_tree, )) }, + None::<&fn(&[u8]) -> Option>, &mut |_, _, _| Ok((false, None)), &mut |_flags, key_bytes_to_remove, value_bytes_to_remove| { Ok(( @@ -171,7 +176,7 @@ pub const fn seq_key(n: u64) -> [u8; 8] { /// Create batch entry with Put op using key n and a fixed value pub fn put_entry(n: u64) -> BatchEntry> { - (seq_key(n).to_vec(), Op::Put(vec![123; 60], BasicMerk)) + (seq_key(n).to_vec(), Op::Put(vec![123; 60], BasicMerkNode)) } /// Create batch entry with Delete op using key n @@ -235,9 +240,9 @@ pub fn make_tree_rand( let value = vec![123; 60]; let feature_type = if is_sum_tree { - SummedMerk(0) + SummedMerkNode(0) } else { - BasicMerk + BasicMerkNode }; let mut tree = TreeNode::new(vec![0; 20], value, None, feature_type).unwrap(); @@ -264,7 +269,7 @@ pub fn make_tree_seq(node_count: u64) -> TreeNode { }; let value = vec![123; 60]; - let mut tree = TreeNode::new(vec![0; 20], value, None, BasicMerk).unwrap(); + let mut tree = TreeNode::new(vec![0; 20], value, None, BasicMerkNode).unwrap(); let batch_count = node_count / batch_size; for i in 0..batch_count { @@ -288,6 +293,7 @@ where .get_storage_context(SubtreePath::empty(), Some(batch)) .unwrap(), false, + None:: Option>, ) .unwrap() .unwrap() @@ -305,6 +311,7 @@ where .get_storage_context(SubtreePath::empty(), None) .unwrap(), false, + None:: Option>, ) .unwrap() .unwrap() diff --git a/merk/src/test_utils/temp_merk.rs b/merk/src/test_utils/temp_merk.rs index 0fb4724ed..25e5b75cc 100644 --- a/merk/src/test_utils/temp_merk.rs +++ b/merk/src/test_utils/temp_merk.rs @@ -39,6 +39,7 @@ use grovedb_storage::{ Storage, }; +use crate::tree::kv::ValueDefinedCostType; #[cfg(feature = "full")] use crate::Merk; @@ -62,7 +63,13 @@ impl TempMerk { .get_storage_context(SubtreePath::empty(), Some(batch)) .unwrap(); - let merk = Merk::open_base(context, false).unwrap().unwrap(); + let merk = Merk::open_base( + context, + false, + None:: Option>, + ) + .unwrap() + .unwrap(); TempMerk { storage, merk, @@ -82,7 +89,13 @@ impl TempMerk { .storage .get_storage_context(SubtreePath::empty(), Some(self.batch)) .unwrap(); - self.merk = Merk::open_base(context, false).unwrap().unwrap(); + self.merk = Merk::open_base( + context, + false, + None:: Option>, + ) + .unwrap() + .unwrap(); } } diff --git a/merk/src/tree/encoding.rs b/merk/src/tree/encoding.rs index d7a56ce1d..e635cd8c5 100644 --- a/merk/src/tree/encoding.rs +++ b/merk/src/tree/encoding.rs @@ -39,6 +39,7 @@ use grovedb_storage::StorageContext; #[cfg(feature = "full")] use super::TreeNode; +use crate::tree::kv::ValueDefinedCostType; #[cfg(feature = "full")] use crate::{ error::{Error, Error::EdError}, @@ -50,12 +51,20 @@ use crate::{ impl TreeNode { /// Decode given bytes and set as Tree fields. Set key to value of given /// key. - pub fn decode_raw(bytes: &[u8], key: Vec) -> Result { - TreeNode::decode(key, bytes).map_err(EdError) + pub fn decode_raw( + bytes: &[u8], + key: Vec, + value_defined_cost_fn: Option Option>, + ) -> Result { + TreeNode::decode(key, bytes, value_defined_cost_fn).map_err(EdError) } /// Get value from storage given key. - pub(crate) fn get<'db, S, K>(storage: &S, key: K) -> CostResult, Error> + pub(crate) fn get<'db, S, K>( + storage: &S, + key: K, + value_defined_cost_fn: Option Option>, + ) -> CostResult, Error> where S: StorageContext<'db>, K: AsRef<[u8]>, @@ -66,7 +75,7 @@ impl TreeNode { let tree_opt = cost_return_on_error_no_add!( &cost, tree_bytes - .map(|x| TreeNode::decode_raw(&x, key.as_ref().to_vec())) + .map(|x| TreeNode::decode_raw(&x, key.as_ref().to_vec(), value_defined_cost_fn)) .transpose() ); @@ -111,18 +120,33 @@ impl TreeNode { #[inline] /// Decode bytes from reader, set as Tree fields and set key to given key - pub fn decode_into(&mut self, key: Vec, input: &[u8]) -> ed::Result<()> { + pub fn decode_into( + &mut self, + key: Vec, + input: &[u8], + value_defined_cost_fn: Option Option>, + ) -> ed::Result<()> { let mut tree_inner: TreeNodeInner = Decode::decode(input)?; tree_inner.kv.key = key; + if let Some(value_defined_cost_fn) = value_defined_cost_fn { + tree_inner.kv.value_defined_cost = value_defined_cost_fn(input); + } self.inner = Box::new(tree_inner); Ok(()) } #[inline] /// Decode input and set as Tree fields. Set the key as the given key. - pub fn decode(key: Vec, input: &[u8]) -> ed::Result { + pub fn decode( + key: Vec, + input: &[u8], + value_defined_cost_fn: Option Option>, + ) -> ed::Result { let mut tree_inner: TreeNodeInner = Decode::decode(input)?; tree_inner.kv.key = key; + if let Some(value_defined_cost_fn) = value_defined_cost_fn { + tree_inner.kv.value_defined_cost = value_defined_cost_fn(input); + } Ok(TreeNode::new_with_tree_inner(tree_inner)) } } @@ -131,12 +155,12 @@ impl TreeNode { #[cfg(test)] mod tests { use super::{super::Link, *}; - use crate::TreeFeatureType::{BasicMerk, SummedMerk}; + use crate::TreeFeatureType::{BasicMerkNode, SummedMerkNode}; #[test] fn encode_leaf_tree() { let tree = - TreeNode::from_fields(vec![0], vec![1], [55; 32], None, None, BasicMerk).unwrap(); + TreeNode::from_fields(vec![0], vec![1], [55; 32], None, None, BasicMerkNode).unwrap(); assert_eq!(tree.encoding_length(), 68); assert_eq!( tree.value_encoding_length_with_parent_to_child_reference(), @@ -163,10 +187,10 @@ mod tests { Some(Link::Modified { pending_writes: 1, child_heights: (123, 124), - tree: TreeNode::new(vec![2], vec![3], None, BasicMerk).unwrap(), + tree: TreeNode::new(vec![2], vec![3], None, BasicMerkNode).unwrap(), }), None, - BasicMerk, + BasicMerkNode, ) .unwrap(); tree.encode(); @@ -182,10 +206,10 @@ mod tests { hash: [66; 32], sum: None, child_heights: (123, 124), - tree: TreeNode::new(vec![2], vec![3], None, BasicMerk).unwrap(), + tree: TreeNode::new(vec![2], vec![3], None, BasicMerkNode).unwrap(), }), None, - BasicMerk, + BasicMerkNode, ) .unwrap(); assert_eq!( @@ -211,10 +235,10 @@ mod tests { hash: [66; 32], sum: Some(10), child_heights: (123, 124), - tree: TreeNode::new(vec![2], vec![3], None, BasicMerk).unwrap(), + tree: TreeNode::new(vec![2], vec![3], None, BasicMerkNode).unwrap(), }), None, - SummedMerk(5), + SummedMerkNode(5), ) .unwrap(); assert_eq!( @@ -243,7 +267,7 @@ mod tests { key: vec![2], }), None, - BasicMerk, + BasicMerkNode, ) .unwrap(); assert_eq!( @@ -276,10 +300,15 @@ mod tests { 131, 208, 25, 73, 98, 245, 209, 227, 170, 26, 72, 212, 134, 166, 126, 39, 98, 166, 199, 149, 144, 21, 1, ]; - let tree = TreeNode::decode(vec![0], bytes.as_slice()).expect("should decode correctly"); + let tree = TreeNode::decode( + vec![0], + bytes.as_slice(), + None::<&fn(&[u8]) -> Option>, + ) + .expect("should decode correctly"); assert_eq!(tree.key(), &[0]); assert_eq!(tree.value_as_slice(), &[1]); - assert_eq!(tree.inner.kv.feature_type, BasicMerk); + assert_eq!(tree.inner.kv.feature_type, BasicMerkNode); } #[test] @@ -291,7 +320,12 @@ mod tests { 55, 55, 55, 55, 55, 55, 32, 34, 236, 157, 87, 27, 167, 116, 207, 158, 131, 208, 25, 73, 98, 245, 209, 227, 170, 26, 72, 212, 134, 166, 126, 39, 98, 166, 199, 149, 144, 21, 1, ]; - let tree = TreeNode::decode(vec![0], bytes.as_slice()).expect("should decode correctly"); + let tree = TreeNode::decode( + vec![0], + bytes.as_slice(), + None::<&fn(&[u8]) -> Option>, + ) + .expect("should decode correctly"); assert_eq!(tree.key(), &[0]); assert_eq!(tree.value_as_slice(), &[1]); if let Some(Link::Reference { @@ -312,7 +346,11 @@ mod tests { #[test] fn decode_invalid_bytes_as_tree() { let bytes = vec![2, 3, 4, 5]; - let tree = TreeNode::decode(vec![0], bytes.as_slice()); + let tree = TreeNode::decode( + vec![0], + bytes.as_slice(), + None::<&fn(&[u8]) -> Option>, + ); assert!(matches!(tree, Err(_))); } } diff --git a/merk/src/tree/kv.rs b/merk/src/tree/kv.rs index 8855855ae..ff020abc5 100644 --- a/merk/src/tree/kv.rs +++ b/merk/src/tree/kv.rs @@ -45,7 +45,7 @@ use crate::tree::kv::ValueDefinedCostType::{LayeredValueDefinedCost, Specialized use crate::{ tree::{ hash::{combine_hash, kv_digest_to_kv_hash, value_hash, HASH_LENGTH_X2}, - tree_feature_type::{TreeFeatureType, TreeFeatureType::BasicMerk}, + tree_feature_type::{TreeFeatureType, TreeFeatureType::BasicMerkNode}, }, Link, HASH_LENGTH_U32, HASH_LENGTH_U32_X2, }; @@ -237,19 +237,6 @@ impl KV { self.wrap_with_cost(cost) } - /// Replaces the `KV`'s value with the given value, updates the hash, - /// value hash and returns the modified `KV`. - /// This is used when we want a fixed cost, for example in sum trees - #[inline] - pub fn put_value_with_fixed_cost_then_update( - mut self, - value: Vec, - value_cost: u32, - ) -> CostContext { - self.value_defined_cost = Some(SpecializedValueDefinedCost(value_cost)); - self.put_value_then_update(value) - } - /// Replaces the `KV`'s value with the given value, does not update the /// hashes, value hash and returns the modified `KV`. /// This is used when we want a fixed cost, for example in sum trees @@ -263,37 +250,6 @@ impl KV { self.put_value_no_update_of_hashes(value) } - /// Replaces the `KV`'s value with the given value and value hash, - /// updates the hash and returns the modified `KV`. - #[inline] - pub fn put_value_and_reference_value_hash_then_update( - mut self, - value: Vec, - reference_value_hash: CryptoHash, - ) -> CostContext { - let mut cost = OperationCost::default(); - let actual_value_hash = value_hash(value.as_slice()).unwrap_add_cost(&mut cost); - let combined_value_hash = - combine_hash(&actual_value_hash, &reference_value_hash).unwrap_add_cost(&mut cost); - self.value = value; - self.value_hash = combined_value_hash; - self.hash = kv_digest_to_kv_hash(self.key(), self.value_hash()).unwrap_add_cost(&mut cost); - self.wrap_with_cost(cost) - } - - /// Replaces the `KV`'s value with the given value and value hash, - /// updates the hash and returns the modified `KV`. - #[inline] - pub fn put_value_with_reference_value_hash_and_value_cost_then_update( - mut self, - value: Vec, - reference_value_hash: CryptoHash, - value_cost: u32, - ) -> CostContext { - self.value_defined_cost = Some(LayeredValueDefinedCost(value_cost)); - self.put_value_and_reference_value_hash_then_update(value, reference_value_hash) - } - /// Returns the key as a slice. #[inline] pub fn key(&self) -> &[u8] { @@ -573,7 +529,7 @@ impl Decode for KV { let mut kv = Self { key: Vec::with_capacity(0), value: Vec::with_capacity(128), - feature_type: BasicMerk, + feature_type: BasicMerkNode, value_defined_cost: None, hash: NULL_HASH, value_hash: NULL_HASH, @@ -604,11 +560,11 @@ impl Terminated for KV {} #[cfg(test)] mod test { use super::*; - use crate::tree::tree_feature_type::TreeFeatureType::SummedMerk; + use crate::tree::tree_feature_type::TreeFeatureType::SummedMerkNode; #[test] fn new_kv() { - let kv = KV::new(vec![1, 2, 3], vec![4, 5, 6], None, BasicMerk).unwrap(); + let kv = KV::new(vec![1, 2, 3], vec![4, 5, 6], None, BasicMerkNode).unwrap(); assert_eq!(kv.key(), &[1, 2, 3]); assert_eq!(kv.value_as_slice(), &[4, 5, 6]); @@ -617,7 +573,7 @@ mod test { #[test] fn with_value() { - let kv = KV::new(vec![1, 2, 3], vec![4, 5, 6], None, BasicMerk) + let kv = KV::new(vec![1, 2, 3], vec![4, 5, 6], None, BasicMerkNode) .unwrap() .put_value_then_update(vec![7, 8, 9]) .unwrap(); @@ -629,7 +585,7 @@ mod test { #[test] fn encode_and_decode_kv() { - let kv = KV::new(vec![1, 2, 3], vec![4, 5, 6], None, BasicMerk).unwrap(); + let kv = KV::new(vec![1, 2, 3], vec![4, 5, 6], None, BasicMerkNode).unwrap(); let mut encoded_kv = vec![]; kv.encode_into(&mut encoded_kv).expect("encoded"); let mut decoded_kv = KV::decode(encoded_kv.as_slice()).unwrap(); @@ -637,7 +593,7 @@ mod test { assert_eq!(kv, decoded_kv); - let kv = KV::new(vec![1, 2, 3], vec![4, 5, 6], None, SummedMerk(20)).unwrap(); + let kv = KV::new(vec![1, 2, 3], vec![4, 5, 6], None, SummedMerkNode(20)).unwrap(); let mut encoded_kv = vec![]; kv.encode_into(&mut encoded_kv).expect("encoded"); let mut decoded_kv = KV::decode(encoded_kv.as_slice()).unwrap(); diff --git a/merk/src/tree/link.rs b/merk/src/tree/link.rs index 8cdb948b4..ab26159b3 100644 --- a/merk/src/tree/link.rs +++ b/merk/src/tree/link.rs @@ -486,11 +486,11 @@ mod test { super::{hash::NULL_HASH, TreeNode}, *, }; - use crate::TreeFeatureType::BasicMerk; + use crate::TreeFeatureType::BasicMerkNode; #[test] fn from_modified_tree() { - let tree = TreeNode::new(vec![0], vec![1], None, BasicMerk).unwrap(); + let tree = TreeNode::new(vec![0], vec![1], None, BasicMerkNode).unwrap(); let link = Link::from_modified_tree(tree); assert!(link.is_modified()); assert_eq!(link.height(), 1); @@ -507,7 +507,7 @@ mod test { let link = Link::maybe_from_modified_tree(None); assert!(link.is_none()); - let tree = TreeNode::new(vec![0], vec![1], None, BasicMerk).unwrap(); + let tree = TreeNode::new(vec![0], vec![1], None, BasicMerkNode).unwrap(); let link = Link::maybe_from_modified_tree(Some(tree)); assert!(link.expect("expected link").is_modified()); } @@ -519,7 +519,7 @@ mod test { let child_heights = (0, 0); let pending_writes = 1; let key = vec![0]; - let tree = || TreeNode::new(vec![0], vec![1], None, BasicMerk).unwrap(); + let tree = || TreeNode::new(vec![0], vec![1], None, BasicMerkNode).unwrap(); let reference = Link::Reference { hash, @@ -585,7 +585,7 @@ mod test { Link::Modified { pending_writes: 1, child_heights: (1, 1), - tree: TreeNode::new(vec![0], vec![1], None, BasicMerk).unwrap(), + tree: TreeNode::new(vec![0], vec![1], None, BasicMerkNode).unwrap(), } .hash(); } @@ -596,7 +596,7 @@ mod test { Link::Modified { pending_writes: 1, child_heights: (1, 1), - tree: TreeNode::new(vec![0], vec![1], None, BasicMerk).unwrap(), + tree: TreeNode::new(vec![0], vec![1], None, BasicMerkNode).unwrap(), } .into_reference(); } @@ -608,7 +608,7 @@ mod test { hash: [1; 32], sum: None, child_heights: (1, 1), - tree: TreeNode::new(vec![0], vec![1], None, BasicMerk).unwrap(), + tree: TreeNode::new(vec![0], vec![1], None, BasicMerkNode).unwrap(), } .into_reference(); } diff --git a/merk/src/tree/mod.rs b/merk/src/tree/mod.rs index 5f15ed945..c57d36447 100644 --- a/merk/src/tree/mod.rs +++ b/merk/src/tree/mod.rs @@ -487,8 +487,8 @@ impl TreeNode { #[inline] pub fn sum(&self) -> Result, Error> { match self.inner.kv.feature_type { - TreeFeatureType::BasicMerk => Ok(None), - TreeFeatureType::SummedMerk(value) => value + TreeFeatureType::BasicMerkNode => Ok(None), + TreeFeatureType::SummedMerkNode(value) => value .checked_add(self.child_sum(true)) .and_then(|a| a.checked_add(self.child_sum(false))) .ok_or(Overflow("sum is overflowing")) @@ -944,7 +944,15 @@ impl TreeNode { /// Fetches the child on the given side using the given data source, and /// places it in the child slot (upgrading the link from `Link::Reference` /// to `Link::Loaded`). - pub fn load(&mut self, left: bool, source: &S) -> CostResult<(), Error> { + pub fn load( + &mut self, + left: bool, + source: &S, + value_defined_cost_fn: Option<&V>, + ) -> CostResult<(), Error> + where + V: Fn(&[u8]) -> Option, + { // TODO: return Err instead of panic? let link = self.link(left).expect("Expected link"); let (child_heights, hash, sum) = match link { @@ -958,7 +966,7 @@ impl TreeNode { }; let mut cost = OperationCost::default(); - let tree = cost_return_on_error!(&mut cost, source.fetch(link)); + let tree = cost_return_on_error!(&mut cost, source.fetch(link, value_defined_cost_fn)); debug_assert_eq!(tree.key(), link.key()); *self.slot_mut(left) = Some(Link::Loaded { tree, @@ -986,11 +994,13 @@ mod test { use grovedb_costs::storage_cost::removal::StorageRemovedBytes::NoStorageRemoval; use super::{commit::NoopCommit, hash::NULL_HASH, TreeNode}; - use crate::tree::{tree_feature_type::TreeFeatureType::SummedMerk, TreeFeatureType::BasicMerk}; + use crate::tree::{ + tree_feature_type::TreeFeatureType::SummedMerkNode, TreeFeatureType::BasicMerkNode, + }; #[test] fn build_tree() { - let tree = TreeNode::new(vec![1], vec![101], None, BasicMerk).unwrap(); + let tree = TreeNode::new(vec![1], vec![101], None, BasicMerkNode).unwrap(); assert_eq!(tree.key(), &[1]); assert_eq!(tree.value_as_slice(), &[101]); assert!(tree.child(true).is_none()); @@ -1002,13 +1012,13 @@ mod test { let tree = tree.attach( true, - Some(TreeNode::new(vec![2], vec![102], None, BasicMerk).unwrap()), + Some(TreeNode::new(vec![2], vec![102], None, BasicMerkNode).unwrap()), ); assert_eq!(tree.key(), &[1]); assert_eq!(tree.child(true).unwrap().key(), &[2]); assert!(tree.child(false).is_none()); - let tree = TreeNode::new(vec![3], vec![103], None, BasicMerk) + let tree = TreeNode::new(vec![3], vec![103], None, BasicMerkNode) .unwrap() .attach(false, Some(tree)); assert_eq!(tree.key(), &[3]); @@ -1019,29 +1029,29 @@ mod test { #[should_panic] #[test] fn attach_existing() { - TreeNode::new(vec![0], vec![1], None, BasicMerk) + TreeNode::new(vec![0], vec![1], None, BasicMerkNode) .unwrap() .attach( true, - Some(TreeNode::new(vec![2], vec![3], None, BasicMerk).unwrap()), + Some(TreeNode::new(vec![2], vec![3], None, BasicMerkNode).unwrap()), ) .attach( true, - Some(TreeNode::new(vec![4], vec![5], None, BasicMerk).unwrap()), + Some(TreeNode::new(vec![4], vec![5], None, BasicMerkNode).unwrap()), ); } #[test] fn modify() { - let tree = TreeNode::new(vec![0], vec![1], None, BasicMerk) + let tree = TreeNode::new(vec![0], vec![1], None, BasicMerkNode) .unwrap() .attach( true, - Some(TreeNode::new(vec![2], vec![3], None, BasicMerk).unwrap()), + Some(TreeNode::new(vec![2], vec![3], None, BasicMerkNode).unwrap()), ) .attach( false, - Some(TreeNode::new(vec![4], vec![5], None, BasicMerk).unwrap()), + Some(TreeNode::new(vec![4], vec![5], None, BasicMerkNode).unwrap()), ); let tree = tree.walk(true, |left_opt| { @@ -1053,7 +1063,7 @@ mod test { let tree = tree.walk(true, |left_opt| { assert!(left_opt.is_none()); - Some(TreeNode::new(vec![2], vec![3], None, BasicMerk).unwrap()) + Some(TreeNode::new(vec![2], vec![3], None, BasicMerkNode).unwrap()) }); assert_eq!(tree.link(true).unwrap().key(), &[2]); @@ -1067,11 +1077,11 @@ mod test { #[test] fn child_and_link() { - let mut tree = TreeNode::new(vec![0], vec![1], None, BasicMerk) + let mut tree = TreeNode::new(vec![0], vec![1], None, BasicMerkNode) .unwrap() .attach( true, - Some(TreeNode::new(vec![2], vec![3], None, BasicMerk).unwrap()), + Some(TreeNode::new(vec![2], vec![3], None, BasicMerkNode).unwrap()), ); assert!(tree.link(true).expect("expected link").is_modified()); assert!(tree.child(true).is_some()); @@ -1095,11 +1105,11 @@ mod test { #[test] fn child_hash() { - let mut tree = TreeNode::new(vec![0], vec![1], None, BasicMerk) + let mut tree = TreeNode::new(vec![0], vec![1], None, BasicMerkNode) .unwrap() .attach( true, - Some(TreeNode::new(vec![2], vec![3], None, BasicMerk).unwrap()), + Some(TreeNode::new(vec![2], vec![3], None, BasicMerkNode).unwrap()), ); tree.commit(&mut NoopCommit {}, &|_, _| Ok(0)) .unwrap() @@ -1116,7 +1126,7 @@ mod test { #[test] fn hash() { - let tree = TreeNode::new(vec![0], vec![1], None, BasicMerk).unwrap(); + let tree = TreeNode::new(vec![0], vec![1], None, BasicMerkNode).unwrap(); assert_eq!( tree.hash().unwrap(), [ @@ -1128,13 +1138,13 @@ mod test { #[test] fn child_pending_writes() { - let tree = TreeNode::new(vec![0], vec![1], None, BasicMerk).unwrap(); + let tree = TreeNode::new(vec![0], vec![1], None, BasicMerkNode).unwrap(); assert_eq!(tree.child_pending_writes(true), 0); assert_eq!(tree.child_pending_writes(false), 0); let tree = tree.attach( true, - Some(TreeNode::new(vec![2], vec![3], None, BasicMerk).unwrap()), + Some(TreeNode::new(vec![2], vec![3], None, BasicMerkNode).unwrap()), ); assert_eq!(tree.child_pending_writes(true), 1); assert_eq!(tree.child_pending_writes(false), 0); @@ -1142,7 +1152,7 @@ mod test { #[test] fn height_and_balance() { - let tree = TreeNode::new(vec![0], vec![1], None, BasicMerk).unwrap(); + let tree = TreeNode::new(vec![0], vec![1], None, BasicMerkNode).unwrap(); assert_eq!(tree.height(), 1); assert_eq!(tree.child_height(true), 0); assert_eq!(tree.child_height(false), 0); @@ -1150,7 +1160,7 @@ mod test { let tree = tree.attach( true, - Some(TreeNode::new(vec![2], vec![3], None, BasicMerk).unwrap()), + Some(TreeNode::new(vec![2], vec![3], None, BasicMerkNode).unwrap()), ); assert_eq!(tree.height(), 2); assert_eq!(tree.child_height(true), 1); @@ -1167,11 +1177,11 @@ mod test { #[test] fn commit() { - let mut tree = TreeNode::new(vec![0], vec![1], None, BasicMerk) + let mut tree = TreeNode::new(vec![0], vec![1], None, BasicMerkNode) .unwrap() .attach( false, - Some(TreeNode::new(vec![2], vec![3], None, BasicMerk).unwrap()), + Some(TreeNode::new(vec![2], vec![3], None, BasicMerkNode).unwrap()), ); tree.commit(&mut NoopCommit {}, &|_, _| Ok(0)) .unwrap() @@ -1182,11 +1192,11 @@ mod test { #[test] fn sum_tree() { - let mut tree = TreeNode::new(vec![0], vec![1], None, SummedMerk(3)) + let mut tree = TreeNode::new(vec![0], vec![1], None, SummedMerkNode(3)) .unwrap() .attach( false, - Some(TreeNode::new(vec![2], vec![3], None, SummedMerk(5)).unwrap()), + Some(TreeNode::new(vec![2], vec![3], None, SummedMerkNode(5)).unwrap()), ); tree.commit(&mut NoopCommit {}, &|_, _| Ok(0)) .unwrap() diff --git a/merk/src/tree/ops.rs b/merk/src/tree/ops.rs index 82055b111..4b3a3d344 100644 --- a/merk/src/tree/ops.rs +++ b/merk/src/tree/ops.rs @@ -150,7 +150,11 @@ pub struct PanicSource {} #[cfg(feature = "full")] impl Fetch for PanicSource { - fn fetch(&self, _link: &Link) -> CostResult { + fn fetch( + &self, + _link: &Link, + value_defined_cost_fn: Option<&impl Fn(&[u8]) -> Option>, + ) -> CostResult { unreachable!("'fetch' should not have been called") } } @@ -165,16 +169,18 @@ where /// not require a non-empty tree. /// /// Keys in batch must be sorted and unique. - pub fn apply_to, C, U, R>( + pub fn apply_to, C, V, U, R>( maybe_tree: Option, batch: &MerkBatch, source: S, old_tree_cost: &C, + value_defined_cost_fn: Option<&V>, update_tree_value_based_on_costs: &mut U, section_removal_bytes: &mut R, ) -> CostContext, KeyUpdates), Error>> where C: Fn(&Vec, &Vec) -> Result, + V: Fn(&[u8]) -> Option, U: FnMut( &StorageCost, &Vec, @@ -201,6 +207,7 @@ where batch, source, old_tree_cost, + value_defined_cost_fn, update_tree_value_based_on_costs, section_removal_bytes, ) @@ -226,6 +233,7 @@ where tree.apply_sorted( batch, old_tree_cost, + value_defined_cost_fn, update_tree_value_based_on_costs, section_removal_bytes ) @@ -241,15 +249,17 @@ where /// Builds a `Tree` from a batch of operations. /// /// Keys in batch must be sorted and unique. - fn build, C, U, R>( + fn build, C, V, U, R>( batch: &MerkBatch, source: S, old_tree_cost: &C, + value_defined_cost_fn: Option<&V>, update_tree_value_based_on_costs: &mut U, section_removal_bytes: &mut R, ) -> CostResult, Error> where C: Fn(&Vec, &Vec) -> Result, + V: Fn(&[u8]) -> Option, U: FnMut( &StorageCost, &Vec, @@ -276,6 +286,7 @@ where left_batch, source.clone(), old_tree_cost, + value_defined_cost_fn, update_tree_value_based_on_costs, section_removal_bytes ) @@ -288,6 +299,7 @@ where tree.apply_sorted( right_batch, old_tree_cost, + value_defined_cost_fn, update_tree_value_based_on_costs, section_removal_bytes ) @@ -300,6 +312,7 @@ where right_batch, source.clone(), old_tree_cost, + value_defined_cost_fn, update_tree_value_based_on_costs, section_removal_bytes ) @@ -370,6 +383,7 @@ where None ), old_tree_cost, + value_defined_cost_fn, update_tree_value_based_on_costs, section_removal_bytes, ) @@ -387,6 +401,7 @@ where self.apply_sorted( batch, &|_, _| Ok(0), + None::<&fn(&[u8]) -> Option>, &mut |_, _, _| Ok((false, None)), &mut |_flags, key_bytes_to_remove, value_bytes_to_remove| { Ok(( @@ -401,15 +416,17 @@ where /// `Walker::apply`_to, but requires a populated tree. /// /// Keys in batch must be sorted and unique. - fn apply_sorted, C, U, R>( + fn apply_sorted, C, V, U, R>( self, batch: &MerkBatch, old_specialized_cost: &C, + value_defined_cost_fn: Option<&V>, update_tree_value_based_on_costs: &mut U, section_removal_bytes: &mut R, ) -> CostResult<(Option, KeyUpdates), Error> where C: Fn(&Vec, &Vec) -> Result, + V: Fn(&[u8]) -> Option, U: FnMut( &StorageCost, &Vec, @@ -527,7 +544,8 @@ where needs_value_verification: false, }; - let maybe_tree = cost_return_on_error!(&mut cost, self.remove()); + let maybe_tree = + cost_return_on_error!(&mut cost, self.remove(value_defined_cost_fn)); #[rustfmt::skip] let (maybe_tree, mut key_updates) @@ -538,6 +556,7 @@ where &batch[..index], source.clone(), old_specialized_cost, + value_defined_cost_fn, update_tree_value_based_on_costs, section_removal_bytes ) @@ -551,6 +570,7 @@ where &batch[index + 1..], source.clone(), old_specialized_cost, + value_defined_cost_fn, update_tree_value_based_on_costs, section_removal_bytes ) @@ -593,6 +613,7 @@ where exclusive, KeyUpdates::new(new_keys, updated_keys, LinkedList::default(), None), old_specialized_cost, + value_defined_cost_fn, update_tree_value_based_on_costs, section_removal_bytes, ) @@ -604,13 +625,14 @@ where /// /// This recursion executes serially in the same thread, but in the future /// will be dispatched to workers in other threads. - fn recurse, C, U, R>( + fn recurse, C, V, U, R>( self, batch: &MerkBatch, mid: usize, exclusive: bool, mut key_updates: KeyUpdates, old_tree_cost: &C, + value_defined_cost_fn: Option<&V>, update_tree_value_based_on_costs: &mut U, section_removal_bytes: &mut R, ) -> CostResult<(Option, KeyUpdates), Error> @@ -621,6 +643,7 @@ where &Vec, &mut Vec, ) -> Result<(bool, Option), Error>, + V: Fn(&[u8]) -> Option, R: FnMut(&Vec, u32, u32) -> Result<(StorageRemovedBytes, StorageRemovedBytes), Error>, { let mut cost = OperationCost::default(); @@ -638,26 +661,31 @@ where let source = self.clone_source(); cost_return_on_error!( &mut cost, - self.walk(true, |maybe_left| { - Self::apply_to( - maybe_left, - left_batch, - source, - old_tree_cost, - update_tree_value_based_on_costs, - section_removal_bytes, - ) - .map_ok(|(maybe_left, mut key_updates_left)| { - key_updates.new_keys.append(&mut key_updates_left.new_keys); - key_updates - .updated_keys - .append(&mut key_updates_left.updated_keys); - key_updates - .deleted_keys - .append(&mut key_updates_left.deleted_keys); - maybe_left - }) - }) + self.walk( + true, + |maybe_left| { + Self::apply_to( + maybe_left, + left_batch, + source, + old_tree_cost, + value_defined_cost_fn, + update_tree_value_based_on_costs, + section_removal_bytes, + ) + .map_ok(|(maybe_left, mut key_updates_left)| { + key_updates.new_keys.append(&mut key_updates_left.new_keys); + key_updates + .updated_keys + .append(&mut key_updates_left.updated_keys); + key_updates + .deleted_keys + .append(&mut key_updates_left.deleted_keys); + maybe_left + }) + }, + value_defined_cost_fn + ) ) } else { self @@ -667,32 +695,37 @@ where let source = tree.clone_source(); cost_return_on_error!( &mut cost, - tree.walk(false, |maybe_right| { - Self::apply_to( - maybe_right, - right_batch, - source, - old_tree_cost, - update_tree_value_based_on_costs, - section_removal_bytes, - ) - .map_ok(|(maybe_right, mut key_updates_right)| { - key_updates.new_keys.append(&mut key_updates_right.new_keys); - key_updates - .updated_keys - .append(&mut key_updates_right.updated_keys); - key_updates - .deleted_keys - .append(&mut key_updates_right.deleted_keys); - maybe_right - }) - }) + tree.walk( + false, + |maybe_right| { + Self::apply_to( + maybe_right, + right_batch, + source, + old_tree_cost, + value_defined_cost_fn, + update_tree_value_based_on_costs, + section_removal_bytes, + ) + .map_ok(|(maybe_right, mut key_updates_right)| { + key_updates.new_keys.append(&mut key_updates_right.new_keys); + key_updates + .updated_keys + .append(&mut key_updates_right.updated_keys); + key_updates + .deleted_keys + .append(&mut key_updates_right.deleted_keys); + maybe_right + }) + }, + value_defined_cost_fn + ) ) } else { tree }; - let tree = cost_return_on_error!(&mut cost, tree.maybe_balance()); + let tree = cost_return_on_error!(&mut cost, tree.maybe_balance(value_defined_cost_fn)); let new_root_key = tree.tree().key(); @@ -715,7 +748,10 @@ where /// Checks if the tree is unbalanced and if so, applies AVL tree rotation(s) /// to rebalance the tree and its subtrees. Returns the root node of the /// balanced tree after applying the rotations. - fn maybe_balance(self) -> CostResult { + fn maybe_balance(self, value_defined_cost_fn: Option<&V>) -> CostResult + where + V: Fn(&[u8]) -> Option, + { let mut cost = OperationCost::default(); let balance_factor = self.balance_factor(); @@ -729,37 +765,55 @@ where let tree = if left == (self.tree().link(left).unwrap().balance_factor() > 0) { cost_return_on_error!( &mut cost, - self.walk_expect(left, |child| child.rotate(!left).map_ok(Option::Some)) + self.walk_expect( + left, + |child| child + .rotate(!left, value_defined_cost_fn) + .map_ok(Option::Some), + value_defined_cost_fn + ) ) } else { self }; - let rotate = tree.rotate(left).unwrap_add_cost(&mut cost); + let rotate = tree + .rotate(left, value_defined_cost_fn) + .unwrap_add_cost(&mut cost); rotate.wrap_with_cost(cost) } /// Applies an AVL tree rotation, a constant-time operation which only needs /// to swap pointers in order to rebalance a tree. - fn rotate(self, left: bool) -> CostResult { + fn rotate(self, left: bool, value_defined_cost_fn: Option<&V>) -> CostResult + where + V: Fn(&[u8]) -> Option, + { let mut cost = OperationCost::default(); - let (tree, child) = cost_return_on_error!(&mut cost, self.detach_expect(left)); - let (child, maybe_grandchild) = cost_return_on_error!(&mut cost, child.detach(!left)); + let (tree, child) = + cost_return_on_error!(&mut cost, self.detach_expect(left, value_defined_cost_fn)); + let (child, maybe_grandchild) = + cost_return_on_error!(&mut cost, child.detach(!left, value_defined_cost_fn)); // attach grandchild to self tree.attach(left, maybe_grandchild) - .maybe_balance() + .maybe_balance(value_defined_cost_fn) .flat_map_ok(|tree| { // attach self to child, return child - child.attach(!left, Some(tree)).maybe_balance() + child + .attach(!left, Some(tree)) + .maybe_balance(value_defined_cost_fn) }) .add_cost(cost) } /// Removes the root node from the tree. Rearranges and rebalances /// descendants (if any) in order to maintain a valid tree. - pub fn remove(self) -> CostResult, Error> { + pub fn remove(self, value_defined_cost_fn: Option<&V>) -> CostResult, Error> + where + V: Fn(&[u8]) -> Option, + { let mut cost = OperationCost::default(); let tree = self.tree(); @@ -769,14 +823,20 @@ where let maybe_tree = if has_left && has_right { // two children, promote edge of taller child - let (tree, tall_child) = cost_return_on_error!(&mut cost, self.detach_expect(left)); - let (_, short_child) = cost_return_on_error!(&mut cost, tree.detach_expect(!left)); - let promoted = - cost_return_on_error!(&mut cost, tall_child.promote_edge(!left, short_child)); + let (tree, tall_child) = + cost_return_on_error!(&mut cost, self.detach_expect(left, value_defined_cost_fn)); + let (_, short_child) = + cost_return_on_error!(&mut cost, tree.detach_expect(!left, value_defined_cost_fn)); + let promoted = cost_return_on_error!( + &mut cost, + tall_child.promote_edge(!left, short_child, value_defined_cost_fn) + ); Some(promoted) } else if has_left || has_right { // single child, promote it - Some(cost_return_on_error!(&mut cost, self.detach_expect(left)).1) + Some( + cost_return_on_error!(&mut cost, self.detach_expect(left, value_defined_cost_fn)).1, + ) } else { // no child None @@ -789,31 +849,49 @@ where /// reattaches it at the top in order to fill in a gap when removing a root /// node from a tree with both left and right children. Attaches `attach` on /// the opposite side. Returns the promoted node. - fn promote_edge(self, left: bool, attach: Self) -> CostResult { - self.remove_edge(left).flat_map_ok(|(edge, maybe_child)| { - edge.attach(!left, maybe_child) - .attach(left, Some(attach)) - .maybe_balance() - }) + fn promote_edge( + self, + left: bool, + attach: Self, + value_defined_cost_fn: Option<&V>, + ) -> CostResult + where + V: Fn(&[u8]) -> Option, + { + self.remove_edge(left, value_defined_cost_fn) + .flat_map_ok(|(edge, maybe_child)| { + edge.attach(!left, maybe_child) + .attach(left, Some(attach)) + .maybe_balance(value_defined_cost_fn) + }) } /// Traverses to the tree's edge on the given side and detaches it /// (reattaching its child, if any, to its former parent). Return value is /// `(edge, maybe_updated_tree)`. - fn remove_edge(self, left: bool) -> CostResult<(Self, Option), Error> { + fn remove_edge( + self, + left: bool, + value_defined_cost_fn: Option<&V>, + ) -> CostResult<(Self, Option), Error> + where + V: Fn(&[u8]) -> Option, + { let mut cost = OperationCost::default(); if self.tree().link(left).is_some() { // this node is not the edge, recurse - let (tree, child) = cost_return_on_error!(&mut cost, self.detach_expect(left)); - let (edge, maybe_child) = cost_return_on_error!(&mut cost, child.remove_edge(left)); + let (tree, child) = + cost_return_on_error!(&mut cost, self.detach_expect(left, value_defined_cost_fn)); + let (edge, maybe_child) = + cost_return_on_error!(&mut cost, child.remove_edge(left, value_defined_cost_fn)); tree.attach(left, maybe_child) - .maybe_balance() + .maybe_balance(value_defined_cost_fn) .map_ok(|tree| (edge, Some(tree))) .add_cost(cost) } else { // this node is the edge, detach its child if present - self.detach(!left) + self.detach(!left, value_defined_cost_fn) } } } @@ -824,13 +902,13 @@ mod test { use super::*; use crate::{ test_utils::{apply_memonly, assert_tree_invariants, del_entry, make_tree_seq, seq_key}, - tree::{tree_feature_type::TreeFeatureType::BasicMerk, *}, + tree::{tree_feature_type::TreeFeatureType::BasicMerkNode, *}, }; #[test] fn simple_insert() { - let batch = [(b"foo2".to_vec(), Op::Put(b"bar2".to_vec(), BasicMerk))]; - let tree = TreeNode::new(b"foo".to_vec(), b"bar".to_vec(), None, BasicMerk).unwrap(); + let batch = [(b"foo2".to_vec(), Op::Put(b"bar2".to_vec(), BasicMerkNode))]; + let tree = TreeNode::new(b"foo".to_vec(), b"bar".to_vec(), None, BasicMerkNode).unwrap(); let (maybe_walker, key_updates) = Walker::new(tree, PanicSource {}) .apply_sorted_without_costs(&batch) .unwrap() @@ -845,8 +923,8 @@ mod test { #[test] fn simple_update() { - let batch = [(b"foo".to_vec(), Op::Put(b"bar2".to_vec(), BasicMerk))]; - let tree = TreeNode::new(b"foo".to_vec(), b"bar".to_vec(), None, BasicMerk).unwrap(); + let batch = [(b"foo".to_vec(), Op::Put(b"bar2".to_vec(), BasicMerkNode))]; + let tree = TreeNode::new(b"foo".to_vec(), b"bar".to_vec(), None, BasicMerkNode).unwrap(); let (maybe_walker, key_updates) = Walker::new(tree, PanicSource {}) .apply_sorted_without_costs(&batch) .unwrap() @@ -872,9 +950,10 @@ mod test { hash: [123; 32], sum: None, child_heights: (0, 0), - tree: TreeNode::new(b"foo2".to_vec(), b"bar2".to_vec(), None, BasicMerk).unwrap(), + tree: TreeNode::new(b"foo2".to_vec(), b"bar2".to_vec(), None, BasicMerkNode) + .unwrap(), }), - BasicMerk, + BasicMerkNode, ) .unwrap(); let (maybe_walker, key_updates) = Walker::new(tree, PanicSource {}) @@ -897,7 +976,7 @@ mod test { #[test] fn delete_non_existent() { let batch = [(b"foo2".to_vec(), Op::Delete)]; - let tree = TreeNode::new(b"foo".to_vec(), b"bar".to_vec(), None, BasicMerk).unwrap(); + let tree = TreeNode::new(b"foo".to_vec(), b"bar".to_vec(), None, BasicMerkNode).unwrap(); Walker::new(tree, PanicSource {}) .apply_sorted_without_costs(&batch) .unwrap() @@ -907,7 +986,7 @@ mod test { #[test] fn delete_only_node() { let batch = [(b"foo".to_vec(), Op::Delete)]; - let tree = TreeNode::new(b"foo".to_vec(), b"bar".to_vec(), None, BasicMerk).unwrap(); + let tree = TreeNode::new(b"foo".to_vec(), b"bar".to_vec(), None, BasicMerkNode).unwrap(); let (maybe_walker, key_updates) = Walker::new(tree, PanicSource {}) .apply_sorted_without_costs(&batch) .unwrap() @@ -977,11 +1056,12 @@ mod test { #[test] fn apply_empty_none() { - let (maybe_tree, key_updates) = Walker::::apply_to::, _, _, _>( + let (maybe_tree, key_updates) = Walker::::apply_to::, _, _, _, _>( None, &[], PanicSource {}, &|_, _| Ok(0), + None::<&fn(&[u8]) -> Option>, &mut |_, _, _| Ok((false, None)), &mut |_flags, key_bytes_to_remove, value_bytes_to_remove| { Ok(( @@ -999,12 +1079,13 @@ mod test { #[test] fn insert_empty_single() { - let batch = vec![(vec![0], Op::Put(vec![1], BasicMerk))]; + let batch = vec![(vec![0], Op::Put(vec![1], BasicMerkNode))]; let (maybe_tree, key_updates) = Walker::::apply_to( None, &batch, PanicSource {}, &|_, _| Ok(0), + None::<&fn(&[u8]) -> Option>, &mut |_, _, _| Ok((false, None)), &mut |_flags, key_bytes_to_remove, value_bytes_to_remove| { Ok(( @@ -1025,12 +1106,13 @@ mod test { #[test] fn insert_updated_single() { - let batch = vec![(vec![0], Op::Put(vec![1], BasicMerk))]; + let batch = vec![(vec![0], Op::Put(vec![1], BasicMerkNode))]; let (maybe_tree, key_updates) = Walker::::apply_to( None, &batch, PanicSource {}, &|_, _| Ok(0), + None::<&fn(&[u8]) -> Option>, &mut |_, _, _| Ok((false, None)), &mut |_flags, key_bytes_to_remove, value_bytes_to_remove| { Ok(( @@ -1046,14 +1128,15 @@ mod test { let maybe_walker = maybe_tree.map(|tree| Walker::::new(tree, PanicSource {})); let batch = vec![ - (vec![0], Op::Put(vec![2], BasicMerk)), - (vec![1], Op::Put(vec![2], BasicMerk)), + (vec![0], Op::Put(vec![2], BasicMerkNode)), + (vec![1], Op::Put(vec![2], BasicMerkNode)), ]; let (maybe_tree, key_updates) = Walker::::apply_to( maybe_walker, &batch, PanicSource {}, &|_, _| Ok(0), + None::<&fn(&[u8]) -> Option>, &mut |_, _, _| Ok((false, None)), &mut |_flags, key_bytes_to_remove, value_bytes_to_remove| { Ok(( @@ -1074,15 +1157,16 @@ mod test { #[test] fn insert_updated_multiple() { let batch = vec![ - (vec![0], Op::Put(vec![1], BasicMerk)), - (vec![1], Op::Put(vec![2], BasicMerk)), - (vec![2], Op::Put(vec![3], BasicMerk)), + (vec![0], Op::Put(vec![1], BasicMerkNode)), + (vec![1], Op::Put(vec![2], BasicMerkNode)), + (vec![2], Op::Put(vec![3], BasicMerkNode)), ]; let (maybe_tree, key_updates) = Walker::::apply_to( None, &batch, PanicSource {}, &|_, _| Ok(0), + None::<&fn(&[u8]) -> Option>, &mut |_, _, _| Ok((false, None)), &mut |_flags, key_bytes_to_remove, value_bytes_to_remove| { Ok(( @@ -1098,8 +1182,8 @@ mod test { let maybe_walker = maybe_tree.map(|tree| Walker::::new(tree, PanicSource {})); let batch = vec![ - (vec![0], Op::Put(vec![5], BasicMerk)), - (vec![1], Op::Put(vec![8], BasicMerk)), + (vec![0], Op::Put(vec![5], BasicMerkNode)), + (vec![1], Op::Put(vec![8], BasicMerkNode)), (vec![2], Op::Delete), ]; let (maybe_tree, key_updates) = Walker::::apply_to( @@ -1107,6 +1191,7 @@ mod test { &batch, PanicSource {}, &|_, _| Ok(0), + None::<&fn(&[u8]) -> Option>, &mut |_, _, _| Ok((false, None)), &mut |_flags, key_bytes_to_remove, value_bytes_to_remove| { Ok(( @@ -1127,8 +1212,8 @@ mod test { #[test] fn insert_root_single() { - let tree = TreeNode::new(vec![5], vec![123], None, BasicMerk).unwrap(); - let batch = vec![(vec![6], Op::Put(vec![123], BasicMerk))]; + let tree = TreeNode::new(vec![5], vec![123], None, BasicMerkNode).unwrap(); + let batch = vec![(vec![6], Op::Put(vec![123], BasicMerkNode))]; let tree = apply_memonly(tree, &batch); assert_eq!(tree.key(), &[5]); assert!(tree.child(true).is_none()); @@ -1137,10 +1222,10 @@ mod test { #[test] fn insert_root_double() { - let tree = TreeNode::new(vec![5], vec![123], None, BasicMerk).unwrap(); + let tree = TreeNode::new(vec![5], vec![123], None, BasicMerkNode).unwrap(); let batch = vec![ - (vec![4], Op::Put(vec![123], BasicMerk)), - (vec![6], Op::Put(vec![123], BasicMerk)), + (vec![4], Op::Put(vec![123], BasicMerkNode)), + (vec![6], Op::Put(vec![123], BasicMerkNode)), ]; let tree = apply_memonly(tree, &batch); assert_eq!(tree.key(), &[5]); @@ -1150,12 +1235,12 @@ mod test { #[test] fn insert_rebalance() { - let tree = TreeNode::new(vec![5], vec![123], None, BasicMerk).unwrap(); + let tree = TreeNode::new(vec![5], vec![123], None, BasicMerkNode).unwrap(); - let batch = vec![(vec![6], Op::Put(vec![123], BasicMerk))]; + let batch = vec![(vec![6], Op::Put(vec![123], BasicMerkNode))]; let tree = apply_memonly(tree, &batch); - let batch = vec![(vec![7], Op::Put(vec![123], BasicMerk))]; + let batch = vec![(vec![7], Op::Put(vec![123], BasicMerkNode))]; let tree = apply_memonly(tree, &batch); assert_eq!(tree.key(), &[6]); @@ -1165,10 +1250,10 @@ mod test { #[test] fn insert_100_sequential() { - let mut tree = TreeNode::new(vec![0], vec![123], None, BasicMerk).unwrap(); + let mut tree = TreeNode::new(vec![0], vec![123], None, BasicMerkNode).unwrap(); for i in 0..100 { - let batch = vec![(vec![i + 1], Op::Put(vec![123], BasicMerk))]; + let batch = vec![(vec![i + 1], Op::Put(vec![123], BasicMerkNode))]; tree = apply_memonly(tree, &batch); } diff --git a/merk/src/tree/tree_feature_type.rs b/merk/src/tree/tree_feature_type.rs index e99ca3106..c1fceed3d 100644 --- a/merk/src/tree/tree_feature_type.rs +++ b/merk/src/tree/tree_feature_type.rs @@ -39,16 +39,16 @@ use ed::{Decode, Encode}; use integer_encoding::{VarInt, VarIntReader, VarIntWriter}; #[cfg(any(feature = "full", feature = "verify"))] -use crate::tree::tree_feature_type::TreeFeatureType::{BasicMerk, SummedMerk}; +use crate::tree::tree_feature_type::TreeFeatureType::{BasicMerkNode, SummedMerkNode}; #[cfg(any(feature = "full", feature = "verify"))] #[derive(Copy, Clone, PartialEq, Eq, Debug)] /// Basic or summed pub enum TreeFeatureType { - /// Basic Merk - BasicMerk, - /// Summed Merk - SummedMerk(i64), + /// Basic Merk Tree Node + BasicMerkNode, + /// Summed Merk Tree Node + SummedMerkNode(i64), } #[cfg(feature = "full")] @@ -57,23 +57,23 @@ impl TreeFeatureType { /// Get length of encoded SummedMerk pub fn sum_length(&self) -> Option { match self { - BasicMerk => None, - SummedMerk(m) => Some(m.encode_var_vec().len() as u32), + BasicMerkNode => None, + SummedMerkNode(m) => Some(m.encode_var_vec().len() as u32), } } #[inline] /// Is sum feature? pub fn is_sum_feature(&self) -> bool { - matches!(self, SummedMerk(_)) + matches!(self, SummedMerkNode(_)) } #[inline] /// Get encoding cost of self pub(crate) fn encoding_cost(&self) -> usize { match self { - BasicMerk => 1, - SummedMerk(_sum) => 9, + BasicMerkNode => 1, + SummedMerkNode(_sum) => 9, } } } @@ -85,11 +85,11 @@ impl Encode for TreeFeatureType { #[inline] fn encode_into(&self, dest: &mut W) -> ed::Result<()> { match self { - BasicMerk => { + BasicMerkNode => { dest.write_all(&[0])?; Ok(()) } - SummedMerk(sum) => { + SummedMerkNode(sum) => { dest.write_all(&[1])?; dest.write_varint(sum.to_owned())?; Ok(()) @@ -100,8 +100,8 @@ impl Encode for TreeFeatureType { #[inline] fn encoding_length(&self) -> ed::Result { match self { - BasicMerk => Ok(1), - SummedMerk(sum) => { + BasicMerkNode => Ok(1), + SummedMerkNode(sum) => { let encoded_sum = sum.encode_var_vec(); // 1 for the enum type // encoded_sum.len() for the length of the encoded vector @@ -118,10 +118,10 @@ impl Decode for TreeFeatureType { let mut feature_type: [u8; 1] = [0]; input.read_exact(&mut feature_type)?; match feature_type { - [0] => Ok(BasicMerk), + [0] => Ok(BasicMerkNode), [1] => { let encoded_sum: i64 = input.read_varint()?; - Ok(SummedMerk(encoded_sum)) + Ok(SummedMerkNode(encoded_sum)) } _ => Err(ed::Error::UnexpectedByte(55)), } diff --git a/merk/src/tree/walk/fetch.rs b/merk/src/tree/walk/fetch.rs index a13be66cf..08b66d999 100644 --- a/merk/src/tree/walk/fetch.rs +++ b/merk/src/tree/walk/fetch.rs @@ -35,6 +35,8 @@ use grovedb_costs::CostResult; use super::super::{Link, TreeNode}; #[cfg(feature = "full")] use crate::error::Error; +#[cfg(feature = "full")] +use crate::tree::kv::ValueDefinedCostType; #[cfg(feature = "full")] /// A source of data to be used by the tree when encountering a pruned node. @@ -43,5 +45,9 @@ use crate::error::Error; pub trait Fetch { /// Called when the tree needs to fetch a node with the given `Link`. The /// `link` value will always be a `Link::Reference` variant. - fn fetch(&self, link: &Link) -> CostResult; + fn fetch( + &self, + link: &Link, + value_defined_cost_fn: Option<&impl Fn(&[u8]) -> Option>, + ) -> CostResult; } diff --git a/merk/src/tree/walk/mod.rs b/merk/src/tree/walk/mod.rs index d4a6a5377..35e476f33 100644 --- a/merk/src/tree/walk/mod.rs +++ b/merk/src/tree/walk/mod.rs @@ -77,7 +77,14 @@ where /// Similar to `Tree#detach`, but yields a `Walker` which fetches from the /// same source as `self`. Returned tuple is `(updated_self, /// maybe_child_walker)`. - pub fn detach(mut self, left: bool) -> CostResult<(Self, Option), Error> { + pub fn detach( + mut self, + left: bool, + value_defined_cost_fn: Option<&V>, + ) -> CostResult<(Self, Option), Error> + where + V: Fn(&[u8]) -> Option, + { let mut cost = OperationCost::default(); let link = match self.tree.link(left) { @@ -96,7 +103,10 @@ where Some(Link::Reference { .. }) => (), _ => unreachable!("Expected Some(Link::Reference)"), } - cost_return_on_error!(&mut cost, self.source.fetch(&link.unwrap())) + cost_return_on_error!( + &mut cost, + self.source.fetch(&link.unwrap(), value_defined_cost_fn) + ) }; let child = self.wrap(child); @@ -106,29 +116,44 @@ where /// Similar to `Tree#detach_expect`, but yields a `Walker` which fetches /// from the same source as `self`. Returned tuple is `(updated_self, /// child_walker)`. - pub fn detach_expect(self, left: bool) -> CostResult<(Self, Self), Error> { - self.detach(left).map_ok(|(walker, maybe_child)| { - if let Some(child) = maybe_child { - (walker, child) - } else { - panic!( - "Expected {} child, got None", - if left { "left" } else { "right" } - ); - } - }) + pub fn detach_expect( + self, + left: bool, + value_defined_cost_fn: Option<&V>, + ) -> CostResult<(Self, Self), Error> + where + V: Fn(&[u8]) -> Option, + { + self.detach(left, value_defined_cost_fn) + .map_ok(|(walker, maybe_child)| { + if let Some(child) = maybe_child { + (walker, child) + } else { + panic!( + "Expected {} child, got None", + if left { "left" } else { "right" } + ); + } + }) } /// Similar to `Tree#walk`, but yields a `Walker` which fetches from the /// same source as `self`. - pub fn walk(self, left: bool, f: F) -> CostResult + pub fn walk( + self, + left: bool, + f: F, + value_defined_cost_fn: Option<&V>, + ) -> CostResult where F: FnOnce(Option) -> CostResult, Error>, T: Into, + V: Fn(&[u8]) -> Option, { let mut cost = OperationCost::default(); - let (mut walker, maybe_child) = cost_return_on_error!(&mut cost, self.detach(left)); + let (mut walker, maybe_child) = + cost_return_on_error!(&mut cost, self.detach(left, value_defined_cost_fn)); let new_child = match f(maybe_child).unwrap_add_cost(&mut cost) { Ok(x) => x.map(|t| t.into()), Err(e) => return Err(e).wrap_with_cost(cost), @@ -139,14 +164,21 @@ where /// Similar to `Tree#walk_expect` but yields a `Walker` which fetches from /// the same source as `self`. - pub fn walk_expect(self, left: bool, f: F) -> CostResult + pub fn walk_expect( + self, + left: bool, + f: F, + value_defined_cost_fn: Option<&V>, + ) -> CostResult where F: FnOnce(Self) -> CostResult, Error>, T: Into, + V: Fn(&[u8]) -> Option, { let mut cost = OperationCost::default(); - let (mut walker, child) = cost_return_on_error!(&mut cost, self.detach_expect(left)); + let (mut walker, child) = + cost_return_on_error!(&mut cost, self.detach_expect(left, value_defined_cost_fn)); let new_child = match f(child).unwrap_add_cost(&mut cost) { Ok(x) => x.map(|t| t.into()), Err(e) => return Err(e).wrap_with_cost(cost), @@ -368,34 +400,42 @@ mod test { use grovedb_costs::{storage_cost::removal::StorageRemovedBytes::NoStorageRemoval, CostsExt}; use super::{super::NoopCommit, *}; - use crate::tree::{TreeFeatureType::BasicMerk, TreeNode}; + use crate::tree::{TreeFeatureType::BasicMerkNode, TreeNode}; #[derive(Clone)] struct MockSource {} impl Fetch for MockSource { - fn fetch(&self, link: &Link) -> CostResult { - TreeNode::new(link.key().to_vec(), b"foo".to_vec(), None, BasicMerk).map(Ok) + fn fetch( + &self, + link: &Link, + _value_defined_cost_fn: Option<&impl Fn(&[u8]) -> Option>, + ) -> CostResult { + TreeNode::new(link.key().to_vec(), b"foo".to_vec(), None, BasicMerkNode).map(Ok) } } #[test] fn walk_modified() { - let tree = TreeNode::new(b"test".to_vec(), b"abc".to_vec(), None, BasicMerk) + let tree = TreeNode::new(b"test".to_vec(), b"abc".to_vec(), None, BasicMerkNode) .unwrap() .attach( true, - Some(TreeNode::new(b"foo".to_vec(), b"bar".to_vec(), None, BasicMerk).unwrap()), + Some(TreeNode::new(b"foo".to_vec(), b"bar".to_vec(), None, BasicMerkNode).unwrap()), ); let source = MockSource {}; let walker = Walker::new(tree, source); let walker = walker - .walk(true, |child| -> CostResult, Error> { - assert_eq!(child.expect("should have child").tree().key(), b"foo"); - Ok(None).wrap_with_cost(Default::default()) - }) + .walk( + true, + |child| -> CostResult, Error> { + assert_eq!(child.expect("should have child").tree().key(), b"foo"); + Ok(None).wrap_with_cost(Default::default()) + }, + None::<&fn(&[u8]) -> Option>, + ) .unwrap() .expect("walk failed"); assert!(walker.into_inner().child(true).is_none()); @@ -403,11 +443,11 @@ mod test { #[test] fn walk_stored() { - let mut tree = TreeNode::new(b"test".to_vec(), b"abc".to_vec(), None, BasicMerk) + let mut tree = TreeNode::new(b"test".to_vec(), b"abc".to_vec(), None, BasicMerkNode) .unwrap() .attach( true, - Some(TreeNode::new(b"foo".to_vec(), b"bar".to_vec(), None, BasicMerk).unwrap()), + Some(TreeNode::new(b"foo".to_vec(), b"bar".to_vec(), None, BasicMerkNode).unwrap()), ); tree.commit(&mut NoopCommit {}, &|_, _| Ok(0)) .unwrap() @@ -417,10 +457,14 @@ mod test { let walker = Walker::new(tree, source); let walker = walker - .walk(true, |child| -> CostResult, Error> { - assert_eq!(child.expect("should have child").tree().key(), b"foo"); - Ok(None).wrap_with_cost(Default::default()) - }) + .walk( + true, + |child| -> CostResult, Error> { + assert_eq!(child.expect("should have child").tree().key(), b"foo"); + Ok(None).wrap_with_cost(Default::default()) + }, + None::<&fn(&[u8]) -> Option>, + ) .unwrap() .expect("walk failed"); assert!(walker.into_inner().child(true).is_none()); @@ -439,7 +483,7 @@ mod test { sum: None, }), None, - BasicMerk, + BasicMerkNode, ) .unwrap(); @@ -447,10 +491,14 @@ mod test { let walker = Walker::new(tree, source); let walker = walker - .walk_expect(true, |child| -> CostResult, Error> { - assert_eq!(child.tree().key(), b"foo"); - Ok(None).wrap_with_cost(Default::default()) - }) + .walk_expect( + true, + |child| -> CostResult, Error> { + assert_eq!(child.tree().key(), b"foo"); + Ok(None).wrap_with_cost(Default::default()) + }, + None::<&fn(&[u8]) -> Option>, + ) .unwrap() .expect("walk failed"); assert!(walker.into_inner().child(true).is_none()); @@ -458,16 +506,20 @@ mod test { #[test] fn walk_none() { - let tree = TreeNode::new(b"test".to_vec(), b"abc".to_vec(), None, BasicMerk).unwrap(); + let tree = TreeNode::new(b"test".to_vec(), b"abc".to_vec(), None, BasicMerkNode).unwrap(); let source = MockSource {}; let walker = Walker::new(tree, source); walker - .walk(true, |child| -> CostResult, Error> { - assert!(child.is_none()); - Ok(None).wrap_with_cost(Default::default()) - }) + .walk( + true, + |child| -> CostResult, Error> { + assert!(child.is_none()); + Ok(None).wrap_with_cost(Default::default()) + }, + None::<&fn(&[u8]) -> Option>, + ) .unwrap() .expect("walk failed"); } diff --git a/merk/src/tree/walk/ref_walker.rs b/merk/src/tree/walk/ref_walker.rs index 26c23b0a6..d9fb1bcd0 100644 --- a/merk/src/tree/walk/ref_walker.rs +++ b/merk/src/tree/walk/ref_walker.rs @@ -36,6 +36,7 @@ use super::{ super::{Link, TreeNode}, Fetch, }; +use crate::tree::kv::ValueDefinedCostType; #[cfg(feature = "full")] use crate::Error; @@ -73,7 +74,14 @@ where /// Traverses to the child on the given side (if any), fetching from the /// source if pruned. When fetching, the link is upgraded from /// `Link::Reference` to `Link::Loaded`. - pub fn walk(&mut self, left: bool) -> CostResult>, Error> { + pub fn walk( + &mut self, + left: bool, + value_defined_cost_fn: Option<&V>, + ) -> CostResult>, Error> + where + V: Fn(&[u8]) -> Option, + { let link = match self.tree.link(left) { None => return Ok(None).wrap_with_cost(Default::default()), Some(link) => link, @@ -84,7 +92,7 @@ where Link::Reference { .. } => { let load_res = self .tree - .load(left, &self.source) + .load(left, &self.source, value_defined_cost_fn) .unwrap_add_cost(&mut cost); if let Err(e) = load_res { return Err(e).wrap_with_cost(cost); From 619f5344252c132edee85ade94b3fa52fc0b90a8 Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Wed, 27 Sep 2023 10:43:30 +0700 Subject: [PATCH 08/18] more fixes --- grovedb/src/batch/estimated_costs/average_case_costs.rs | 2 +- grovedb/src/batch/single_insert_cost_tests.rs | 5 +---- grovedb/src/batch/single_sum_item_insert_cost_tests.rs | 4 ++-- grovedb/src/operations/insert/mod.rs | 6 +++--- merk/src/tree/encoding.rs | 6 ++++-- 5 files changed, 11 insertions(+), 12 deletions(-) diff --git a/grovedb/src/batch/estimated_costs/average_case_costs.rs b/grovedb/src/batch/estimated_costs/average_case_costs.rs index e6e58975c..f3421e87b 100644 --- a/grovedb/src/batch/estimated_costs/average_case_costs.rs +++ b/grovedb/src/batch/estimated_costs/average_case_costs.rs @@ -576,7 +576,7 @@ mod tests { seek_count: 5, // todo: why is this 5 storage_cost: StorageCost { added_bytes: 115, - replaced_bytes: 106, + replaced_bytes: 75, removed_bytes: NoStorageRemoval, }, storage_loaded_bytes: 109, diff --git a/grovedb/src/batch/single_insert_cost_tests.rs b/grovedb/src/batch/single_insert_cost_tests.rs index c7f791464..1dd2d43c2 100644 --- a/grovedb/src/batch/single_insert_cost_tests.rs +++ b/grovedb/src/batch/single_insert_cost_tests.rs @@ -335,9 +335,6 @@ mod tests { // Replaced bytes // 37 + 36 = 74 (key is not replaced) //needs update - // We instead are getting 106, because we are paying for (+ hash - key byte - // size) this means 31 extra bytes. - // In reality though we really are replacing 106 bytes. TBD what to do. // Hash node calls 8 // 1 to get tree hash @@ -359,7 +356,7 @@ mod tests { seek_count: 5, storage_cost: StorageCost { added_bytes: 115, - replaced_bytes: 106, // todo: this should actually be less + replaced_bytes: 75, removed_bytes: NoStorageRemoval, }, storage_loaded_bytes: 71, // todo: verify and explain diff --git a/grovedb/src/batch/single_sum_item_insert_cost_tests.rs b/grovedb/src/batch/single_sum_item_insert_cost_tests.rs index 09ca5ee20..d1e13fead 100644 --- a/grovedb/src/batch/single_sum_item_insert_cost_tests.rs +++ b/grovedb/src/batch/single_sum_item_insert_cost_tests.rs @@ -209,7 +209,7 @@ mod tests { seek_count: 5, storage_cost: StorageCost { added_bytes: 124, - replaced_bytes: 106, // todo: this should actually be less + replaced_bytes: 75, removed_bytes: NoStorageRemoval, }, storage_loaded_bytes: 71, // todo: verify and explain @@ -287,7 +287,7 @@ mod tests { seek_count: 5, storage_cost: StorageCost { added_bytes: 124, - replaced_bytes: 107, // todo: this should actually be less + replaced_bytes: 84, removed_bytes: NoStorageRemoval, }, storage_loaded_bytes: 72, // todo: verify and explain diff --git a/grovedb/src/operations/insert/mod.rs b/grovedb/src/operations/insert/mod.rs index d691ea346..513e20981 100644 --- a/grovedb/src/operations/insert/mod.rs +++ b/grovedb/src/operations/insert/mod.rs @@ -886,7 +886,7 @@ mod tests { seek_count: 7, storage_cost: StorageCost { added_bytes: 170, - replaced_bytes: 209, // todo: verify + replaced_bytes: 217, removed_bytes: NoStorageRemoval }, storage_loaded_bytes: 232, @@ -955,7 +955,7 @@ mod tests { seek_count: 7, storage_cost: StorageCost { added_bytes: 170, - replaced_bytes: 211, // todo: verify + replaced_bytes: 217, // todo: verify removed_bytes: NoStorageRemoval }, storage_loaded_bytes: 237, @@ -1720,7 +1720,7 @@ mod tests { seek_count: 9, // todo: verify this storage_cost: StorageCost { added_bytes: 0, - replaced_bytes: 405, // todo: verify this + replaced_bytes: 409, // todo: verify this removed_bytes: NoStorageRemoval }, storage_loaded_bytes: 487, // todo verify this diff --git a/merk/src/tree/encoding.rs b/merk/src/tree/encoding.rs index e635cd8c5..057880999 100644 --- a/merk/src/tree/encoding.rs +++ b/merk/src/tree/encoding.rs @@ -129,7 +129,8 @@ impl TreeNode { let mut tree_inner: TreeNodeInner = Decode::decode(input)?; tree_inner.kv.key = key; if let Some(value_defined_cost_fn) = value_defined_cost_fn { - tree_inner.kv.value_defined_cost = value_defined_cost_fn(input); + tree_inner.kv.value_defined_cost = + value_defined_cost_fn(tree_inner.kv.value.as_slice()); } self.inner = Box::new(tree_inner); Ok(()) @@ -145,7 +146,8 @@ impl TreeNode { let mut tree_inner: TreeNodeInner = Decode::decode(input)?; tree_inner.kv.key = key; if let Some(value_defined_cost_fn) = value_defined_cost_fn { - tree_inner.kv.value_defined_cost = value_defined_cost_fn(input); + tree_inner.kv.value_defined_cost = + value_defined_cost_fn(tree_inner.kv.value.as_slice()); } Ok(TreeNode::new_with_tree_inner(tree_inner)) } From e3d4866543fed54b594147633b0ef49c296ab5a3 Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Wed, 27 Sep 2023 11:22:40 +0700 Subject: [PATCH 09/18] grovedb working again --- .../src/batch/estimated_costs/average_case_costs.rs | 2 +- merk/src/estimated_costs/average_case_costs.rs | 10 +++++----- merk/src/merk/apply.rs | 9 +++++++-- 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/grovedb/src/batch/estimated_costs/average_case_costs.rs b/grovedb/src/batch/estimated_costs/average_case_costs.rs index f3421e87b..646537930 100644 --- a/grovedb/src/batch/estimated_costs/average_case_costs.rs +++ b/grovedb/src/batch/estimated_costs/average_case_costs.rs @@ -723,7 +723,7 @@ mod tests { seek_count: 41, storage_cost: StorageCost { added_bytes: 0, - replaced_bytes: 5625, + replaced_bytes: 5594, removed_bytes: NoStorageRemoval, }, storage_loaded_bytes: 7669, diff --git a/merk/src/estimated_costs/average_case_costs.rs b/merk/src/estimated_costs/average_case_costs.rs index 53d6cc9b5..099978faa 100644 --- a/merk/src/estimated_costs/average_case_costs.rs +++ b/merk/src/estimated_costs/average_case_costs.rs @@ -461,16 +461,16 @@ pub fn add_average_case_merk_propagate( estimated_sum_trees, average_flags_size, ) => { - let flags_len = average_flags_size.unwrap_or(0); - // it is normal to have LAYER_COST_SIZE here, as we add estimated sum tree // additions right after - let value_len = LAYER_COST_SIZE + flags_len; + let value_len = LAYER_COST_SIZE + + average_flags_size + .map_or(0, |flags_len| flags_len + flags_len.required_space() as u32); // in order to simplify calculations we get the estimated size and remove the // cost for the basic merk let sum_tree_addition = estimated_sum_trees.estimated_size()?; nodes_updated - * (KV::value_byte_cost_size_for_key_and_raw_value_lengths( + * (KV::layered_value_byte_cost_size_for_key_and_value_lengths( *average_key_size as u32, value_len, *is_sum_tree, @@ -520,7 +520,7 @@ pub fn add_average_case_merk_propagate( let flags_len = average_flags_size.unwrap_or(0); let value_len = LAYER_COST_SIZE + flags_len; let sum_tree_addition = estimated_sum_trees.estimated_size()?; - let cost = KV::value_byte_cost_size_for_key_and_raw_value_lengths( + let cost = KV::layered_value_byte_cost_size_for_key_and_value_lengths( *average_key_size as u32, value_len, in_sum_tree, diff --git a/merk/src/merk/apply.rs b/merk/src/merk/apply.rs index 4418dfc11..a33dfcc4e 100644 --- a/merk/src/merk/apply.rs +++ b/merk/src/merk/apply.rs @@ -146,12 +146,14 @@ where /// &[], /// None, /// &|k, v| Ok(0), + /// None::<&fn(&[u8]) -> Option>, /// &mut |s, v, o| Ok((false, None)), /// &mut |s, k, v| Ok((NoStorageRemoval, NoStorageRemoval)) /// ).unwrap().expect(""); /// /// use grovedb_costs::storage_cost::removal::StorageRemovedBytes::NoStorageRemoval; /// use grovedb_merk::Op; + /// use grovedb_merk::tree::kv::ValueDefinedCostType; /// use grovedb_merk::TreeFeatureType::BasicMerkNode; /// /// let batch = &[ @@ -166,6 +168,7 @@ where /// &[], /// None, /// &|k, v| Ok(0), + /// None::<&fn(&[u8]) -> Option>, /// &mut |s, v, o| Ok((false, None)), /// &mut |s, k, v| Ok((NoStorageRemoval, NoStorageRemoval)) /// ).unwrap().expect(""); @@ -244,12 +247,14 @@ where /// &[], /// None, /// &|k, v| Ok(0), + /// None::<&fn(&[u8]) -> Option>, /// &mut |s, o, v| Ok((false, None)), /// &mut |s, k, v| Ok((NoStorageRemoval, NoStorageRemoval)) /// ).unwrap().expect(""); /// /// use grovedb_costs::storage_cost::removal::StorageRemovedBytes::NoStorageRemoval; /// use grovedb_merk::Op; + /// use grovedb_merk::tree::kv::ValueDefinedCostType; /// use grovedb_merk::TreeFeatureType::BasicMerkNode; /// /// let batch = &[ @@ -258,12 +263,12 @@ where /// // deletes key [4,5,6] /// (vec![4, 5, 6], Op::Delete), /// ]; - /// unsafe { store.apply_unchecked::<_, Vec<_>, _, _, _>( /// /// /// + /// unsafe { store.apply_unchecked::<_, Vec<_>, _, _, _, _>( /// /// /// /// batch, /// &[], /// None, /// &|k, v| Ok(0), - /// &|v| None, + /// None::<&fn(&[u8]) -> Option>, /// &mut |s, o, v| Ok((false, None)), /// &mut |s, k, v| Ok((NoStorageRemoval, NoStorageRemoval)) /// ).unwrap().expect(""); From 79f8334e647b19348ab5ade373178c3479590bca Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Wed, 27 Sep 2023 13:13:30 +0700 Subject: [PATCH 10/18] fixed proof feature --- grovedb/src/element/helpers.rs | 1 + grovedb/src/lib.rs | 1 + merk/src/proofs/query/mod.rs | 1 + merk/src/tree/mod.rs | 1 + 4 files changed, 4 insertions(+) diff --git a/grovedb/src/element/helpers.rs b/grovedb/src/element/helpers.rs index 85ce8f4e4..91d57fe33 100644 --- a/grovedb/src/element/helpers.rs +++ b/grovedb/src/element/helpers.rs @@ -29,6 +29,7 @@ //! Helpers //! Implements helper functions in Element +#[cfg(feature = "full")] use grovedb_merk::tree::kv::{ ValueDefinedCostType, ValueDefinedCostType::{LayeredValueDefinedCost, SpecializedValueDefinedCost}, diff --git a/grovedb/src/lib.rs b/grovedb/src/lib.rs index 30ca2e916..8868b5983 100644 --- a/grovedb/src/lib.rs +++ b/grovedb/src/lib.rs @@ -193,6 +193,7 @@ pub use grovedb_merk::estimated_costs::{ pub use grovedb_merk::proofs::query::query_item::QueryItem; #[cfg(any(feature = "full", feature = "verify"))] pub use grovedb_merk::proofs::Query; +#[cfg(feature = "full")] use grovedb_merk::tree::kv::ValueDefinedCostType; #[cfg(feature = "full")] use grovedb_merk::{ diff --git a/merk/src/proofs/query/mod.rs b/merk/src/proofs/query/mod.rs index 40aea9059..7a122f23a 100644 --- a/merk/src/proofs/query/mod.rs +++ b/merk/src/proofs/query/mod.rs @@ -67,6 +67,7 @@ use {super::Op, std::collections::LinkedList}; use super::Node; #[cfg(any(feature = "full", feature = "verify"))] use crate::error::Error; +#[cfg(feature = "full")] use crate::tree::kv::ValueDefinedCostType; #[cfg(feature = "full")] use crate::tree::{Fetch, Link, RefWalker}; diff --git a/merk/src/tree/mod.rs b/merk/src/tree/mod.rs index c57d36447..b257d8db1 100644 --- a/merk/src/tree/mod.rs +++ b/merk/src/tree/mod.rs @@ -92,6 +92,7 @@ pub use walk::{Fetch, RefWalker, Walker}; #[cfg(feature = "full")] use crate::tree::kv::ValueDefinedCostType; +#[cfg(feature = "full")] use crate::tree::kv::ValueDefinedCostType::{LayeredValueDefinedCost, SpecializedValueDefinedCost}; #[cfg(feature = "full")] use crate::{error::Error, Error::Overflow}; From 6bef9c9af81d39a1da9328521dcdaa4b9cea892d Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Wed, 27 Sep 2023 14:00:54 +0700 Subject: [PATCH 11/18] fixed bench --- grovedb/src/operations/proof/verify.rs | 2 +- grovedb/src/tests/mod.rs | 2 +- grovedb/src/versioning.rs | 8 +-- merk/benches/merk.rs | 68 ++++++++++++++++++-------- merk/benches/ops.rs | 7 ++- merk/src/merk/committer.rs | 2 - merk/src/merk/mod.rs | 11 ++--- merk/src/proofs/chunk.rs | 1 - merk/src/proofs/query/mod.rs | 1 - merk/src/tree/commit.rs | 4 -- merk/src/tree/mod.rs | 1 - merk/src/tree/ops.rs | 2 +- merk/src/tree/walk/mod.rs | 4 +- storage/src/rocksdb_storage/storage.rs | 5 +- 14 files changed, 69 insertions(+), 49 deletions(-) diff --git a/grovedb/src/operations/proof/verify.rs b/grovedb/src/operations/proof/verify.rs index ac13306cf..9e8c6e442 100644 --- a/grovedb/src/operations/proof/verify.rs +++ b/grovedb/src/operations/proof/verify.rs @@ -260,7 +260,7 @@ impl ProofVerifier { query: &PathQuery, is_verbose: bool, ) -> Result<[u8; 32], Error> { - let (proof_version, proof) = read_and_consume_proof_version(proof)?; + let (_proof_version, proof) = read_and_consume_proof_version(proof)?; let mut proof_reader = ProofReader::new_with_verbose_status(proof, is_verbose); let path_slices = query.path.iter().map(|x| x.as_slice()).collect::>(); diff --git a/grovedb/src/tests/mod.rs b/grovedb/src/tests/mod.rs index 922f5e9b0..451b2307a 100644 --- a/grovedb/src/tests/mod.rs +++ b/grovedb/src/tests/mod.rs @@ -2929,7 +2929,7 @@ fn test_tree_value_exists_method_tx() { #[test] fn test_storage_wipe() { let db = make_test_grovedb(); - let path = db._tmp_dir.path(); + let _path = db._tmp_dir.path(); // Test keys in non-root tree db.insert( diff --git a/grovedb/src/versioning.rs b/grovedb/src/versioning.rs index ca9a29247..a041b3d8a 100644 --- a/grovedb/src/versioning.rs +++ b/grovedb/src/versioning.rs @@ -1,8 +1,8 @@ use std::io::Cursor; -use integer_encoding::{VarInt, VarIntReader, VarIntWriter}; +use integer_encoding::{VarInt, VarIntReader}; -use crate::{Error, Error::InternalError}; +use crate::Error; pub(crate) const PROOF_VERSION: u32 = 1; @@ -44,11 +44,11 @@ mod tests { #[test] fn read_correct_version() { - let mut data = vec![1, 2, 3]; + let data = vec![1, 2, 3]; let version = 500_u32; // prepend the version information to the data vector - let mut new_data = prepend_version_to_bytes(data, version).unwrap(); + let new_data = prepend_version_to_bytes(data, version).unwrap(); assert_eq!(new_data, [244, 3, 1, 2, 3]); // show that read_version doesn't consume diff --git a/merk/benches/merk.rs b/merk/benches/merk.rs index 7846a78ce..b0f9cca44 100644 --- a/merk/benches/merk.rs +++ b/merk/benches/merk.rs @@ -30,9 +30,14 @@ use criterion::{criterion_group, criterion_main, BatchSize, Criterion}; use grovedb_costs::storage_cost::removal::StorageRemovedBytes::BasicStorageRemoval; +use grovedb_merk::{ + proofs, + test_utils::{make_batch_rand, make_batch_seq, make_del_batch_rand, TempMerk}, + tree::kv::ValueDefinedCostType, + Merk, +}; use grovedb_path::SubtreePath; use grovedb_storage::{rocksdb_storage::test_utils::TempStorage, Storage}; -use merk::{proofs::encode_into as encode_proof_into, test_utils::*, Merk}; use rand::prelude::*; /// 1 million gets in 2k batches @@ -46,11 +51,12 @@ pub fn get(c: &mut Criterion) { let mut batches = vec![]; for i in 0..num_batches { let batch = make_batch_rand(batch_size, i); - merk.apply_unchecked::<_, Vec, _, _, _>( + merk.apply_unchecked::<_, Vec, _, _, _, _>( &batch, &[], None, &|_k, _v| Ok(0), + None::<&fn(&[u8]) -> Option>, &mut |_costs, _old_value, _value| Ok((false, None)), &mut |_a, key_bytes_to_remove, value_bytes_to_remove| { Ok(( @@ -72,7 +78,9 @@ pub fn get(c: &mut Criterion) { let key_index = (i / num_batches) as usize; let key = &batches[batch_index][key_index].0; - merk.get(key, true).unwrap().expect("get failed"); + merk.get(key, true, None:: Option>) + .unwrap() + .expect("get failed"); i = (i + 1) % initial_size; }) @@ -98,11 +106,12 @@ pub fn insert_1m_2k_seq(c: &mut Criterion) { b.iter_with_large_drop(|| { let batch = &batches[i % n_batches]; - merk.apply_unchecked::<_, Vec, _, _, _>( + merk.apply_unchecked::<_, Vec, _, _, _, _>( batch, &[], None, &|_k, _v| Ok(0), + None::<&fn(&[u8]) -> Option>, &mut |_costs, _old_value, _value| Ok((false, None)), &mut |_a, key_bytes_to_remove, value_bytes_to_remove| { Ok(( @@ -137,11 +146,12 @@ pub fn insert_1m_2k_rand(c: &mut Criterion) { b.iter_with_large_drop(|| { let batch = &batches[i % n_batches]; - merk.apply_unchecked::<_, Vec, _, _, _>( + merk.apply_unchecked::<_, Vec, _, _, _, _>( batch, &[], None, &|_k, _v| Ok(0), + None::<&fn(&[u8]) -> Option>, &mut |_costs, _old_value, _value| Ok((false, None)), &mut |_a, key_bytes_to_remove, value_bytes_to_remove| { Ok(( @@ -169,11 +179,12 @@ pub fn update_1m_2k_seq(c: &mut Criterion) { for i in 0..n_batches { let batch = make_batch_seq(((i * batch_size) as u64)..((i + 1) * batch_size) as u64); - merk.apply_unchecked::<_, Vec, _, _, _>( + merk.apply_unchecked::<_, Vec, _, _, _, _>( &batch, &[], None, &|_k, _v| Ok(0), + None::<&fn(&[u8]) -> Option>, &mut |_costs, _old_value, _value| Ok((false, None)), &mut |_a, key_bytes_to_remove, value_bytes_to_remove| { Ok(( @@ -193,11 +204,12 @@ pub fn update_1m_2k_seq(c: &mut Criterion) { b.iter_with_large_drop(|| { let batch = &batches[i % n_batches]; - merk.apply_unchecked::<_, Vec, _, _, _>( + merk.apply_unchecked::<_, Vec, _, _, _, _>( batch, &[], None, &|_k, _v| Ok(0), + None::<&fn(&[u8]) -> Option>, &mut |_costs, _old_value, _value| Ok((false, None)), &mut |_a, key_bytes_to_remove, value_bytes_to_remove| { Ok(( @@ -225,11 +237,12 @@ pub fn update_1m_2k_rand(c: &mut Criterion) { for i in 0..n_batches { let batch = make_batch_rand(batch_size as u64, i as u64); - merk.apply_unchecked::<_, Vec, _, _, _>( + merk.apply_unchecked::<_, Vec, _, _, _, _>( &batch, &[], None, &|_k, _v| Ok(0), + None::<&fn(&[u8]) -> Option>, &mut |_costs, _old_value, _value| Ok((false, None)), &mut |_a, key_bytes_to_remove, value_bytes_to_remove| { Ok(( @@ -249,11 +262,12 @@ pub fn update_1m_2k_rand(c: &mut Criterion) { b.iter_with_large_drop(|| { let batch = &batches[i % n_batches]; - merk.apply_unchecked::<_, Vec, _, _, _>( + merk.apply_unchecked::<_, Vec, _, _, _, _>( batch, &[], None, &|_k, _v| Ok(0), + None::<&fn(&[u8]) -> Option>, &mut |_costs, _old_value, _value| Ok((false, None)), &mut |_a, key_bytes_to_remove, value_bytes_to_remove| { Ok(( @@ -283,11 +297,12 @@ pub fn delete_1m_2k_rand(c: &mut Criterion) { for i in 0..n_batches { let batch = make_batch_rand(batch_size as u64, i as u64); let delete_batch = make_del_batch_rand(batch_size as u64, i as u64); - merk.apply_unchecked::<_, Vec, _, _, _>( + merk.apply_unchecked::<_, Vec, _, _, _, _>( &batch, &[], None, &|_k, _v| Ok(0), + None::<&fn(&[u8]) -> Option>, &mut |_costs, _old_value, _value| Ok((false, None)), &mut |_a, key_bytes_to_remove, value_bytes_to_remove| { Ok(( @@ -311,11 +326,12 @@ pub fn delete_1m_2k_rand(c: &mut Criterion) { // Merk tree is kept with 1m elements before each bench iteration for more or // less same inputs. - merk.apply_unchecked::<_, Vec, _, _, _>( + merk.apply_unchecked::<_, Vec, _, _, _, _>( insert_batch, &[], None, &|_k, _v| Ok(0), + None::<&fn(&[u8]) -> Option>, &mut |_costs, _old_value, _value| Ok((false, None)), &mut |_a, key_bytes_to_remove, value_bytes_to_remove| { Ok(( @@ -328,11 +344,12 @@ pub fn delete_1m_2k_rand(c: &mut Criterion) { .expect("apply failed"); b.iter_with_large_drop(|| { - merk.apply_unchecked::<_, Vec, _, _, _>( + merk.apply_unchecked::<_, Vec, _, _, _, _>( delete_batch, &[], None, &|_k, _v| Ok(0), + None::<&fn(&[u8]) -> Option>, &mut |_costs, _old_value, _value| Ok((false, None)), &mut |_a, key_bytes_to_remove, value_bytes_to_remove| { Ok(( @@ -361,11 +378,12 @@ pub fn prove_1m_2k_rand(c: &mut Criterion) { for i in 0..n_batches { let batch = make_batch_rand(batch_size as u64, i as u64); - merk.apply_unchecked::<_, Vec, _, _, _>( + merk.apply_unchecked::<_, Vec, _, _, _, _>( &batch, &[], None, &|_k, _v| Ok(0), + None::<&fn(&[u8]) -> Option>, &mut |_costs, _old_value, _value| Ok((false, None)), &mut |_a, key_bytes_to_remove, value_bytes_to_remove| { Ok(( @@ -378,7 +396,7 @@ pub fn prove_1m_2k_rand(c: &mut Criterion) { .expect("apply failed"); let mut prove_keys = Vec::with_capacity(batch_size); for (key, _) in batch.iter() { - prove_keys.push(merk::proofs::query::query_item::QueryItem::Key(key.clone())); + prove_keys.push(proofs::query::query_item::QueryItem::Key(key.clone())); } prove_keys_per_batch.push(prove_keys); batches.push(batch); @@ -409,11 +427,12 @@ pub fn build_trunk_chunk_1m_2k_rand(c: &mut Criterion) { for i in 0..n_batches { let batch = make_batch_rand(batch_size as u64, i as u64); - merk.apply_unchecked::<_, Vec, _, _, _>( + merk.apply_unchecked::<_, Vec, _, _, _, _>( &batch, &[], None, &|_k, _v| Ok(0), + None::<&fn(&[u8]) -> Option>, &mut |_costs, _old_value, _value| Ok((false, None)), &mut |_a, key_bytes_to_remove, value_bytes_to_remove| { Ok(( @@ -434,7 +453,7 @@ pub fn build_trunk_chunk_1m_2k_rand(c: &mut Criterion) { let (ops, _) = merk.walk(|walker| walker.unwrap().create_trunk_proof().unwrap().unwrap()); - encode_proof_into(ops.iter(), &mut bytes); + proofs::encode_into(ops.iter(), &mut bytes); }); }); } @@ -450,11 +469,12 @@ pub fn chunkproducer_rand_1m_1_rand(c: &mut Criterion) { for i in 0..n_batches { let batch = make_batch_rand(batch_size as u64, i as u64); - merk.apply_unchecked::<_, Vec, _, _, _>( + merk.apply_unchecked::<_, Vec, _, _, _, _>( &batch, &[], None, &|_k, _v| Ok(0), + None::<&fn(&[u8]) -> Option>, &mut |_costs, _old_value, _value| Ok((false, None)), &mut |_a, key_bytes_to_remove, value_bytes_to_remove| { Ok(( @@ -489,11 +509,12 @@ pub fn chunk_iter_1m_1(c: &mut Criterion) { for i in 0..n_batches { let batch = make_batch_rand(batch_size as u64, i as u64); - merk.apply_unchecked::<_, Vec, _, _, _>( + merk.apply_unchecked::<_, Vec, _, _, _, _>( &batch, &[], None, &|_k, _v| Ok(0), + None::<&fn(&[u8]) -> Option>, &mut |_costs, _old_value, _value| Ok((false, None)), &mut |_a, key_bytes_to_remove, value_bytes_to_remove| { Ok(( @@ -530,11 +551,12 @@ pub fn restore_500_1(c: &mut Criterion) { let mut merk = TempMerk::new(); let batch = make_batch_rand(merk_size as u64, 0_u64); - merk.apply_unchecked::<_, Vec, _, _, _>( + merk.apply_unchecked::<_, Vec, _, _, _, _>( &batch, &[], None, &|_k, _v| Ok(0), + None::<&fn(&[u8]) -> Option>, &mut |_costs, _old_value, _value| Ok((false, None)), &mut |_a, key_bytes_to_remove, value_bytes_to_remove| { Ok(( @@ -560,7 +582,13 @@ pub fn restore_500_1(c: &mut Criterion) { .0 .get_immediate_storage_context(SubtreePath::empty(), &tx) .unwrap(); - let m = Merk::open_standalone(ctx, false).unwrap().unwrap(); + let m = Merk::open_standalone( + ctx, + false, + None::<&fn(&[u8]) -> Option>, + ) + .unwrap() + .unwrap(); let mut restorer = Merk::restore(m, root_hash); for chunk in data.1 { diff --git a/merk/benches/ops.rs b/merk/benches/ops.rs index a1fa8bf26..f9576fbac 100644 --- a/merk/benches/ops.rs +++ b/merk/benches/ops.rs @@ -29,7 +29,12 @@ //! Merk benches ops use criterion::{criterion_group, criterion_main, Criterion}; -use merk::{owner::Owner, test_utils::*}; +use grovedb_merk::{ + owner::Owner, + test_utils::{ + apply_memonly_unchecked, make_batch_rand, make_batch_seq, make_tree_rand, make_tree_seq, + }, +}; /// 1m sequential inserts in 10k batches, memonly fn insert_1m_10k_seq_memonly(c: &mut Criterion) { diff --git a/merk/src/merk/committer.rs b/merk/src/merk/committer.rs index af460bd80..9fb029875 100644 --- a/merk/src/merk/committer.rs +++ b/merk/src/merk/committer.rs @@ -1,5 +1,3 @@ -use grovedb_costs::storage_cost::key_value_cost::KeyValueStorageCost; - use crate::{ merk::BatchValue, tree::{Commit, TreeNode}, diff --git a/merk/src/merk/mod.rs b/merk/src/merk/mod.rs index 9c025dcb9..92f7cc93d 100644 --- a/merk/src/merk/mod.rs +++ b/merk/src/merk/mod.rs @@ -52,10 +52,8 @@ use std::{ use committer::MerkCommitter; use grovedb_costs::{ cost_return_on_error, cost_return_on_error_default, cost_return_on_error_no_add, - storage_cost::{ - key_value_cost::KeyValueStorageCost, removal::StorageRemovedBytes, StorageCost, - }, - ChildrenSizesWithValue, CostContext, CostResult, CostsExt, FeatureSumLength, OperationCost, + storage_cost::key_value_cost::KeyValueStorageCost, ChildrenSizesWithValue, CostContext, + CostResult, CostsExt, FeatureSumLength, OperationCost, }; use grovedb_storage::{self, Batch, RawIterator, StorageContext}; use source::MerkSource; @@ -63,10 +61,9 @@ use source::MerkSource; use crate::{ error::Error, merk::{defaults::ROOT_KEY_KEY, options::MerkOptions}, - proofs::{query::query_item::QueryItem, Op as ProofOp, Query}, + proofs::{query::query_item::QueryItem, Query}, tree::{ - kv::ValueDefinedCostType, AuxMerkBatch, Commit, CryptoHash, Fetch, Op, RefWalker, TreeNode, - NULL_HASH, + kv::ValueDefinedCostType, AuxMerkBatch, CryptoHash, Op, RefWalker, TreeNode, NULL_HASH, }, Error::{CostsError, EdError, StorageError}, MerkType::{BaseMerk, LayeredMerk, StandaloneMerk}, diff --git a/merk/src/proofs/chunk.rs b/merk/src/proofs/chunk.rs index 5e852ec13..1e3b9fb12 100644 --- a/merk/src/proofs/chunk.rs +++ b/merk/src/proofs/chunk.rs @@ -397,7 +397,6 @@ pub(crate) fn verify_trunk>>( mod tests { use std::usize; - use grovedb_costs::storage_cost::removal::StorageRemovedBytes::NoStorageRemoval; use grovedb_storage::StorageContext; use super::{super::tree::Tree, *}; diff --git a/merk/src/proofs/query/mod.rs b/merk/src/proofs/query/mod.rs index 7a122f23a..4ecc808be 100644 --- a/merk/src/proofs/query/mod.rs +++ b/merk/src/proofs/query/mod.rs @@ -782,7 +782,6 @@ where #[allow(deprecated)] #[cfg(test)] mod test { - use grovedb_costs::storage_cost::removal::StorageRemovedBytes::NoStorageRemoval; use super::{ super::{encoding::encode_into, *}, diff --git a/merk/src/tree/commit.rs b/merk/src/tree/commit.rs index e7bcc66a6..24c1d996d 100644 --- a/merk/src/tree/commit.rs +++ b/merk/src/tree/commit.rs @@ -28,14 +28,10 @@ //! Merk tree commit -#[cfg(feature = "full")] -use grovedb_costs::storage_cost::{removal::StorageRemovedBytes, StorageCost}; - #[cfg(feature = "full")] use super::TreeNode; #[cfg(feature = "full")] use crate::error::Error; -use crate::tree::kv::ValueDefinedCostType; #[cfg(feature = "full")] /// To be used when committing a tree (writing it to a store after applying the diff --git a/merk/src/tree/mod.rs b/merk/src/tree/mod.rs index b257d8db1..cb732b568 100644 --- a/merk/src/tree/mod.rs +++ b/merk/src/tree/mod.rs @@ -992,7 +992,6 @@ pub const fn side_to_str(left: bool) -> &'static str { #[cfg(feature = "full")] #[cfg(test)] mod test { - use grovedb_costs::storage_cost::removal::StorageRemovedBytes::NoStorageRemoval; use super::{commit::NoopCommit, hash::NULL_HASH, TreeNode}; use crate::tree::{ diff --git a/merk/src/tree/ops.rs b/merk/src/tree/ops.rs index 4b3a3d344..a55adbe9f 100644 --- a/merk/src/tree/ops.rs +++ b/merk/src/tree/ops.rs @@ -153,7 +153,7 @@ impl Fetch for PanicSource { fn fetch( &self, _link: &Link, - value_defined_cost_fn: Option<&impl Fn(&[u8]) -> Option>, + _value_defined_cost_fn: Option<&impl Fn(&[u8]) -> Option>, ) -> CostResult { unreachable!("'fetch' should not have been called") } diff --git a/merk/src/tree/walk/mod.rs b/merk/src/tree/walk/mod.rs index 35e476f33..e54018146 100644 --- a/merk/src/tree/walk/mod.rs +++ b/merk/src/tree/walk/mod.rs @@ -36,7 +36,7 @@ mod ref_walker; #[cfg(feature = "full")] pub use fetch::Fetch; #[cfg(feature = "full")] -use grovedb_costs::{cost_return_on_error, CostContext, CostResult, CostsExt, OperationCost}; +use grovedb_costs::{cost_return_on_error, CostResult, CostsExt, OperationCost}; use grovedb_costs::{ cost_return_on_error_no_add, storage_cost::{removal::StorageRemovedBytes, StorageCost}, @@ -397,7 +397,7 @@ where #[cfg(feature = "full")] #[cfg(test)] mod test { - use grovedb_costs::{storage_cost::removal::StorageRemovedBytes::NoStorageRemoval, CostsExt}; + use grovedb_costs::CostsExt; use super::{super::NoopCommit, *}; use crate::tree::{TreeFeatureType::BasicMerkNode, TreeNode}; diff --git a/storage/src/rocksdb_storage/storage.rs b/storage/src/rocksdb_storage/storage.rs index b386ecfc6..b4b1b1d40 100644 --- a/storage/src/rocksdb_storage/storage.rs +++ b/storage/src/rocksdb_storage/storage.rs @@ -40,8 +40,8 @@ use grovedb_path::SubtreePath; use integer_encoding::VarInt; use lazy_static::lazy_static; use rocksdb::{ - checkpoint::Checkpoint, ColumnFamily, ColumnFamilyDescriptor, OptimisticTransactionDB, Options, - Transaction, WriteBatchWithTransaction, DB, DEFAULT_COLUMN_FAMILY_NAME, + checkpoint::Checkpoint, ColumnFamily, ColumnFamilyDescriptor, OptimisticTransactionDB, + Transaction, WriteBatchWithTransaction, DEFAULT_COLUMN_FAMILY_NAME, }; use super::{ @@ -53,7 +53,6 @@ use crate::{ error::Error::{CostError, RocksDBError}, storage::AbstractBatchOperation, worst_case_costs::WorstKeyLength, - Error::StorageError, Storage, StorageBatch, }; From 2db0b083f065f95f3b81911431cb95eaf8b742ae Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Wed, 27 Sep 2023 14:04:30 +0700 Subject: [PATCH 12/18] clippy fixes --- costs/src/storage_cost/removal.rs | 8 +++---- .../estimated_costs/average_case_costs.rs | 8 ++----- grovedb/src/batch/mod.rs | 24 ++++++++----------- grovedb/src/operations/delete/average_case.rs | 2 +- grovedb/src/operations/delete/mod.rs | 4 ++-- grovedb/src/operations/delete/worst_case.rs | 2 +- .../src/estimated_costs/average_case_costs.rs | 8 +++---- merk/src/merk/mod.rs | 4 ++-- merk/src/tree/encoding.rs | 2 +- path/src/lib.rs | 2 +- .../storage_context/context_no_tx.rs | 4 ++-- .../storage_context/context_tx.rs | 4 ++-- .../storage_context/raw_iterator.rs | 4 ++-- 13 files changed, 32 insertions(+), 44 deletions(-) diff --git a/costs/src/storage_cost/removal.rs b/costs/src/storage_cost/removal.rs index 6d36a57be..1ef9d28b6 100644 --- a/costs/src/storage_cost/removal.rs +++ b/costs/src/storage_cost/removal.rs @@ -50,8 +50,10 @@ pub type StorageRemovalPerEpochByIdentifier = BTreeMap>; /// Removal bytes #[derive(Debug, PartialEq, Clone, Eq)] +#[derive(Default)] pub enum StorageRemovedBytes { /// No storage removal + #[default] NoStorageRemoval, /// Basic storage removal BasicStorageRemoval(u32), @@ -59,11 +61,7 @@ pub enum StorageRemovedBytes { SectionedStorageRemoval(StorageRemovalPerEpochByIdentifier), } -impl Default for StorageRemovedBytes { - fn default() -> Self { - NoStorageRemoval - } -} + impl Add for StorageRemovedBytes { type Output = Self; diff --git a/grovedb/src/batch/estimated_costs/average_case_costs.rs b/grovedb/src/batch/estimated_costs/average_case_costs.rs index 646537930..c8f99eebf 100644 --- a/grovedb/src/batch/estimated_costs/average_case_costs.rs +++ b/grovedb/src/batch/estimated_costs/average_case_costs.rs @@ -211,9 +211,7 @@ impl TreeCache for AverageCaseTreeCacheKnownPaths { &cost, self.paths.get(path).ok_or_else(|| { let paths = self - .paths - .iter() - .map(|(k, _v)| k.0.iter().map(|k| hex::encode(k.as_slice())).join("/")) + .paths.keys().map(|k| k.0.iter().map(|k| hex::encode(k.as_slice())).join("/")) .join(" | "); Error::PathNotFoundInCacheForEstimatedCosts(format!( "required path {} not found in paths {}", @@ -233,9 +231,7 @@ impl TreeCache for AverageCaseTreeCacheKnownPaths { &cost, self.paths.get(path).ok_or_else(|| { let paths = self - .paths - .iter() - .map(|(k, _v)| k.0.iter().map(|k| hex::encode(k.as_slice())).join("/")) + .paths.keys().map(|k| k.0.iter().map(|k| hex::encode(k.as_slice())).join("/")) .join(" | "); Error::PathNotFoundInCacheForEstimatedCosts(format!( "required path for estimated merk caching {} not found in paths {}", diff --git a/grovedb/src/batch/mod.rs b/grovedb/src/batch/mod.rs index 5d463f91a..6c94ddc71 100644 --- a/grovedb/src/batch/mod.rs +++ b/grovedb/src/batch/mod.rs @@ -1581,9 +1581,7 @@ impl GroveDb { // we need to pause the batch execution return Ok(Some(ops_by_level_paths)).wrap_with_cost(cost); } - if current_level > 0 { - current_level -= 1; - } + current_level = current_level.saturating_sub(1); } Ok(None).wrap_with_cost(cost) } @@ -1809,18 +1807,16 @@ impl GroveDb { .wrap_with_cost(OperationCost::default()) } } + } else if new_merk { + Ok(Merk::open_empty(storage, MerkType::BaseMerk, false)).wrap_with_cost(cost) } else { - if new_merk { - Ok(Merk::open_empty(storage, MerkType::BaseMerk, false)).wrap_with_cost(cost) - } else { - Merk::open_base( - storage, - false, - Some(&Element::value_defined_cost_for_serialized_value), - ) - .map_err(|_| Error::CorruptedData("cannot open a the root subtree".to_owned())) - .add_cost(cost) - } + Merk::open_base( + storage, + false, + Some(&Element::value_defined_cost_for_serialized_value), + ) + .map_err(|_| Error::CorruptedData("cannot open a the root subtree".to_owned())) + .add_cost(cost) } } diff --git a/grovedb/src/operations/delete/average_case.rs b/grovedb/src/operations/delete/average_case.rs index ce3d141a1..5b1dba7c0 100644 --- a/grovedb/src/operations/delete/average_case.rs +++ b/grovedb/src/operations/delete/average_case.rs @@ -73,7 +73,7 @@ impl GroveDb { let mut used_path = path.0.as_slice(); let mut ops = vec![]; let path_len = path.len() as u16; - for height in (stop_path_height..(path_len as u16)).rev() { + for height in (stop_path_height..path_len).rev() { let ( path_at_level, key_at_level, diff --git a/grovedb/src/operations/delete/mod.rs b/grovedb/src/operations/delete/mod.rs index 590926979..512a33eff 100644 --- a/grovedb/src/operations/delete/mod.rs +++ b/grovedb/src/operations/delete/mod.rs @@ -269,7 +269,7 @@ impl GroveDb { .map_err(|e| MerkError::ClientCorruptionError(e.to_string())), } }, - &batch, + batch, ) } @@ -522,7 +522,7 @@ impl GroveDb { cost_return_on_error!( &mut cost, self.propagate_changes_with_batch_transaction( - &batch, + batch, merk_cache, &path, transaction diff --git a/grovedb/src/operations/delete/worst_case.rs b/grovedb/src/operations/delete/worst_case.rs index 60699f590..a887a469e 100644 --- a/grovedb/src/operations/delete/worst_case.rs +++ b/grovedb/src/operations/delete/worst_case.rs @@ -68,7 +68,7 @@ impl GroveDb { let mut used_path = path.0.as_slice(); let mut ops = vec![]; let path_len = path.len() as u16; - for height in (stop_path_height..(path_len as u16)).rev() { + for height in (stop_path_height..path_len).rev() { let ( path_at_level, key_at_level, diff --git a/merk/src/estimated_costs/average_case_costs.rs b/merk/src/estimated_costs/average_case_costs.rs index 099978faa..93c51847f 100644 --- a/merk/src/estimated_costs/average_case_costs.rs +++ b/merk/src/estimated_costs/average_case_costs.rs @@ -57,8 +57,10 @@ pub type Weight = u8; #[cfg(feature = "full")] #[derive(Clone, PartialEq, Eq, Debug)] /// Estimated number of sum trees +#[derive(Default)] pub enum EstimatedSumTrees { /// No sum trees + #[default] NoSumTrees, /// Some sum trees SomeSumTrees { @@ -72,11 +74,7 @@ pub enum EstimatedSumTrees { } #[cfg(feature = "full")] -impl Default for EstimatedSumTrees { - fn default() -> Self { - EstimatedSumTrees::NoSumTrees - } -} + #[cfg(feature = "full")] impl EstimatedSumTrees { diff --git a/merk/src/merk/mod.rs b/merk/src/merk/mod.rs index 92f7cc93d..17c7aa7e7 100644 --- a/merk/src/merk/mod.rs +++ b/merk/src/merk/mod.rs @@ -871,8 +871,8 @@ mod test { #[test] fn reopen_iter() { - fn collect<'db, 'ctx>( - iter: PrefixedStorageIter<'db, 'ctx>, + fn collect( + iter: PrefixedStorageIter<'_, '_>, nodes: &mut Vec<(Vec, Vec)>, ) { while iter.valid().unwrap() { diff --git a/merk/src/tree/encoding.rs b/merk/src/tree/encoding.rs index 057880999..29307246e 100644 --- a/merk/src/tree/encoding.rs +++ b/merk/src/tree/encoding.rs @@ -353,6 +353,6 @@ mod tests { bytes.as_slice(), None::<&fn(&[u8]) -> Option>, ); - assert!(matches!(tree, Err(_))); + assert!(tree.is_err()); } } diff --git a/path/src/lib.rs b/path/src/lib.rs index b79c7deb9..0691874ee 100644 --- a/path/src/lib.rs +++ b/path/src/lib.rs @@ -44,7 +44,7 @@ mod tests { use super::*; use crate::util::calculate_hash; - fn assert_path_properties<'b, B>(path: SubtreePath<'b, B>, reference: Vec>) + fn assert_path_properties(path: SubtreePath<'_, B>, reference: Vec>) where B: AsRef<[u8]> + std::fmt::Debug, { diff --git a/storage/src/rocksdb_storage/storage_context/context_no_tx.rs b/storage/src/rocksdb_storage/storage_context/context_no_tx.rs index 20cb65c17..fd639a5a6 100644 --- a/storage/src/rocksdb_storage/storage_context/context_no_tx.rs +++ b/storage/src/rocksdb_storage/storage_context/context_no_tx.rs @@ -265,7 +265,7 @@ impl<'db> StorageContext<'db> for PrefixedRocksDbStorageContext<'db> { fn new_batch(&self) -> Self::Batch { PrefixedMultiContextBatchPart { - prefix: self.prefix.clone(), + prefix: self.prefix, batch: StorageBatch::new(), } } @@ -279,7 +279,7 @@ impl<'db> StorageContext<'db> for PrefixedRocksDbStorageContext<'db> { fn raw_iter(&self) -> Self::RawIterator { PrefixedRocksDbRawIterator { - prefix: self.prefix.clone(), + prefix: self.prefix, raw_iterator: self.storage.raw_iterator(), } } diff --git a/storage/src/rocksdb_storage/storage_context/context_tx.rs b/storage/src/rocksdb_storage/storage_context/context_tx.rs index 045cd982d..d5a480c38 100644 --- a/storage/src/rocksdb_storage/storage_context/context_tx.rs +++ b/storage/src/rocksdb_storage/storage_context/context_tx.rs @@ -296,7 +296,7 @@ impl<'db> StorageContext<'db> for PrefixedRocksDbTransactionContext<'db> { fn new_batch(&self) -> Self::Batch { PrefixedMultiContextBatchPart { - prefix: self.prefix.clone(), + prefix: self.prefix, batch: StorageBatch::new(), } } @@ -311,7 +311,7 @@ impl<'db> StorageContext<'db> for PrefixedRocksDbTransactionContext<'db> { fn raw_iter(&self) -> Self::RawIterator { PrefixedRocksDbRawIterator { - prefix: self.prefix.clone(), + prefix: self.prefix, raw_iterator: self.transaction.raw_iterator(), } } diff --git a/storage/src/rocksdb_storage/storage_context/raw_iterator.rs b/storage/src/rocksdb_storage/storage_context/raw_iterator.rs index 4ee36510d..a9d6cf4fe 100644 --- a/storage/src/rocksdb_storage/storage_context/raw_iterator.rs +++ b/storage/src/rocksdb_storage/storage_context/raw_iterator.rs @@ -48,7 +48,7 @@ pub struct PrefixedRocksDbRawIterator { impl<'a> RawIterator for PrefixedRocksDbRawIterator> { fn seek_to_first(&mut self) -> CostContext<()> { - self.raw_iterator.seek(&self.prefix); + self.raw_iterator.seek(self.prefix); ().wrap_with_cost(OperationCost::with_seek_count(1)) } @@ -169,7 +169,7 @@ impl<'a> RawIterator for PrefixedRocksDbRawIterator RawIterator for PrefixedRocksDbRawIterator>> { fn seek_to_first(&mut self) -> CostContext<()> { - self.raw_iterator.seek(&self.prefix); + self.raw_iterator.seek(self.prefix); ().wrap_with_cost(OperationCost::with_seek_count(1)) } From 89628d711e28601cdee04e5dc19b5b73868e9a4d Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Wed, 27 Sep 2023 14:04:51 +0700 Subject: [PATCH 13/18] fmt fixes --- costs/src/storage_cost/removal.rs | 5 +---- grovedb/src/batch/estimated_costs/average_case_costs.rs | 8 ++++++-- merk/src/estimated_costs/average_case_costs.rs | 2 -- merk/src/merk/mod.rs | 5 +---- 4 files changed, 8 insertions(+), 12 deletions(-) diff --git a/costs/src/storage_cost/removal.rs b/costs/src/storage_cost/removal.rs index 1ef9d28b6..9fa7af991 100644 --- a/costs/src/storage_cost/removal.rs +++ b/costs/src/storage_cost/removal.rs @@ -49,8 +49,7 @@ pub const UNKNOWN_EPOCH: u64 = u64::MAX; pub type StorageRemovalPerEpochByIdentifier = BTreeMap>; /// Removal bytes -#[derive(Debug, PartialEq, Clone, Eq)] -#[derive(Default)] +#[derive(Debug, PartialEq, Clone, Eq, Default)] pub enum StorageRemovedBytes { /// No storage removal #[default] @@ -61,8 +60,6 @@ pub enum StorageRemovedBytes { SectionedStorageRemoval(StorageRemovalPerEpochByIdentifier), } - - impl Add for StorageRemovedBytes { type Output = Self; diff --git a/grovedb/src/batch/estimated_costs/average_case_costs.rs b/grovedb/src/batch/estimated_costs/average_case_costs.rs index c8f99eebf..0a8d573db 100644 --- a/grovedb/src/batch/estimated_costs/average_case_costs.rs +++ b/grovedb/src/batch/estimated_costs/average_case_costs.rs @@ -211,7 +211,9 @@ impl TreeCache for AverageCaseTreeCacheKnownPaths { &cost, self.paths.get(path).ok_or_else(|| { let paths = self - .paths.keys().map(|k| k.0.iter().map(|k| hex::encode(k.as_slice())).join("/")) + .paths + .keys() + .map(|k| k.0.iter().map(|k| hex::encode(k.as_slice())).join("/")) .join(" | "); Error::PathNotFoundInCacheForEstimatedCosts(format!( "required path {} not found in paths {}", @@ -231,7 +233,9 @@ impl TreeCache for AverageCaseTreeCacheKnownPaths { &cost, self.paths.get(path).ok_or_else(|| { let paths = self - .paths.keys().map(|k| k.0.iter().map(|k| hex::encode(k.as_slice())).join("/")) + .paths + .keys() + .map(|k| k.0.iter().map(|k| hex::encode(k.as_slice())).join("/")) .join(" | "); Error::PathNotFoundInCacheForEstimatedCosts(format!( "required path for estimated merk caching {} not found in paths {}", diff --git a/merk/src/estimated_costs/average_case_costs.rs b/merk/src/estimated_costs/average_case_costs.rs index 93c51847f..b92222ac9 100644 --- a/merk/src/estimated_costs/average_case_costs.rs +++ b/merk/src/estimated_costs/average_case_costs.rs @@ -74,8 +74,6 @@ pub enum EstimatedSumTrees { } #[cfg(feature = "full")] - - #[cfg(feature = "full")] impl EstimatedSumTrees { fn estimated_size(&self) -> Result { diff --git a/merk/src/merk/mod.rs b/merk/src/merk/mod.rs index 17c7aa7e7..93c052a4c 100644 --- a/merk/src/merk/mod.rs +++ b/merk/src/merk/mod.rs @@ -871,10 +871,7 @@ mod test { #[test] fn reopen_iter() { - fn collect( - iter: PrefixedStorageIter<'_, '_>, - nodes: &mut Vec<(Vec, Vec)>, - ) { + fn collect(iter: PrefixedStorageIter<'_, '_>, nodes: &mut Vec<(Vec, Vec)>) { while iter.valid().unwrap() { nodes.push(( iter.key().unwrap().unwrap().to_vec(), From 78fcce0c1a2dafd140e318a299578498f8bef077 Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Wed, 27 Sep 2023 14:14:31 +0700 Subject: [PATCH 14/18] updated to rc2 --- costs/Cargo.toml | 2 +- grovedb/Cargo.toml | 12 ++++++------ merk/Cargo.toml | 10 +++++----- path/Cargo.toml | 2 +- storage/Cargo.toml | 8 ++++---- visualize/Cargo.toml | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/costs/Cargo.toml b/costs/Cargo.toml index 5d9784d9d..6aaa6ece2 100644 --- a/costs/Cargo.toml +++ b/costs/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "grovedb-costs" -version = "1.0.0-rc.1" +version = "1.0.0-rc.2" edition = "2021" license = "MIT" description = "Costs extension crate for GroveDB" diff --git a/grovedb/Cargo.toml b/grovedb/Cargo.toml index a03f48e2e..a7fb382bc 100644 --- a/grovedb/Cargo.toml +++ b/grovedb/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "grovedb" description = "Fully featured database using balanced hierarchical authenticated data structures" -version = "1.0.0-rc.1" +version = "1.0.0-rc.2" authors = ["Samuel Westrich ", "Wisdom Ogwu "] edition = "2021" license = "MIT" @@ -12,21 +12,21 @@ documentation = "https://docs.rs/grovedb" [dependencies] -grovedb-merk = { version = "1.0.0-rc.1", path = "../merk", optional = true, default-features = false } +grovedb-merk = { version = "1.0.0-rc.2", path = "../merk", optional = true, default-features = false } thiserror = { version = "1.0.37", optional = true } tempfile = { version = "3.3.0", optional = true } bincode = { version = "1.3.3", optional = true } serde = { version = "1.0.149", optional = true } -grovedb-storage = { version = "1.0.0-rc.1", path = "../storage", optional = true } -grovedb-visualize = { version = "1.0.0-rc.1", path = "../visualize", optional = true } +grovedb-storage = { version = "1.0.0-rc.2", path = "../storage", optional = true } +grovedb-visualize = { version = "1.0.0-rc.2", path = "../visualize", optional = true } hex = { version = "0.4.3", optional = true } itertools = { version = "0.10.5", optional = true } integer-encoding = { version = "3.0.4", optional = true } -grovedb-costs = { version = "1.0.0-rc.1", path = "../costs", optional = true } +grovedb-costs = { version = "1.0.0-rc.2", path = "../costs", optional = true } nohash-hasher = { version = "0.2.0", optional = true } indexmap = { version = "1.9.2", optional = true } intmap = { version = "2.0.0", optional = true } -grovedb-path = { version = "1.0.0-rc.1", path = "../path" } +grovedb-path = { version = "1.0.0-rc.2", path = "../path" } [dev-dependencies] rand = "0.8.5" diff --git a/merk/Cargo.toml b/merk/Cargo.toml index 4fec012ee..8cfab2db2 100644 --- a/merk/Cargo.toml +++ b/merk/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "grovedb-merk" description = "Merkle key/value store adapted for GroveDB" -version = "1.0.0-rc.1" +version = "1.0.0-rc.2" authors = ["Samuel Westrich ", "Wisdom Ogwu ", "Matt Bell "] edition = "2021" license = "MIT" @@ -12,13 +12,13 @@ documentation = "https://docs.rs/grovedb-merk" [dependencies] thiserror = "1.0.37" -grovedb-storage = { version = "1.0.0-rc.1", path = "../storage", optional = true } +grovedb-storage = { version = "1.0.0-rc.2", path = "../storage", optional = true } failure = "0.1.8" integer-encoding = "3.0.4" indexmap = "1.9.2" -grovedb-costs = { version = "1.0.0-rc.1", path = "../costs" } -grovedb-visualize = { version = "1.0.0-rc.1", path = "../visualize" } -grovedb-path = { version = "1.0.0-rc.1", path = "../path" } +grovedb-costs = { version = "1.0.0-rc.2", path = "../costs" } +grovedb-visualize = { version = "1.0.0-rc.2", path = "../visualize" } +grovedb-path = { version = "1.0.0-rc.2", path = "../path" } [dependencies.time] version = "0.3.17" diff --git a/path/Cargo.toml b/path/Cargo.toml index c627f855c..bae126750 100644 --- a/path/Cargo.toml +++ b/path/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "grovedb-path" -version = "1.0.0-rc.1" +version = "1.0.0-rc.2" edition = "2021" license = "MIT" description = "Path extension crate for GroveDB" diff --git a/storage/Cargo.toml b/storage/Cargo.toml index 96c7522a2..856888a79 100644 --- a/storage/Cargo.toml +++ b/storage/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "grovedb-storage" -version = "1.0.0-rc.1" +version = "1.0.0-rc.2" edition = "2021" license = "MIT" description = "Storage extension crate for GroveDB" @@ -14,13 +14,13 @@ num_cpus = { version = "1.14.0", optional = true } tempfile = { version = "3.3.0", optional = true } blake3 = { version = "1.3.3", optional = true } integer-encoding = { version = "3.0.4", optional = true } -grovedb-visualize = { version = "1.0.0-rc.1", path = "../visualize" } +grovedb-visualize = { version = "1.0.0-rc.2", path = "../visualize" } strum = { version = "0.24.1", features = ["derive"] } -grovedb-costs = { version = "1.0.0-rc.1", path = "../costs" } +grovedb-costs = { version = "1.0.0-rc.2", path = "../costs" } thiserror = "1.0.37" rocksdb = { version = "0.21.0", optional = true } hex = "0.4.3" -grovedb-path = { version = "1.0.0-rc.1", path = "../path" } +grovedb-path = { version = "1.0.0-rc.2", path = "../path" } [features] rocksdb_storage = ["rocksdb", "num_cpus", "lazy_static", "tempfile", "blake3", "integer-encoding"] diff --git a/visualize/Cargo.toml b/visualize/Cargo.toml index a1fc2fcbb..f27fe48ed 100644 --- a/visualize/Cargo.toml +++ b/visualize/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "grovedb-visualize" -version = "1.0.0-rc.1" +version = "1.0.0-rc.2" edition = "2021" license = "MIT" description = "Visualizer extension crate for GroveDB" From e4891b7ac6c799fb32ec9e5f7d7f062ecc215e80 Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Wed, 27 Sep 2023 14:22:01 +0700 Subject: [PATCH 15/18] updated caching system --- .github/workflows/grovedb.yml | 8 +++++ .github/workflows/nodejs.yml | 64 ----------------------------------- 2 files changed, 8 insertions(+), 64 deletions(-) delete mode 100644 .github/workflows/nodejs.yml diff --git a/.github/workflows/grovedb.yml b/.github/workflows/grovedb.yml index af4242def..118beafb6 100644 --- a/.github/workflows/grovedb.yml +++ b/.github/workflows/grovedb.yml @@ -27,6 +27,8 @@ jobs: - name: Enable Rust cache uses: Swatinem/rust-cache@v2 + with: + cache-on-failure: "false" - run: cargo test --workspace --all-features @@ -52,6 +54,8 @@ jobs: - name: Enable Rust cache uses: Swatinem/rust-cache@v2 + with: + cache-on-failure: "false" - uses: actions-rs/clippy-check@v1 with: @@ -78,6 +82,8 @@ jobs: - name: Enable Rust cache uses: Swatinem/rust-cache@v2 + with: + cache-on-failure: "false" - run: exit `cargo +nightly fmt --check | wc -l` @@ -100,6 +106,8 @@ jobs: - name: Enable Rust cache uses: Swatinem/rust-cache@v2 + with: + cache-on-failure: "false" - run: cargo check diff --git a/.github/workflows/nodejs.yml b/.github/workflows/nodejs.yml deleted file mode 100644 index bac693cbf..000000000 --- a/.github/workflows/nodejs.yml +++ /dev/null @@ -1,64 +0,0 @@ -on: - workflow_dispatch: - pull_request: - branches: - - master - -name: Node.JS binding - -jobs: - test: - name: Tests - runs-on: ubuntu-latest - steps: - - name: Cancel previous runs - uses: styfle/cancel-workflow-action@0.9.1 - with: - access_token: ${{ github.token }} - - - uses: actions/checkout@v2 - - - name: Setup Node.JS - uses: actions/setup-node@v2 - with: - node-version: '16' - - - name: Setup Rust toolchain - uses: actions-rs/toolchain@v1 - with: - toolchain: stable - default: true - - - name: Install NPM deps - run: npm ci - - - name: Run tests - run: npm test - - linting: - name: Linting - runs-on: ubuntu-latest - steps: - - name: Cancel previous runs - uses: styfle/cancel-workflow-action@0.9.1 - with: - access_token: ${{ github.token }} - - - uses: actions/checkout@v2 - - - name: Setup Node.JS - uses: actions/setup-node@v2 - with: - node-version: '16' - - - name: Setup Rust toolchain - uses: actions-rs/toolchain@v1 - with: - toolchain: stable - default: true - - - name: Install NPM deps - run: npm ci - - - name: Run ES linter - run: npm run lint \ No newline at end of file From aa5490fa4e0618becb136c68b72910b49e0fc261 Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Wed, 27 Sep 2023 14:26:15 +0700 Subject: [PATCH 16/18] removed cache --- .github/workflows/grovedb.yml | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/.github/workflows/grovedb.yml b/.github/workflows/grovedb.yml index 118beafb6..0a8ded9de 100644 --- a/.github/workflows/grovedb.yml +++ b/.github/workflows/grovedb.yml @@ -25,10 +25,10 @@ jobs: with: toolchain: stable - - name: Enable Rust cache - uses: Swatinem/rust-cache@v2 - with: - cache-on-failure: "false" +# - name: Enable Rust cache +# uses: Swatinem/rust-cache@v2 +# with: +# cache-on-failure: "false" - run: cargo test --workspace --all-features @@ -52,10 +52,10 @@ jobs: default: true components: clippy - - name: Enable Rust cache - uses: Swatinem/rust-cache@v2 - with: - cache-on-failure: "false" +# - name: Enable Rust cache +# uses: Swatinem/rust-cache@v2 +# with: +# cache-on-failure: "false" - uses: actions-rs/clippy-check@v1 with: @@ -80,10 +80,10 @@ jobs: default: true components: rustfmt - - name: Enable Rust cache - uses: Swatinem/rust-cache@v2 - with: - cache-on-failure: "false" +# - name: Enable Rust cache +# uses: Swatinem/rust-cache@v2 +# with: +# cache-on-failure: "false" - run: exit `cargo +nightly fmt --check | wc -l` @@ -104,10 +104,10 @@ jobs: toolchain: stable default: true - - name: Enable Rust cache - uses: Swatinem/rust-cache@v2 - with: - cache-on-failure: "false" +# - name: Enable Rust cache +# uses: Swatinem/rust-cache@v2 +# with: +# cache-on-failure: "false" - run: cargo check From 46297f07728f51fc585d9466b80e010614479d9c Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Wed, 27 Sep 2023 14:37:43 +0700 Subject: [PATCH 17/18] readded cache --- .github/workflows/grovedb.yml | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/.github/workflows/grovedb.yml b/.github/workflows/grovedb.yml index 0a8ded9de..118beafb6 100644 --- a/.github/workflows/grovedb.yml +++ b/.github/workflows/grovedb.yml @@ -25,10 +25,10 @@ jobs: with: toolchain: stable -# - name: Enable Rust cache -# uses: Swatinem/rust-cache@v2 -# with: -# cache-on-failure: "false" + - name: Enable Rust cache + uses: Swatinem/rust-cache@v2 + with: + cache-on-failure: "false" - run: cargo test --workspace --all-features @@ -52,10 +52,10 @@ jobs: default: true components: clippy -# - name: Enable Rust cache -# uses: Swatinem/rust-cache@v2 -# with: -# cache-on-failure: "false" + - name: Enable Rust cache + uses: Swatinem/rust-cache@v2 + with: + cache-on-failure: "false" - uses: actions-rs/clippy-check@v1 with: @@ -80,10 +80,10 @@ jobs: default: true components: rustfmt -# - name: Enable Rust cache -# uses: Swatinem/rust-cache@v2 -# with: -# cache-on-failure: "false" + - name: Enable Rust cache + uses: Swatinem/rust-cache@v2 + with: + cache-on-failure: "false" - run: exit `cargo +nightly fmt --check | wc -l` @@ -104,10 +104,10 @@ jobs: toolchain: stable default: true -# - name: Enable Rust cache -# uses: Swatinem/rust-cache@v2 -# with: -# cache-on-failure: "false" + - name: Enable Rust cache + uses: Swatinem/rust-cache@v2 + with: + cache-on-failure: "false" - run: cargo check From 3fc63a4f8dcd620d374d64e9d57349d0dedec767 Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Thu, 28 Sep 2023 00:43:01 +0700 Subject: [PATCH 18/18] merge --- grovedb/src/lib.rs | 18 +++++++++++++++--- storage/src/rocksdb_storage/storage.rs | 2 +- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/grovedb/src/lib.rs b/grovedb/src/lib.rs index a34ed5c40..9ea955134 100644 --- a/grovedb/src/lib.rs +++ b/grovedb/src/lib.rs @@ -912,7 +912,11 @@ impl GroveDb { issues.extend(self.verify_merk_and_submerks(inner_merk, &new_path_ref, batch)?); } else if element.is_item() { let (kv_value, element_value_hash) = merk - .get_value_and_value_hash(&key, true) + .get_value_and_value_hash( + &key, + true, + None::<&fn(&[u8]) -> Option>, + ) .unwrap() .map_err(MerkError)? .ok_or(Error::CorruptedData( @@ -948,7 +952,11 @@ impl GroveDb { let element = raw_decode(&element_value)?; if element.is_tree() { let (kv_value, element_value_hash) = merk - .get_value_and_value_hash(&key, true) + .get_value_and_value_hash( + &key, + true, + None::<&fn(&[u8]) -> Option>, + ) .unwrap() .map_err(MerkError)? .ok_or(Error::CorruptedData( @@ -979,7 +987,11 @@ impl GroveDb { )?); } else if element.is_item() { let (kv_value, element_value_hash) = merk - .get_value_and_value_hash(&key, true) + .get_value_and_value_hash( + &key, + true, + None::<&fn(&[u8]) -> Option>, + ) .unwrap() .map_err(MerkError)? .ok_or(Error::CorruptedData( diff --git a/storage/src/rocksdb_storage/storage.rs b/storage/src/rocksdb_storage/storage.rs index b4b1b1d40..a396b75fe 100644 --- a/storage/src/rocksdb_storage/storage.rs +++ b/storage/src/rocksdb_storage/storage.rs @@ -427,7 +427,7 @@ impl RocksDbStorage { let mut iter = self.db.raw_iterator_cf(&cf_handle); iter.seek_to_first(); while iter.valid() { - self.db.delete(iter.key().expect("should have key")); + self.db.delete(iter.key().expect("should have key"))?; iter.next() } Ok(())