From db0ef634ac0b637dd4907cf6e322b1411c1568e5 Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Thu, 22 Aug 2024 17:09:06 +0700 Subject: [PATCH 01/15] trial --- merk/src/tree/just_in_time_value_update.rs | 5 ++++- merk/src/tree/mod.rs | 5 +++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/merk/src/tree/just_in_time_value_update.rs b/merk/src/tree/just_in_time_value_update.rs index 20861ec4..468ac024 100644 --- a/merk/src/tree/just_in_time_value_update.rs +++ b/merk/src/tree/just_in_time_value_update.rs @@ -37,11 +37,13 @@ impl TreeNode { if let Some(old_value) = self.old_value.clone() { // At this point the tree value can be updated based on client requirements // For example to store the costs + let original_new_value = self.value_ref(); loop { + let mut to_update_value = original_new_value.clone(); let (flags_changed, value_defined_cost) = update_tree_value_based_on_costs( &storage_costs.value_storage_cost, &old_value, - self.value_mut_ref(), + &mut to_update_value, )?; if !flags_changed { break; @@ -50,6 +52,7 @@ impl TreeNode { let after_update_tree_plus_hook_size = self.value_encoding_length_with_parent_to_child_reference(); if after_update_tree_plus_hook_size == current_tree_plus_hook_size { + self.set_value(to_update_value); break; } let new_size_and_storage_costs = diff --git a/merk/src/tree/mod.rs b/merk/src/tree/mod.rs index dc0cbaf2..5c9fcce9 100644 --- a/merk/src/tree/mod.rs +++ b/merk/src/tree/mod.rs @@ -335,6 +335,11 @@ impl TreeNode { self.inner.kv.key = key; } + /// Set value of Tree + pub fn set_value(&mut self, value: Vec) { + self.inner.kv.value = value; + } + /// Consumes the tree and returns its root node's key, without having to /// clone or allocate. #[inline] From dea5cd859793edac61baa3703d838b8d687d7e68 Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Thu, 22 Aug 2024 17:11:13 +0700 Subject: [PATCH 02/15] trial --- merk/src/tree/just_in_time_value_update.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/merk/src/tree/just_in_time_value_update.rs b/merk/src/tree/just_in_time_value_update.rs index 468ac024..3a072e1e 100644 --- a/merk/src/tree/just_in_time_value_update.rs +++ b/merk/src/tree/just_in_time_value_update.rs @@ -37,7 +37,8 @@ impl TreeNode { if let Some(old_value) = self.old_value.clone() { // At this point the tree value can be updated based on client requirements // For example to store the costs - let original_new_value = self.value_ref(); + // todo: clean up clones + let original_new_value = self.value_ref().clone(); loop { let mut to_update_value = original_new_value.clone(); let (flags_changed, value_defined_cost) = update_tree_value_based_on_costs( From 071016b70246be841b22714bee490b283be157e8 Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Thu, 22 Aug 2024 17:27:04 +0700 Subject: [PATCH 03/15] another try --- merk/src/tree/just_in_time_value_update.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/merk/src/tree/just_in_time_value_update.rs b/merk/src/tree/just_in_time_value_update.rs index 3a072e1e..6f59eda4 100644 --- a/merk/src/tree/just_in_time_value_update.rs +++ b/merk/src/tree/just_in_time_value_update.rs @@ -40,11 +40,10 @@ impl TreeNode { // todo: clean up clones let original_new_value = self.value_ref().clone(); loop { - let mut to_update_value = original_new_value.clone(); let (flags_changed, value_defined_cost) = update_tree_value_based_on_costs( &storage_costs.value_storage_cost, &old_value, - &mut to_update_value, + self.value_mut_ref(), )?; if !flags_changed { break; @@ -53,13 +52,13 @@ impl TreeNode { let after_update_tree_plus_hook_size = self.value_encoding_length_with_parent_to_child_reference(); if after_update_tree_plus_hook_size == current_tree_plus_hook_size { - self.set_value(to_update_value); break; } let new_size_and_storage_costs = self.kv_with_parent_hook_size_and_storage_cost(old_specialized_cost)?; current_tree_plus_hook_size = new_size_and_storage_costs.0; storage_costs = new_size_and_storage_costs.1; + self.set_value(original_new_value.clone()) } if i > MAX_UPDATE_VALUE_BASED_ON_COSTS_TIMES { return Err(Error::CyclicError( From 8fbc18402c9e4ff0a571d1f8e55cfc35a5011a96 Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Thu, 22 Aug 2024 19:42:03 +0700 Subject: [PATCH 04/15] fix --- grovedb/src/batch/just_in_time_cost_tests.rs | 4 +- .../batch/just_in_time_reference_update.rs | 116 ++++++++++++++++++ grovedb/src/batch/mod.rs | 67 +++------- grovedb/src/batch/multi_insert_cost_tests.rs | 7 +- grovedb/src/batch/single_insert_cost_tests.rs | 54 ++++++-- grovedb/src/error.rs | 4 + 6 files changed, 189 insertions(+), 63 deletions(-) create mode 100644 grovedb/src/batch/just_in_time_reference_update.rs diff --git a/grovedb/src/batch/just_in_time_cost_tests.rs b/grovedb/src/batch/just_in_time_cost_tests.rs index a3fa9aec..9b907843 100644 --- a/grovedb/src/batch/just_in_time_cost_tests.rs +++ b/grovedb/src/batch/just_in_time_cost_tests.rs @@ -389,7 +389,6 @@ mod tests { new_flags[1] = old_flags.unwrap()[1]; new_flags.push(new_flags_epoch); new_flags.extend(cost.added_bytes.encode_var_vec()); - assert_eq!(new_flags, &vec![1u8, 0, 1, 2]); Ok(true) } else { assert_eq!(new_flags[0], 1); @@ -411,7 +410,8 @@ mod tests { Some(&tx), grove_version, ) - .cost; + .cost_as_result() + .expect("expected to not error"); let issues = db .visualize_verify_grovedb(Some(&tx), true, false, &Default::default()) diff --git a/grovedb/src/batch/just_in_time_reference_update.rs b/grovedb/src/batch/just_in_time_reference_update.rs new file mode 100644 index 00000000..8e77a5fa --- /dev/null +++ b/grovedb/src/batch/just_in_time_reference_update.rs @@ -0,0 +1,116 @@ +use std::borrow::Cow; + +use grovedb_costs::{ + cost_return_on_error_no_add, storage_cost::StorageCost, CostResult, CostsExt, OperationCost, +}; +use grovedb_merk::{ + tree::{kv::KV, value_hash, TreeNode}, + CryptoHash, Merk, +}; +use grovedb_storage::StorageContext; +use grovedb_version::version::GroveVersion; + +use crate::{ + batch::{MerkError, TreeCacheMerkByPath}, + Element, ElementFlags, Error, +}; + +impl<'db, S, F> TreeCacheMerkByPath +where + F: FnMut(&[Vec], bool) -> CostResult, Error>, + S: StorageContext<'db>, +{ + pub(crate) fn process_old_element_flags( + key: &[u8], + serialized: &[u8], + new_element: &mut Element, + old_element: Element, + old_serialized_element: &[u8], + is_in_sum_tree: bool, + flags_update: &mut G, + grove_version: &GroveVersion, + ) -> CostResult + where + G: FnMut(&StorageCost, Option, &mut ElementFlags) -> Result, + { + let maybe_old_flags = old_element.get_flags_owned(); + + let mut cost = OperationCost::default(); + + let old_storage_cost = KV::node_byte_cost_size_for_key_and_raw_value_lengths( + key.len() as u32, + old_serialized_element.len() as u32, + is_in_sum_tree, + ); + + let original_new_element = new_element.clone(); + + let mut serialization_to_use = Cow::Borrowed(serialized); + + let mut new_storage_cost = KV::node_byte_cost_size_for_key_and_raw_value_lengths( + key.len() as u32, + serialized.len() as u32, + is_in_sum_tree, + ); + + let mut i = 0; + + loop { + // Calculate storage costs + let storage_costs = + TreeNode::storage_cost_for_update(new_storage_cost, old_storage_cost); + + let mut new_element_cloned = original_new_element.clone(); + + let changed = cost_return_on_error_no_add!( + &cost, + (flags_update)( + &storage_costs, + maybe_old_flags.clone(), + new_element_cloned.get_flags_mut().as_mut().unwrap() + ) + .map_err(|e| match e { + Error::JustInTimeElementFlagsClientError(_) => { + MerkError::ClientCorruptionError(e.to_string()).into() + } + _ => MerkError::ClientCorruptionError("non client error".to_string(),).into(), + }) + ); + if !changed { + // There are no storage flags, we can just hash new element + + let val_hash = value_hash(&serialization_to_use).unwrap_add_cost(&mut cost); + return Ok(val_hash).wrap_with_cost(cost); + } else { + // There are no storage flags, we can just hash new element + let new_serialized_bytes = cost_return_on_error_no_add!( + &cost, + new_element_cloned.serialize(grove_version) + ); + + new_storage_cost = KV::node_byte_cost_size_for_key_and_raw_value_lengths( + key.len() as u32, + new_serialized_bytes.len() as u32, + is_in_sum_tree, + ); + + if serialization_to_use == new_serialized_bytes { + // it hasn't actually changed, let's do the value hash of it + let val_hash = value_hash(&serialization_to_use).unwrap_add_cost(&mut cost); + return Ok(val_hash).wrap_with_cost(cost); + } + + serialization_to_use = Cow::Owned(new_serialized_bytes); + } + + // Prevent potential infinite loop + if i > 8 { + return Err(Error::CyclicError( + "updated value based on costs too many times in reference", + )) + .wrap_with_cost(cost); + } + i += 1; + } + } +} diff --git a/grovedb/src/batch/mod.rs b/grovedb/src/batch/mod.rs index c4dc6bf1..04142b60 100644 --- a/grovedb/src/batch/mod.rs +++ b/grovedb/src/batch/mod.rs @@ -13,6 +13,7 @@ mod multi_insert_cost_tests; #[cfg(test)] mod just_in_time_cost_tests; +pub mod just_in_time_reference_update; mod options; #[cfg(test)] mod single_deletion_cost_tests; @@ -1094,7 +1095,6 @@ where Ok(val_hash).wrap_with_cost(cost) } else { let mut new_element = element.clone(); - let new_flags = new_element.get_flags_mut().as_mut().unwrap(); // it can be unmerged, let's get the value on disk let (key, reference_path) = qualified_path.split_last().unwrap(); @@ -1109,61 +1109,24 @@ where if let Some((old_element, old_serialized_element, is_in_sum_tree)) = serialized_element_result { - let maybe_old_flags = old_element.get_flags_owned(); - - let old_storage_cost = - KV::node_byte_cost_size_for_key_and_raw_value_lengths( - key.len() as u32, - old_serialized_element.len() as u32, - is_in_sum_tree, - ); - let new_storage_cost = - KV::node_byte_cost_size_for_key_and_raw_value_lengths( - key.len() as u32, - serialized.len() as u32, + let value_hash = cost_return_on_error!( + &mut cost, + Self::process_old_element_flags( + key, + &serialized, + &mut new_element, + old_element, + &old_serialized_element, is_in_sum_tree, - ); - - // There are storage flags - let storage_costs = TreeNode::storage_cost_for_update( - new_storage_cost, - old_storage_cost, - ); - - let changed = cost_return_on_error_no_add!( - &cost, - (flags_update)(&storage_costs, maybe_old_flags, new_flags) - .map_err(|e| match e { - Error::JustInTimeElementFlagsClientError(_) => { - MerkError::ClientCorruptionError(e.to_string()) - .into() - } - _ => MerkError::ClientCorruptionError( - "non client error".to_string(), - ) - .into(), - }) + flags_update, + grove_version, + ) ); - if changed { - // There are no storage flags, we can just hash new element - let serialized = cost_return_on_error_no_add!( - &cost, - new_element.serialize(grove_version) - ); - let val_hash = - value_hash(&serialized).unwrap_add_cost(&mut cost); - Ok(val_hash).wrap_with_cost(cost) - } else { - // There are no storage flags, we can just hash new element - - let val_hash = - value_hash(&serialized).unwrap_add_cost(&mut cost); - Ok(val_hash).wrap_with_cost(cost) - } + Ok(value_hash).wrap_with_cost(cost) } else { - let val_hash = + let value_hash = value_hash(&serialized).unwrap_add_cost(&mut cost); - Ok(val_hash).wrap_with_cost(cost) + Ok(value_hash).wrap_with_cost(cost) } } } diff --git a/grovedb/src/batch/multi_insert_cost_tests.rs b/grovedb/src/batch/multi_insert_cost_tests.rs index 8a1200e6..666de224 100644 --- a/grovedb/src/batch/multi_insert_cost_tests.rs +++ b/grovedb/src/batch/multi_insert_cost_tests.rs @@ -405,7 +405,12 @@ mod tests { new_flags[1] = old_flags.unwrap()[1]; new_flags.push(new_flags_epoch); new_flags.extend(cost.added_bytes.encode_var_vec()); - assert_eq!(new_flags, &vec![1u8, 0, 1, 2]); + // first pass will be vec![1u8, 0, 1, 2], second pass will be vec![1u8, + // 0, 1, 4] + assert!( + new_flags == &vec![1u8, 0, 1, 2] + || new_flags == &vec![1u8, 0, 1, 4] + ); Ok(true) } else { assert_eq!(new_flags[0], 1); diff --git a/grovedb/src/batch/single_insert_cost_tests.rs b/grovedb/src/batch/single_insert_cost_tests.rs index 2a269159..d86a2bf2 100644 --- a/grovedb/src/batch/single_insert_cost_tests.rs +++ b/grovedb/src/batch/single_insert_cost_tests.rs @@ -816,7 +816,12 @@ mod tests { new_flags[1] = old_flags.unwrap()[1]; new_flags.push(new_flags_epoch); new_flags.extend(cost.added_bytes.encode_var_vec()); - assert_eq!(new_flags, &vec![1u8, 0, 1, 2]); + // first pass will be vec![1u8, 0, 1, 2], second pass will be vec![1u8, + // 0, 1, 4] + assert!( + new_flags == &vec![1u8, 0, 1, 2] + || new_flags == &vec![1u8, 0, 1, 4] + ); Ok(true) } else { assert_eq!(new_flags[0], 1); @@ -926,7 +931,12 @@ mod tests { new_flags[1] = old_flags.unwrap()[1]; new_flags.push(new_flags_epoch); new_flags.extend(cost.added_bytes.encode_var_vec()); - assert_eq!(new_flags, &vec![1u8, 0, 1, 2]); + // first pass will be vec![1u8, 0, 1, 2], second pass will be vec![1u8, + // 0, 1, 4] + assert!( + new_flags == &vec![1u8, 0, 1, 2] + || new_flags == &vec![1u8, 0, 1, 4] + ); Ok(true) } else { assert_eq!(new_flags[0], 1); @@ -1039,7 +1049,12 @@ mod tests { new_flags[1] = old_flags.unwrap()[1]; new_flags.push(new_flags_epoch); new_flags.extend(cost.added_bytes.encode_var_vec()); - assert_eq!(new_flags, &vec![1u8, 0, 1, 2]); + // first pass will be vec![1u8, 0, 1, 2], second pass will be vec![1u8, + // 0, 1, 4] + assert!( + new_flags == &vec![1u8, 0, 1, 2] + || new_flags == &vec![1u8, 0, 1, 4] + ); Ok(true) } else { assert_eq!(new_flags[0], 1); @@ -1207,7 +1222,12 @@ mod tests { new_flags[1] = old_flags.unwrap()[1]; new_flags.push(new_flags_epoch); new_flags.extend(cost.added_bytes.encode_var_vec()); - assert_eq!(new_flags, &vec![1u8, 0, 1, 2]); + // first pass will be vec![1u8, 0, 1, 2], second pass will be vec![1u8, + // 0, 1, 4] + assert!( + new_flags == &vec![1u8, 0, 1, 2] + || new_flags == &vec![1u8, 0, 1, 4] + ); Ok(true) } else { assert_eq!(new_flags[0], 1); @@ -1318,7 +1338,12 @@ mod tests { new_flags[1] = old_flags.unwrap()[1]; new_flags.push(new_flags_epoch); new_flags.extend(cost.added_bytes.encode_var_vec()); - assert_eq!(new_flags, &vec![1u8, 0, 1, 2]); + // first pass will be vec![1u8, 0, 1, 2], second pass will be vec![1u8, + // 0, 1, 4] + assert!( + new_flags == &vec![1u8, 0, 1, 2] + || new_flags == &vec![1u8, 0, 1, 4] + ); Ok(true) } else { assert_eq!(new_flags[0], 1); @@ -1423,7 +1448,10 @@ mod tests { new_flags[1] = old_flags[1]; new_flags.push(new_flags_epoch); new_flags.extend(cost.added_bytes.encode_var_vec()); - assert_eq!(new_flags, &vec![1u8, 0, 1, 1, 1]); + assert!( + new_flags == &vec![1u8, 0, 1, 1, 1] + || new_flags == &vec![1u8, 0, 1, 1, 3] + ); Ok(true) } else { assert_eq!(new_flags[0], 1); @@ -1507,7 +1535,12 @@ mod tests { new_flags[1] = old_flags.unwrap()[1]; new_flags.push(new_flags_epoch); new_flags.extend(cost.added_bytes.encode_var_vec()); - assert_eq!(new_flags, &vec![1u8, 0, 1, 2]); + // first pass will be vec![1u8, 0, 1, 2], second pass will be vec![1u8, + // 0, 1, 4] + assert!( + new_flags == &vec![1u8, 0, 1, 2] + || new_flags == &vec![1u8, 0, 1, 4] + ); Ok(true) } else { assert_eq!(new_flags[0], 1); @@ -1601,7 +1634,12 @@ mod tests { new_flags[1] = old_flags.unwrap()[1]; new_flags.push(new_flags_epoch); new_flags.extend(cost.added_bytes.encode_var_vec()); - assert_eq!(new_flags, &vec![1u8, 0, 1, 2]); + // first pass will be vec![1u8, 0, 1, 2], second pass will be vec![1u8, + // 0, 1, 4] + assert!( + new_flags == &vec![1u8, 0, 1, 2] + || new_flags == &vec![1u8, 0, 1, 4] + ); Ok(true) } else { assert_eq!(new_flags[0], 1); diff --git a/grovedb/src/error.rs b/grovedb/src/error.rs index 0f6cd5d1..92343935 100644 --- a/grovedb/src/error.rs +++ b/grovedb/src/error.rs @@ -152,6 +152,10 @@ pub enum Error { #[error(transparent)] /// Version error VersionError(grovedb_version::error::GroveVersionError), + + #[error("cyclic error")] + /// Cyclic reference + CyclicError(&'static str), } impl From for Error { From 540d120d1bacaf80797c56a0896ac6f57e7f05ec Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Fri, 23 Aug 2024 15:51:56 +0700 Subject: [PATCH 05/15] update --- costs/Cargo.toml | 2 +- grovedb/Cargo.toml | 3 +- grovedb/src/batch/just_in_time_cost_tests.rs | 381 +++++++++++++++++- .../batch/just_in_time_reference_update.rs | 29 +- grovedb/src/batch/mod.rs | 1 + merk/src/tree/just_in_time_value_update.rs | 8 + 6 files changed, 416 insertions(+), 8 deletions(-) diff --git a/costs/Cargo.toml b/costs/Cargo.toml index 4ee1a55b..6fa8ed0c 100644 --- a/costs/Cargo.toml +++ b/costs/Cargo.toml @@ -12,4 +12,4 @@ repository = "https://github.com/dashpay/grovedb" [dependencies] thiserror = "1.0.59" intmap = "2.0.0" -integer-encoding = "4.0.0" +integer-encoding = "4.0.2" diff --git a/grovedb/Cargo.toml b/grovedb/Cargo.toml index 73cd34f4..28b6d048 100644 --- a/grovedb/Cargo.toml +++ b/grovedb/Cargo.toml @@ -40,6 +40,7 @@ rand = "0.8.5" criterion = "0.5.1" hex = "0.4.3" pretty_assertions = "1.4.0" +drive-storage-flags = { git = "https://github.com/dashpay/platform", branch = "fix/just_in_time_fee_update_fixes"} [[bench]] name = "insertion_benchmark" @@ -58,7 +59,7 @@ full = [ "integer-encoding", "grovedb-costs", "nohash-hasher", - "intmap" + "intmap", ] visualize = [ "grovedb-visualize", diff --git a/grovedb/src/batch/just_in_time_cost_tests.rs b/grovedb/src/batch/just_in_time_cost_tests.rs index 9b907843..32385b07 100644 --- a/grovedb/src/batch/just_in_time_cost_tests.rs +++ b/grovedb/src/batch/just_in_time_cost_tests.rs @@ -5,8 +5,9 @@ mod tests { use std::option::Option::None; + use drive_storage_flags::StorageFlags; use grovedb_costs::storage_cost::{ - removal::StorageRemovedBytes::BasicStorageRemoval, + removal::StorageRemovedBytes::{BasicStorageRemoval, NoStorageRemoval}, transition::OperationStorageTransitionType, }; use grovedb_version::version::GroveVersion; @@ -320,7 +321,104 @@ mod tests { } #[test] - fn test_batch_root_one_update_tree_bigger_flags_with_reference() { + fn test_batch_root_one_update_bigger_item_same_epoch_with_reference() { + let grove_version = GroveVersion::latest(); + let db = make_empty_grovedb(); + let tx = db.start_transaction(); + db.insert( + EMPTY_PATH, + b"tree", + Element::empty_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("expected to insert tree"); + + db.insert( + EMPTY_PATH, + b"refs", + Element::empty_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("expected to insert tree"); + + db.insert( + [b"tree".as_slice()].as_ref(), + b"key1", + Element::new_item_with_flags(b"value1".to_vec(), Some(vec![0, 0])), + None, + None, + grove_version, + ) + .unwrap() + .expect("expected to insert item"); + + // We are adding 2 bytes + let ops = vec![ + QualifiedGroveDbOp::insert_or_replace_op( + vec![b"tree".to_vec()], + b"key1".to_vec(), + Element::new_item_with_flags(b"value100".to_vec(), Some(vec![0, 0])), + ), + QualifiedGroveDbOp::insert_only_op( + vec![b"refs".to_vec()], + b"ref_key".to_vec(), + Element::new_reference_with_hops( + ReferencePathType::AbsolutePathReference(vec![ + b"tree".to_vec(), + b"key1".to_vec(), + ]), + Some(1), + ), + ), + ]; + + let _ = db + .apply_batch_with_element_flags_update( + ops, + None, + |cost, old_flags, new_flags| match cost.transition_type() { + OperationStorageTransitionType::OperationUpdateBiggerSize => Ok(false), + OperationStorageTransitionType::OperationUpdateSmallerSize => { + new_flags.extend(vec![1, 2]); + Ok(true) + } + _ => Ok(false), + }, + |_flags, removed_key_bytes, removed_value_bytes| { + Ok(( + BasicStorageRemoval(removed_key_bytes), + BasicStorageRemoval(removed_value_bytes), + )) + }, + Some(&tx), + grove_version, + ) + .cost_as_result() + .expect("expected to not error"); + + let issues = db + .visualize_verify_grovedb(Some(&tx), true, false, &Default::default()) + .unwrap(); + assert_eq!( + issues.len(), + 0, + "reference issue: {}", + issues + .iter() + .map(|(hash, (a, b, c))| format!("{}: {} {} {}", hash, a, b, c)) + .collect::>() + .join(" | ") + ); + } + + #[test] + fn test_batch_root_one_update_bigger_item_different_epoch_with_reference() { let grove_version = GroveVersion::latest(); let db = make_empty_grovedb(); let tx = db.start_transaction(); @@ -427,4 +525,283 @@ mod tests { .join(" | ") ); } + + #[test] + fn test_batch_root_one_update_bigger_item_different_base_epoch_with_bytes_in_epoch_with_reference( + ) { + let grove_version = GroveVersion::latest(); + let db = make_empty_grovedb(); + let tx = db.start_transaction(); + db.insert( + EMPTY_PATH, + b"tree", + Element::empty_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("expected to insert tree"); + + db.insert( + EMPTY_PATH, + b"refs", + Element::empty_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("expected to insert tree"); + + db.insert( + [b"tree".as_slice()].as_ref(), + b"key1", + Element::new_item_with_flags(b"value1".to_vec(), Some(vec![1, 0, 1, 4])), + None, + None, + grove_version, + ) + .unwrap() + .expect("expected to insert item"); + + // We are adding 2 bytes + let ops = vec![ + QualifiedGroveDbOp::insert_or_replace_op( + vec![b"tree".to_vec()], + b"key1".to_vec(), + Element::new_item_with_flags(b"value100".to_vec(), Some(vec![0, 1])), + ), + QualifiedGroveDbOp::insert_only_op( + vec![b"refs".to_vec()], + b"ref_key".to_vec(), + Element::new_reference_with_hops( + ReferencePathType::AbsolutePathReference(vec![ + b"tree".to_vec(), + b"key1".to_vec(), + ]), + Some(1), + ), + ), + ]; + + let _ = db + .apply_batch_with_element_flags_update( + ops, + None, + |cost, old_flags, new_flags| { + StorageFlags::update_element_flags(cost, old_flags, new_flags) + }, + |_flags, removed_key_bytes, removed_value_bytes| { + Ok(( + BasicStorageRemoval(removed_key_bytes), + BasicStorageRemoval(removed_value_bytes), + )) + }, + Some(&tx), + grove_version, + ) + .cost_as_result() + .expect("expected to not error"); + + let issues = db + .visualize_verify_grovedb(Some(&tx), true, false, &Default::default()) + .unwrap(); + assert_eq!( + issues.len(), + 0, + "reference issue: {}", + issues + .iter() + .map(|(hash, (a, b, c))| format!("{}: {} {} {}", hash, a, b, c)) + .collect::>() + .join(" | ") + ); + } + + #[test] + fn test_batch_root_one_update_smaller_item_same_base_epoch_with_reference() { + let grove_version = GroveVersion::latest(); + let db = make_empty_grovedb(); + let tx = db.start_transaction(); + db.insert( + EMPTY_PATH, + b"tree", + Element::empty_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("expected to insert tree"); + + db.insert( + EMPTY_PATH, + b"refs", + Element::empty_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("expected to insert tree"); + + db.insert( + [b"tree".as_slice()].as_ref(), + b"key1", + Element::new_item_with_flags(b"value15".to_vec(), Some(vec![0])), + None, + None, + grove_version, + ) + .unwrap() + .expect("expected to insert item"); + + // We are removing 2 bytes + let ops = vec![ + QualifiedGroveDbOp::insert_or_replace_op( + vec![b"tree".to_vec()], + b"key1".to_vec(), + Element::new_item_with_flags(b"value".to_vec(), Some(vec![0])), + ), + QualifiedGroveDbOp::insert_only_op( + vec![b"refs".to_vec()], + b"ref_key".to_vec(), + Element::new_reference_with_hops( + ReferencePathType::AbsolutePathReference(vec![ + b"tree".to_vec(), + b"key1".to_vec(), + ]), + Some(1), + ), + ), + ]; + + let _ = db + .apply_batch_with_element_flags_update( + ops, + None, + |cost, old_flags, new_flags| match cost.transition_type() { + OperationStorageTransitionType::OperationUpdateBiggerSize => { + unreachable!(); + } + OperationStorageTransitionType::OperationUpdateSmallerSize => Ok(false), + _ => unreachable!(), + }, + |_flags, removed_key_bytes, removed_value_bytes| { + Ok((NoStorageRemoval, BasicStorageRemoval(removed_value_bytes))) + }, + Some(&tx), + grove_version, + ) + .cost_as_result() + .expect("expected to not error"); + + let issues = db + .visualize_verify_grovedb(Some(&tx), true, false, &Default::default()) + .unwrap(); + assert_eq!( + issues.len(), + 0, + "reference issue: {}", + issues + .iter() + .map(|(hash, (a, b, c))| format!("{}: {} {} {}", hash, a, b, c)) + .collect::>() + .join(" | ") + ); + } + + #[test] + fn test_batch_root_one_update_smaller_item_different_base_epoch_with_reference() { + let grove_version = GroveVersion::latest(); + let db = make_empty_grovedb(); + let tx = db.start_transaction(); + db.insert( + EMPTY_PATH, + b"tree", + Element::empty_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("expected to insert tree"); + + db.insert( + EMPTY_PATH, + b"refs", + Element::empty_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("expected to insert tree"); + + db.insert( + [b"tree".as_slice()].as_ref(), + b"key1", + Element::new_item_with_flags(b"value15".to_vec(), Some(vec![0])), + None, + None, + grove_version, + ) + .unwrap() + .expect("expected to insert item"); + + // We are removing 2 bytes + let ops = vec![ + QualifiedGroveDbOp::insert_or_replace_op( + vec![b"tree".to_vec()], + b"key1".to_vec(), + Element::new_item_with_flags(b"value".to_vec(), Some(vec![1])), + ), + QualifiedGroveDbOp::insert_only_op( + vec![b"refs".to_vec()], + b"ref_key".to_vec(), + Element::new_reference_with_hops( + ReferencePathType::AbsolutePathReference(vec![ + b"tree".to_vec(), + b"key1".to_vec(), + ]), + Some(1), + ), + ), + ]; + + let _ = db + .apply_batch_with_element_flags_update( + ops, + None, + |cost, old_flags, new_flags| match cost.transition_type() { + OperationStorageTransitionType::OperationUpdateBiggerSize => { + unreachable!(); + } + OperationStorageTransitionType::OperationUpdateSmallerSize => Ok(false), + _ => unreachable!(), + }, + |_flags, removed_key_bytes, removed_value_bytes| { + Ok((NoStorageRemoval, BasicStorageRemoval(removed_value_bytes))) + }, + Some(&tx), + grove_version, + ) + .cost_as_result() + .expect("expected to not error"); + + let issues = db + .visualize_verify_grovedb(Some(&tx), true, false, &Default::default()) + .unwrap(); + assert_eq!( + issues.len(), + 0, + "reference issue: {}", + issues + .iter() + .map(|(hash, (a, b, c))| format!("{}: {} {} {}", hash, a, b, c)) + .collect::>() + .join(" | ") + ); + } } diff --git a/grovedb/src/batch/just_in_time_reference_update.rs b/grovedb/src/batch/just_in_time_reference_update.rs index 8e77a5fa..18042f2e 100644 --- a/grovedb/src/batch/just_in_time_reference_update.rs +++ b/grovedb/src/batch/just_in_time_reference_update.rs @@ -1,7 +1,12 @@ use std::borrow::Cow; use grovedb_costs::{ - cost_return_on_error_no_add, storage_cost::StorageCost, CostResult, CostsExt, OperationCost, + cost_return_on_error_no_add, + storage_cost::{ + removal::{StorageRemovedBytes, StorageRemovedBytes::BasicStorageRemoval}, + StorageCost, + }, + CostResult, CostsExt, OperationCost, }; use grovedb_merk::{ tree::{kv::KV, value_hash, TreeNode}, @@ -20,7 +25,7 @@ where F: FnMut(&[Vec], bool) -> CostResult, Error>, S: StorageContext<'db>, { - pub(crate) fn process_old_element_flags( + pub(crate) fn process_old_element_flags( key: &[u8], serialized: &[u8], new_element: &mut Element, @@ -28,12 +33,18 @@ where old_serialized_element: &[u8], is_in_sum_tree: bool, flags_update: &mut G, + split_removal_bytes: &mut SR, grove_version: &GroveVersion, ) -> CostResult where G: FnMut(&StorageCost, Option, &mut ElementFlags) -> Result, + SR: FnMut( + &mut ElementFlags, + u32, + u32, + ) -> Result<(StorageRemovedBytes, StorageRemovedBytes), Error>, { - let maybe_old_flags = old_element.get_flags_owned(); + let mut maybe_old_flags = old_element.get_flags_owned(); let mut cost = OperationCost::default(); @@ -57,9 +68,19 @@ where loop { // Calculate storage costs - let storage_costs = + let mut storage_costs = TreeNode::storage_cost_for_update(new_storage_cost, old_storage_cost); + if let Some(old_element_flags) = maybe_old_flags.as_mut() { + if let BasicStorageRemoval(removed_bytes) = storage_costs.removed_bytes { + let (_, value_removed_bytes) = cost_return_on_error_no_add!( + &cost, + split_removal_bytes(old_element_flags, 0, removed_bytes) + ); + storage_costs.removed_bytes = value_removed_bytes; + } + } + let mut new_element_cloned = original_new_element.clone(); let changed = cost_return_on_error_no_add!( diff --git a/grovedb/src/batch/mod.rs b/grovedb/src/batch/mod.rs index 04142b60..22596e28 100644 --- a/grovedb/src/batch/mod.rs +++ b/grovedb/src/batch/mod.rs @@ -1119,6 +1119,7 @@ where &old_serialized_element, is_in_sum_tree, flags_update, + split_removal_bytes, grove_version, ) ); diff --git a/merk/src/tree/just_in_time_value_update.rs b/merk/src/tree/just_in_time_value_update.rs index 6f59eda4..1702361f 100644 --- a/merk/src/tree/just_in_time_value_update.rs +++ b/merk/src/tree/just_in_time_value_update.rs @@ -40,6 +40,14 @@ impl TreeNode { // todo: clean up clones let original_new_value = self.value_ref().clone(); loop { + if let BasicStorageRemoval(removed_bytes) = + storage_costs.value_storage_cost.removed_bytes + { + let (_, value_removed_bytes) = + section_removal_bytes(&old_value, 0, removed_bytes)?; + storage_costs.value_storage_cost.removed_bytes = value_removed_bytes; + } + let (flags_changed, value_defined_cost) = update_tree_value_based_on_costs( &storage_costs.value_storage_cost, &old_value, From 34ee8204dcd84eee789037f34c19b8b9d84c1ab5 Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Fri, 23 Aug 2024 16:04:03 +0700 Subject: [PATCH 06/15] costs --- costs/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/costs/Cargo.toml b/costs/Cargo.toml index 6fa8ed0c..4ee1a55b 100644 --- a/costs/Cargo.toml +++ b/costs/Cargo.toml @@ -12,4 +12,4 @@ repository = "https://github.com/dashpay/grovedb" [dependencies] thiserror = "1.0.59" intmap = "2.0.0" -integer-encoding = "4.0.2" +integer-encoding = "4.0.0" From 614e5dbca1f66fc6134c1753fb89e551a300ea7c Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Fri, 23 Aug 2024 18:27:03 +0700 Subject: [PATCH 07/15] more add tests --- grovedb/Cargo.toml | 2 +- grovedb/src/batch/just_in_time_cost_tests.rs | 524 ++++++++++++------- merk/Cargo.toml | 2 +- storage/Cargo.toml | 2 +- 4 files changed, 327 insertions(+), 203 deletions(-) diff --git a/grovedb/Cargo.toml b/grovedb/Cargo.toml index 28b6d048..f34fb9eb 100644 --- a/grovedb/Cargo.toml +++ b/grovedb/Cargo.toml @@ -21,7 +21,7 @@ hex = "0.4.3" itertools = { version = "0.12.1", optional = true } derive_more = "0.99.18" integer-encoding = { version = "4.0.0", optional = true } -grovedb-costs = { version = "1.0.0", path = "../costs", optional = true } +grovedb-costs = { version = "1.0.0", git = "https://github.com/dashpay/grovedb", branch = "fix/JITfix", optional = true } nohash-hasher = { version = "0.2.0", optional = true } indexmap = "2.2.6" intmap = { version = "2.0.0", optional = true } diff --git a/grovedb/src/batch/just_in_time_cost_tests.rs b/grovedb/src/batch/just_in_time_cost_tests.rs index 32385b07..c2781b8d 100644 --- a/grovedb/src/batch/just_in_time_cost_tests.rs +++ b/grovedb/src/batch/just_in_time_cost_tests.rs @@ -3,25 +3,102 @@ #[cfg(feature = "full")] mod tests { - use std::option::Option::None; + use std::{collections::BTreeMap, option::Option::None}; use drive_storage_flags::StorageFlags; - use grovedb_costs::storage_cost::{ - removal::StorageRemovedBytes::{BasicStorageRemoval, NoStorageRemoval}, - transition::OperationStorageTransitionType, - }; use grovedb_version::version::GroveVersion; - use integer_encoding::VarInt; use crate::{ batch::QualifiedGroveDbOp, reference_path::{ ReferencePathType, ReferencePathType::UpstreamFromElementHeightReference, }, - tests::{common::EMPTY_PATH, make_empty_grovedb}, - Element, + tests::{common::EMPTY_PATH, make_empty_grovedb, TempGroveDb}, + Element, Error, Transaction, }; + fn apply_batch( + grove_db: &TempGroveDb, + ops: Vec, + tx: &Transaction, + grove_version: &GroveVersion, + ) { + grove_db + .apply_batch_with_element_flags_update( + ops, + None, + |cost, old_flags, new_flags| { + StorageFlags::update_element_flags(cost, old_flags, new_flags) + .map_err(|e| Error::JustInTimeElementFlagsClientError(e.to_string())) + }, + |flags, removed_key_bytes, removed_value_bytes| { + StorageFlags::split_removal_bytes(flags, removed_key_bytes, removed_value_bytes) + .map_err(|e| Error::SplitRemovalBytesClientError(e.to_string())) + }, + Some(&tx), + grove_version, + ) + .cost_as_result() + .expect("expected to not error"); + } + + fn expect_storage_flags( + grove_db: &TempGroveDb, + tx: &Transaction, + expected_storage_flags: StorageFlags, + grove_version: &GroveVersion, + ) { + let element = grove_db + .get( + [b"tree".as_slice()].as_ref(), + b"key1", + Some(tx), + grove_version, + ) + .unwrap() + .expect("expected element"); + let storage_flags = StorageFlags::from_element_flags_ref( + element.get_flags().as_ref().expect("expected flags"), + ) + .expect("expected to get storage flags") + .expect("expected storage flags"); + assert_eq!(storage_flags, expected_storage_flags); + } + + fn verify_references(grove_db: &TempGroveDb, tx: &Transaction) { + let issues = grove_db + .visualize_verify_grovedb(Some(&tx), true, false, &Default::default()) + .unwrap(); + assert_eq!( + issues.len(), + 0, + "reference issue: {}", + issues + .iter() + .map(|(hash, (a, b, c))| format!("{}: {} {} {}", hash, a, b, c)) + .collect::>() + .join(" | ") + ); + } + + fn create_epoch_map(epoch: u16, bytes: u32) -> BTreeMap { + let mut map = BTreeMap::new(); + map.insert(epoch, bytes); + map + } + + fn create_two_epoch_map( + first_epoch: u16, + first_epoch_bytes: u32, + second_epoch: u16, + second_epoch_bytes: u32, + ) -> BTreeMap { + let mut map = BTreeMap::new(); + map.insert(first_epoch, first_epoch_bytes); + map.insert(second_epoch, second_epoch_bytes); + map + } + #[test] fn test_partial_costs_with_no_new_operations_are_same_as_apply_batch() { let grove_version = GroveVersion::latest(); @@ -325,6 +402,8 @@ mod tests { let grove_version = GroveVersion::latest(); let db = make_empty_grovedb(); let tx = db.start_transaction(); + + let owner_id = [1; 32]; db.insert( EMPTY_PATH, b"tree", @@ -350,7 +429,10 @@ mod tests { db.insert( [b"tree".as_slice()].as_ref(), b"key1", - Element::new_item_with_flags(b"value1".to_vec(), Some(vec![0, 0])), + Element::new_item_with_flags( + b"value1".to_vec(), + Some(StorageFlags::new_single_epoch(0, Some(owner_id)).to_element_flags()), + ), None, None, grove_version, @@ -363,7 +445,10 @@ mod tests { QualifiedGroveDbOp::insert_or_replace_op( vec![b"tree".to_vec()], b"key1".to_vec(), - Element::new_item_with_flags(b"value100".to_vec(), Some(vec![0, 0])), + Element::new_item_with_flags( + b"value100".to_vec(), + Some(StorageFlags::new_single_epoch(0, Some(owner_id)).to_element_flags()), + ), ), QualifiedGroveDbOp::insert_only_op( vec![b"refs".to_vec()], @@ -378,50 +463,25 @@ mod tests { ), ]; - let _ = db - .apply_batch_with_element_flags_update( - ops, - None, - |cost, old_flags, new_flags| match cost.transition_type() { - OperationStorageTransitionType::OperationUpdateBiggerSize => Ok(false), - OperationStorageTransitionType::OperationUpdateSmallerSize => { - new_flags.extend(vec![1, 2]); - Ok(true) - } - _ => Ok(false), - }, - |_flags, removed_key_bytes, removed_value_bytes| { - Ok(( - BasicStorageRemoval(removed_key_bytes), - BasicStorageRemoval(removed_value_bytes), - )) - }, - Some(&tx), - grove_version, - ) - .cost_as_result() - .expect("expected to not error"); + apply_batch(&db, ops, &tx, grove_version); - let issues = db - .visualize_verify_grovedb(Some(&tx), true, false, &Default::default()) - .unwrap(); - assert_eq!( - issues.len(), - 0, - "reference issue: {}", - issues - .iter() - .map(|(hash, (a, b, c))| format!("{}: {} {} {}", hash, a, b, c)) - .collect::>() - .join(" | ") + expect_storage_flags( + &db, + &tx, + StorageFlags::new_single_epoch(0, Some(owner_id)), + grove_version, ); + + verify_references(&db, &tx); } #[test] fn test_batch_root_one_update_bigger_item_different_epoch_with_reference() { let grove_version = GroveVersion::latest(); let db = make_empty_grovedb(); - let tx = db.start_transaction(); + + let owner_id = [1; 32]; + db.insert( EMPTY_PATH, b"tree", @@ -447,7 +507,10 @@ mod tests { db.insert( [b"tree".as_slice()].as_ref(), b"key1", - Element::new_item_with_flags(b"value1".to_vec(), Some(vec![0, 0])), + Element::new_item_with_flags( + b"value1".to_vec(), + Some(StorageFlags::new_single_epoch(0, Some(owner_id)).to_element_flags()), + ), None, None, grove_version, @@ -455,83 +518,68 @@ mod tests { .unwrap() .expect("expected to insert item"); - // We are adding 2 bytes - let ops = vec![ - QualifiedGroveDbOp::insert_or_replace_op( - vec![b"tree".to_vec()], - b"key1".to_vec(), - Element::new_item_with_flags(b"value100".to_vec(), Some(vec![0, 1])), - ), - QualifiedGroveDbOp::insert_only_op( - vec![b"refs".to_vec()], - b"ref_key".to_vec(), - Element::new_reference_with_hops( - ReferencePathType::AbsolutePathReference(vec![ - b"tree".to_vec(), - b"key1".to_vec(), - ]), - Some(1), + let base_item = b"value1".to_vec(); + + for n in 1..150 { + let tx = db.start_transaction(); + let mut item = base_item.clone(); + item.extend(std::iter::repeat(0).take(n)); + // We are adding n bytes + let ops = vec![ + QualifiedGroveDbOp::insert_or_replace_op( + vec![b"tree".to_vec()], + b"key1".to_vec(), + Element::new_item_with_flags( + item, // value100 if n was 2 + Some(StorageFlags::new_single_epoch(1, Some(owner_id)).to_element_flags()), + ), + ), + QualifiedGroveDbOp::insert_only_op( + vec![b"refs".to_vec()], + b"ref_key".to_vec(), + Element::new_reference_with_hops( + ReferencePathType::AbsolutePathReference(vec![ + b"tree".to_vec(), + b"key1".to_vec(), + ]), + Some(1), + ), + ), + ]; + + apply_batch(&db, ops, &tx, grove_version); + + let expected_added_bytes = if n < 15 { + n as u32 + 3 + } else if n < 124 { + n as u32 + 4 // the varint requires an extra byte + } else { + n as u32 + 5 // the varint requires an extra byte + }; + expect_storage_flags( + &db, + &tx, + StorageFlags::MultiEpochOwned( + 0, + create_epoch_map(1, expected_added_bytes), + owner_id, ), - ), - ]; - - let _ = db - .apply_batch_with_element_flags_update( - ops, - None, - |cost, old_flags, new_flags| match cost.transition_type() { - OperationStorageTransitionType::OperationUpdateBiggerSize => { - if new_flags[0] == 0 { - new_flags[0] = 1; - let new_flags_epoch = new_flags[1]; - new_flags[1] = old_flags.unwrap()[1]; - new_flags.push(new_flags_epoch); - new_flags.extend(cost.added_bytes.encode_var_vec()); - Ok(true) - } else { - assert_eq!(new_flags[0], 1); - Ok(false) - } - } - OperationStorageTransitionType::OperationUpdateSmallerSize => { - new_flags.extend(vec![1, 2]); - Ok(true) - } - _ => Ok(false), - }, - |_flags, removed_key_bytes, removed_value_bytes| { - Ok(( - BasicStorageRemoval(removed_key_bytes), - BasicStorageRemoval(removed_value_bytes), - )) - }, - Some(&tx), grove_version, - ) - .cost_as_result() - .expect("expected to not error"); + ); - let issues = db - .visualize_verify_grovedb(Some(&tx), true, false, &Default::default()) - .unwrap(); - assert_eq!( - issues.len(), - 0, - "reference issue: {}", - issues - .iter() - .map(|(hash, (a, b, c))| format!("{}: {} {} {}", hash, a, b, c)) - .collect::>() - .join(" | ") - ); + verify_references(&db, &tx); + } } #[test] - fn test_batch_root_one_update_bigger_item_different_base_epoch_with_bytes_in_epoch_with_reference( + fn test_batch_root_one_update_bigger_item_different_base_epoch_with_bytes_in_last_epoch_with_reference( ) { let grove_version = GroveVersion::latest(); let db = make_empty_grovedb(); let tx = db.start_transaction(); + + let owner_id = [1; 32]; + db.insert( EMPTY_PATH, b"tree", @@ -557,7 +605,13 @@ mod tests { db.insert( [b"tree".as_slice()].as_ref(), b"key1", - Element::new_item_with_flags(b"value1".to_vec(), Some(vec![1, 0, 1, 4])), + Element::new_item_with_flags( + b"value1".to_vec(), + Some( + StorageFlags::MultiEpochOwned(0, create_epoch_map(1, 4), owner_id) + .to_element_flags(), + ), + ), None, None, grove_version, @@ -565,58 +619,158 @@ mod tests { .unwrap() .expect("expected to insert item"); - // We are adding 2 bytes - let ops = vec![ - QualifiedGroveDbOp::insert_or_replace_op( - vec![b"tree".to_vec()], - b"key1".to_vec(), - Element::new_item_with_flags(b"value100".to_vec(), Some(vec![0, 1])), - ), - QualifiedGroveDbOp::insert_only_op( - vec![b"refs".to_vec()], - b"ref_key".to_vec(), - Element::new_reference_with_hops( - ReferencePathType::AbsolutePathReference(vec![ - b"tree".to_vec(), - b"key1".to_vec(), - ]), - Some(1), + let base_item = b"value1".to_vec(); + + for n in 1..150 { + let tx = db.start_transaction(); + let mut item = base_item.clone(); + item.extend(std::iter::repeat(0).take(n)); + // We are adding n bytes + let ops = vec![ + QualifiedGroveDbOp::insert_or_replace_op( + vec![b"tree".to_vec()], + b"key1".to_vec(), + Element::new_item_with_flags( + item, // value100 if n was 2 + Some(StorageFlags::new_single_epoch(1, Some(owner_id)).to_element_flags()), + ), + ), + QualifiedGroveDbOp::insert_only_op( + vec![b"refs".to_vec()], + b"ref_key".to_vec(), + Element::new_reference_with_hops( + ReferencePathType::AbsolutePathReference(vec![ + b"tree".to_vec(), + b"key1".to_vec(), + ]), + Some(1), + ), + ), + ]; + + apply_batch(&db, ops, &tx, grove_version); + + let expected_added_bytes = if n < 15 { + n as u32 + 4 + } else if n < 123 { + n as u32 + 5 // the varint requires an extra byte + } else { + n as u32 + 6 // the varint requires an extra byte + }; + expect_storage_flags( + &db, + &tx, + StorageFlags::MultiEpochOwned( + 0, + create_epoch_map(1, expected_added_bytes), + owner_id, + ), + grove_version, + ); + + verify_references(&db, &tx); + } + } + + #[test] + fn test_batch_root_one_update_bigger_item_different_base_epoch_with_bytes_in_future_epoch_with_reference( + ) { + let grove_version = GroveVersion::latest(); + let db = make_empty_grovedb(); + let tx = db.start_transaction(); + + let owner_id = [1; 32]; + + db.insert( + EMPTY_PATH, + b"tree", + Element::empty_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("expected to insert tree"); + + db.insert( + EMPTY_PATH, + b"refs", + Element::empty_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("expected to insert tree"); + + db.insert( + [b"tree".as_slice()].as_ref(), + b"key1", + Element::new_item_with_flags( + b"value1".to_vec(), + Some( + StorageFlags::MultiEpochOwned(0, create_epoch_map(1, 4), owner_id) + .to_element_flags(), ), ), - ]; + None, + None, + grove_version, + ) + .unwrap() + .expect("expected to insert item"); - let _ = db - .apply_batch_with_element_flags_update( - ops, - None, - |cost, old_flags, new_flags| { - StorageFlags::update_element_flags(cost, old_flags, new_flags) - }, - |_flags, removed_key_bytes, removed_value_bytes| { - Ok(( - BasicStorageRemoval(removed_key_bytes), - BasicStorageRemoval(removed_value_bytes), - )) - }, - Some(&tx), + let base_item = b"value1".to_vec(); + + for n in 1..150 { + let tx = db.start_transaction(); + let mut item = base_item.clone(); + item.extend(std::iter::repeat(0).take(n)); + // We are adding n bytes + let ops = vec![ + QualifiedGroveDbOp::insert_or_replace_op( + vec![b"tree".to_vec()], + b"key1".to_vec(), + Element::new_item_with_flags( + item, // value100 if n was 2 + Some(StorageFlags::new_single_epoch(2, Some(owner_id)).to_element_flags()), + ), + ), + QualifiedGroveDbOp::insert_only_op( + vec![b"refs".to_vec()], + b"ref_key".to_vec(), + Element::new_reference_with_hops( + ReferencePathType::AbsolutePathReference(vec![ + b"tree".to_vec(), + b"key1".to_vec(), + ]), + Some(1), + ), + ), + ]; + + apply_batch(&db, ops, &tx, grove_version); + + let expected_added_bytes = if n < 12 { + n as u32 + 3 + } else if n < 124 { + n as u32 + 4 // the varint requires an extra byte + } else { + n as u32 + 5 // the varint requires an extra byte + }; + expect_storage_flags( + &db, + &tx, + StorageFlags::MultiEpochOwned( + 0, + create_two_epoch_map(1, 4, 2, expected_added_bytes), + owner_id, + ), grove_version, - ) - .cost_as_result() - .expect("expected to not error"); + ); - let issues = db - .visualize_verify_grovedb(Some(&tx), true, false, &Default::default()) - .unwrap(); - assert_eq!( - issues.len(), - 0, - "reference issue: {}", - issues - .iter() - .map(|(hash, (a, b, c))| format!("{}: {} {} {}", hash, a, b, c)) - .collect::>() - .join(" | ") - ); + verify_references(&db, &tx); + } } #[test] @@ -624,6 +778,9 @@ mod tests { let grove_version = GroveVersion::latest(); let db = make_empty_grovedb(); let tx = db.start_transaction(); + + let owner_id = [1; 32]; + db.insert( EMPTY_PATH, b"tree", @@ -677,25 +834,7 @@ mod tests { ), ]; - let _ = db - .apply_batch_with_element_flags_update( - ops, - None, - |cost, old_flags, new_flags| match cost.transition_type() { - OperationStorageTransitionType::OperationUpdateBiggerSize => { - unreachable!(); - } - OperationStorageTransitionType::OperationUpdateSmallerSize => Ok(false), - _ => unreachable!(), - }, - |_flags, removed_key_bytes, removed_value_bytes| { - Ok((NoStorageRemoval, BasicStorageRemoval(removed_value_bytes))) - }, - Some(&tx), - grove_version, - ) - .cost_as_result() - .expect("expected to not error"); + apply_batch(&db, ops, &tx, grove_version); let issues = db .visualize_verify_grovedb(Some(&tx), true, false, &Default::default()) @@ -717,6 +856,9 @@ mod tests { let grove_version = GroveVersion::latest(); let db = make_empty_grovedb(); let tx = db.start_transaction(); + + let owner_id = [1; 32]; + db.insert( EMPTY_PATH, b"tree", @@ -770,25 +912,7 @@ mod tests { ), ]; - let _ = db - .apply_batch_with_element_flags_update( - ops, - None, - |cost, old_flags, new_flags| match cost.transition_type() { - OperationStorageTransitionType::OperationUpdateBiggerSize => { - unreachable!(); - } - OperationStorageTransitionType::OperationUpdateSmallerSize => Ok(false), - _ => unreachable!(), - }, - |_flags, removed_key_bytes, removed_value_bytes| { - Ok((NoStorageRemoval, BasicStorageRemoval(removed_value_bytes))) - }, - Some(&tx), - grove_version, - ) - .cost_as_result() - .expect("expected to not error"); + apply_batch(&db, ops, &tx, grove_version); let issues = db .visualize_verify_grovedb(Some(&tx), true, false, &Default::default()) diff --git a/merk/Cargo.toml b/merk/Cargo.toml index 689d57e4..950eda38 100644 --- a/merk/Cargo.toml +++ b/merk/Cargo.toml @@ -17,7 +17,7 @@ grovedb-storage = { version = "1.0.0", path = "../storage", optional = true } failure = "0.1.8" integer-encoding = "4.0.0" indexmap = "2.2.6" -grovedb-costs = { version = "1.0.0", path = "../costs" } +grovedb-costs = { version = "1.0.0", git = "https://github.com/dashpay/grovedb", branch = "fix/JITfix" } grovedb-visualize = { version = "1.0.0", path = "../visualize" } grovedb-path = { version = "1.0.0", path = "../path" } hex = "0.4.3" diff --git a/storage/Cargo.toml b/storage/Cargo.toml index 7db2b599..b2c3e72b 100644 --- a/storage/Cargo.toml +++ b/storage/Cargo.toml @@ -16,7 +16,7 @@ blake3 = { version = "1.5.1", optional = true } integer-encoding = { version = "4.0.0", optional = true } grovedb-visualize = { version = "1.0.0", path = "../visualize" } strum = { version = "0.26.2", features = ["derive"] } -grovedb-costs = { version = "1.0.0", path = "../costs" } +grovedb-costs = { version = "1.0.0", git = "https://github.com/dashpay/grovedb", branch = "fix/JITfix" } thiserror = "1.0.59" rocksdb = { version = "0.22.0", optional = true } hex = "0.4.3" From 473804e4580aef868af0596e60abf59e2c942661 Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Sat, 24 Aug 2024 03:04:53 +0700 Subject: [PATCH 08/15] fixes --- grovedb/src/batch/just_in_time_cost_tests.rs | 292 +++++++++++++++--- .../batch/just_in_time_reference_update.rs | 12 +- grovedb/src/batch/mod.rs | 14 +- grovedb/src/element/delete.rs | 1 + grovedb/src/element/helpers.rs | 12 + merk/src/merk/apply.rs | 18 +- merk/src/test_utils/mod.rs | 2 + merk/src/tree/just_in_time_value_update.rs | 30 +- merk/src/tree/mod.rs | 61 +++- merk/src/tree/ops.rs | 45 ++- merk/src/tree/walk/mod.rs | 20 ++ 11 files changed, 452 insertions(+), 55 deletions(-) diff --git a/grovedb/src/batch/just_in_time_cost_tests.rs b/grovedb/src/batch/just_in_time_cost_tests.rs index c2781b8d..f1503df0 100644 --- a/grovedb/src/batch/just_in_time_cost_tests.rs +++ b/grovedb/src/batch/just_in_time_cost_tests.rs @@ -6,7 +6,12 @@ mod tests { use std::{collections::BTreeMap, option::Option::None}; use drive_storage_flags::StorageFlags; + use grovedb_costs::{ + storage_cost::removal::{StorageRemovalPerEpochByIdentifier, StorageRemovedBytes}, + OperationCost, + }; use grovedb_version::version::GroveVersion; + use intmap::IntMap; use crate::{ batch::QualifiedGroveDbOp, @@ -17,12 +22,24 @@ mod tests { Element, Error, Transaction, }; + fn single_epoch_removed_bytes_map( + owner_id: [u8; 32], + epoch_index: u64, + bytes_removed: u32, + ) -> StorageRemovalPerEpochByIdentifier { + let mut removed_bytes = StorageRemovalPerEpochByIdentifier::default(); + let mut removed_bytes_for_identity = IntMap::new(); + removed_bytes_for_identity.insert(epoch_index, bytes_removed); + removed_bytes.insert(owner_id, removed_bytes_for_identity); + removed_bytes + } + fn apply_batch( grove_db: &TempGroveDb, ops: Vec, tx: &Transaction, grove_version: &GroveVersion, - ) { + ) -> OperationCost { grove_db .apply_batch_with_element_flags_update( ops, @@ -39,7 +56,7 @@ mod tests { grove_version, ) .cost_as_result() - .expect("expected to not error"); + .expect("expected to not error") } fn expect_storage_flags( @@ -542,7 +559,7 @@ mod tests { b"tree".to_vec(), b"key1".to_vec(), ]), - Some(1), + None, ), ), ]; @@ -576,7 +593,6 @@ mod tests { ) { let grove_version = GroveVersion::latest(); let db = make_empty_grovedb(); - let tx = db.start_transaction(); let owner_id = [1; 32]; @@ -643,7 +659,7 @@ mod tests { b"tree".to_vec(), b"key1".to_vec(), ]), - Some(1), + None, ), ), ]; @@ -677,7 +693,6 @@ mod tests { ) { let grove_version = GroveVersion::latest(); let db = make_empty_grovedb(); - let tx = db.start_transaction(); let owner_id = [1; 32]; @@ -744,7 +759,7 @@ mod tests { b"tree".to_vec(), b"key1".to_vec(), ]), - Some(1), + None, ), ), ]; @@ -777,6 +792,198 @@ mod tests { fn test_batch_root_one_update_smaller_item_same_base_epoch_with_reference() { let grove_version = GroveVersion::latest(); let db = make_empty_grovedb(); + + let owner_id = [1; 32]; + + db.insert( + EMPTY_PATH, + b"tree", + Element::empty_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("expected to insert tree"); + + db.insert( + EMPTY_PATH, + b"refs", + Element::empty_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("expected to insert tree"); + + let base_item = b"value1".to_vec(); + let mut original_item = base_item.clone(); + original_item.extend(std::iter::repeat(0).take(150)); + + db.insert( + [b"tree".as_slice()].as_ref(), + b"key1", + Element::new_item_with_flags( + original_item, + Some(StorageFlags::new_single_epoch(0, Some(owner_id)).to_element_flags()), + ), + None, + None, + grove_version, + ) + .unwrap() + .expect("expected to insert item"); + + let to = 150usize; + + for n in (0..to).rev() { + let tx = db.start_transaction(); + let mut item = base_item.clone(); + item.extend(std::iter::repeat(0).take(n)); + // We are adding n bytes + let ops = vec![ + QualifiedGroveDbOp::insert_or_replace_op( + vec![b"tree".to_vec()], + b"key1".to_vec(), + Element::new_item_with_flags( + item, // value1 if n was 1 + Some(StorageFlags::new_single_epoch(0, Some(owner_id)).to_element_flags()), + ), + ), + QualifiedGroveDbOp::insert_only_op( + vec![b"refs".to_vec()], + b"ref_key".to_vec(), + Element::new_reference_with_hops( + ReferencePathType::AbsolutePathReference(vec![ + b"tree".to_vec(), + b"key1".to_vec(), + ]), + None, + ), + ), + ]; + + let removed_bytes = if n > 17 { + to as u32 - n as u32 + } else { + to as u32 - n as u32 + 1 // we remove an extra byte + }; + + let storage_removed_bytes = apply_batch(&db, ops, &tx, grove_version) + .storage_cost + .removed_bytes; + + let expected_storage_removed_bytes = + single_epoch_removed_bytes_map(owner_id, 0, removed_bytes); + + assert_eq!( + storage_removed_bytes, + StorageRemovedBytes::SectionedStorageRemoval(expected_storage_removed_bytes) + ); + + expect_storage_flags( + &db, + &tx, + StorageFlags::SingleEpochOwned(0, owner_id), + grove_version, + ); + + verify_references(&db, &tx); + } + } + + #[test] + fn test_batch_root_one_update_smaller_item_different_base_epoch_with_reference() { + let grove_version = GroveVersion::latest(); + let db = make_empty_grovedb(); + + let owner_id = [1; 32]; + + db.insert( + EMPTY_PATH, + b"tree", + Element::empty_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("expected to insert tree"); + + db.insert( + EMPTY_PATH, + b"refs", + Element::empty_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("expected to insert tree"); + + let base_item = b"value1".to_vec(); + let mut original_item = base_item.clone(); + original_item.extend(std::iter::repeat(0).take(150)); + + db.insert( + [b"tree".as_slice()].as_ref(), + b"key1", + Element::new_item_with_flags( + original_item, + Some(StorageFlags::new_single_epoch(0, Some(owner_id)).to_element_flags()), + ), + None, + None, + grove_version, + ) + .unwrap() + .expect("expected to insert item"); + + for n in 0..150 { + let tx = db.start_transaction(); + let mut item = base_item.clone(); + item.extend(std::iter::repeat(0).take(n)); + // We are adding n bytes + let ops = vec![ + QualifiedGroveDbOp::insert_or_replace_op( + vec![b"tree".to_vec()], + b"key1".to_vec(), + Element::new_item_with_flags( + item, // value1 if n was 1 + Some(StorageFlags::new_single_epoch(1, Some(owner_id)).to_element_flags()), + ), + ), + QualifiedGroveDbOp::insert_only_op( + vec![b"refs".to_vec()], + b"ref_key".to_vec(), + Element::new_reference_with_hops( + ReferencePathType::AbsolutePathReference(vec![ + b"tree".to_vec(), + b"key1".to_vec(), + ]), + None, + ), + ), + ]; + + apply_batch(&db, ops, &tx, grove_version); + + expect_storage_flags( + &db, + &tx, + StorageFlags::SingleEpochOwned(0, owner_id), + grove_version, + ); + + verify_references(&db, &tx); + } + } + + #[test] + fn test_batch_root_one_update_smaller_item_different_base_epoch_with_previous_flags() { + let grove_version = GroveVersion::latest(); + let db = make_empty_grovedb(); let tx = db.start_transaction(); let owner_id = [1; 32]; @@ -806,7 +1013,13 @@ mod tests { db.insert( [b"tree".as_slice()].as_ref(), b"key1", - Element::new_item_with_flags(b"value15".to_vec(), Some(vec![0])), + Element::new_item_with_flags( + b"value1500".to_vec(), // the 1500 is 4 bytes + Some( + StorageFlags::MultiEpochOwned(0, create_epoch_map(1, 7), owner_id) + .to_element_flags(), + ), + ), None, None, grove_version, @@ -819,7 +1032,10 @@ mod tests { QualifiedGroveDbOp::insert_or_replace_op( vec![b"tree".to_vec()], b"key1".to_vec(), - Element::new_item_with_flags(b"value".to_vec(), Some(vec![0])), + Element::new_item_with_flags( + b"value15".to_vec(), + Some(StorageFlags::new_single_epoch(1, Some(owner_id)).to_element_flags()), + ), ), QualifiedGroveDbOp::insert_only_op( vec![b"refs".to_vec()], @@ -829,30 +1045,26 @@ mod tests { b"tree".to_vec(), b"key1".to_vec(), ]), - Some(1), + None, ), ), ]; apply_batch(&db, ops, &tx, grove_version); - let issues = db - .visualize_verify_grovedb(Some(&tx), true, false, &Default::default()) - .unwrap(); - assert_eq!( - issues.len(), - 0, - "reference issue: {}", - issues - .iter() - .map(|(hash, (a, b, c))| format!("{}: {} {} {}", hash, a, b, c)) - .collect::>() - .join(" | ") + expect_storage_flags( + &db, + &tx, + StorageFlags::MultiEpochOwned(0, create_epoch_map(1, 5), owner_id), + grove_version, ); + + verify_references(&db, &tx); } #[test] - fn test_batch_root_one_update_smaller_item_different_base_epoch_with_reference() { + fn test_batch_root_one_update_smaller_item_different_base_epoch_with_previous_flags_all_multi_epoch_removal( + ) { let grove_version = GroveVersion::latest(); let db = make_empty_grovedb(); let tx = db.start_transaction(); @@ -884,7 +1096,13 @@ mod tests { db.insert( [b"tree".as_slice()].as_ref(), b"key1", - Element::new_item_with_flags(b"value15".to_vec(), Some(vec![0])), + Element::new_item_with_flags( + b"value1500".to_vec(), // the 1500 is 4 bytes + Some( + StorageFlags::MultiEpochOwned(0, create_epoch_map(1, 7), owner_id) + .to_element_flags(), + ), + ), None, None, grove_version, @@ -897,7 +1115,10 @@ mod tests { QualifiedGroveDbOp::insert_or_replace_op( vec![b"tree".to_vec()], b"key1".to_vec(), - Element::new_item_with_flags(b"value".to_vec(), Some(vec![1])), + Element::new_item_with_flags( + b"value".to_vec(), + Some(StorageFlags::new_single_epoch(1, Some(owner_id)).to_element_flags()), + ), ), QualifiedGroveDbOp::insert_only_op( vec![b"refs".to_vec()], @@ -907,25 +1128,20 @@ mod tests { b"tree".to_vec(), b"key1".to_vec(), ]), - Some(1), + None, ), ), ]; apply_batch(&db, ops, &tx, grove_version); - let issues = db - .visualize_verify_grovedb(Some(&tx), true, false, &Default::default()) - .unwrap(); - assert_eq!( - issues.len(), - 0, - "reference issue: {}", - issues - .iter() - .map(|(hash, (a, b, c))| format!("{}: {} {} {}", hash, a, b, c)) - .collect::>() - .join(" | ") + expect_storage_flags( + &db, + &tx, + StorageFlags::SingleEpochOwned(0, owner_id), + grove_version, ); + + verify_references(&db, &tx); } } diff --git a/grovedb/src/batch/just_in_time_reference_update.rs b/grovedb/src/batch/just_in_time_reference_update.rs index 18042f2e..f50b233e 100644 --- a/grovedb/src/batch/just_in_time_reference_update.rs +++ b/grovedb/src/batch/just_in_time_reference_update.rs @@ -58,9 +58,19 @@ where let mut serialization_to_use = Cow::Borrowed(serialized); + // we need to get the new storage_cost as if it had the same storage flags as + // before + let mut updated_new_element_with_old_flags = original_new_element.clone(); + updated_new_element_with_old_flags.set_flags(maybe_old_flags.clone()); + + let serialized_with_old_flags = cost_return_on_error_no_add!( + &cost, + updated_new_element_with_old_flags.serialize(grove_version) + ); + let mut new_storage_cost = KV::node_byte_cost_size_for_key_and_raw_value_lengths( key.len() as u32, - serialized.len() as u32, + serialized_with_old_flags.len() as u32, is_in_sum_tree, ); diff --git a/grovedb/src/batch/mod.rs b/grovedb/src/batch/mod.rs index 22596e28..2f55753f 100644 --- a/grovedb/src/batch/mod.rs +++ b/grovedb/src/batch/mod.rs @@ -1588,7 +1588,7 @@ where cost_return_on_error!( &mut cost, - merk.apply_unchecked::<_, Vec, _, _, _, _>( + merk.apply_unchecked::<_, Vec, _, _, _, _, _>( &batch_operations, &[], Some(batch_apply_options.as_merk_options()), @@ -1597,6 +1597,18 @@ where .map_err(|e| MerkError::ClientCorruptionError(e.to_string())) }, Some(&Element::value_defined_cost_for_serialized_value), + &|old_value, new_value| { + let old_element = Element::deserialize(old_value.as_slice(), grove_version) + .map_err(|e| MerkError::ClientCorruptionError(e.to_string()))?; + let maybe_old_flags = old_element.get_flags_owned(); + let mut new_element = Element::deserialize(new_value.as_slice(), grove_version) + .map_err(|e| MerkError::ClientCorruptionError(e.to_string()))?; + new_element.set_flags(maybe_old_flags); + new_element + .serialize(grove_version) + .map(Some) + .map_err(|e| MerkError::ClientCorruptionError(e.to_string())) + }, &mut |storage_costs, old_value, new_value| { // todo: change the flags without full deserialization let old_element = Element::deserialize(old_value.as_slice(), grove_version) diff --git a/grovedb/src/element/delete.rs b/grovedb/src/element/delete.rs index 9c0879a7..ced24e27 100644 --- a/grovedb/src/element/delete.rs +++ b/grovedb/src/element/delete.rs @@ -95,6 +95,7 @@ impl Element { .map_err(|e| MerkError::ClientCorruptionError(e.to_string())) }, Some(&Element::value_defined_cost_for_serialized_value), + &|_, _| Ok(None), &mut |_costs, _old_value, _value| Ok((false, None)), sectioned_removal, grove_version, diff --git a/grovedb/src/element/helpers.rs b/grovedb/src/element/helpers.rs index 2d2db076..86c523f2 100644 --- a/grovedb/src/element/helpers.rs +++ b/grovedb/src/element/helpers.rs @@ -189,6 +189,18 @@ impl Element { } } + #[cfg(feature = "full")] + /// Sets the optional flag stored in an element + pub fn set_flags(&mut self, new_flags: Option) { + match self { + Element::Tree(_, flags) + | Element::Item(_, flags) + | Element::Reference(_, _, flags) + | Element::SumTree(.., flags) + | Element::SumItem(_, flags) => *flags = new_flags, + } + } + #[cfg(feature = "full")] /// Get the required item space pub fn required_item_space( diff --git a/merk/src/merk/apply.rs b/merk/src/merk/apply.rs index 84b4cb9a..2a0e8040 100644 --- a/merk/src/merk/apply.rs +++ b/merk/src/merk/apply.rs @@ -77,6 +77,7 @@ where )) }, None::<&fn(&[u8], &GroveVersion) -> Option>, + &|_old_value, _value| Ok(None), &mut |_costs, _old_value, _value| Ok((false, None)), &mut |_a, key_bytes_to_remove, value_bytes_to_remove| { Ok(( @@ -140,6 +141,7 @@ where options, old_specialized_cost, value_defined_cost_fn, + &|_, _| Ok(None), &mut |_costs, _old_value, _value| Ok((false, None)), &mut |_a, key_bytes_to_remove, value_bytes_to_remove| { Ok(( @@ -169,6 +171,7 @@ where /// None, /// &|k, v| Ok(0), /// None::<&fn(&[u8], &GroveVersion) -> Option>, + /// &|k, v| Ok(None), /// &mut |s, v, o| Ok((false, None)), /// &mut |s, k, v| Ok((NoStorageRemoval, NoStorageRemoval)), /// grove_version, @@ -193,6 +196,7 @@ where /// None, /// &|k, v| Ok(0), /// None::<&fn(&[u8], &GroveVersion) -> Option>, + /// &|k, v| Ok(None), /// &mut |s, v, o| Ok((false, None)), /// &mut |s, k, v| Ok((NoStorageRemoval, NoStorageRemoval)), /// grove_version, @@ -207,6 +211,10 @@ where value_defined_cost_fn: Option< &impl Fn(&[u8], &GroveVersion) -> Option, >, + get_temp_new_value_with_old_flags: &impl Fn( + &Vec, + &Vec, + ) -> Result>, Error>, update_tree_value_based_on_costs: &mut impl FnMut( &StorageCost, &Vec, @@ -254,6 +262,7 @@ where options, old_specialized_cost, value_defined_cost_fn, + get_temp_new_value_with_old_flags, update_tree_value_based_on_costs, section_removal_bytes, grove_version, @@ -278,6 +287,7 @@ where /// None, /// &|k, v| Ok(0), /// None::<&fn(&[u8], &GroveVersion) -> Option>, + /// &|k, v| Ok(None), /// &mut |s, o, v| Ok((false, None)), /// &mut |s, k, v| Ok((NoStorageRemoval, NoStorageRemoval)), /// grove_version, @@ -295,25 +305,27 @@ where /// // deletes key [4,5,6] /// (vec![4, 5, 6], Op::Delete), /// ]; - /// unsafe { store.apply_unchecked::<_, Vec<_>, _, _, _, _>( /// /// /// /// /// ////// + /// unsafe { store.apply_unchecked::<_, Vec<_>, _, _, _, _, _>( /// /// /// /// /// ////// /// batch, /// &[], /// None, /// &|k, v| Ok(0), /// None::<&fn(&[u8], &GroveVersion) -> Option>, + /// &|o, v| Ok(v.clone()), /// &mut |s, o, v| Ok((false, None)), /// &mut |s, k, v| Ok((NoStorageRemoval, NoStorageRemoval)), /// grove_version, /// ).unwrap().expect(""); /// } /// ``` - pub fn apply_unchecked( + pub fn apply_unchecked( &mut self, batch: &MerkBatch, aux: &AuxMerkBatch, options: Option, old_specialized_cost: &C, value_defined_cost_fn: Option<&V>, + get_temp_new_value_with_old_flags: &T, update_tree_value_based_on_costs: &mut U, section_removal_bytes: &mut R, grove_version: &GroveVersion, @@ -323,6 +335,7 @@ where KA: AsRef<[u8]>, C: Fn(&Vec, &Vec) -> Result, V: Fn(&[u8], &GroveVersion) -> Option, + T: Fn(&Vec, &Vec) -> Result>, Error>, U: FnMut( &StorageCost, &Vec, @@ -342,6 +355,7 @@ where self.source(), old_specialized_cost, value_defined_cost_fn, + get_temp_new_value_with_old_flags, update_tree_value_based_on_costs, section_removal_bytes, grove_version, diff --git a/merk/src/test_utils/mod.rs b/merk/src/test_utils/mod.rs index 397ad13a..45beda4f 100644 --- a/merk/src/test_utils/mod.rs +++ b/merk/src/test_utils/mod.rs @@ -94,6 +94,7 @@ pub fn apply_memonly_unchecked( )) }, None::<&fn(&[u8], &GroveVersion) -> Option>, + &|_, _| Ok(None), &mut |_, _, _| Ok((false, None)), &mut |_flags, key_bytes_to_remove, value_bytes_to_remove| { Ok(( @@ -153,6 +154,7 @@ pub fn apply_to_memonly( )) }, None::<&fn(&[u8], &GroveVersion) -> Option>, + &|_, _| Ok(None), &mut |_, _, _| Ok((false, None)), &mut |_flags, key_bytes_to_remove, value_bytes_to_remove| { Ok(( diff --git a/merk/src/tree/just_in_time_value_update.rs b/merk/src/tree/just_in_time_value_update.rs index 1702361f..293c232d 100644 --- a/merk/src/tree/just_in_time_value_update.rs +++ b/merk/src/tree/just_in_time_value_update.rs @@ -5,7 +5,10 @@ use grovedb_costs::storage_cost::{ use crate::{ merk::defaults::MAX_UPDATE_VALUE_BASED_ON_COSTS_TIMES, - tree::{kv::ValueDefinedCostType, TreeNode}, + tree::{ + kv::{ValueDefinedCostType, KV}, + TreeNode, + }, Error, }; @@ -13,6 +16,10 @@ impl TreeNode { pub(in crate::tree) fn just_in_time_tree_node_value_update( &mut self, old_specialized_cost: &impl Fn(&Vec, &Vec) -> Result, + get_temp_new_value_with_old_flags: &impl Fn( + &Vec, + &Vec, + ) -> Result>, Error>, update_tree_value_based_on_costs: &mut impl FnMut( &StorageCost, &Vec, @@ -30,8 +37,6 @@ impl TreeNode { Error, >, ) -> Result<(), Error> { - let (mut current_tree_plus_hook_size, mut storage_costs) = - self.kv_with_parent_hook_size_and_storage_cost(old_specialized_cost)?; let mut i = 0; if let Some(old_value) = self.old_value.clone() { @@ -39,6 +44,16 @@ impl TreeNode { // For example to store the costs // todo: clean up clones let original_new_value = self.value_ref().clone(); + + let new_value_with_old_flags = + get_temp_new_value_with_old_flags(&old_value, &original_new_value)?; + + let (mut current_tree_plus_hook_size, mut storage_costs) = self + .kv_with_parent_hook_size_and_storage_cost_change_for_value( + old_specialized_cost, + new_value_with_old_flags, + )?; + loop { if let BasicStorageRemoval(removed_bytes) = storage_costs.value_storage_cost.removed_bytes @@ -62,6 +77,8 @@ impl TreeNode { if after_update_tree_plus_hook_size == current_tree_plus_hook_size { break; } + // we are calling this with merged flags that are were put in through value mut + // ref let new_size_and_storage_costs = self.kv_with_parent_hook_size_and_storage_cost(old_specialized_cost)?; current_tree_plus_hook_size = new_size_and_storage_costs.0; @@ -82,11 +99,14 @@ impl TreeNode { let (_, value_removed_bytes) = section_removal_bytes(&old_value, 0, removed_bytes)?; storage_costs.value_storage_cost.removed_bytes = value_removed_bytes; } + self.known_storage_cost = Some(storage_costs); + } else { + let (_, storage_costs) = + self.kv_with_parent_hook_size_and_storage_cost(old_specialized_cost)?; + self.known_storage_cost = Some(storage_costs); } - // Update old tree size after generating value storage_cost cost self.old_value = Some(self.value_ref().clone()); - self.known_storage_cost = Some(storage_costs); Ok(()) } diff --git a/merk/src/tree/mod.rs b/merk/src/tree/mod.rs index 5c9fcce9..24b568be 100644 --- a/merk/src/tree/mod.rs +++ b/merk/src/tree/mod.rs @@ -63,13 +63,13 @@ pub use tree_feature_type::TreeFeatureType; #[cfg(feature = "full")] pub use walk::{Fetch, RefWalker, Walker}; +use crate::tree::hash::HASH_LENGTH_X2; #[cfg(feature = "full")] use crate::tree::kv::ValueDefinedCostType; #[cfg(feature = "full")] use crate::tree::kv::ValueDefinedCostType::{LayeredValueDefinedCost, SpecializedValueDefinedCost}; #[cfg(feature = "full")] use crate::{error::Error, Error::Overflow}; - // TODO: remove need for `TreeInner`, and just use `Box` receiver for // relevant methods @@ -227,6 +227,45 @@ impl TreeNode { ) } + /// The point of this function is to get the cost change when we create a + /// temp value that's a partial merger between the old value and the new + /// value. Basically it is the new value with the old values flags + /// For example if we had an old value "Sam" with 40 bytes of flags + /// and a new value "Samuel" with 2 bytes of flags, the cost is probably + /// going to go up, As when we merge we will have Samuel with at least + /// 40 bytes of flags/ + pub fn kv_with_parent_hook_size_and_storage_cost_change_for_value( + &self, + old_tree_cost: &impl Fn(&Vec, &Vec) -> Result, + value: Option>, + ) -> Result<(u32, KeyValueStorageCost), Error> { + let current_value_byte_cost = if let Some(value_cost) = &self.inner.kv.value_defined_cost { + self.inner.kv.predefined_value_byte_cost_size(value_cost) + } else if let Some(value) = value { + let key_len = self.inner.kv.key.len() as u32; + let value_len = + HASH_LENGTH_X2 + value.len() + self.inner.kv.feature_type.encoding_cost(); + KV::value_byte_cost_size_for_key_and_value_lengths( + key_len, + value_len as u32, + self.inner.kv.feature_type.is_sum_feature(), + ) + } else { + self.inner.kv.value_byte_cost_size() + }; + + let old_cost = if let Some(old_value) = self.old_value.as_ref() { + old_tree_cost(self.key_as_ref(), old_value) + } else { + Ok(0) // there was no old value, hence old cost would be 0 + }?; + + self.kv_with_parent_hook_size_and_storage_cost_from_old_cost( + current_value_byte_cost, + old_cost, + ) + } + /// Creates a new `Tree` with the given key, value and value hash, and no /// children. /// @@ -644,6 +683,10 @@ impl TreeNode { value: Vec, feature_type: TreeFeatureType, old_specialized_cost: &impl Fn(&Vec, &Vec) -> Result, + get_temp_new_value_with_old_flags: &impl Fn( + &Vec, + &Vec, + ) -> Result>, Error>, update_tree_value_based_on_costs: &mut impl FnMut( &StorageCost, &Vec, @@ -674,6 +717,7 @@ impl TreeNode { &cost, self.just_in_time_tree_node_value_update( old_specialized_cost, + get_temp_new_value_with_old_flags, update_tree_value_based_on_costs, section_removal_bytes ) @@ -693,6 +737,10 @@ impl TreeNode { value_fixed_cost: u32, feature_type: TreeFeatureType, old_specialized_cost: &impl Fn(&Vec, &Vec) -> Result, + get_temp_new_value_with_old_flags: &impl Fn( + &Vec, + &Vec, + ) -> Result>, Error>, update_tree_value_based_on_costs: &mut impl FnMut( &StorageCost, &Vec, @@ -725,6 +773,7 @@ impl TreeNode { &cost, self.just_in_time_tree_node_value_update( old_specialized_cost, + get_temp_new_value_with_old_flags, update_tree_value_based_on_costs, section_removal_bytes ) @@ -744,6 +793,10 @@ impl TreeNode { value_hash: CryptoHash, feature_type: TreeFeatureType, old_specialized_cost: &impl Fn(&Vec, &Vec) -> Result, + get_temp_new_value_with_old_flags: &impl Fn( + &Vec, + &Vec, + ) -> Result>, Error>, update_tree_value_based_on_costs: &mut impl FnMut( &StorageCost, &Vec, @@ -774,6 +827,7 @@ impl TreeNode { &cost, self.just_in_time_tree_node_value_update( old_specialized_cost, + get_temp_new_value_with_old_flags, update_tree_value_based_on_costs, section_removal_bytes ) @@ -798,6 +852,10 @@ impl TreeNode { value_cost: u32, feature_type: TreeFeatureType, old_specialized_cost: &impl Fn(&Vec, &Vec) -> Result, + get_temp_new_value_with_old_flags: &impl Fn( + &Vec, + &Vec, + ) -> Result>, Error>, update_tree_value_based_on_costs: &mut impl FnMut( &StorageCost, &Vec, @@ -831,6 +889,7 @@ impl TreeNode { &cost, self.just_in_time_tree_node_value_update( old_specialized_cost, + get_temp_new_value_with_old_flags, update_tree_value_based_on_costs, section_removal_bytes ) diff --git a/merk/src/tree/ops.rs b/merk/src/tree/ops.rs index 156c7e60..3e10b2c8 100644 --- a/merk/src/tree/ops.rs +++ b/merk/src/tree/ops.rs @@ -145,12 +145,13 @@ where /// not require a non-empty tree. /// /// Keys in batch must be sorted and unique. - pub fn apply_to, C, V, U, R>( + pub fn apply_to, C, V, T, U, R>( maybe_tree: Option, batch: &MerkBatch, source: S, old_tree_cost: &C, value_defined_cost_fn: Option<&V>, + get_temp_new_value_with_old_flags: &T, update_tree_value_based_on_costs: &mut U, section_removal_bytes: &mut R, grove_version: &GroveVersion, @@ -158,6 +159,7 @@ where where C: Fn(&Vec, &Vec) -> Result, V: Fn(&[u8], &GroveVersion) -> Option, + T: Fn(&Vec, &Vec) -> Result>, Error>, U: FnMut( &StorageCost, &Vec, @@ -185,6 +187,7 @@ where source, old_tree_cost, value_defined_cost_fn, + get_temp_new_value_with_old_flags, update_tree_value_based_on_costs, section_removal_bytes, grove_version, @@ -212,6 +215,7 @@ where batch, old_tree_cost, value_defined_cost_fn, + get_temp_new_value_with_old_flags, update_tree_value_based_on_costs, section_removal_bytes, grove_version @@ -228,11 +232,12 @@ where /// Builds a `Tree` from a batch of operations. /// /// Keys in batch must be sorted and unique. - fn build, C, V, U, R>( + fn build, C, V, T, U, R>( batch: &MerkBatch, source: S, old_tree_cost: &C, value_defined_cost_fn: Option<&V>, + get_temp_new_value_with_old_flags: &T, update_tree_value_based_on_costs: &mut U, section_removal_bytes: &mut R, grove_version: &GroveVersion, @@ -240,6 +245,7 @@ where where C: Fn(&Vec, &Vec) -> Result, V: Fn(&[u8], &GroveVersion) -> Option, + T: Fn(&Vec, &Vec) -> Result>, Error>, U: FnMut( &StorageCost, &Vec, @@ -267,6 +273,7 @@ where source.clone(), old_tree_cost, value_defined_cost_fn, + get_temp_new_value_with_old_flags, update_tree_value_based_on_costs, section_removal_bytes, grove_version @@ -281,6 +288,7 @@ where right_batch, old_tree_cost, value_defined_cost_fn, + get_temp_new_value_with_old_flags, update_tree_value_based_on_costs, section_removal_bytes, grove_version @@ -295,6 +303,7 @@ where source.clone(), old_tree_cost, value_defined_cost_fn, + get_temp_new_value_with_old_flags, update_tree_value_based_on_costs, section_removal_bytes, grove_version @@ -367,6 +376,7 @@ where ), old_tree_cost, value_defined_cost_fn, + get_temp_new_value_with_old_flags, update_tree_value_based_on_costs, section_removal_bytes, grove_version, @@ -377,8 +387,7 @@ where .wrap_with_cost(cost) } - #[allow(dead_code)] - fn apply_sorted_without_costs>( + pub(crate) fn apply_sorted_without_costs>( self, batch: &MerkBatch, grove_version: &GroveVersion, @@ -387,6 +396,7 @@ where batch, &|_, _| Ok(0), None::<&fn(&[u8], &GroveVersion) -> Option>, + &|_, _| Ok(None), &mut |_, _, _| Ok((false, None)), &mut |_flags, key_bytes_to_remove, value_bytes_to_remove| { Ok(( @@ -402,11 +412,12 @@ where /// `Walker::apply`_to, but requires a populated tree. /// /// Keys in batch must be sorted and unique. - fn apply_sorted, C, V, U, R>( + fn apply_sorted, C, V, T, U, R>( self, batch: &MerkBatch, old_specialized_cost: &C, value_defined_cost_fn: Option<&V>, + get_temp_new_value_with_old_flags: &T, update_tree_value_based_on_costs: &mut U, section_removal_bytes: &mut R, grove_version: &GroveVersion, @@ -414,6 +425,7 @@ where where C: Fn(&Vec, &Vec) -> Result, V: Fn(&[u8], &GroveVersion) -> Option, + T: Fn(&Vec, &Vec) -> Result>, Error>, U: FnMut( &StorageCost, &Vec, @@ -441,6 +453,7 @@ where value.to_vec(), feature_type.to_owned(), old_specialized_cost, + get_temp_new_value_with_old_flags, update_tree_value_based_on_costs, section_removal_bytes, ) @@ -455,6 +468,7 @@ where *value_cost, feature_type.to_owned(), old_specialized_cost, + get_temp_new_value_with_old_flags, update_tree_value_based_on_costs, section_removal_bytes ) @@ -468,6 +482,7 @@ where referenced_value.to_owned(), feature_type.to_owned(), old_specialized_cost, + get_temp_new_value_with_old_flags, update_tree_value_based_on_costs, section_removal_bytes, ) @@ -483,6 +498,7 @@ where *value_cost, feature_type.to_owned(), old_specialized_cost, + get_temp_new_value_with_old_flags, update_tree_value_based_on_costs, section_removal_bytes, ) @@ -565,6 +581,7 @@ where source.clone(), old_specialized_cost, value_defined_cost_fn, + get_temp_new_value_with_old_flags, update_tree_value_based_on_costs, section_removal_bytes, grove_version, @@ -591,6 +608,7 @@ where &batch[..index], old_specialized_cost, value_defined_cost_fn, + get_temp_new_value_with_old_flags, update_tree_value_based_on_costs, section_removal_bytes, grove_version @@ -623,6 +641,7 @@ where source.clone(), old_specialized_cost, value_defined_cost_fn, + get_temp_new_value_with_old_flags, update_tree_value_based_on_costs, section_removal_bytes, grove_version, @@ -649,6 +668,7 @@ where &batch[index + 1..], old_specialized_cost, value_defined_cost_fn, + get_temp_new_value_with_old_flags, update_tree_value_based_on_costs, section_removal_bytes, grove_version @@ -697,6 +717,7 @@ where KeyUpdates::new(new_keys, updated_keys, LinkedList::default(), None), old_specialized_cost, value_defined_cost_fn, + get_temp_new_value_with_old_flags, update_tree_value_based_on_costs, section_removal_bytes, grove_version, @@ -709,7 +730,7 @@ where /// /// This recursion executes serially in the same thread, but in the future /// will be dispatched to workers in other threads. - fn recurse, C, V, U, R>( + fn recurse, C, V, T, U, R>( self, batch: &MerkBatch, mid: usize, @@ -717,12 +738,14 @@ where mut key_updates: KeyUpdates, old_tree_cost: &C, value_defined_cost_fn: Option<&V>, + get_temp_new_value_with_old_flags: &T, update_tree_value_based_on_costs: &mut U, section_removal_bytes: &mut R, grove_version: &GroveVersion, ) -> CostResult<(Option, KeyUpdates), Error> where C: Fn(&Vec, &Vec) -> Result, + T: Fn(&Vec, &Vec) -> Result>, Error>, U: FnMut( &StorageCost, &Vec, @@ -755,6 +778,7 @@ where source, old_tree_cost, value_defined_cost_fn, + get_temp_new_value_with_old_flags, update_tree_value_based_on_costs, section_removal_bytes, grove_version, @@ -791,6 +815,7 @@ where source, old_tree_cost, value_defined_cost_fn, + get_temp_new_value_with_old_flags, update_tree_value_based_on_costs, section_removal_bytes, grove_version, @@ -1189,12 +1214,13 @@ mod test { #[test] fn apply_empty_none() { let grove_version = GroveVersion::latest(); - let (maybe_tree, key_updates) = Walker::::apply_to::, _, _, _, _>( + let (maybe_tree, key_updates) = Walker::::apply_to::, _, _, _, _, _>( None, &[], PanicSource {}, &|_, _| Ok(0), None::<&fn(&[u8], &GroveVersion) -> Option>, + &|_, _| Ok(None), &mut |_, _, _| Ok((false, None)), &mut |_flags, key_bytes_to_remove, value_bytes_to_remove| { Ok(( @@ -1221,6 +1247,7 @@ mod test { PanicSource {}, &|_, _| Ok(0), None::<&fn(&[u8], &GroveVersion) -> Option>, + &|_, _| Ok(None), &mut |_, _, _| Ok((false, None)), &mut |_flags, key_bytes_to_remove, value_bytes_to_remove| { Ok(( @@ -1250,6 +1277,7 @@ mod test { PanicSource {}, &|_, _| Ok(0), None::<&fn(&[u8], &GroveVersion) -> Option>, + &|_, _| Ok(None), &mut |_, _, _| Ok((false, None)), &mut |_flags, key_bytes_to_remove, value_bytes_to_remove| { Ok(( @@ -1275,6 +1303,7 @@ mod test { PanicSource {}, &|_, _| Ok(0), None::<&fn(&[u8], &GroveVersion) -> Option>, + &|_, _| Ok(None), &mut |_, _, _| Ok((false, None)), &mut |_flags, key_bytes_to_remove, value_bytes_to_remove| { Ok(( @@ -1307,6 +1336,7 @@ mod test { PanicSource {}, &|_, _| Ok(0), None::<&fn(&[u8], &GroveVersion) -> Option>, + &|_, _| Ok(None), &mut |_, _, _| Ok((false, None)), &mut |_flags, key_bytes_to_remove, value_bytes_to_remove| { Ok(( @@ -1333,6 +1363,7 @@ mod test { PanicSource {}, &|_, _| Ok(0), None::<&fn(&[u8], &GroveVersion) -> Option>, + &|_, _| Ok(None), &mut |_, _, _| Ok((false, None)), &mut |_flags, key_bytes_to_remove, value_bytes_to_remove| { Ok(( diff --git a/merk/src/tree/walk/mod.rs b/merk/src/tree/walk/mod.rs index a84a1d4c..4b67bb60 100644 --- a/merk/src/tree/walk/mod.rs +++ b/merk/src/tree/walk/mod.rs @@ -207,6 +207,10 @@ where value: Vec, feature_type: TreeFeatureType, old_specialized_cost: &impl Fn(&Vec, &Vec) -> Result, + get_temp_new_value_with_old_flags: &impl Fn( + &Vec, + &Vec, + ) -> Result>, Error>, update_tree_value_based_on_costs: &mut impl FnMut( &StorageCost, &Vec, @@ -232,6 +236,7 @@ where value, feature_type, old_specialized_cost, + get_temp_new_value_with_old_flags, update_tree_value_based_on_costs, section_removal_bytes ) @@ -247,6 +252,10 @@ where value_fixed_cost: u32, feature_type: TreeFeatureType, old_specialized_cost: &impl Fn(&Vec, &Vec) -> Result, + get_temp_new_value_with_old_flags: &impl Fn( + &Vec, + &Vec, + ) -> Result>, Error>, update_tree_value_based_on_costs: &mut impl FnMut( &StorageCost, &Vec, @@ -273,6 +282,7 @@ where value_fixed_cost, feature_type, old_specialized_cost, + get_temp_new_value_with_old_flags, update_tree_value_based_on_costs, section_removal_bytes ) @@ -288,6 +298,10 @@ where value_hash: CryptoHash, feature_type: TreeFeatureType, old_specialized_cost: &impl Fn(&Vec, &Vec) -> Result, + get_temp_new_value_with_old_flags: &impl Fn( + &Vec, + &Vec, + ) -> Result>, Error>, update_tree_value_based_on_costs: &mut impl FnMut( &StorageCost, &Vec, @@ -314,6 +328,7 @@ where value_hash, feature_type, old_specialized_cost, + get_temp_new_value_with_old_flags, update_tree_value_based_on_costs, section_removal_bytes ) @@ -330,6 +345,10 @@ where value_fixed_cost: u32, feature_type: TreeFeatureType, old_specialized_cost: &impl Fn(&Vec, &Vec) -> Result, + get_temp_new_value_with_old_flags: &impl Fn( + &Vec, + &Vec, + ) -> Result>, Error>, update_tree_value_based_on_costs: &mut impl FnMut( &StorageCost, &Vec, @@ -357,6 +376,7 @@ where value_fixed_cost, feature_type, old_specialized_cost, + get_temp_new_value_with_old_flags, update_tree_value_based_on_costs, section_removal_bytes ) From 9242fd43368ddf39763de526ed6781aee15e0ec4 Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Sat, 24 Aug 2024 03:09:51 +0700 Subject: [PATCH 09/15] ok it works --- grovedb/src/batch/just_in_time_cost_tests.rs | 20 +++++++++----------- merk/src/merk/apply.rs | 4 ++-- 2 files changed, 11 insertions(+), 13 deletions(-) diff --git a/grovedb/src/batch/just_in_time_cost_tests.rs b/grovedb/src/batch/just_in_time_cost_tests.rs index f1503df0..094670ce 100644 --- a/grovedb/src/batch/just_in_time_cost_tests.rs +++ b/grovedb/src/batch/just_in_time_cost_tests.rs @@ -415,7 +415,7 @@ mod tests { } #[test] - fn test_batch_root_one_update_bigger_item_same_epoch_with_reference() { + fn test_one_update_bigger_item_same_epoch_with_reference() { let grove_version = GroveVersion::latest(); let db = make_empty_grovedb(); let tx = db.start_transaction(); @@ -493,7 +493,7 @@ mod tests { } #[test] - fn test_batch_root_one_update_bigger_item_different_epoch_with_reference() { + fn test_one_update_bigger_item_different_epoch_with_reference() { let grove_version = GroveVersion::latest(); let db = make_empty_grovedb(); @@ -589,8 +589,7 @@ mod tests { } #[test] - fn test_batch_root_one_update_bigger_item_different_base_epoch_with_bytes_in_last_epoch_with_reference( - ) { + fn test_one_update_bigger_item_different_base_epoch_with_bytes_in_last_epoch_with_reference() { let grove_version = GroveVersion::latest(); let db = make_empty_grovedb(); @@ -689,8 +688,8 @@ mod tests { } #[test] - fn test_batch_root_one_update_bigger_item_different_base_epoch_with_bytes_in_future_epoch_with_reference( - ) { + fn test_one_update_bigger_item_different_base_epoch_with_bytes_in_future_epoch_with_reference() + { let grove_version = GroveVersion::latest(); let db = make_empty_grovedb(); @@ -789,7 +788,7 @@ mod tests { } #[test] - fn test_batch_root_one_update_smaller_item_same_base_epoch_with_reference() { + fn test_one_update_smaller_item_same_base_epoch_with_reference() { let grove_version = GroveVersion::latest(); let db = make_empty_grovedb(); @@ -894,7 +893,7 @@ mod tests { } #[test] - fn test_batch_root_one_update_smaller_item_different_base_epoch_with_reference() { + fn test_one_update_smaller_item_different_base_epoch_with_reference() { let grove_version = GroveVersion::latest(); let db = make_empty_grovedb(); @@ -981,7 +980,7 @@ mod tests { } #[test] - fn test_batch_root_one_update_smaller_item_different_base_epoch_with_previous_flags() { + fn test_one_update_smaller_item_different_base_epoch_with_previous_flags() { let grove_version = GroveVersion::latest(); let db = make_empty_grovedb(); let tx = db.start_transaction(); @@ -1063,8 +1062,7 @@ mod tests { } #[test] - fn test_batch_root_one_update_smaller_item_different_base_epoch_with_previous_flags_all_multi_epoch_removal( - ) { + fn test_one_update_smaller_item_different_base_epoch_with_previous_flags_all_multi_epoch() { let grove_version = GroveVersion::latest(); let db = make_empty_grovedb(); let tx = db.start_transaction(); diff --git a/merk/src/merk/apply.rs b/merk/src/merk/apply.rs index 2a0e8040..9c5c9ec9 100644 --- a/merk/src/merk/apply.rs +++ b/merk/src/merk/apply.rs @@ -305,13 +305,13 @@ where /// // deletes key [4,5,6] /// (vec![4, 5, 6], Op::Delete), /// ]; - /// unsafe { store.apply_unchecked::<_, Vec<_>, _, _, _, _, _>( /// /// /// /// /// ////// + /// unsafe { store.apply_unchecked::<_, Vec<_>, _, _, _, _, _>( /// batch, /// &[], /// None, /// &|k, v| Ok(0), /// None::<&fn(&[u8], &GroveVersion) -> Option>, - /// &|o, v| Ok(v.clone()), + /// &|o, v| Ok(None), /// &mut |s, o, v| Ok((false, None)), /// &mut |s, k, v| Ok((NoStorageRemoval, NoStorageRemoval)), /// grove_version, From ab97fcdd9a76e463f585fc484928d18300e6f9b7 Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Sat, 24 Aug 2024 14:30:07 +0700 Subject: [PATCH 10/15] added more tests --- grovedb/src/batch/just_in_time_cost_tests.rs | 351 ++++++++++++++++++ .../batch/just_in_time_reference_update.rs | 71 +++- grovedb/src/batch/mod.rs | 19 +- grovedb/src/element/get.rs | 6 +- grovedb/src/element/helpers.rs | 12 +- merk/src/tree/just_in_time_value_update.rs | 9 +- merk/src/tree/kv.rs | 25 +- 7 files changed, 426 insertions(+), 67 deletions(-) diff --git a/grovedb/src/batch/just_in_time_cost_tests.rs b/grovedb/src/batch/just_in_time_cost_tests.rs index 094670ce..4fddc922 100644 --- a/grovedb/src/batch/just_in_time_cost_tests.rs +++ b/grovedb/src/batch/just_in_time_cost_tests.rs @@ -1142,4 +1142,355 @@ mod tests { verify_references(&db, &tx); } + + #[test] + fn test_one_update_bigger_sum_item_same_epoch_with_reference() { + let grove_version = GroveVersion::latest(); + let db = make_empty_grovedb(); + let tx = db.start_transaction(); + + let owner_id = [1; 32]; + db.insert( + EMPTY_PATH, + b"tree", + Element::empty_sum_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("expected to insert tree"); + + db.insert( + EMPTY_PATH, + b"refs", + Element::empty_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("expected to insert tree"); + + db.insert( + [b"tree".as_slice()].as_ref(), + b"key1", + Element::new_sum_item_with_flags( + 1, + Some(StorageFlags::new_single_epoch(0, Some(owner_id)).to_element_flags()), + ), + None, + None, + grove_version, + ) + .unwrap() + .expect("expected to insert item"); + + // We are adding 2 bytes + let ops = vec![ + QualifiedGroveDbOp::insert_or_replace_op( + vec![b"tree".to_vec()], + b"key1".to_vec(), + Element::new_sum_item_with_flags( + 100000000000, + Some(StorageFlags::new_single_epoch(0, Some(owner_id)).to_element_flags()), + ), + ), + QualifiedGroveDbOp::insert_only_op( + vec![b"refs".to_vec()], + b"ref_key".to_vec(), + Element::new_reference_with_hops( + ReferencePathType::AbsolutePathReference(vec![ + b"tree".to_vec(), + b"key1".to_vec(), + ]), + Some(1), + ), + ), + ]; + + apply_batch(&db, ops, &tx, grove_version); + + expect_storage_flags( + &db, + &tx, + StorageFlags::new_single_epoch(0, Some(owner_id)), + grove_version, + ); + + verify_references(&db, &tx); + } + + #[test] + fn test_one_update_bigger_sum_item_different_epoch_with_reference() { + let grove_version = GroveVersion::latest(); + let db = make_empty_grovedb(); + + let owner_id = [1; 32]; + + db.insert( + EMPTY_PATH, + b"tree", + Element::empty_sum_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("expected to insert tree"); + + db.insert( + EMPTY_PATH, + b"refs", + Element::empty_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("expected to insert tree"); + + db.insert( + [b"tree".as_slice()].as_ref(), + b"key1", + Element::new_sum_item_with_flags( + 1, + Some(StorageFlags::new_single_epoch(0, Some(owner_id)).to_element_flags()), + ), + None, + None, + grove_version, + ) + .unwrap() + .expect("expected to insert item"); + + let tx = db.start_transaction(); + // We are adding n bytes + let ops = vec![ + QualifiedGroveDbOp::insert_or_replace_op( + vec![b"tree".to_vec()], + b"key1".to_vec(), + Element::new_sum_item_with_flags( + 10000000000, + Some(StorageFlags::new_single_epoch(1, Some(owner_id)).to_element_flags()), + ), + ), + QualifiedGroveDbOp::insert_only_op( + vec![b"refs".to_vec()], + b"ref_key".to_vec(), + Element::new_reference_with_hops( + ReferencePathType::AbsolutePathReference(vec![ + b"tree".to_vec(), + b"key1".to_vec(), + ]), + None, + ), + ), + ]; + + apply_batch(&db, ops, &tx, grove_version); + + expect_storage_flags( + &db, + &tx, + StorageFlags::SingleEpochOwned(0, owner_id), // no change + grove_version, + ); + + verify_references(&db, &tx); + } + + #[test] + fn test_one_update_bigger_item_add_flags() { + let grove_version = GroveVersion::latest(); + let db = make_empty_grovedb(); + + let owner_id = [1; 32]; + + db.insert( + EMPTY_PATH, + b"tree", + Element::empty_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("expected to insert tree"); + + db.insert( + EMPTY_PATH, + b"refs", + Element::empty_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("expected to insert tree"); + + db.insert( + [b"tree".as_slice()].as_ref(), + b"key1", + Element::new_item_with_flags(b"value1".to_vec(), None), + None, + None, + grove_version, + ) + .unwrap() + .expect("expected to insert item"); + + let base_item = b"value1".to_vec(); + + for n in 1..150 { + let tx = db.start_transaction(); + let mut item = base_item.clone(); + item.extend(std::iter::repeat(0).take(n)); + // We are adding n bytes + let ops = vec![ + QualifiedGroveDbOp::insert_or_replace_op( + vec![b"tree".to_vec()], + b"key1".to_vec(), + Element::new_item_with_flags( + item, // value100 if n was 2 + Some(StorageFlags::new_single_epoch(1, Some(owner_id)).to_element_flags()), + ), + ), + QualifiedGroveDbOp::insert_only_op( + vec![b"refs".to_vec()], + b"ref_key".to_vec(), + Element::new_reference_with_hops( + ReferencePathType::AbsolutePathReference(vec![ + b"tree".to_vec(), + b"key1".to_vec(), + ]), + None, + ), + ), + ]; + + apply_batch(&db, ops, &tx, grove_version); + + let expected_added_bytes = if n < 15 { + n as u32 + 3 + } else if n < 124 { + n as u32 + 4 // the varint requires an extra byte + } else { + n as u32 + 5 // the varint requires an extra byte + }; + expect_storage_flags( + &db, + &tx, + StorageFlags::SingleEpochOwned(1, owner_id), + grove_version, + ); + + verify_references(&db, &tx); + } + } + #[test] + fn test_one_update_smaller_item_add_flags() { + let grove_version = GroveVersion::latest(); + let db = make_empty_grovedb(); + + let owner_id = [1; 32]; + + db.insert( + EMPTY_PATH, + b"tree", + Element::empty_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("expected to insert tree"); + + db.insert( + EMPTY_PATH, + b"refs", + Element::empty_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("expected to insert tree"); + + let base_item = b"value1".to_vec(); + let mut original_item = base_item.clone(); + original_item.extend(std::iter::repeat(0).take(150)); + + db.insert( + [b"tree".as_slice()].as_ref(), + b"key1", + Element::new_item_with_flags(original_item, None), + None, + None, + grove_version, + ) + .unwrap() + .expect("expected to insert item"); + + let to = 150usize; + + for n in (0..to).rev() { + let tx = db.start_transaction(); + let mut item = base_item.clone(); + item.extend(std::iter::repeat(0).take(n)); + // We are adding n bytes + let ops = vec![ + QualifiedGroveDbOp::insert_or_replace_op( + vec![b"tree".to_vec()], + b"key1".to_vec(), + Element::new_item_with_flags( + item, // value1 if n was 1 + Some(StorageFlags::new_single_epoch(0, Some(owner_id)).to_element_flags()), + ), + ), + QualifiedGroveDbOp::insert_only_op( + vec![b"refs".to_vec()], + b"ref_key".to_vec(), + Element::new_reference_with_hops( + ReferencePathType::AbsolutePathReference(vec![ + b"tree".to_vec(), + b"key1".to_vec(), + ]), + None, + ), + ), + ]; + + let storage_removed_bytes = apply_batch(&db, ops, &tx, grove_version) + .storage_cost + .removed_bytes; + + println!("{} {:?}", n, storage_removed_bytes); + + if n > 113 { + assert_eq!(storage_removed_bytes, StorageRemovedBytes::NoStorageRemoval); + } else if n > 17 { + let removed_bytes = 114 - n as u32; + assert_eq!( + storage_removed_bytes, + StorageRemovedBytes::BasicStorageRemoval(removed_bytes) + ); + } else { + let removed_bytes = 114 - n as u32 + 1; // because of varint + assert_eq!( + storage_removed_bytes, + StorageRemovedBytes::BasicStorageRemoval(removed_bytes) + ); + }; + + expect_storage_flags( + &db, + &tx, + StorageFlags::SingleEpochOwned(0, owner_id), + grove_version, + ); + + verify_references(&db, &tx); + } + } } diff --git a/grovedb/src/batch/just_in_time_reference_update.rs b/grovedb/src/batch/just_in_time_reference_update.rs index f50b233e..7be6d220 100644 --- a/grovedb/src/batch/just_in_time_reference_update.rs +++ b/grovedb/src/batch/just_in_time_reference_update.rs @@ -1,7 +1,7 @@ use std::borrow::Cow; use grovedb_costs::{ - cost_return_on_error_no_add, + cost_return_on_error_default, cost_return_on_error_no_add, storage_cost::{ removal::{StorageRemovedBytes, StorageRemovedBytes::BasicStorageRemoval}, StorageCost, @@ -14,9 +14,11 @@ use grovedb_merk::{ }; use grovedb_storage::StorageContext; use grovedb_version::version::GroveVersion; +use integer_encoding::VarInt; use crate::{ batch::{MerkError, TreeCacheMerkByPath}, + element::SUM_ITEM_COST_SIZE, Element, ElementFlags, Error, }; @@ -44,11 +46,39 @@ where u32, ) -> Result<(StorageRemovedBytes, StorageRemovedBytes), Error>, { - let mut maybe_old_flags = old_element.get_flags_owned(); - let mut cost = OperationCost::default(); + if old_element.is_sum_item() { + return if new_element.is_sum_item() { + let mut maybe_old_flags = old_element.get_flags_owned(); + if maybe_old_flags.is_some() { + let mut updated_new_element_with_old_flags = new_element.clone(); + updated_new_element_with_old_flags.set_flags(maybe_old_flags.clone()); + // There are no storage flags, we can just hash new element + let new_serialized_bytes = cost_return_on_error_no_add!( + &cost, + updated_new_element_with_old_flags.serialize(grove_version) + ); + let val_hash = value_hash(&new_serialized_bytes).unwrap_add_cost(&mut cost); + Ok(val_hash).wrap_with_cost(cost) + } else { + let val_hash = value_hash(&serialized).unwrap_add_cost(&mut cost); + Ok(val_hash).wrap_with_cost(cost) + } + } else { + Err(Error::NotSupported( + "going from a sum item to a not sum item is not supported".to_string(), + )) + .wrap_with_cost(cost) + }; + } else if new_element.is_sum_item() { + return Err(Error::NotSupported( + "going from an item to a sum item is not supported".to_string(), + )) + .wrap_with_cost(cost); + } + let mut maybe_old_flags = old_element.get_flags_owned(); - let old_storage_cost = KV::node_byte_cost_size_for_key_and_raw_value_lengths( + let old_storage_cost = KV::node_value_byte_cost_size( key.len() as u32, old_serialized_element.len() as u32, is_in_sum_tree, @@ -58,21 +88,24 @@ where let mut serialization_to_use = Cow::Borrowed(serialized); - // we need to get the new storage_cost as if it had the same storage flags as - // before - let mut updated_new_element_with_old_flags = original_new_element.clone(); - updated_new_element_with_old_flags.set_flags(maybe_old_flags.clone()); - - let serialized_with_old_flags = cost_return_on_error_no_add!( - &cost, - updated_new_element_with_old_flags.serialize(grove_version) - ); + let mut new_storage_cost = if maybe_old_flags.is_some() { + // we need to get the new storage_cost as if it had the same storage flags as + // before + let mut updated_new_element_with_old_flags = original_new_element.clone(); + updated_new_element_with_old_flags.set_flags(maybe_old_flags.clone()); - let mut new_storage_cost = KV::node_byte_cost_size_for_key_and_raw_value_lengths( - key.len() as u32, - serialized_with_old_flags.len() as u32, - is_in_sum_tree, - ); + let serialized_with_old_flags = cost_return_on_error_no_add!( + &cost, + updated_new_element_with_old_flags.serialize(grove_version) + ); + KV::node_value_byte_cost_size( + key.len() as u32, + serialized_with_old_flags.len() as u32, + is_in_sum_tree, + ) + } else { + KV::node_value_byte_cost_size(key.len() as u32, serialized.len() as u32, is_in_sum_tree) + }; let mut i = 0; @@ -119,7 +152,7 @@ where new_element_cloned.serialize(grove_version) ); - new_storage_cost = KV::node_byte_cost_size_for_key_and_raw_value_lengths( + new_storage_cost = KV::node_value_byte_cost_size( key.len() as u32, new_serialized_bytes.len() as u32, is_in_sum_tree, diff --git a/grovedb/src/batch/mod.rs b/grovedb/src/batch/mod.rs index 2f55753f..822de9dc 100644 --- a/grovedb/src/batch/mod.rs +++ b/grovedb/src/batch/mod.rs @@ -1601,13 +1601,18 @@ where let old_element = Element::deserialize(old_value.as_slice(), grove_version) .map_err(|e| MerkError::ClientCorruptionError(e.to_string()))?; let maybe_old_flags = old_element.get_flags_owned(); - let mut new_element = Element::deserialize(new_value.as_slice(), grove_version) - .map_err(|e| MerkError::ClientCorruptionError(e.to_string()))?; - new_element.set_flags(maybe_old_flags); - new_element - .serialize(grove_version) - .map(Some) - .map_err(|e| MerkError::ClientCorruptionError(e.to_string())) + if maybe_old_flags.is_some() { + let mut new_element = + Element::deserialize(new_value.as_slice(), grove_version) + .map_err(|e| MerkError::ClientCorruptionError(e.to_string()))?; + new_element.set_flags(maybe_old_flags); + new_element + .serialize(grove_version) + .map(Some) + .map_err(|e| MerkError::ClientCorruptionError(e.to_string())) + } else { + Ok(None) + } }, &mut |storage_costs, old_value, new_value| { // todo: change the flags without full deserialization diff --git a/grovedb/src/element/get.rs b/grovedb/src/element/get.rs index 1fda91dd..6ec625e7 100644 --- a/grovedb/src/element/get.rs +++ b/grovedb/src/element/get.rs @@ -173,11 +173,7 @@ impl Element { }); let value_len = cost_size + flags_len; cost.storage_loaded_bytes = - KV::specialized_value_byte_cost_size_for_key_and_value_lengths( - key_ref.len() as u32, - value_len, - false, - ) + KV::node_value_byte_cost_size(key_ref.len() as u32, value_len, false) } Some(Element::Tree(_, flags)) | Some(Element::SumTree(_, _, flags)) => { let tree_cost_size = if element.as_ref().unwrap().is_sum_tree() { diff --git a/grovedb/src/element/helpers.rs b/grovedb/src/element/helpers.rs index 86c523f2..fd0f17b1 100644 --- a/grovedb/src/element/helpers.rs +++ b/grovedb/src/element/helpers.rs @@ -297,17 +297,9 @@ impl Element { }); let value_len = SUM_ITEM_COST_SIZE + flags_len; let key_len = key.len() as u32; - KV::specialized_value_byte_cost_size_for_key_and_value_lengths( - key_len, - value_len, - is_sum_node, - ) + KV::node_value_byte_cost_size(key_len, value_len, is_sum_node) } - _ => KV::specialized_value_byte_cost_size_for_key_and_value_lengths( - key.len() as u32, - value.len() as u32, - is_sum_node, - ), + _ => KV::node_value_byte_cost_size(key.len() as u32, value.len() as u32, is_sum_node), }; Ok(cost) } diff --git a/merk/src/tree/just_in_time_value_update.rs b/merk/src/tree/just_in_time_value_update.rs index 293c232d..8620cc97 100644 --- a/merk/src/tree/just_in_time_value_update.rs +++ b/merk/src/tree/just_in_time_value_update.rs @@ -45,8 +45,13 @@ impl TreeNode { // todo: clean up clones let original_new_value = self.value_ref().clone(); - let new_value_with_old_flags = - get_temp_new_value_with_old_flags(&old_value, &original_new_value)?; + let new_value_with_old_flags = if self.inner.kv.value_defined_cost.is_none() { + // for items + get_temp_new_value_with_old_flags(&old_value, &original_new_value)? + } else { + // don't do this for sum items or trees + None + }; let (mut current_tree_plus_hook_size, mut storage_costs) = self .kv_with_parent_hook_size_and_storage_cost_change_for_value( diff --git a/merk/src/tree/kv.rs b/merk/src/tree/kv.rs index 4fd3f7f7..f4a0e224 100644 --- a/merk/src/tree/kv.rs +++ b/merk/src/tree/kv.rs @@ -356,25 +356,6 @@ impl KV { node_value_size + parent_to_child_cost } - /// Get the costs for the node, this has the parent to child hooks - #[inline] - pub fn specialized_value_byte_cost_size_for_key_and_value_lengths( - not_prefixed_key_len: u32, - inner_value_len: u32, - is_sum_node: bool, - ) -> u32 { - // Sum trees are either 1 or 9 bytes. While they might be more or less on disk, - // costs can not take advantage of the varint aspect of the feature. - let feature_len = if is_sum_node { 9 } else { 1 }; - // Each node stores the key and value, and the node hash and the value hash - let node_value_size = inner_value_len + feature_len + HASH_LENGTH_U32_X2; - let node_value_size = node_value_size + node_value_size.required_space() as u32; - // The node will be a child of another node which stores it's key and hash - // That will be added during propagation - let parent_to_child_cost = Link::encoded_link_size(not_prefixed_key_len, is_sum_node); - node_value_size + parent_to_child_cost - } - /// Get the costs for the value with known value_len and non prefixed key /// len sizes, this has the parent to child hooks #[inline] @@ -452,11 +433,7 @@ impl KV { let key_len = self.key.len() as u32; let is_sum_node = self.feature_type.is_sum_feature(); - Self::specialized_value_byte_cost_size_for_key_and_value_lengths( - key_len, - value_cost, - is_sum_node, - ) + Self::node_value_byte_cost_size(key_len, value_cost, is_sum_node) } /// Costs based on predefined types (Trees, SumTrees, SumItems) that behave From f58ff63afb41761c1c188f8fef0b4d60d060f739 Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Sat, 24 Aug 2024 15:15:04 +0700 Subject: [PATCH 11/15] cleaned up grovedb --- Cargo.toml | 2 +- grovedb-epoch-based-storage-flags/Cargo.toml | 13 + .../src/error.rs | 45 + grovedb-epoch-based-storage-flags/src/lib.rs | 1588 +++++++++++++++++ .../src/split_removal_bytes.rs | 30 + .../src/update_element_flags.rs | 138 ++ grovedb/Cargo.toml | 4 +- grovedb/src/batch/just_in_time_cost_tests.rs | 4 +- merk/Cargo.toml | 2 +- storage/Cargo.toml | 2 +- .../rocksdb_storage/storage_context/batch.rs | 28 - tutorials/Cargo.toml | 1 + tutorials/src/bin/delete.rs | 2 + tutorials/src/bin/insert.rs | 3 + tutorials/src/bin/proofs.rs | 8 +- tutorials/src/bin/query-complex.rs | 8 +- tutorials/src/bin/query-simple.rs | 8 +- tutorials/src/bin/replication.rs | 55 +- 18 files changed, 1876 insertions(+), 65 deletions(-) create mode 100644 grovedb-epoch-based-storage-flags/Cargo.toml create mode 100644 grovedb-epoch-based-storage-flags/src/error.rs create mode 100644 grovedb-epoch-based-storage-flags/src/lib.rs create mode 100644 grovedb-epoch-based-storage-flags/src/split_removal_bytes.rs create mode 100644 grovedb-epoch-based-storage-flags/src/update_element_flags.rs diff --git a/Cargo.toml b/Cargo.toml index 17e25e98..bd532171 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,4 +11,4 @@ members = [ "path", "grovedbg-types", "grovedb-version" -] +, "grovedb-epoch-based-storage-flags"] diff --git a/grovedb-epoch-based-storage-flags/Cargo.toml b/grovedb-epoch-based-storage-flags/Cargo.toml new file mode 100644 index 00000000..65026a6a --- /dev/null +++ b/grovedb-epoch-based-storage-flags/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "grovedb-epoch-based-storage-flags" +authors = ["Samuel Westrich "] +description = "Epoch based storage flags for GroveDB" +version = "1.1.0" +edition = "2021" + +[dependencies] +thiserror = { version = "1.0.63" } +grovedb-costs = { version = "1.0.0", path = "../costs" } +intmap = { version = "2.0.0", features = ["serde"]} +integer-encoding = { version = "4.0.0" } +hex = { version = "0.4.3" } \ No newline at end of file diff --git a/grovedb-epoch-based-storage-flags/src/error.rs b/grovedb-epoch-based-storage-flags/src/error.rs new file mode 100644 index 00000000..006c918d --- /dev/null +++ b/grovedb-epoch-based-storage-flags/src/error.rs @@ -0,0 +1,45 @@ +/// Storage flag errors +#[derive(Debug, thiserror::Error)] +pub enum StorageFlagsError { + /// Error + #[error("deserialize unknown storage flags type error: {0}")] + DeserializeUnknownStorageFlagsType(String), + /// Error + #[error("storage flags wrong size error: {0}")] + StorageFlagsWrongSize(String), + /// Error + #[error("removing at epoch with no associated storage error: {0}")] + RemovingAtEpochWithNoAssociatedStorage(String), + /// Error + #[error("storage flags overflow error: {0}")] + StorageFlagsOverflow(String), + /// Error + #[error("removing flags error: {0}")] + RemovingFlagsError(String), + /// Error + #[error("merging storage flags from different owners error: {0}")] + MergingStorageFlagsFromDifferentOwners(String), + /// Error + #[error("merging storage flags with different base epoch: {0}")] + MergingStorageFlagsWithDifferentBaseEpoch(String), +} + +impl StorageFlagsError { + /// Gets a mutable reference to the inner string of the error variant + pub(crate) fn get_mut_info(&mut self) -> &mut String { + match self { + StorageFlagsError::DeserializeUnknownStorageFlagsType(ref mut msg) + | StorageFlagsError::StorageFlagsWrongSize(ref mut msg) + | StorageFlagsError::RemovingAtEpochWithNoAssociatedStorage(ref mut msg) + | StorageFlagsError::StorageFlagsOverflow(ref mut msg) + | StorageFlagsError::RemovingFlagsError(ref mut msg) + | StorageFlagsError::MergingStorageFlagsFromDifferentOwners(ref mut msg) + | StorageFlagsError::MergingStorageFlagsWithDifferentBaseEpoch(ref mut msg) => msg, + } + } + + /// adds info to the storage flags error + pub(crate) fn add_info(&mut self, info: &str) { + self.get_mut_info().push_str(format!(": {}", info).as_str()); + } +} diff --git a/grovedb-epoch-based-storage-flags/src/lib.rs b/grovedb-epoch-based-storage-flags/src/lib.rs new file mode 100644 index 00000000..add8765b --- /dev/null +++ b/grovedb-epoch-based-storage-flags/src/lib.rs @@ -0,0 +1,1588 @@ +//! Flags + +pub mod error; +mod split_removal_bytes; +mod update_element_flags; + +use crate::{ + error::StorageFlagsError, + StorageFlags::{MultiEpoch, MultiEpochOwned, SingleEpoch, SingleEpochOwned}, +}; + +const DEFAULT_HASH_SIZE_U32: u32 = 32; + +/// Optional meta-data to be stored per element +pub type ElementFlags = Vec; + +use std::{borrow::Cow, cmp::Ordering, collections::BTreeMap, fmt}; + +use grovedb_costs::storage_cost::removal::{ + StorageRemovalPerEpochByIdentifier, StorageRemovedBytes, + StorageRemovedBytes::{NoStorageRemoval, SectionedStorageRemoval}, +}; +use integer_encoding::VarInt; +use intmap::IntMap; + +type EpochIndex = u16; + +type BaseEpoch = EpochIndex; + +type BytesAddedInEpoch = u32; + +type OwnerId = [u8; 32]; + +/// The size of single epoch flags +pub const SINGLE_EPOCH_FLAGS_SIZE: u32 = 3; + +/// The minimum size of the non-base flags +pub const MINIMUM_NON_BASE_FLAGS_SIZE: u32 = 3; + +/// Storage flags +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum StorageFlags { + /// Single epoch + /// represented as byte 0 + SingleEpoch(BaseEpoch), + + /// Multi epoch + /// represented as byte 1 + MultiEpoch(BaseEpoch, BTreeMap), + + /// Single epoch owned + /// represented as byte 2 + SingleEpochOwned(BaseEpoch, OwnerId), + + /// Multi epoch owned + /// represented as byte 3 + MultiEpochOwned(BaseEpoch, BTreeMap, OwnerId), +} + +impl fmt::Display for StorageFlags { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + StorageFlags::SingleEpoch(base_epoch) => { + write!(f, "SingleEpoch(BaseEpoch: {})", base_epoch) + } + StorageFlags::MultiEpoch(base_epoch, epochs) => { + write!(f, "MultiEpoch(BaseEpoch: {}, Epochs: ", base_epoch)?; + for (index, bytes) in epochs { + write!(f, "[EpochIndex: {}, BytesAdded: {}] ", index, bytes)?; + } + write!(f, ")") + } + StorageFlags::SingleEpochOwned(base_epoch, owner_id) => { + write!( + f, + "SingleEpochOwned(BaseEpoch: {}, OwnerId: {})", + base_epoch, + hex::encode(owner_id) + ) + } + StorageFlags::MultiEpochOwned(base_epoch, epochs, owner_id) => { + write!(f, "MultiEpochOwned(BaseEpoch: {}, Epochs: ", base_epoch)?; + for (index, bytes) in epochs { + write!(f, "[EpochIndex: {}, BytesAdded: {}] ", index, bytes)?; + } + write!(f, ", OwnerId: {})", hex::encode(owner_id)) + } + } + } +} + +/// MergingOwnersStrategy decides which owner to keep during a merge +#[derive(Clone, Copy, Debug, PartialEq, Eq, Default)] +pub enum MergingOwnersStrategy { + #[default] + /// Raise an issue that owners of nodes are different + RaiseIssue, + /// Use the original owner id + UseOurs, + /// Use the new owner id + UseTheirs, +} + +impl StorageFlags { + /// Create new single epoch storage flags + pub fn new_single_epoch(epoch: BaseEpoch, maybe_owner_id: Option) -> Self { + match maybe_owner_id { + None => SingleEpoch(epoch), + Some(owner_id) => SingleEpochOwned(epoch, owner_id), + } + } + + /// Sets the owner id if we have owned storage flags + pub fn set_owner_id(&mut self, owner_id: OwnerId) { + match self { + SingleEpochOwned(_, previous_owner_id) | MultiEpochOwned(_, _, previous_owner_id) => { + *previous_owner_id = owner_id; + } + _ => {} + } + } + + fn combine_owner_id<'a>( + &'a self, + rhs: &'a Self, + merging_owners_strategy: MergingOwnersStrategy, + ) -> Result, StorageFlagsError> { + if let Some(our_owner_id) = self.owner_id() { + if let Some(other_owner_id) = rhs.owner_id() { + if our_owner_id != other_owner_id { + match merging_owners_strategy { + MergingOwnersStrategy::RaiseIssue => { + Err(StorageFlagsError::MergingStorageFlagsFromDifferentOwners( + "can not merge from different owners".to_string(), + )) + } + MergingOwnersStrategy::UseOurs => Ok(Some(our_owner_id)), + MergingOwnersStrategy::UseTheirs => Ok(Some(other_owner_id)), + } + } else { + Ok(Some(our_owner_id)) + } + } else { + Ok(Some(our_owner_id)) + } + } else if let Some(other_owner_id) = rhs.owner_id() { + Ok(Some(other_owner_id)) + } else { + Ok(None) + } + } + + fn combine_non_base_epoch_bytes( + &self, + rhs: &Self, + ) -> Option> { + if let Some(our_epoch_index_map) = self.epoch_index_map() { + if let Some(other_epoch_index_map) = rhs.epoch_index_map() { + let mut combined_index_map = our_epoch_index_map.clone(); + other_epoch_index_map + .iter() + .for_each(|(epoch_index, bytes_added)| { + // Simply insert the value from rhs, overwriting any existing value + combined_index_map.insert(*epoch_index, *bytes_added); + }); + // println!( + // " >combine_non_base_epoch_bytes: self:{:?} & rhs:{:?} -> {:?}", + // our_epoch_index_map, other_epoch_index_map, combined_index_map + // ); + Some(combined_index_map) + } else { + Some(our_epoch_index_map.clone()) + } + } else { + rhs.epoch_index_map().cloned() + } + } + + fn combine_same_base_epoch( + &self, + rhs: Self, + merging_owners_strategy: MergingOwnersStrategy, + ) -> Result { + let base_epoch = *self.base_epoch(); + let owner_id = self.combine_owner_id(&rhs, merging_owners_strategy)?; + let other_epoch_bytes = self.combine_non_base_epoch_bytes(&rhs); + + match (owner_id, other_epoch_bytes) { + (None, None) => Ok(SingleEpoch(base_epoch)), + (Some(owner_id), None) => Ok(SingleEpochOwned(base_epoch, *owner_id)), + (None, Some(other_epoch_bytes)) => Ok(MultiEpoch(base_epoch, other_epoch_bytes)), + (Some(owner_id), Some(other_epoch_bytes)) => { + Ok(MultiEpochOwned(base_epoch, other_epoch_bytes, *owner_id)) + } + } + } + + fn combine_with_higher_base_epoch( + &self, + rhs: Self, + added_bytes: u32, + merging_owners_strategy: MergingOwnersStrategy, + ) -> Result { + let base_epoch = *self.base_epoch(); + let epoch_with_adding_bytes = rhs.base_epoch(); + let owner_id = self.combine_owner_id(&rhs, merging_owners_strategy)?; + let mut other_epoch_bytes = self.combine_non_base_epoch_bytes(&rhs).unwrap_or_default(); + let original_value = other_epoch_bytes.remove(epoch_with_adding_bytes); + match original_value { + None => other_epoch_bytes.insert(*epoch_with_adding_bytes, added_bytes), + Some(original_bytes) => { + other_epoch_bytes.insert(*epoch_with_adding_bytes, original_bytes + added_bytes) + } + }; + // println!( + // " >combine_with_higher_base_epoch added_bytes:{} self:{:?} & + // rhs:{:?} -> {:?}", added_bytes, + // self.epoch_index_map(), + // rhs.epoch_index_map(), + // other_epoch_bytes + // ); + + match owner_id { + None => Ok(MultiEpoch(base_epoch, other_epoch_bytes)), + Some(owner_id) => Ok(MultiEpochOwned(base_epoch, other_epoch_bytes, *owner_id)), + } + } + + fn combine_with_higher_base_epoch_remove_bytes( + self, + rhs: Self, + removed_bytes: &StorageRemovedBytes, + merging_owners_strategy: MergingOwnersStrategy, + ) -> Result { + if matches!(&self, &SingleEpoch(_) | &SingleEpochOwned(..)) { + return Ok(self); + } + let base_epoch = *self.base_epoch(); + let owner_id = self.combine_owner_id(&rhs, merging_owners_strategy)?; + let mut other_epoch_bytes = self.combine_non_base_epoch_bytes(&rhs).unwrap_or_default(); + if let SectionedStorageRemoval(sectioned_bytes_by_identifier) = removed_bytes { + if sectioned_bytes_by_identifier.len() > 1 { + return Err(StorageFlagsError::MergingStorageFlagsFromDifferentOwners( + "can not remove bytes when there is no epoch".to_string(), + )); + } + let identifier = owner_id.copied().unwrap_or_default(); + let sectioned_bytes = sectioned_bytes_by_identifier.get(&identifier).ok_or( + StorageFlagsError::MergingStorageFlagsFromDifferentOwners( + "can not remove bytes when there is no epoch".to_string(), + ), + )?; + let mut keys_to_remove = Vec::new(); // To store the keys that need to be removed + + sectioned_bytes + .iter() + .try_for_each(|(epoch, removed_bytes)| { + if *epoch == base_epoch as u64 { + return Ok::<(), StorageFlagsError>(()); + } + let bytes_added_in_epoch = other_epoch_bytes.get_mut(&(*epoch as u16)).ok_or( + StorageFlagsError::RemovingAtEpochWithNoAssociatedStorage(format!( + "can not remove bytes when there is no epoch number [{}]", + *epoch + )), + )?; + + let desired_bytes_in_epoch = bytes_added_in_epoch + .checked_sub(*removed_bytes) + .ok_or(StorageFlagsError::StorageFlagsOverflow( + "can't remove more bytes than exist at that epoch".to_string(), + ))?; + + if desired_bytes_in_epoch <= MINIMUM_NON_BASE_FLAGS_SIZE { + // Collect the key to remove later + keys_to_remove.push(*epoch as u16); + } else { + *bytes_added_in_epoch = desired_bytes_in_epoch; + } + + Ok::<(), StorageFlagsError>(()) + })?; + + // Now remove the keys after the iteration + for key in keys_to_remove { + other_epoch_bytes.remove(&key); + } + } + // println!( + // " >combine_with_higher_base_epoch_remove_bytes: self:{:?} & + // rhs:{:?} -> {:?}", self.epoch_index_map(), + // rhs.epoch_index_map(), + // other_epoch_bytes + // ); + + if other_epoch_bytes.is_empty() { + match owner_id { + None => Ok(SingleEpoch(base_epoch)), + Some(owner_id) => Ok(SingleEpochOwned(base_epoch, *owner_id)), + } + } else { + match owner_id { + None => Ok(MultiEpoch(base_epoch, other_epoch_bytes)), + Some(owner_id) => Ok(MultiEpochOwned(base_epoch, other_epoch_bytes, *owner_id)), + } + } + } + + /// Optional combine added bytes + pub fn optional_combine_added_bytes( + ours: Option, + theirs: Self, + added_bytes: u32, + merging_owners_strategy: MergingOwnersStrategy, + ) -> Result { + match ours { + None => Ok(theirs), + Some(ours) => { + Ok(ours.combine_added_bytes(theirs, added_bytes, merging_owners_strategy)?) + } + } + } + + /// Optional combine removed bytes + pub fn optional_combine_removed_bytes( + ours: Option, + theirs: Self, + removed_bytes: &StorageRemovedBytes, + merging_owners_strategy: MergingOwnersStrategy, + ) -> Result { + match ours { + None => Ok(theirs), + Some(ours) => { + Ok(ours.combine_removed_bytes(theirs, removed_bytes, merging_owners_strategy)?) + } + } + } + + /// Combine added bytes + pub fn combine_added_bytes( + self, + rhs: Self, + added_bytes: u32, + merging_owners_strategy: MergingOwnersStrategy, + ) -> Result { + match self.base_epoch().cmp(rhs.base_epoch()) { + Ordering::Equal => self.combine_same_base_epoch(rhs, merging_owners_strategy), + Ordering::Less => { + self.combine_with_higher_base_epoch(rhs, added_bytes, merging_owners_strategy) + } + Ordering::Greater => Err( + StorageFlagsError::MergingStorageFlagsWithDifferentBaseEpoch( + "can not merge with new item in older base epoch".to_string(), + ), + ), + } + } + + /// Combine removed bytes + pub fn combine_removed_bytes( + self, + rhs: Self, + removed_bytes: &StorageRemovedBytes, + merging_owners_strategy: MergingOwnersStrategy, + ) -> Result { + match self.base_epoch().cmp(rhs.base_epoch()) { + Ordering::Equal => self.combine_same_base_epoch(rhs, merging_owners_strategy), + Ordering::Less => self.combine_with_higher_base_epoch_remove_bytes( + rhs, + removed_bytes, + merging_owners_strategy, + ), + Ordering::Greater => Err( + StorageFlagsError::MergingStorageFlagsWithDifferentBaseEpoch( + "can not merge with new item in older base epoch".to_string(), + ), + ), + } + } + + /// Returns base epoch + pub fn base_epoch(&self) -> &BaseEpoch { + match self { + SingleEpoch(base_epoch) + | MultiEpoch(base_epoch, _) + | SingleEpochOwned(base_epoch, _) + | MultiEpochOwned(base_epoch, ..) => base_epoch, + } + } + + /// Returns owner id + pub fn owner_id(&self) -> Option<&OwnerId> { + match self { + SingleEpochOwned(_, owner_id) | MultiEpochOwned(_, _, owner_id) => Some(owner_id), + _ => None, + } + } + + /// Returns epoch index map + pub fn epoch_index_map(&self) -> Option<&BTreeMap> { + match self { + MultiEpoch(_, epoch_int_map) | MultiEpochOwned(_, epoch_int_map, _) => { + Some(epoch_int_map) + } + _ => None, + } + } + + /// Returns optional default storage flags + pub fn optional_default() -> Option { + None + } + + /// Returns default optional storage flag as ref + pub fn optional_default_as_ref() -> Option<&'static Self> { + None + } + + /// Returns default optional storage flag as ref + pub fn optional_default_as_cow() -> Option> { + None + } + + /// Returns type byte + pub fn type_byte(&self) -> u8 { + match self { + SingleEpoch(_) => 0, + MultiEpoch(..) => 1, + SingleEpochOwned(..) => 2, + MultiEpochOwned(..) => 3, + } + } + + fn append_to_vec_base_epoch(&self, buffer: &mut Vec) { + match self { + SingleEpoch(base_epoch) + | MultiEpoch(base_epoch, ..) + | SingleEpochOwned(base_epoch, ..) + | MultiEpochOwned(base_epoch, ..) => buffer.extend(base_epoch.to_be_bytes()), + } + } + + fn maybe_append_to_vec_epoch_map(&self, buffer: &mut Vec) { + match self { + MultiEpoch(_, epoch_map) | MultiEpochOwned(_, epoch_map, _) => { + if epoch_map.is_empty() { + panic!("this should not be empty"); + } + epoch_map.iter().for_each(|(epoch_index, bytes_added)| { + buffer.extend(epoch_index.to_be_bytes()); + buffer.extend(bytes_added.encode_var_vec()); + }) + } + _ => {} + } + } + + fn maybe_epoch_map_size(&self) -> u32 { + let mut size = 0; + match self { + MultiEpoch(_, epoch_map) | MultiEpochOwned(_, epoch_map, _) => { + epoch_map.iter().for_each(|(_epoch_index, bytes_added)| { + size += 2; + size += bytes_added.encode_var_vec().len() as u32; + }) + } + _ => {} + } + size + } + + fn maybe_append_to_vec_owner_id(&self, buffer: &mut Vec) { + match self { + SingleEpochOwned(_, owner_id) | MultiEpochOwned(_, _, owner_id) => { + buffer.extend(owner_id); + } + _ => {} + } + } + + fn maybe_owner_id_size(&self) -> u32 { + match self { + SingleEpochOwned(..) | MultiEpochOwned(..) => DEFAULT_HASH_SIZE_U32, + _ => 0, + } + } + + /// ApproximateSize + pub fn approximate_size( + has_owner_id: bool, + approximate_changes_and_bytes_count: Option<(u16, u8)>, + ) -> u32 { + let mut size = 3; // 1 for type byte, 2 for epoch number + if has_owner_id { + size += DEFAULT_HASH_SIZE_U32; + } + if let Some((approximate_change_count, bytes_changed_required_size)) = + approximate_changes_and_bytes_count + { + size += (approximate_change_count as u32) * (2 + bytes_changed_required_size as u32) + } + size + } + + /// Serialize storage flags + pub fn serialize(&self) -> Vec { + let mut buffer = vec![self.type_byte()]; + self.maybe_append_to_vec_owner_id(&mut buffer); + self.append_to_vec_base_epoch(&mut buffer); + self.maybe_append_to_vec_epoch_map(&mut buffer); + buffer + } + + /// Serialize storage flags + pub fn serialized_size(&self) -> u32 { + let mut buffer_len = 3; // for type byte and base epoch + buffer_len += self.maybe_owner_id_size(); + buffer_len += self.maybe_epoch_map_size(); + buffer_len + } + + /// Deserialize single epoch storage flags from bytes + pub fn deserialize_single_epoch(data: &[u8]) -> Result { + if data.len() != 3 { + Err(StorageFlagsError::StorageFlagsWrongSize( + "single epoch must be 3 bytes total".to_string(), + )) + } else { + let epoch = u16::from_be_bytes(data[1..3].try_into().map_err(|_| { + StorageFlagsError::StorageFlagsWrongSize( + "single epoch must be 3 bytes total".to_string(), + ) + })?); + Ok(SingleEpoch(epoch)) + } + } + + /// Deserialize multi epoch storage flags from bytes + pub fn deserialize_multi_epoch(data: &[u8]) -> Result { + let len = data.len(); + if len < 6 { + Err(StorageFlagsError::StorageFlagsWrongSize( + "multi epoch must be at least 6 bytes total".to_string(), + )) + } else { + let base_epoch = u16::from_be_bytes(data[1..3].try_into().map_err(|_| { + StorageFlagsError::StorageFlagsWrongSize( + "multi epoch must have enough bytes for the base epoch".to_string(), + ) + })?); + let mut offset = 3; + let mut bytes_per_epoch: BTreeMap = BTreeMap::default(); + while offset + 2 < len { + // 2 for epoch size + let epoch_index = + u16::from_be_bytes(data[offset..offset + 2].try_into().map_err(|_| { + StorageFlagsError::StorageFlagsWrongSize( + "multi epoch must have enough bytes epoch indexes".to_string(), + ) + })?); + offset += 2; + let (bytes_at_epoch, bytes_used) = u32::decode_var(&data[offset..]).ok_or( + StorageFlagsError::StorageFlagsWrongSize( + "multi epoch must have enough bytes for the amount of bytes used" + .to_string(), + ), + )?; + offset += bytes_used; + bytes_per_epoch.insert(epoch_index, bytes_at_epoch); + } + Ok(MultiEpoch(base_epoch, bytes_per_epoch)) + } + } + + /// Deserialize single epoch owned storage flags from bytes + pub fn deserialize_single_epoch_owned(data: &[u8]) -> Result { + if data.len() != 35 { + Err(StorageFlagsError::StorageFlagsWrongSize( + "single epoch owned must be 35 bytes total".to_string(), + )) + } else { + let owner_id: OwnerId = data[1..33].try_into().map_err(|_| { + StorageFlagsError::StorageFlagsWrongSize( + "single epoch owned must be 35 bytes total for owner id".to_string(), + ) + })?; + let epoch = u16::from_be_bytes(data[33..35].try_into().map_err(|_| { + StorageFlagsError::StorageFlagsWrongSize( + "single epoch owned must be 35 bytes total for epoch".to_string(), + ) + })?); + Ok(SingleEpochOwned(epoch, owner_id)) + } + } + + /// Deserialize multi epoch owned storage flags from bytes + pub fn deserialize_multi_epoch_owned(data: &[u8]) -> Result { + let len = data.len(); + if len < 38 { + Err(StorageFlagsError::StorageFlagsWrongSize( + "multi epoch owned must be at least 38 bytes total".to_string(), + )) + } else { + let owner_id: OwnerId = data[1..33].try_into().map_err(|_| { + StorageFlagsError::StorageFlagsWrongSize( + "multi epoch owned must be 38 bytes total for owner id".to_string(), + ) + })?; + let base_epoch = u16::from_be_bytes(data[33..35].try_into().map_err(|_| { + StorageFlagsError::StorageFlagsWrongSize( + "multi epoch must have enough bytes for the base epoch".to_string(), + ) + })?); + let mut offset = 35; + let mut bytes_per_epoch: BTreeMap = BTreeMap::default(); + while offset + 2 < len { + // 2 for epoch size + let epoch_index = + u16::from_be_bytes(data[offset..offset + 2].try_into().map_err(|_| { + StorageFlagsError::StorageFlagsWrongSize( + "multi epoch must have enough bytes epoch indexes".to_string(), + ) + })?); + offset += 2; + let (bytes_at_epoch, bytes_used) = u32::decode_var(&data[offset..]).ok_or( + StorageFlagsError::StorageFlagsWrongSize( + "multi epoch must have enough bytes for the amount of bytes used" + .to_string(), + ), + )?; + offset += bytes_used; + bytes_per_epoch.insert(epoch_index, bytes_at_epoch); + } + Ok(MultiEpochOwned(base_epoch, bytes_per_epoch, owner_id)) + } + } + + /// Deserialize storage flags from bytes + pub fn deserialize(data: &[u8]) -> Result, StorageFlagsError> { + let first_byte = data.first(); + match first_byte { + None => Ok(None), + Some(first_byte) => match *first_byte { + 0 => Ok(Some(Self::deserialize_single_epoch(data)?)), + 1 => Ok(Some(Self::deserialize_multi_epoch(data)?)), + 2 => Ok(Some(Self::deserialize_single_epoch_owned(data)?)), + 3 => Ok(Some(Self::deserialize_multi_epoch_owned(data)?)), + _ => Err(StorageFlagsError::DeserializeUnknownStorageFlagsType( + "unknown storage flags serialization".to_string(), + )), + }, + } + } + + /// Creates storage flags from a slice. + pub fn from_slice(data: &[u8]) -> Result, StorageFlagsError> { + Self::deserialize(data) + } + + /// Creates storage flags from element flags. + pub fn from_element_flags_ref(data: &ElementFlags) -> Result, StorageFlagsError> { + Self::from_slice(data.as_slice()) + } + + /// Create Storage flags from optional element flags ref + pub fn map_some_element_flags_ref( + data: &Option, + ) -> Result, StorageFlagsError> { + match data { + None => Ok(None), + Some(data) => Self::from_slice(data.as_slice()), + } + } + + /// Create Storage flags from optional element flags ref + pub fn map_cow_some_element_flags_ref( + data: &Option, + ) -> Result>, StorageFlagsError> { + match data { + None => Ok(None), + Some(data) => Self::from_slice(data.as_slice()).map(|option| option.map(Cow::Owned)), + } + } + + /// Map to owned optional element flags + pub fn map_owned_to_element_flags(maybe_storage_flags: Option) -> ElementFlags { + maybe_storage_flags + .map(|storage_flags| storage_flags.serialize()) + .unwrap_or_default() + } + + /// Map to optional element flags + pub fn map_to_some_element_flags(maybe_storage_flags: Option<&Self>) -> Option { + maybe_storage_flags.map(|storage_flags| storage_flags.serialize()) + } + + /// Map to optional element flags + pub fn map_cow_to_some_element_flags( + maybe_storage_flags: Option>, + ) -> Option { + maybe_storage_flags.map(|storage_flags| storage_flags.serialize()) + } + + /// Map to optional element flags + pub fn map_borrowed_cow_to_some_element_flags( + maybe_storage_flags: &Option>, + ) -> Option { + maybe_storage_flags + .as_ref() + .map(|storage_flags| storage_flags.serialize()) + } + + /// Creates optional element flags + pub fn to_some_element_flags(&self) -> Option { + Some(self.serialize()) + } + + /// Creates element flags. + pub fn to_element_flags(&self) -> ElementFlags { + self.serialize() + } + + /// split_storage_removed_bytes removes bytes as LIFO + pub fn split_storage_removed_bytes( + &self, + removed_key_bytes: u32, + removed_value_bytes: u32, + ) -> (StorageRemovedBytes, StorageRemovedBytes) { + fn single_storage_removal( + removed_bytes: u32, + base_epoch: &BaseEpoch, + owner_id: Option<&OwnerId>, + ) -> StorageRemovedBytes { + if removed_bytes == 0 { + return NoStorageRemoval; + } + let bytes_left = removed_bytes; + let mut sectioned_storage_removal: IntMap = IntMap::default(); + if bytes_left > 0 { + // We need to take some from the base epoch + sectioned_storage_removal.insert(*base_epoch as u64, removed_bytes); + } + let mut sectioned_storage_removal_by_identifier: StorageRemovalPerEpochByIdentifier = + BTreeMap::new(); + if let Some(owner_id) = owner_id { + sectioned_storage_removal_by_identifier + .insert(*owner_id, sectioned_storage_removal); + } else { + let default = [0u8; 32]; + sectioned_storage_removal_by_identifier.insert(default, sectioned_storage_removal); + } + SectionedStorageRemoval(sectioned_storage_removal_by_identifier) + } + + fn sectioned_storage_removal( + removed_bytes: u32, + base_epoch: &BaseEpoch, + other_epoch_bytes: &BTreeMap, + owner_id: Option<&OwnerId>, + ) -> StorageRemovedBytes { + if removed_bytes == 0 { + return NoStorageRemoval; + } + let mut bytes_left = removed_bytes; + let mut rev_iter = other_epoch_bytes.iter().rev(); + let mut sectioned_storage_removal: IntMap = IntMap::default(); + + while bytes_left > 0 { + if let Some((epoch_index, bytes_in_epoch)) = rev_iter.next() { + if *bytes_in_epoch <= bytes_left + MINIMUM_NON_BASE_FLAGS_SIZE { + sectioned_storage_removal.insert( + *epoch_index as u64, + *bytes_in_epoch - MINIMUM_NON_BASE_FLAGS_SIZE, + ); + bytes_left -= *bytes_in_epoch - MINIMUM_NON_BASE_FLAGS_SIZE; + } else { + // Correctly take only the required bytes_left from this epoch + sectioned_storage_removal.insert(*epoch_index as u64, bytes_left); + bytes_left = 0; // All required bytes have been removed, stop processing + break; // Exit the loop as there's no need to process + // further epochs + } + } else { + break; + } + } + + if bytes_left > 0 { + // If there are still bytes left, take them from the base epoch + sectioned_storage_removal.insert(*base_epoch as u64, bytes_left); + } + + let mut sectioned_storage_removal_by_identifier: StorageRemovalPerEpochByIdentifier = + BTreeMap::new(); + + if let Some(owner_id) = owner_id { + sectioned_storage_removal_by_identifier + .insert(*owner_id, sectioned_storage_removal); + } else { + let default = [0u8; 32]; + sectioned_storage_removal_by_identifier.insert(default, sectioned_storage_removal); + } + + SectionedStorageRemoval(sectioned_storage_removal_by_identifier) + } + + // If key bytes are being removed, it implies a delete; thus, we should remove + // all relevant storage bytes + let key_storage_removal = if removed_key_bytes > 0 { + match self { + // For any variant, always take the key's removed bytes from the base epoch + SingleEpoch(base_epoch) | MultiEpoch(base_epoch, _) => { + single_storage_removal(removed_key_bytes, base_epoch, None) + } + SingleEpochOwned(base_epoch, owner_id) + | MultiEpochOwned(base_epoch, _, owner_id) => { + single_storage_removal(removed_key_bytes, base_epoch, Some(owner_id)) + } + } + } else { + StorageRemovedBytes::default() + }; + + // For normal logic, we only need to process the value-related bytes. + let value_storage_removal = match self { + SingleEpoch(base_epoch) => { + single_storage_removal(removed_value_bytes, base_epoch, None) + } + SingleEpochOwned(base_epoch, owner_id) => { + single_storage_removal(removed_value_bytes, base_epoch, Some(owner_id)) + } + MultiEpoch(base_epoch, other_epoch_bytes) => { + sectioned_storage_removal(removed_value_bytes, base_epoch, other_epoch_bytes, None) + } + MultiEpochOwned(base_epoch, other_epoch_bytes, owner_id) => sectioned_storage_removal( + removed_value_bytes, + base_epoch, + other_epoch_bytes, + Some(owner_id), + ), + }; + + // For key removal, simply return the empty removal since it's an update does + // not modify the key. + (key_storage_removal, value_storage_removal) + } + + /// Wrap Storage Flags into optional owned cow + pub fn into_optional_cow<'a>(self) -> Option> { + Some(Cow::Owned(self)) + } +} + +#[cfg(test)] +mod storage_flags_tests { + use std::collections::BTreeMap; + + use grovedb_costs::storage_cost::removal::{ + StorageRemovalPerEpochByIdentifier, StorageRemovedBytes, + }; + use intmap::IntMap; + + use crate::{ + BaseEpoch, BytesAddedInEpoch, MergingOwnersStrategy, OwnerId, StorageFlags, + MINIMUM_NON_BASE_FLAGS_SIZE, + }; + #[test] + fn test_storage_flags_combine() { + { + // Same SingleEpoch - AdditionBytes + let common_base_index: BaseEpoch = 1; + let left_flag = StorageFlags::new_single_epoch(common_base_index, None); + let right_flag = StorageFlags::new_single_epoch(common_base_index, None); + + let added_bytes: BytesAddedInEpoch = 10; + let combined_flag = left_flag.clone().combine_added_bytes( + right_flag.clone(), + added_bytes, + MergingOwnersStrategy::UseOurs, + ); + // println!( + // "{:?} & {:?} added_bytes:{} --> {:?}\n", + // left_flag, right_flag, added_bytes, combined_flag + // ); + } + // { + // Same SingleEpoch - RemovedBytes + // let common_base_index: BaseEpoch = 1; + // let left_flag = StorageFlags::new_single_epoch(common_base_index, None); + // let right_flag = StorageFlags::new_single_epoch(common_base_index, None); + // + // let removed_bytes = StorageRemovedBytes::BasicStorageRemoval(10); + // let combined_flag = + // left_flag.clone().combine_removed_bytes(right_flag.clone(), &removed_bytes, + // MergingOwnersStrategy::UseOurs); println!("{:?} & {:?} + // removed_bytes:{:?} --> {:?}\n", left_flag, right_flag, removed_bytes, + // combined_flag); } + { + // Different-Higher SingleEpoch - AdditionBytes + let left_base_index: BaseEpoch = 1; + let right_base_index: BaseEpoch = 2; + let left_flag = StorageFlags::new_single_epoch(left_base_index, None); + let right_flag = StorageFlags::new_single_epoch(right_base_index, None); + + let added_bytes: BytesAddedInEpoch = 10; + let combined_flag = left_flag.clone().combine_added_bytes( + right_flag.clone(), + added_bytes, + MergingOwnersStrategy::UseOurs, + ); + // println!( + // "{:?} & {:?} added_bytes:{} --> {:?}\n", + // left_flag, right_flag, added_bytes, combined_flag + // ); + } + { + // Different-Lesser SingleEpoch - AdditionBytes + let left_base_index: BaseEpoch = 2; + let right_base_index: BaseEpoch = 1; + let left_flag = StorageFlags::new_single_epoch(left_base_index, None); + let right_flag = StorageFlags::new_single_epoch(right_base_index, None); + + let added_bytes: BytesAddedInEpoch = 10; + let combined_flag = left_flag.clone().combine_added_bytes( + right_flag.clone(), + added_bytes, + MergingOwnersStrategy::UseOurs, + ); + // println!( + // "{:?} & {:?} added_bytes:{} --> {:?}\n", + // left_flag, right_flag, added_bytes, combined_flag + // ); + } + { + // SingleEpoch-MultiEpoch same BaseEpoch - AdditionBytes + let common_base_index: BaseEpoch = 1; + let left_flag = StorageFlags::new_single_epoch(common_base_index, None); + let right_flag = StorageFlags::MultiEpoch( + common_base_index, + [(common_base_index + 1, 5)].iter().cloned().collect(), + ); + + let added_bytes: BytesAddedInEpoch = 10; + let combined_flag = left_flag.clone().combine_added_bytes( + right_flag.clone(), + added_bytes, + MergingOwnersStrategy::UseOurs, + ); + // println!( + // "{:?} & {:?} added_bytes:{} --> {:?}\n", + // left_flag, right_flag, added_bytes, combined_flag + // ); + } + { + // SingleEpoch-MultiEpoch higher BaseEpoch - AdditionBytes + let left_base_index: BaseEpoch = 1; + let right_base_index: BaseEpoch = 2; + let left_flag = StorageFlags::new_single_epoch(left_base_index, None); + let right_flag = StorageFlags::MultiEpoch( + right_base_index, + [(right_base_index + 1, 5)].iter().cloned().collect(), + ); + + let added_bytes: BytesAddedInEpoch = 10; + let combined_flag = left_flag.clone().combine_added_bytes( + right_flag.clone(), + added_bytes, + MergingOwnersStrategy::UseOurs, + ); + // println!( + // "{:?} & {:?} added_bytes:{} --> {:?}\n", + // left_flag, right_flag, added_bytes, combined_flag + // ); + } + // { + // SingleEpoch-MultiEpoch same BaseEpoch - RemovedBytes (positive difference) + // let common_base_index: BaseEpoch = 1; + // let left_flag = StorageFlags::new_single_epoch(common_base_index, None); + // let right_flag = StorageFlags::MultiEpoch(common_base_index, + // [(common_base_index + 1, 10)].iter().cloned().collect()); + // + // let removed_bytes = StorageRemovedBytes::BasicStorageRemoval(3); + // let combined_flag = + // left_flag.clone().combine_removed_bytes(right_flag.clone(), &removed_bytes, + // MergingOwnersStrategy::UseOurs); println!("{:?} & {:?} + // removed_bytes:{:?} --> {:?}\n", left_flag, right_flag, &removed_bytes, + // combined_flag); } + // { + // SingleEpoch-MultiEpoch same BaseEpoch - RemovedBytes (negative difference) + // let common_base_index: BaseEpoch = 1; + // let left_flag = StorageFlags::new_single_epoch(common_base_index, None); + // let right_flag = StorageFlags::MultiEpoch(common_base_index, + // [(common_base_index + 1, 10)].iter().cloned().collect()); + // + // let removed_bytes = StorageRemovedBytes::BasicStorageRemoval(13); + // let combined_flag = + // left_flag.clone().combine_removed_bytes(right_flag.clone(), &removed_bytes, + // MergingOwnersStrategy::UseOurs); println!("{:?} & {:?} + // removed_bytes:{:?} --> {:?}\n", left_flag, right_flag, &removed_bytes, + // combined_flag); } + // { + // SingleEpoch-MultiEpoch higher BaseEpoch - RemovedBytes (positive difference) + // let left_base_index: BaseEpoch = 1; + // let right_base_index: BaseEpoch = 2; + // let left_flag = StorageFlags::new_single_epoch(left_base_index, None); + // let right_flag = StorageFlags::MultiEpoch(right_base_index, + // [(right_base_index + 1, 10)].iter().cloned().collect()); + // + // let removed_bytes = StorageRemovedBytes::BasicStorageRemoval(3); + // let combined_flag = + // left_flag.clone().combine_removed_bytes(right_flag.clone(), &removed_bytes, + // MergingOwnersStrategy::UseOurs); println!("{:?} & {:?} + // removed_bytes:{:?} --> {:?}\n", left_flag, right_flag, &removed_bytes, + // combined_flag); } + // { + // SingleEpoch-MultiEpoch higher BaseEpoch - RemovedBytes (negative difference) + // let left_base_index: BaseEpoch = 1; + // let right_base_index: BaseEpoch = 2; + // let left_flag = StorageFlags::new_single_epoch(left_base_index, None); + // let right_flag = StorageFlags::MultiEpoch(right_base_index, + // [(right_base_index + 1, 5)].iter().cloned().collect()); + // + // let removed_bytes = StorageRemovedBytes::BasicStorageRemoval(7); + // let combined_flag = + // left_flag.clone().combine_removed_bytes(right_flag.clone(), &removed_bytes, + // MergingOwnersStrategy::UseOurs); println!("{:?} & {:?} + // removed_bytes:{:?} --> {:?}\n", left_flag, right_flag, &removed_bytes, + // combined_flag); } + { + // MultiEpochs same BaseEpoch - AdditionBytes #1 + let common_base_index: BaseEpoch = 1; + let left_flag = StorageFlags::MultiEpoch( + common_base_index, + [(common_base_index + 1, 7)].iter().cloned().collect(), + ); + let right_flag = StorageFlags::MultiEpoch( + common_base_index, + [(common_base_index + 1, 5)].iter().cloned().collect(), + ); + + let added_bytes: BytesAddedInEpoch = 10; + let combined_flag = left_flag.clone().combine_added_bytes( + right_flag.clone(), + added_bytes, + MergingOwnersStrategy::UseOurs, + ); + println!( + "{:?} & {:?} added_bytes:{} --> {:?}\n", + left_flag, right_flag, added_bytes, combined_flag + ); + } + { + // MultiEpochs same BaseEpoch - AdditionBytes #2 + let common_base_index: BaseEpoch = 1; + let left_flag = StorageFlags::MultiEpoch( + common_base_index, + [(common_base_index + 1, 7)].iter().cloned().collect(), + ); + let right_flag = StorageFlags::MultiEpoch( + common_base_index, + [(common_base_index + 2, 5)].iter().cloned().collect(), + ); + + let added_bytes: BytesAddedInEpoch = 10; + let combined_flag = left_flag.clone().combine_added_bytes( + right_flag.clone(), + added_bytes, + MergingOwnersStrategy::UseOurs, + ); + println!( + "{:?} & {:?} added_bytes:{} --> {:?}\n", + left_flag, right_flag, added_bytes, combined_flag + ); + } + { + // MultiEpochs same BaseEpoch - AdditionBytes #3 + let common_base_index: BaseEpoch = 1; + let left_flag = StorageFlags::MultiEpoch( + common_base_index, + [(common_base_index + 1, 7)].iter().cloned().collect(), + ); + let right_flag = StorageFlags::MultiEpoch( + common_base_index, + [(common_base_index + 1, 3), (common_base_index + 2, 5)] + .iter() + .cloned() + .collect(), + ); + + let added_bytes: BytesAddedInEpoch = 10; + let combined_flag = left_flag.clone().combine_added_bytes( + right_flag.clone(), + added_bytes, + MergingOwnersStrategy::UseOurs, + ); + println!( + "{:?} & {:?} added_bytes:{} --> {:?}\n", + left_flag, right_flag, added_bytes, combined_flag + ); + } + { + // MultiEpochs higher BaseEpoch - AdditionBytes #1 + let common_base_index: BaseEpoch = 1; + let left_flag = StorageFlags::MultiEpoch( + common_base_index, + [(common_base_index + 1, 7)].iter().cloned().collect(), + ); + let right_flag = StorageFlags::MultiEpoch( + common_base_index + 1, + [(common_base_index + 1, 5)].iter().cloned().collect(), + ); + + let added_bytes: BytesAddedInEpoch = 10; + let combined_flag = left_flag.clone().combine_added_bytes( + right_flag.clone(), + added_bytes, + MergingOwnersStrategy::UseOurs, + ); + println!( + "{:?} & {:?} added_bytes:{} --> {:?}\n", + left_flag, right_flag, added_bytes, combined_flag + ); + } + { + // MultiEpochs higher BaseEpoch - AdditionBytes #2 + let common_base_index: BaseEpoch = 1; + let left_flag = StorageFlags::MultiEpoch( + common_base_index, + [(common_base_index + 1, 7)].iter().cloned().collect(), + ); + let right_flag = StorageFlags::MultiEpoch( + common_base_index + 1, + [(common_base_index + 2, 5)].iter().cloned().collect(), + ); + + let added_bytes: BytesAddedInEpoch = 10; + let combined_flag = left_flag.clone().combine_added_bytes( + right_flag.clone(), + added_bytes, + MergingOwnersStrategy::UseOurs, + ); + println!( + "{:?} & {:?} added_bytes:{} --> {:?}\n", + left_flag, right_flag, added_bytes, combined_flag + ); + } + { + // MultiEpochs higher BaseEpoch - AdditionBytes #3 + let common_base_index: BaseEpoch = 1; + let left_flag = StorageFlags::MultiEpoch( + common_base_index, + [(common_base_index + 1, 7)].iter().cloned().collect(), + ); + let right_flag = StorageFlags::MultiEpoch( + common_base_index + 1, + [(common_base_index + 2, 3), (common_base_index + 3, 5)] + .iter() + .cloned() + .collect(), + ); + + let added_bytes: BytesAddedInEpoch = 10; + let combined_flag = left_flag.clone().combine_added_bytes( + right_flag.clone(), + added_bytes, + MergingOwnersStrategy::UseOurs, + ); + println!( + "{:?} & {:?} added_bytes:{} --> {:?}\n", + left_flag, right_flag, added_bytes, combined_flag + ); + } + } + + fn create_epoch_map(epoch: u16, bytes: u32) -> BTreeMap { + let mut map = BTreeMap::new(); + map.insert(epoch, bytes); + map + } + + fn default_owner_id() -> OwnerId { + [0u8; 32] + } + + fn single_epoch_removed_bytes_map( + owner_id: [u8; 32], + epoch_index: u64, + bytes_removed: u32, + ) -> StorageRemovalPerEpochByIdentifier { + let mut removed_bytes = StorageRemovalPerEpochByIdentifier::default(); + let mut removed_bytes_for_identity = IntMap::new(); + removed_bytes_for_identity.insert(epoch_index, bytes_removed); + removed_bytes.insert(owner_id, removed_bytes_for_identity); + removed_bytes + } + + fn multi_epoch_removed_bytes_map( + owner_id: [u8; 32], + removed_bytes_per_epoch: IntMap, + ) -> StorageRemovalPerEpochByIdentifier { + let mut removed_bytes = StorageRemovalPerEpochByIdentifier::default(); + removed_bytes.insert(owner_id, removed_bytes_per_epoch); + removed_bytes + } + + #[test] + fn test_combine_that_would_remove_the_epoch_completely() { + let owner_id = [2; 32]; + let left_base_index: BaseEpoch = 0; + let right_base_index: BaseEpoch = 1; + let other_epochs = create_epoch_map(1, 5); + let left_flag = StorageFlags::MultiEpochOwned(left_base_index, other_epochs, owner_id); + let right_flag = StorageFlags::new_single_epoch(right_base_index, Some(owner_id)); + + let removed_bytes = single_epoch_removed_bytes_map(owner_id, 1, 3); + let combined_flag = left_flag + .clone() + .combine_removed_bytes( + right_flag.clone(), + &StorageRemovedBytes::SectionedStorageRemoval(removed_bytes.clone()), + MergingOwnersStrategy::UseOurs, + ) + .expect("expected to combine flags"); + + assert_eq!( + combined_flag, + StorageFlags::SingleEpochOwned(left_base_index, owner_id) + ); + println!( + "{:?} & {:?} removed:{:?} --> {:?}\n", + left_flag, right_flag, removed_bytes, combined_flag + ); + } + + #[test] + fn test_combine_that_would_remove_the_epoch_completely_with_many_entries() { + let owner_id = [2; 32]; + let left_base_index: BaseEpoch = 0; + let right_base_index: BaseEpoch = 1; + let mut other_epochs = BTreeMap::new(); + let mut removed_bytes = IntMap::new(); + for i in 1..200 { + other_epochs.insert(i, MINIMUM_NON_BASE_FLAGS_SIZE + 1); + removed_bytes.insert(i as u64, 1); // anything between 1 and + // MINIMUM_NON_BASE_FLAGS_SIZE + + // 1 would be the same + } + + let left_flag = StorageFlags::MultiEpochOwned(left_base_index, other_epochs, owner_id); + let right_flag = StorageFlags::new_single_epoch(right_base_index, Some(owner_id)); + + let removed_bytes = multi_epoch_removed_bytes_map(owner_id, removed_bytes); + let combined_flag = left_flag + .clone() + .combine_removed_bytes( + right_flag.clone(), + &StorageRemovedBytes::SectionedStorageRemoval(removed_bytes.clone()), + MergingOwnersStrategy::UseOurs, + ) + .expect("expected to combine flags"); + + assert_eq!( + combined_flag, + StorageFlags::SingleEpochOwned(left_base_index, owner_id) + ); + println!( + "{:?} & {:?} removed:{:?} --> {:?}\n", + left_flag, right_flag, removed_bytes, combined_flag + ); + } + + /// Tests the case when using SingleEpoch flags, ensuring that the correct + /// storage removal is calculated. + #[test] + fn test_single_epoch_removal() { + let flags = StorageFlags::SingleEpoch(5); + let (key_removal, value_removal) = flags.split_storage_removed_bytes(100, 200); + + assert_eq!( + key_removal, + StorageRemovedBytes::SectionedStorageRemoval({ + let mut map = BTreeMap::new(); + map.insert(default_owner_id(), IntMap::from_iter([(5u64, 100)])); + map + }) + ); + assert_eq!( + value_removal, + StorageRemovedBytes::SectionedStorageRemoval({ + let mut map = BTreeMap::new(); + map.insert(default_owner_id(), IntMap::from_iter([(5u64, 200)])); + map + }) + ); + } + + /// Tests SingleEpochOwned flags where the removal is done under an OwnerId + #[test] + fn test_single_epoch_owned_removal() { + let owner_id = [1u8; 32]; + let flags = StorageFlags::SingleEpochOwned(5, owner_id); + let (key_removal, value_removal) = flags.split_storage_removed_bytes(50, 150); + + assert_eq!( + key_removal, + StorageRemovedBytes::SectionedStorageRemoval({ + let mut map = BTreeMap::new(); + map.insert(owner_id, IntMap::from_iter([(5u64, 50)])); + map + }) + ); + assert_eq!( + value_removal, + StorageRemovedBytes::SectionedStorageRemoval({ + let mut map = BTreeMap::new(); + map.insert(owner_id, IntMap::from_iter([(5u64, 150)])); + map + }) + ); + } + + /// Tests the case where multiple epochs are used and the total removal + /// doesn’t exceed the extra epoch bytes + #[test] + fn test_multi_epoch_removal_no_remaining_base() { + let mut other_epochs = create_epoch_map(6, 100); + other_epochs.insert(7, 200); + + let flags = StorageFlags::MultiEpoch(5, other_epochs); + let (_key_removal, value_removal) = flags.split_storage_removed_bytes(0, 250); + + assert_eq!( + value_removal, + StorageRemovedBytes::SectionedStorageRemoval({ + let mut map = BTreeMap::new(); + map.insert( + default_owner_id(), + IntMap::from_iter([(7u64, 197), (6u64, 53)]), + ); + map + }) + ); + } + + /// Similar to the previous test, but this time the base epoch is also used + /// due to insufficient bytes in the extra epochs + #[test] + fn test_multi_epoch_removal_with_remaining_base() { + let mut other_epochs = create_epoch_map(6, 100); + other_epochs.insert(7, 50); + + let flags = StorageFlags::MultiEpoch(5, other_epochs); + let (key_removal, value_removal) = flags.split_storage_removed_bytes(250, 250); + + assert_eq!( + key_removal, + StorageRemovedBytes::SectionedStorageRemoval({ + let mut map = BTreeMap::new(); + map.insert(default_owner_id(), IntMap::from_iter([(5u64, 250)])); + map + }) + ); + assert_eq!( + value_removal, + StorageRemovedBytes::SectionedStorageRemoval({ + let mut map = BTreeMap::new(); + map.insert( + default_owner_id(), + IntMap::from_iter([(7u64, 47), (6u64, 97), (5u64, 106)]), + ); + map + }) + ); + } + + /// Same as last test but for owned flags with OwnerId + #[test] + fn test_multi_epoch_owned_removal_with_remaining_base() { + let owner_id = [2u8; 32]; + let mut other_epochs = create_epoch_map(6, 100); + other_epochs.insert(7, 50); + + let flags = StorageFlags::MultiEpochOwned(5, other_epochs, owner_id); + let (key_removal, value_removal) = flags.split_storage_removed_bytes(250, 250); + + assert_eq!( + key_removal, + StorageRemovedBytes::SectionedStorageRemoval({ + let mut map = BTreeMap::new(); + map.insert(owner_id, IntMap::from_iter([(5u64, 250)])); + map + }) + ); + assert_eq!( + value_removal, + StorageRemovedBytes::SectionedStorageRemoval({ + let mut map = BTreeMap::new(); + map.insert( + owner_id, + IntMap::from_iter([(7u64, 47), (6u64, 97), (5u64, 106)]), + ); + map + }) + ); + } + + /// Tests the function when zero bytes are to be removed, expecting an empty + /// removal result + #[test] + fn test_single_epoch_removal_zero_bytes() { + let flags = StorageFlags::SingleEpoch(5); + let (key_removal, value_removal) = flags.split_storage_removed_bytes(0, 0); + + assert_eq!(key_removal, StorageRemovedBytes::NoStorageRemoval); + assert_eq!(value_removal, StorageRemovedBytes::NoStorageRemoval); + } + + /// Tests the removal of only part of the bytes using SingleEpochOwned + #[test] + fn test_single_epoch_owned_removal_partial_bytes() { + let owner_id = [3u8; 32]; + let flags = StorageFlags::SingleEpochOwned(5, owner_id); + let (key_removal, value_removal) = flags.split_storage_removed_bytes(100, 50); + + assert_eq!( + key_removal, + StorageRemovedBytes::SectionedStorageRemoval({ + let mut map = BTreeMap::new(); + map.insert(owner_id, IntMap::from_iter([(5u64, 100)])); + map + }) + ); + assert_eq!( + value_removal, + StorageRemovedBytes::SectionedStorageRemoval({ + let mut map = BTreeMap::new(); + map.insert(owner_id, IntMap::from_iter([(5u64, 50)])); + map + }) + ); + } + + /// Ensures that the function correctly handles when there are more bytes to + /// be removed than are available in the epoch map, requiring the base epoch + /// to be used + #[test] + fn test_multi_epoch_removal_excess_bytes() { + let mut other_epochs = create_epoch_map(6, 100); + other_epochs.insert(7, 200); + + let flags = StorageFlags::MultiEpoch(5, other_epochs); + let (key_removal, value_removal) = flags.split_storage_removed_bytes(400, 300); + + assert_eq!( + key_removal, + StorageRemovedBytes::SectionedStorageRemoval({ + let mut map = BTreeMap::new(); + map.insert(default_owner_id(), IntMap::from_iter([(5u64, 400)])); + map + }) + ); + assert_eq!( + value_removal, + StorageRemovedBytes::SectionedStorageRemoval({ + let mut map = BTreeMap::new(); + map.insert( + default_owner_id(), + IntMap::from_iter([(7u64, 197), (6u64, 97), (5u64, 6)]), + ); + map + }) + ); + } + + /// Similar to the previous test, but for owned flags with OwnerId + #[test] + fn test_multi_epoch_owned_removal_excess_bytes() { + let owner_id = [4u8; 32]; + let mut other_epochs = create_epoch_map(6, 100); + other_epochs.insert(7, 200); + + let flags = StorageFlags::MultiEpochOwned(5, other_epochs, owner_id); + let (key_removal, value_removal) = flags.split_storage_removed_bytes(450, 350); + + assert_eq!( + key_removal, + StorageRemovedBytes::SectionedStorageRemoval({ + let mut map = BTreeMap::new(); + map.insert(owner_id, IntMap::from_iter([(5u64, 450)])); + map + }) + ); + assert_eq!( + value_removal, + StorageRemovedBytes::SectionedStorageRemoval({ + let mut map = BTreeMap::new(); + map.insert( + owner_id, + IntMap::from_iter([(7u64, 197), (6u64, 97), (5u64, 56)]), + ); + map + }) + ); + } + + #[test] + /// This test verifies the `split_storage_removed_bytes` function when all + /// required bytes are taken from non-base epochs during the removal + /// process. + /// + /// The scenario: + /// - The test initializes a `StorageFlags::MultiEpochOwned` with a + /// `BaseEpoch` of 5. + /// - Two additional epochs, 6 and 7, are provided with 300 and 400 bytes + /// respectively. + /// - The function is then called to remove 700 bytes from the value, while + /// no bytes are removed from the key. + /// + /// The expected behavior: + /// - For key removal: No bytes should be removed since the key removal + /// request is zero. + /// - For value removal: It should consume all 400 bytes from epoch 7 (LIFO + /// order) and the remaining 300 bytes from epoch 6. + fn test_multi_epoch_owned_removal_all_bytes_taken_from_non_base_epoch() { + // Define the owner ID as a 32-byte array filled with 5s. + let owner_id = [5u8; 32]; + + // Create a map for additional epochs with 300 bytes in epoch 6. + let mut other_epochs = create_epoch_map(6, 300); + + // Insert 400 bytes for epoch 7 into the map. + other_epochs.insert(7, 400); + + // Initialize the `StorageFlags::MultiEpochOwned` with base epoch 5, additional + // epochs, and the owner ID. + let flags = StorageFlags::MultiEpochOwned(5, other_epochs, owner_id); + + // Call the function to split the storage removal bytes, expecting to remove 700 + // bytes from the value. + let (key_removal, value_removal) = flags.split_storage_removed_bytes(0, 700); + + // Verify that no bytes are removed from the key. + assert_eq!(key_removal, StorageRemovedBytes::NoStorageRemoval); + + // Verify that 700 bytes are removed from the value, consuming 400 bytes from + // epoch 7 and 300 bytes from epoch 6. + assert_eq!( + value_removal, + StorageRemovedBytes::SectionedStorageRemoval({ + let mut map = BTreeMap::new(); + map.insert( + owner_id, + IntMap::from_iter([(5u64, 6), (6u64, 297), (7u64, 397)]), + ); + map + }) + ); + } + + #[test] + fn test_multi_epoch_removal_remaining_base_epoch() { + let mut other_epochs = create_epoch_map(6, 300); + other_epochs.insert(7, 100); + + let flags = StorageFlags::MultiEpoch(5, other_epochs); + let (key_removal, value_removal) = flags.split_storage_removed_bytes(400, 500); + + assert_eq!( + key_removal, + StorageRemovedBytes::SectionedStorageRemoval({ + let mut map = BTreeMap::new(); + map.insert(default_owner_id(), IntMap::from_iter([(5u64, 400)])); + map + }) + ); + assert_eq!( + value_removal, + StorageRemovedBytes::SectionedStorageRemoval({ + let mut map = BTreeMap::new(); + map.insert( + default_owner_id(), + IntMap::from_iter([(7u64, 97), (6u64, 297), (5u64, 106)]), + ); + map + }) + ); + } +} diff --git a/grovedb-epoch-based-storage-flags/src/split_removal_bytes.rs b/grovedb-epoch-based-storage-flags/src/split_removal_bytes.rs new file mode 100644 index 00000000..f9fb351f --- /dev/null +++ b/grovedb-epoch-based-storage-flags/src/split_removal_bytes.rs @@ -0,0 +1,30 @@ +use grovedb_costs::storage_cost::removal::{ + StorageRemovedBytes, StorageRemovedBytes::BasicStorageRemoval, +}; + +use crate::{error::StorageFlagsError, ElementFlags, StorageFlags}; + +impl StorageFlags { + pub fn split_removal_bytes( + flags: &mut ElementFlags, + removed_key_bytes: u32, + removed_value_bytes: u32, + ) -> Result<(StorageRemovedBytes, StorageRemovedBytes), StorageFlagsError> { + let maybe_storage_flags = + StorageFlags::from_element_flags_ref(flags).map_err(|mut e| { + e.add_info("drive did not understand flags of item being updated"); + e + })?; + // if we removed key bytes then we removed the entire value + match maybe_storage_flags { + None => Ok(( + BasicStorageRemoval(removed_key_bytes), + BasicStorageRemoval(removed_value_bytes), + )), + Some(storage_flags) => { + Ok(storage_flags + .split_storage_removed_bytes(removed_key_bytes, removed_value_bytes)) + } + } + } +} diff --git a/grovedb-epoch-based-storage-flags/src/update_element_flags.rs b/grovedb-epoch-based-storage-flags/src/update_element_flags.rs new file mode 100644 index 00000000..932979b1 --- /dev/null +++ b/grovedb-epoch-based-storage-flags/src/update_element_flags.rs @@ -0,0 +1,138 @@ +use grovedb_costs::storage_cost::{transition::OperationStorageTransitionType, StorageCost}; + +use crate::{error::StorageFlagsError, ElementFlags, MergingOwnersStrategy, StorageFlags}; + +impl StorageFlags { + pub fn update_element_flags( + cost: &StorageCost, + old_flags: Option, + new_flags: &mut ElementFlags, + ) -> Result { + // if there were no flags before then the new flags are used + let Some(old_flags) = old_flags else { + return Ok(false); + }; + + // This could be none only because the old element didn't exist + // If they were empty we get an error + let maybe_old_storage_flags = + StorageFlags::from_element_flags_ref(&old_flags).map_err(|mut e| { + e.add_info("drive did not understand flags of old item being updated"); + e + })?; + let new_storage_flags = StorageFlags::from_element_flags_ref(new_flags) + .map_err(|mut e| { + e.add_info("drive did not understand updated item flag information"); + e + })? + .ok_or(StorageFlagsError::RemovingFlagsError( + "removing flags from an item with flags is not allowed".to_string(), + ))?; + let binding = maybe_old_storage_flags.clone().unwrap(); + let old_epoch_index_map = binding.epoch_index_map(); + let new_epoch_index_map = new_storage_flags.epoch_index_map(); + if old_epoch_index_map.is_some() || new_epoch_index_map.is_some() { + // println!("> old:{:?} new:{:?}", old_epoch_index_map, + // new_epoch_index_map); + } + + match &cost.transition_type() { + OperationStorageTransitionType::OperationUpdateBiggerSize => { + // In the case that the owners do not match up this means that there has been a + // transfer of ownership of the underlying document, the value + // held is transferred to the new owner + // println!(">---------------------combine_added_bytes:{}", cost.added_bytes); + // println!(">---------------------apply_batch_with_add_costs old_flags:{:?} + // new_flags:{:?}", maybe_old_storage_flags, new_storage_flags); + let combined_storage_flags = StorageFlags::optional_combine_added_bytes( + maybe_old_storage_flags.clone(), + new_storage_flags.clone(), + cost.added_bytes, + MergingOwnersStrategy::UseTheirs, + ) + .map_err(|mut e| { + e.add_info("drive could not combine storage flags (new flags were bigger)"); + e + })?; + // println!( + // ">added_bytes:{} old:{} new:{} --> combined:{}", + // cost.added_bytes, + // if maybe_old_storage_flags.is_some() { + // maybe_old_storage_flags.as_ref().unwrap().to_string() + // } else { + // "None".to_string() + // }, + // new_storage_flags, + // combined_storage_flags + // ); + // if combined_storage_flags.epoch_index_map().is_some() { + // //println!(" --------> bigger_combined_flags:{:?}", + // combined_storage_flags.epoch_index_map()); } + let combined_flags = combined_storage_flags.to_element_flags(); + // it's possible they got bigger in the same epoch + if combined_flags == *new_flags { + // they are the same there was no update + Ok(false) + } else { + *new_flags = combined_flags; + Ok(true) + } + } + OperationStorageTransitionType::OperationUpdateSmallerSize => { + // println!( + // ">removing_bytes:{:?} old:{} new:{}", + // cost.removed_bytes, + // if maybe_old_storage_flags.is_some() { + // maybe_old_storage_flags.as_ref().unwrap().to_string() + // } else { + // "None".to_string() + // }, + // new_storage_flags, + // ); + // In the case that the owners do not match up this means that there has been a + // transfer of ownership of the underlying document, the value + // held is transferred to the new owner + let combined_storage_flags = StorageFlags::optional_combine_removed_bytes( + maybe_old_storage_flags.clone(), + new_storage_flags.clone(), + &cost.removed_bytes, + MergingOwnersStrategy::UseTheirs, + ) + .map_err(|mut e| { + e.add_info("drive could not combine storage flags (new flags were smaller)"); + e + })?; + // println!( + // ">removed_bytes:{:?} old:{:?} new:{:?} --> combined:{:?}", + // cost.removed_bytes, + // maybe_old_storage_flags, + // new_storage_flags, + // combined_storage_flags + // ); + if combined_storage_flags.epoch_index_map().is_some() { + // println!(" --------> smaller_combined_flags:{:?}", + // combined_storage_flags.epoch_index_map()); + } + let combined_flags = combined_storage_flags.to_element_flags(); + // it's possible they got bigger in the same epoch + if combined_flags == *new_flags { + // they are the same there was no update + Ok(false) + } else { + *new_flags = combined_flags; + Ok(true) + } + } + OperationStorageTransitionType::OperationUpdateSameSize => { + if let Some(old_storage_flags) = maybe_old_storage_flags { + // if there were old storage flags we should just keep them + *new_flags = old_storage_flags.to_element_flags(); + Ok(true) + } else { + Ok(false) + } + } + _ => Ok(false), + } + } +} diff --git a/grovedb/Cargo.toml b/grovedb/Cargo.toml index f34fb9eb..a52372a5 100644 --- a/grovedb/Cargo.toml +++ b/grovedb/Cargo.toml @@ -21,7 +21,7 @@ hex = "0.4.3" itertools = { version = "0.12.1", optional = true } derive_more = "0.99.18" integer-encoding = { version = "4.0.0", optional = true } -grovedb-costs = { version = "1.0.0", git = "https://github.com/dashpay/grovedb", branch = "fix/JITfix", optional = true } +grovedb-costs = { version = "1.0.0", path = "../costs" , optional = true } nohash-hasher = { version = "0.2.0", optional = true } indexmap = "2.2.6" intmap = { version = "2.0.0", optional = true } @@ -40,7 +40,7 @@ rand = "0.8.5" criterion = "0.5.1" hex = "0.4.3" pretty_assertions = "1.4.0" -drive-storage-flags = { git = "https://github.com/dashpay/platform", branch = "fix/just_in_time_fee_update_fixes"} +grovedb-epoch-based-storage-flags = { version = "1.0.0", path = "../grovedb-epoch-based-storage-flags" } [[bench]] name = "insertion_benchmark" diff --git a/grovedb/src/batch/just_in_time_cost_tests.rs b/grovedb/src/batch/just_in_time_cost_tests.rs index 4fddc922..9c45680b 100644 --- a/grovedb/src/batch/just_in_time_cost_tests.rs +++ b/grovedb/src/batch/just_in_time_cost_tests.rs @@ -5,11 +5,11 @@ mod tests { use std::{collections::BTreeMap, option::Option::None}; - use drive_storage_flags::StorageFlags; use grovedb_costs::{ storage_cost::removal::{StorageRemovalPerEpochByIdentifier, StorageRemovedBytes}, OperationCost, }; + use grovedb_epoch_based_storage_flags::StorageFlags; use grovedb_version::version::GroveVersion; use intmap::IntMap; @@ -1465,8 +1465,6 @@ mod tests { .storage_cost .removed_bytes; - println!("{} {:?}", n, storage_removed_bytes); - if n > 113 { assert_eq!(storage_removed_bytes, StorageRemovedBytes::NoStorageRemoval); } else if n > 17 { diff --git a/merk/Cargo.toml b/merk/Cargo.toml index 950eda38..b20be3b4 100644 --- a/merk/Cargo.toml +++ b/merk/Cargo.toml @@ -17,7 +17,7 @@ grovedb-storage = { version = "1.0.0", path = "../storage", optional = true } failure = "0.1.8" integer-encoding = "4.0.0" indexmap = "2.2.6" -grovedb-costs = { version = "1.0.0", git = "https://github.com/dashpay/grovedb", branch = "fix/JITfix" } +grovedb-costs = { version = "1.0.0" , path = "../costs" } grovedb-visualize = { version = "1.0.0", path = "../visualize" } grovedb-path = { version = "1.0.0", path = "../path" } hex = "0.4.3" diff --git a/storage/Cargo.toml b/storage/Cargo.toml index b2c3e72b..7db2b599 100644 --- a/storage/Cargo.toml +++ b/storage/Cargo.toml @@ -16,7 +16,7 @@ blake3 = { version = "1.5.1", optional = true } integer-encoding = { version = "4.0.0", optional = true } grovedb-visualize = { version = "1.0.0", path = "../visualize" } strum = { version = "0.26.2", features = ["derive"] } -grovedb-costs = { version = "1.0.0", git = "https://github.com/dashpay/grovedb", branch = "fix/JITfix" } +grovedb-costs = { version = "1.0.0", path = "../costs" } thiserror = "1.0.59" rocksdb = { version = "0.22.0", optional = true } hex = "0.4.3" diff --git a/storage/src/rocksdb_storage/storage_context/batch.rs b/storage/src/rocksdb_storage/storage_context/batch.rs index 8cb9987e..bcf58372 100644 --- a/storage/src/rocksdb_storage/storage_context/batch.rs +++ b/storage/src/rocksdb_storage/storage_context/batch.rs @@ -1,31 +1,3 @@ -// MIT LICENSE -// -// Copyright (c) 2021 Dash Core Group -// -// Permission is hereby granted, free of charge, to any -// person obtaining a copy of this software and associated -// documentation files (the "Software"), to deal in the -// Software without restriction, including without -// limitation the rights to use, copy, modify, merge, -// publish, distribute, sublicense, and/or sell copies of -// the Software, and to permit persons to whom the Software -// is furnished to do so, subject to the following -// conditions: -// -// The above copyright notice and this permission notice -// shall be included in all copies or substantial portions -// of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF -// ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED -// TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A -// PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -// SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY -// CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -// OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR -// IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -// DEALINGS IN THE SOFTWARE. - //! Prefixed storage batch implementation for RocksDB backend. use grovedb_costs::{ diff --git a/tutorials/Cargo.toml b/tutorials/Cargo.toml index 409a1c64..12f1a87e 100644 --- a/tutorials/Cargo.toml +++ b/tutorials/Cargo.toml @@ -12,6 +12,7 @@ grovedb = { path = "../grovedb" } grovedb-merk = { path = "../merk" } grovedb-storage = { path = "../storage" } grovedb-visualize = { path = "../visualize" } +grovedb-version = { path = "../grovedb-version" } grovedb-path = { path = "../path" } rand = "0.8.5" hex = "0.4" diff --git a/tutorials/src/bin/delete.rs b/tutorials/src/bin/delete.rs index 063243fd..0655c722 100644 --- a/tutorials/src/bin/delete.rs +++ b/tutorials/src/bin/delete.rs @@ -1,7 +1,9 @@ use grovedb::{Element, GroveDb}; +use grovedb_version::version::GroveVersion; fn main() { let root_path: &[&[u8]] = &[]; + let grove_version = GroveVersion::latest(); // Specify a path and open GroveDB at the path as db let path = String::from("../tutorial-storage"); diff --git a/tutorials/src/bin/insert.rs b/tutorials/src/bin/insert.rs index 3d9f9b2a..3f478281 100644 --- a/tutorials/src/bin/insert.rs +++ b/tutorials/src/bin/insert.rs @@ -1,10 +1,13 @@ use grovedb::{Element, GroveDb}; +use grovedb_version::version::GroveVersion; fn main() { // Specify a path and open GroveDB at the path as db let path = String::from("../tutorial-storage"); let db = GroveDb::open(path).unwrap(); + let grove_version = GroveVersion::latest(); + // Define key-values for insertion let key1 = b"hello"; let val1 = b"world"; diff --git a/tutorials/src/bin/proofs.rs b/tutorials/src/bin/proofs.rs index 02596919..3499a635 100644 --- a/tutorials/src/bin/proofs.rs +++ b/tutorials/src/bin/proofs.rs @@ -1,4 +1,5 @@ use grovedb::{operations::insert::InsertOptions, Element, GroveDb, PathQuery, Query}; +use grovedb_version::version::GroveVersion; const KEY1: &[u8] = b"key1"; const KEY2: &[u8] = b"key2"; @@ -16,6 +17,9 @@ fn main() { let path = String::from("../tutorial-storage"); // Open GroveDB as db let db = GroveDb::open(path).unwrap(); + + let grove_version = GroveVersion::latest(); + // Populate GroveDB with values. This function is defined below. populate(&db); // Define the path to the subtree we want to query. @@ -28,7 +32,7 @@ fn main() { let path_query = PathQuery::new_unsized(path, query.clone()); // Execute the query and collect the result items in "elements". let (_elements, _) = db - .query_item_value(&path_query, true, false, true, None) + .query_item_value(&path_query, true, false, true, None, grove_version) .unwrap() .expect("expected successful get_path_query"); @@ -50,6 +54,8 @@ fn main() { fn populate(db: &GroveDb) { let root_path: &[&[u8]] = &[]; + let grove_version = GroveVersion::latest(); + // Put an empty subtree into the root tree nodes at KEY1. // Call this SUBTREE1. db.insert(root_path, KEY1, Element::empty_tree(), INSERT_OPTIONS, None, grove_version) diff --git a/tutorials/src/bin/query-complex.rs b/tutorials/src/bin/query-complex.rs index b4fb78cf..fa72b085 100644 --- a/tutorials/src/bin/query-complex.rs +++ b/tutorials/src/bin/query-complex.rs @@ -2,6 +2,7 @@ use grovedb::{ operations::insert::InsertOptions, Element, GroveDb, PathQuery, Query, QueryItem, SizedQuery, }; use rand::Rng; +use grovedb_version::version::GroveVersion; const KEY1: &[u8] = b"key1"; const KEY2: &[u8] = b"key2"; @@ -19,6 +20,8 @@ fn main() { // Specify the path where the GroveDB instance exists. let path = String::from("../tutorial-storage"); + let grove_version = GroveVersion::latest(); + // Open GroveDB at the path. let db = GroveDb::open(path).unwrap(); @@ -66,7 +69,7 @@ fn main() { // Execute the path query and collect the result items in "elements". let (elements, _) = db - .query_item_value(&path_query, true, false, true, None) + .query_item_value(&path_query, true, false, true, None, grove_version) .unwrap() .expect("expected successful get_path_query"); @@ -76,6 +79,9 @@ fn main() { fn populate(db: &GroveDb) { let root_path: &[&[u8]] = &[]; + + let grove_version = GroveVersion::latest(); + // Put an empty subtree into the root tree nodes at KEY1. // Call this SUBTREE1. db.insert(root_path, KEY1, Element::empty_tree(), INSERT_OPTIONS, None, grove_version) diff --git a/tutorials/src/bin/query-simple.rs b/tutorials/src/bin/query-simple.rs index ab888873..87d01ca6 100644 --- a/tutorials/src/bin/query-simple.rs +++ b/tutorials/src/bin/query-simple.rs @@ -1,4 +1,5 @@ use grovedb::{operations::insert::InsertOptions, Element, GroveDb, PathQuery, Query}; +use grovedb_version::version::GroveVersion; const KEY1: &[u8] = b"key1"; const KEY2: &[u8] = b"key2"; @@ -15,6 +16,8 @@ fn main() { // Specify the path where the GroveDB instance exists. let path = String::from("../tutorial-storage"); + let grove_version = GroveVersion::latest(); + // Open GroveDB at the path. let db = GroveDb::open(path).unwrap(); @@ -36,7 +39,7 @@ fn main() { // Execute the query and collect the result items in "elements". let (elements, _) = db - .query_item_value(&path_query, true, false, true,None) + .query_item_value(&path_query, true, false, true,None, &grove_version) .unwrap() .expect("expected successful get_path_query"); @@ -46,6 +49,9 @@ fn main() { fn populate(db: &GroveDb) { let root_path: &[&[u8]] = &[]; + + let grove_version = GroveVersion::latest(); + // Put an empty subtree into the root tree nodes at KEY1. // Call this SUBTREE1. db.insert(root_path, KEY1, Element::empty_tree(), INSERT_OPTIONS, None, grove_version) diff --git a/tutorials/src/bin/replication.rs b/tutorials/src/bin/replication.rs index 5ed6ab5b..ceeec2f2 100644 --- a/tutorials/src/bin/replication.rs +++ b/tutorials/src/bin/replication.rs @@ -6,6 +6,7 @@ use rand::{distributions::Alphanumeric, Rng, }; use grovedb::element::SumValue; use grovedb::replication::CURRENT_STATE_SYNC_VERSION; use grovedb::replication::MultiStateSyncInfo; +use grovedb_version::version::GroveVersion; const MAIN_ΚΕΥ: &[u8] = b"key_main"; const MAIN_ΚΕΥ_EMPTY: &[u8] = b"key_main_empty"; @@ -25,47 +26,47 @@ const INSERT_OPTIONS: Option = Some(InsertOptions { base_root_storage_is_free: true, }); -fn populate_db(grovedb_path: String) -> GroveDb { +fn populate_db(grovedb_path: String, grove_version: &GroveVersion) -> GroveDb { let db = GroveDb::open(grovedb_path).unwrap(); - insert_empty_tree_db(&db, ROOT_PATH, MAIN_ΚΕΥ); - insert_empty_tree_db(&db, ROOT_PATH, MAIN_ΚΕΥ_EMPTY); - insert_empty_tree_db(&db, &[MAIN_ΚΕΥ], KEY_INT_0); - insert_empty_tree_db(&db, &[MAIN_ΚΕΥ], KEY_INT_1); - insert_empty_tree_db(&db, &[MAIN_ΚΕΥ], KEY_INT_2); + insert_empty_tree_db(&db, ROOT_PATH, MAIN_ΚΕΥ, &grove_version); + insert_empty_tree_db(&db, ROOT_PATH, MAIN_ΚΕΥ_EMPTY, &grove_version); + insert_empty_tree_db(&db, &[MAIN_ΚΕΥ], KEY_INT_0, &grove_version); + insert_empty_tree_db(&db, &[MAIN_ΚΕΥ], KEY_INT_1, &grove_version); + insert_empty_tree_db(&db, &[MAIN_ΚΕΥ], KEY_INT_2, &grove_version); let tx = db.start_transaction(); let batch_size = 50; for i in 0..=5 { - insert_range_values_db(&db, &[MAIN_ΚΕΥ, KEY_INT_0], i * batch_size, i * batch_size + batch_size - 1, &tx); + insert_range_values_db(&db, &[MAIN_ΚΕΥ, KEY_INT_0], i * batch_size, i * batch_size + batch_size - 1, &tx, &grove_version); } let _ = db.commit_transaction(tx); let tx = db.start_transaction(); let batch_size = 50; for i in 0..=5 { - insert_range_values_db(&db, &[MAIN_ΚΕΥ, KEY_INT_1], i * batch_size, i * batch_size + batch_size - 1, &tx); + insert_range_values_db(&db, &[MAIN_ΚΕΥ, KEY_INT_1], i * batch_size, i * batch_size + batch_size - 1, &tx, &grove_version); } let _ = db.commit_transaction(tx); let tx = db.start_transaction(); let batch_size = 50; for i in 0..=5 { - insert_range_values_db(&db, &[MAIN_ΚΕΥ, KEY_INT_2], i * batch_size, i * batch_size + batch_size - 1, &tx); + insert_range_values_db(&db, &[MAIN_ΚΕΥ, KEY_INT_2], i * batch_size, i * batch_size + batch_size - 1, &tx, &grove_version); } let _ = db.commit_transaction(tx); - insert_empty_tree_db(&db, &[MAIN_ΚΕΥ], KEY_INT_REF_0); + insert_empty_tree_db(&db, &[MAIN_ΚΕΥ], KEY_INT_REF_0, &grove_version); let tx_2 = db.start_transaction(); - insert_range_ref_double_values_db(&db, &[MAIN_ΚΕΥ, KEY_INT_REF_0], KEY_INT_0, 1, 50, &tx_2); + insert_range_ref_double_values_db(&db, &[MAIN_ΚΕΥ, KEY_INT_REF_0], KEY_INT_0, 1, 50, &tx_2, &grove_version); let _ = db.commit_transaction(tx_2); - insert_empty_sum_tree_db(&db, &[MAIN_ΚΕΥ], KEY_INT_A); + insert_empty_sum_tree_db(&db, &[MAIN_ΚΕΥ], KEY_INT_A, &grove_version); let tx_3 = db.start_transaction(); - insert_range_values_db(&db, &[MAIN_ΚΕΥ, KEY_INT_A], 1, 500, &tx_3); - insert_sum_element_db(&db, &[MAIN_ΚΕΥ, KEY_INT_A], 501, 550, &tx_3); + insert_range_values_db(&db, &[MAIN_ΚΕΥ, KEY_INT_A], 1, 500, &tx_3, &grove_version); + insert_sum_element_db(&db, &[MAIN_ΚΕΥ, KEY_INT_A], 501, 550, &tx_3, &grove_version); let _ = db.commit_transaction(tx_3); db } @@ -76,8 +77,9 @@ fn create_empty_db(grovedb_path: String) -> GroveDb { } fn main() { + let grove_version = GroveVersion::latest(); let path_source = generate_random_path("../tutorial-storage/", "/db_0", 24); - let db_source = populate_db(path_source.clone()); + let db_source = populate_db(path_source.clone(), &grove_version); let checkpoint_dir = path_source + "/checkpoint"; let path_checkpoint = Path::new(checkpoint_dir.as_str()); @@ -103,11 +105,11 @@ fn main() { println!("\n######### db_checkpoint_0 -> db_destination state sync"); let state_info = MultiStateSyncInfo::default(); let tx = db_destination.start_transaction(); - sync_db_demo(&db_checkpoint_0, &db_destination, state_info, &tx).unwrap(); + sync_db_demo(&db_checkpoint_0, &db_destination, state_info, &tx, &grove_version).unwrap(); db_destination.commit_transaction(tx).unwrap().expect("expected to commit transaction"); println!("\n######### verify db_destination"); - let incorrect_hashes = db_destination.verify_grovedb(None, grove_version).unwrap(); + let incorrect_hashes = db_destination.verify_grovedb(None, true, false, grove_version).unwrap(); if incorrect_hashes.len() > 0 { println!("DB verification failed!"); } @@ -126,21 +128,21 @@ fn main() { let query_path = &[MAIN_ΚΕΥ, KEY_INT_0]; let query_key = (20487u32).to_be_bytes().to_vec(); println!("\n######## Query on db_checkpoint_0:"); - query_db(&db_checkpoint_0, query_path, query_key.clone()); + query_db(&db_checkpoint_0, query_path, query_key.clone(), &grove_version); println!("\n######## Query on db_destination:"); - query_db(&db_destination, query_path, query_key.clone()); + query_db(&db_destination, query_path, query_key.clone(), &grove_version); return; } -fn insert_empty_tree_db(db: &GroveDb, path: &[&[u8]], key: &[u8]) +fn insert_empty_tree_db(db: &GroveDb, path: &[&[u8]], key: &[u8], grove_version: &GroveVersion) { db.insert(path, key, Element::empty_tree(), INSERT_OPTIONS, None, grove_version) .unwrap() .expect("successfully inserted tree"); } -fn insert_range_values_db(db: &GroveDb, path: &[&[u8]], min_i: u32, max_i: u32, transaction: &Transaction) +fn insert_range_values_db(db: &GroveDb, path: &[&[u8]], min_i: u32, max_i: u32, transaction: &Transaction, grove_version: &GroveVersion) { for i in min_i..=max_i { let i_vec = i.to_be_bytes().to_vec(); @@ -157,7 +159,7 @@ fn insert_range_values_db(db: &GroveDb, path: &[&[u8]], min_i: u32, max_i: u32, } } -fn insert_range_ref_double_values_db(db: &GroveDb, path: &[&[u8]], ref_key: &[u8], min_i: u32, max_i: u32, transaction: &Transaction) +fn insert_range_ref_double_values_db(db: &GroveDb, path: &[&[u8]], ref_key: &[u8], min_i: u32, max_i: u32, transaction: &Transaction, grove_version: &GroveVersion) { for i in min_i..=max_i { let i_vec = i.to_be_bytes().to_vec(); @@ -180,13 +182,13 @@ fn insert_range_ref_double_values_db(db: &GroveDb, path: &[&[u8]], ref_key: &[u8 } } -fn insert_empty_sum_tree_db(db: &GroveDb, path: &[&[u8]], key: &[u8]) +fn insert_empty_sum_tree_db(db: &GroveDb, path: &[&[u8]], key: &[u8], grove_version: &GroveVersion) { db.insert(path, key, Element::empty_sum_tree(), INSERT_OPTIONS, None, grove_version) .unwrap() .expect("successfully inserted tree"); } -fn insert_sum_element_db(db: &GroveDb, path: &[&[u8]], min_i: u32, max_i: u32, transaction: &Transaction) +fn insert_sum_element_db(db: &GroveDb, path: &[&[u8]], min_i: u32, max_i: u32, transaction: &Transaction, grove_version: &GroveVersion) { for i in min_i..=max_i { //let value : u32 = i; @@ -214,7 +216,7 @@ fn generate_random_path(prefix: &str, suffix: &str, len: usize) -> String { format!("{}{}{}", prefix, random_string, suffix) } -fn query_db(db: &GroveDb, path: &[&[u8]], key: Vec) { +fn query_db(db: &GroveDb, path: &[&[u8]], key: Vec, grove_version: &GroveVersion) { let path_vec: Vec> = path.iter() .map(|&slice| slice.to_vec()) .collect(); @@ -225,7 +227,7 @@ fn query_db(db: &GroveDb, path: &[&[u8]], key: Vec) { let path_query = PathQuery::new_unsized(path_vec, query.clone()); let (elements, _) = db - .query_item_value(&path_query, true, false, true, None) + .query_item_value(&path_query, true, false, true, None, grove_version) .unwrap() .expect("expected successful get_path_query"); for e in elements.into_iter() { @@ -246,6 +248,7 @@ fn sync_db_demo( target_db: &GroveDb, state_sync_info: MultiStateSyncInfo, target_tx: &Transaction, + grove_version: &GroveVersion, ) -> Result<(), grovedb::Error> { let app_hash = source_db.root_hash(None, grove_version).value.unwrap(); let mut state_sync_info = target_db.start_snapshot_syncing(state_sync_info, app_hash, target_tx, CURRENT_STATE_SYNC_VERSION, grove_version)?; From 7d03ad0819401d869cc1d8a6999cf0efb15dd914 Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Sat, 24 Aug 2024 15:20:42 +0700 Subject: [PATCH 12/15] clean --- Cargo.toml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index bd532171..76a014d2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,5 +10,6 @@ members = [ "visualize", "path", "grovedbg-types", - "grovedb-version" -, "grovedb-epoch-based-storage-flags"] + "grovedb-version", + "grovedb-epoch-based-storage-flags" +] From 69d25f6df5ecb210d76b971927cb73b81779d1f1 Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Sat, 24 Aug 2024 15:23:48 +0700 Subject: [PATCH 13/15] clippy warnings --- grovedb/src/operations/proof/util.rs | 2 +- grovedb/src/query_result_type.rs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/grovedb/src/operations/proof/util.rs b/grovedb/src/operations/proof/util.rs index d6b34ecc..a33954d1 100644 --- a/grovedb/src/operations/proof/util.rs +++ b/grovedb/src/operations/proof/util.rs @@ -306,7 +306,7 @@ pub fn path_hex_to_ascii(path: &Path) -> String { } pub fn path_as_slices_hex_to_ascii(path: &[&[u8]]) -> String { - path.into_iter() + path.iter() .map(|e| hex_to_ascii(e)) .collect::>() .join("/") diff --git a/grovedb/src/query_result_type.rs b/grovedb/src/query_result_type.rs index 02035238..1850544e 100644 --- a/grovedb/src/query_result_type.rs +++ b/grovedb/src/query_result_type.rs @@ -283,7 +283,7 @@ impl QueryResultElements { result_item { if let Some(last) = path.pop() { - map.entry(last).or_insert_with(Vec::new).push(key); + map.entry(last).or_default().push(key); } } } @@ -333,7 +333,7 @@ impl QueryResultElements { result_item { if let Some(last) = path.pop() { - map.entry(last).or_insert_with(Vec::new).push(element); + map.entry(last).or_default().push(element); } } } From 7f6d1a291286690825be1fc3dc01a274ee00010d Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Sat, 24 Aug 2024 15:29:37 +0700 Subject: [PATCH 14/15] chore: update to v1.1.0 --- costs/Cargo.toml | 2 +- grovedb-epoch-based-storage-flags/Cargo.toml | 2 +- grovedb-version/Cargo.toml | 2 +- grovedb/Cargo.toml | 18 +++++++++--------- grovedbg-types/Cargo.toml | 2 +- merk/Cargo.toml | 12 ++++++------ node-grove/Cargo.toml | 2 +- path/Cargo.toml | 2 +- storage/Cargo.toml | 8 ++++---- visualize/Cargo.toml | 2 +- 10 files changed, 26 insertions(+), 26 deletions(-) diff --git a/costs/Cargo.toml b/costs/Cargo.toml index 4ee1a55b..ef9954d3 100644 --- a/costs/Cargo.toml +++ b/costs/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "grovedb-costs" -version = "1.0.0" +version = "1.1.0" edition = "2021" license = "MIT" description = "Costs extension crate for GroveDB" diff --git a/grovedb-epoch-based-storage-flags/Cargo.toml b/grovedb-epoch-based-storage-flags/Cargo.toml index 65026a6a..7638b292 100644 --- a/grovedb-epoch-based-storage-flags/Cargo.toml +++ b/grovedb-epoch-based-storage-flags/Cargo.toml @@ -7,7 +7,7 @@ edition = "2021" [dependencies] thiserror = { version = "1.0.63" } -grovedb-costs = { version = "1.0.0", path = "../costs" } +grovedb-costs = { version = "1.1.0", path = "../costs" } intmap = { version = "2.0.0", features = ["serde"]} integer-encoding = { version = "4.0.0" } hex = { version = "0.4.3" } \ No newline at end of file diff --git a/grovedb-version/Cargo.toml b/grovedb-version/Cargo.toml index 9802a0cf..52806b78 100644 --- a/grovedb-version/Cargo.toml +++ b/grovedb-version/Cargo.toml @@ -2,7 +2,7 @@ name = "grovedb-version" authors = ["Samuel Westrich "] description = "Versioning library for Platform" -version = "1.0.0" +version = "1.1.0" edition = "2021" license = "MIT" diff --git a/grovedb/Cargo.toml b/grovedb/Cargo.toml index a52372a5..70acaaab 100644 --- a/grovedb/Cargo.toml +++ b/grovedb/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "grovedb" description = "Fully featured database using balanced hierarchical authenticated data structures" -version = "1.0.0" +version = "1.1.0" authors = ["Samuel Westrich ", "Wisdom Ogwu "] edition = "2021" license = "MIT" @@ -11,36 +11,36 @@ readme = "../README.md" documentation = "https://docs.rs/grovedb" [dependencies] -grovedb-merk = { version = "1.0.0", path = "../merk", optional = true, default-features = false } +grovedb-merk = { version = "1.1.0", path = "../merk", optional = true, default-features = false } thiserror = { version = "1.0.59", optional = true } tempfile = { version = "3.10.1", optional = true } bincode = { version = "2.0.0-rc.3" } -grovedb-storage = { version = "1.0.0", path = "../storage", optional = true } -grovedb-visualize = { version = "1.0.0", path = "../visualize", optional = true } +grovedb-storage = { version = "1.1.0", path = "../storage", optional = true } +grovedb-visualize = { version = "1.1.0", path = "../visualize", optional = true } hex = "0.4.3" itertools = { version = "0.12.1", optional = true } derive_more = "0.99.18" integer-encoding = { version = "4.0.0", optional = true } -grovedb-costs = { version = "1.0.0", path = "../costs" , optional = true } +grovedb-costs = { version = "1.1.0", path = "../costs" , optional = true } nohash-hasher = { version = "0.2.0", optional = true } indexmap = "2.2.6" intmap = { version = "2.0.0", optional = true } -grovedb-path = { version = "1.0.0", path = "../path" } -grovedbg-types = { version = "1.0.0", path = "../grovedbg-types", optional = true } +grovedb-path = { version = "1.1.0", path = "../path" } +grovedbg-types = { version = "1.1.0", path = "../grovedbg-types", optional = true } tokio = { version = "1.37.0", features = ["rt-multi-thread", "net"], optional = true } axum = { version = "0.7.5", features = ["macros"], optional = true } tower-http = { version = "0.5.2", features = ["fs"], optional = true } blake3 = "1.4.0" bitvec = "1" zip-extensions = { version ="0.6.2", optional = true } -grovedb-version = { path = "../grovedb-version", version = "1.0.0" } +grovedb-version = { path = "../grovedb-version", version = "1.1.0" } [dev-dependencies] rand = "0.8.5" criterion = "0.5.1" hex = "0.4.3" pretty_assertions = "1.4.0" -grovedb-epoch-based-storage-flags = { version = "1.0.0", path = "../grovedb-epoch-based-storage-flags" } +grovedb-epoch-based-storage-flags = { version = "1.1.0", path = "../grovedb-epoch-based-storage-flags" } [[bench]] name = "insertion_benchmark" diff --git a/grovedbg-types/Cargo.toml b/grovedbg-types/Cargo.toml index f1ab09d6..65ffc893 100644 --- a/grovedbg-types/Cargo.toml +++ b/grovedbg-types/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "grovedbg-types" -version = "1.0.0" +version = "1.1.0" edition = "2021" description = "Common type definitions for data exchange over GroveDBG protocol" authors = ["Evgeny Fomin "] diff --git a/merk/Cargo.toml b/merk/Cargo.toml index b20be3b4..2ee7ed9a 100644 --- a/merk/Cargo.toml +++ b/merk/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "grovedb-merk" description = "Merkle key/value store adapted for GroveDB" -version = "1.0.0" +version = "1.1.0" authors = ["Samuel Westrich ", "Wisdom Ogwu ", "Matt Bell "] edition = "2021" license = "MIT" @@ -13,15 +13,15 @@ documentation = "https://docs.rs/grovedb-merk" [dependencies] thiserror = "1.0.58" bincode = { version = "2.0.0-rc.3" } -grovedb-storage = { version = "1.0.0", path = "../storage", optional = true } +grovedb-storage = { version = "1.1.0", path = "../storage", optional = true } failure = "0.1.8" integer-encoding = "4.0.0" indexmap = "2.2.6" -grovedb-costs = { version = "1.0.0" , path = "../costs" } -grovedb-visualize = { version = "1.0.0", path = "../visualize" } -grovedb-path = { version = "1.0.0", path = "../path" } +grovedb-costs = { version = "1.1.0" , path = "../costs" } +grovedb-visualize = { version = "1.1.0", path = "../visualize" } +grovedb-path = { version = "1.1.0", path = "../path" } hex = "0.4.3" -grovedb-version = { version = "1.0.0", path = "../grovedb-version" } +grovedb-version = { version = "1.1.0", path = "../grovedb-version" } [dependencies.time] version = "0.3.34" diff --git a/node-grove/Cargo.toml b/node-grove/Cargo.toml index 4a4f5281..f73517b1 100644 --- a/node-grove/Cargo.toml +++ b/node-grove/Cargo.toml @@ -11,7 +11,7 @@ crate-type = ["cdylib"] [dependencies] grovedb = { path = "../grovedb", features = ["full", "estimated_costs"] } -grovedb-version = { path = "../grovedb-version", version = "1.0.0" } +grovedb-version = { path = "../grovedb-version", version = "1.1.0" } [dependencies.neon] version = "0.10.1" diff --git a/path/Cargo.toml b/path/Cargo.toml index 111bc474..7861de8e 100644 --- a/path/Cargo.toml +++ b/path/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "grovedb-path" -version = "1.0.0" +version = "1.1.0" edition = "2021" license = "MIT" description = "Path extension crate for GroveDB" diff --git a/storage/Cargo.toml b/storage/Cargo.toml index 7db2b599..299bddac 100644 --- a/storage/Cargo.toml +++ b/storage/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "grovedb-storage" -version = "1.0.0" +version = "1.1.0" edition = "2021" license = "MIT" description = "Storage extension crate for GroveDB" @@ -14,13 +14,13 @@ num_cpus = { version = "1.16.0", optional = true } tempfile = { version = "3.10.1", optional = true } blake3 = { version = "1.5.1", optional = true } integer-encoding = { version = "4.0.0", optional = true } -grovedb-visualize = { version = "1.0.0", path = "../visualize" } +grovedb-visualize = { version = "1.1.0", path = "../visualize" } strum = { version = "0.26.2", features = ["derive"] } -grovedb-costs = { version = "1.0.0", path = "../costs" } +grovedb-costs = { version = "1.1.0", path = "../costs" } thiserror = "1.0.59" rocksdb = { version = "0.22.0", optional = true } hex = "0.4.3" -grovedb-path = { version = "1.0.0", path = "../path" } +grovedb-path = { version = "1.1.0", path = "../path" } [features] rocksdb_storage = ["rocksdb", "num_cpus", "lazy_static", "tempfile", "blake3", "integer-encoding"] diff --git a/visualize/Cargo.toml b/visualize/Cargo.toml index 821bf443..14d320bd 100644 --- a/visualize/Cargo.toml +++ b/visualize/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "grovedb-visualize" -version = "1.0.0" +version = "1.1.0" edition = "2021" license = "MIT" description = "Debug prints extension crate for GroveDB" From 008d729851ebc5b7778b4f046ccc7de91e0ac151 Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Sat, 24 Aug 2024 15:51:37 +0700 Subject: [PATCH 15/15] clean up --- grovedb/src/batch/just_in_time_reference_update.rs | 6 ++---- grovedb/src/batch/mod.rs | 7 ++----- grovedb/src/element/helpers.rs | 1 + grovedb/src/element/query.rs | 11 +++++++---- grovedb/src/lib.rs | 3 +++ merk/src/proofs/query/mod.rs | 1 + merk/src/tree/just_in_time_value_update.rs | 5 +---- merk/src/tree/mod.rs | 2 ++ 8 files changed, 19 insertions(+), 17 deletions(-) diff --git a/grovedb/src/batch/just_in_time_reference_update.rs b/grovedb/src/batch/just_in_time_reference_update.rs index 7be6d220..f4385b89 100644 --- a/grovedb/src/batch/just_in_time_reference_update.rs +++ b/grovedb/src/batch/just_in_time_reference_update.rs @@ -1,7 +1,7 @@ use std::borrow::Cow; use grovedb_costs::{ - cost_return_on_error_default, cost_return_on_error_no_add, + cost_return_on_error_no_add, storage_cost::{ removal::{StorageRemovedBytes, StorageRemovedBytes::BasicStorageRemoval}, StorageCost, @@ -14,11 +14,9 @@ use grovedb_merk::{ }; use grovedb_storage::StorageContext; use grovedb_version::version::GroveVersion; -use integer_encoding::VarInt; use crate::{ batch::{MerkError, TreeCacheMerkByPath}, - element::SUM_ITEM_COST_SIZE, Element, ElementFlags, Error, }; @@ -49,7 +47,7 @@ where let mut cost = OperationCost::default(); if old_element.is_sum_item() { return if new_element.is_sum_item() { - let mut maybe_old_flags = old_element.get_flags_owned(); + let maybe_old_flags = old_element.get_flags_owned(); if maybe_old_flags.is_some() { let mut updated_new_element_with_old_flags = new_element.clone(); updated_new_element_with_old_flags.set_flags(maybe_old_flags.clone()); diff --git a/grovedb/src/batch/mod.rs b/grovedb/src/batch/mod.rs index 822de9dc..2767ab2b 100644 --- a/grovedb/src/batch/mod.rs +++ b/grovedb/src/batch/mod.rs @@ -49,11 +49,8 @@ use grovedb_costs::{ }; use grovedb_merk::{ tree::{ - kv::{ - ValueDefinedCostType::{LayeredValueDefinedCost, SpecializedValueDefinedCost}, - KV, - }, - value_hash, TreeNode, NULL_HASH, + kv::ValueDefinedCostType::{LayeredValueDefinedCost, SpecializedValueDefinedCost}, + value_hash, NULL_HASH, }, CryptoHash, Error as MerkError, Merk, MerkType, Op, RootHashKeyAndSum, TreeFeatureType::{BasicMerkNode, SummedMerkNode}, diff --git a/grovedb/src/element/helpers.rs b/grovedb/src/element/helpers.rs index fd0f17b1..5b3662df 100644 --- a/grovedb/src/element/helpers.rs +++ b/grovedb/src/element/helpers.rs @@ -12,6 +12,7 @@ use grovedb_merk::{ TreeFeatureType, TreeFeatureType::{BasicMerkNode, SummedMerkNode}, }; +#[cfg(feature = "full")] use grovedb_version::{check_grovedb_v0, error::GroveVersionError, version::GroveVersion}; #[cfg(feature = "full")] use integer_encoding::VarInt; diff --git a/grovedb/src/element/query.rs b/grovedb/src/element/query.rs index 39c0494c..7226db55 100644 --- a/grovedb/src/element/query.rs +++ b/grovedb/src/element/query.rs @@ -12,18 +12,21 @@ use grovedb_costs::{ use grovedb_merk::proofs::query::query_item::QueryItem; #[cfg(feature = "full")] use grovedb_merk::proofs::query::SubqueryBranch; -#[cfg(any(feature = "full", feature = "verify"))] +#[cfg(feature = "full")] use grovedb_merk::proofs::Query; #[cfg(feature = "full")] use grovedb_path::SubtreePath; #[cfg(feature = "full")] use grovedb_storage::{rocksdb_storage::RocksDbStorage, RawIterator, StorageContext}; +#[cfg(feature = "full")] use grovedb_version::{ check_grovedb_v0, check_grovedb_v0_with_cost, error::GroveVersionError, version::GroveVersion, }; #[cfg(feature = "full")] use crate::operations::proof::util::hex_to_ascii; +#[cfg(any(feature = "full", feature = "verify"))] +use crate::Element; #[cfg(feature = "full")] use crate::{ element::helpers::raw_decode, @@ -37,8 +40,8 @@ use crate::{ util::{merk_optional_tx, merk_optional_tx_internal_error, storage_context_optional_tx}, Error, PathQuery, TransactionArg, }; -#[cfg(any(feature = "full", feature = "verify"))] -use crate::{query_result_type::Path, Element, SizedQuery}; +#[cfg(feature = "full")] +use crate::{query_result_type::Path, SizedQuery}; #[cfg(any(feature = "full", feature = "verify"))] #[derive(Copy, Clone, Debug)] @@ -681,7 +684,7 @@ impl Element { Ok(()).wrap_with_cost(cost) } - #[cfg(any(feature = "full", feature = "verify"))] + #[cfg(feature = "full")] /// Takes a sized query and a key and returns subquery key and subquery as /// tuple fn subquery_paths_and_value_for_sized_query( diff --git a/grovedb/src/lib.rs b/grovedb/src/lib.rs index feef0721..57f68d33 100644 --- a/grovedb/src/lib.rs +++ b/grovedb/src/lib.rs @@ -199,11 +199,13 @@ use grovedb_storage::{ }; #[cfg(feature = "full")] use grovedb_storage::{Storage, StorageContext}; +#[cfg(feature = "full")] use grovedb_version::version::GroveVersion; #[cfg(feature = "full")] use grovedb_visualize::DebugByteVectors; #[cfg(any(feature = "full", feature = "verify"))] pub use query::{PathQuery, SizedQuery}; +#[cfg(feature = "full")] use reference_path::path_from_reference_path_type; #[cfg(feature = "grovedbg")] use tokio::net::ToSocketAddrs; @@ -212,6 +214,7 @@ use tokio::net::ToSocketAddrs; use crate::element::helpers::raw_decode; #[cfg(any(feature = "full", feature = "verify"))] pub use crate::error::Error; +#[cfg(feature = "full")] use crate::operations::proof::util::hex_to_ascii; #[cfg(feature = "full")] use crate::util::{root_merk_optional_tx, storage_context_optional_tx}; diff --git a/merk/src/proofs/query/mod.rs b/merk/src/proofs/query/mod.rs index 6c543950..6f1e506a 100644 --- a/merk/src/proofs/query/mod.rs +++ b/merk/src/proofs/query/mod.rs @@ -26,6 +26,7 @@ use bincode::{ }; #[cfg(feature = "full")] use grovedb_costs::{cost_return_on_error, CostContext, CostResult, CostsExt, OperationCost}; +#[cfg(feature = "full")] use grovedb_version::version::GroveVersion; #[cfg(any(feature = "full", feature = "verify"))] use indexmap::IndexMap; diff --git a/merk/src/tree/just_in_time_value_update.rs b/merk/src/tree/just_in_time_value_update.rs index 8620cc97..fd202670 100644 --- a/merk/src/tree/just_in_time_value_update.rs +++ b/merk/src/tree/just_in_time_value_update.rs @@ -5,10 +5,7 @@ use grovedb_costs::storage_cost::{ use crate::{ merk::defaults::MAX_UPDATE_VALUE_BASED_ON_COSTS_TIMES, - tree::{ - kv::{ValueDefinedCostType, KV}, - TreeNode, - }, + tree::{kv::ValueDefinedCostType, TreeNode}, Error, }; diff --git a/merk/src/tree/mod.rs b/merk/src/tree/mod.rs index 24b568be..91eebf52 100644 --- a/merk/src/tree/mod.rs +++ b/merk/src/tree/mod.rs @@ -42,6 +42,7 @@ use grovedb_costs::{ }, CostContext, CostResult, CostsExt, OperationCost, }; +#[cfg(feature = "full")] use grovedb_version::version::GroveVersion; #[cfg(any(feature = "full", feature = "verify"))] pub use hash::{ @@ -63,6 +64,7 @@ pub use tree_feature_type::TreeFeatureType; #[cfg(feature = "full")] pub use walk::{Fetch, RefWalker, Walker}; +#[cfg(feature = "full")] use crate::tree::hash::HASH_LENGTH_X2; #[cfg(feature = "full")] use crate::tree::kv::ValueDefinedCostType;