From fe66ee75303a33b95586c85e8b6d82a5734757fd Mon Sep 17 00:00:00 2001 From: Evgeny Fomin Date: Fri, 27 Dec 2024 12:44:21 +0100 Subject: [PATCH] bidirectional reference insertion --- grovedb/src/batch/mod.rs | 55 +- grovedb/src/bidirectional_references.rs | 1890 +++++++++++++++++ grovedb/src/debugger.rs | 86 +- grovedb/src/element/constructor.rs | 33 + grovedb/src/element/get.rs | 8 +- grovedb/src/element/helpers.rs | 24 +- grovedb/src/element/mod.rs | 49 +- .../src/estimated_costs/average_case_costs.rs | 4 +- .../src/estimated_costs/worst_case_costs.rs | 4 +- grovedb/src/lib.rs | 17 +- grovedb/src/operations/get/query.rs | 79 +- grovedb/src/operations/insert/mod.rs | 66 +- grovedb/src/operations/proof/verify.rs | 5 +- grovedb/src/util.rs | 10 +- grovedb/src/visualize.rs | 22 + 15 files changed, 2286 insertions(+), 66 deletions(-) create mode 100644 grovedb/src/bidirectional_references.rs diff --git a/grovedb/src/batch/mod.rs b/grovedb/src/batch/mod.rs index be7a570d..bc77a7a6 100644 --- a/grovedb/src/batch/mod.rs +++ b/grovedb/src/batch/mod.rs @@ -71,6 +71,7 @@ pub use crate::batch::batch_structure::{OpsByLevelPath, OpsByPath}; use crate::batch::estimated_costs::EstimatedCostsType; use crate::{ batch::{batch_structure::BatchStructure, mode::BatchRunMode}, + bidirectional_references::BidirectionalReference, element::{MaxReferenceHop, SUM_ITEM_COST_SIZE, SUM_TREE_COST_SIZE, TREE_COST_SIZE}, operations::{get::MAX_REFERENCE_HOPS, proof::util::hex_to_ascii}, reference_path::{ @@ -1005,13 +1006,23 @@ where }; match element { - Element::Item(..) | Element::SumItem(..) => { + Element::Item(..) + | Element::SumItem(..) + | Element::ItemWithBackwardsReferences(..) + | Element::SumItemWithBackwardsReferences(..) => { let serialized = cost_return_on_error_no_add!(cost, element.serialize(grove_version)); let val_hash = value_hash(&serialized).unwrap_add_cost(&mut cost); Ok(val_hash).wrap_with_cost(cost) } - Element::Reference(path, ..) => { + Element::Reference(path, ..) + | Element::BidirectionalReference( + BidirectionalReference { + forward_reference_path: path, + .. + }, + .., + ) => { let path = cost_return_on_error_no_add!( cost, path_from_reference_qualified_path_type(path, qualified_path) @@ -1079,7 +1090,10 @@ where | GroveOp::Replace { element } | GroveOp::Patch { element, .. } => { match element { - Element::Item(..) | Element::SumItem(..) => { + Element::Item(..) + | Element::SumItem(..) + | Element::ItemWithBackwardsReferences(..) + | Element::SumItemWithBackwardsReferences(..) => { let serialized = cost_return_on_error_no_add!( cost, element.serialize(grove_version) @@ -1126,7 +1140,14 @@ where } } } - Element::Reference(path, ..) => { + Element::Reference(path, ..) + | Element::BidirectionalReference( + BidirectionalReference { + forward_reference_path: path, + .. + }, + .., + ) => { let path = cost_return_on_error_no_add!( cost, path_from_reference_qualified_path_type( @@ -1152,13 +1173,23 @@ where } } GroveOp::InsertOnly { element } => match element { - Element::Item(..) | Element::SumItem(..) => { + Element::Item(..) + | Element::SumItem(..) + | Element::ItemWithBackwardsReferences(..) + | Element::SumItemWithBackwardsReferences(..) => { let serialized = cost_return_on_error_no_add!(cost, element.serialize(grove_version)); let val_hash = value_hash(&serialized).unwrap_add_cost(&mut cost); Ok(val_hash).wrap_with_cost(cost) } - Element::Reference(path, ..) => { + Element::Reference(path, ..) + | Element::BidirectionalReference( + BidirectionalReference { + forward_reference_path: path, + .. + }, + .., + ) => { let path = cost_return_on_error_no_add!( cost, path_from_reference_qualified_path_type(path.clone(), qualified_path) @@ -1300,7 +1331,12 @@ where | GroveOp::InsertOrReplace { element } | GroveOp::Replace { element } | GroveOp::Patch { element, .. } => match &element { - Element::Reference(path_reference, element_max_reference_hop, _) => { + Element::Reference(path_reference, element_max_reference_hop, _) + | Element::BidirectionalReference(BidirectionalReference { + forward_reference_path: path_reference, + max_hop: element_max_reference_hop, + .. + }) => { let merk_feature_type = cost_return_on_error!( &mut cost, element @@ -1365,7 +1401,10 @@ where ) ); } - Element::Item(..) | Element::SumItem(..) => { + Element::Item(..) + | Element::SumItem(..) + | Element::ItemWithBackwardsReferences(..) + | Element::SumItemWithBackwardsReferences(..) => { let merk_feature_type = cost_return_on_error!( &mut cost, element diff --git a/grovedb/src/bidirectional_references.rs b/grovedb/src/bidirectional_references.rs new file mode 100644 index 00000000..62c1efca --- /dev/null +++ b/grovedb/src/bidirectional_references.rs @@ -0,0 +1,1890 @@ +//! Bidirectional references management module. + +use std::{collections::VecDeque, io::Write}; + +use bincode::{config, Decode, Encode}; +use bitvec::{array::BitArray, order::Lsb0}; +use grovedb_costs::{cost_return_on_error, cost_return_on_error_no_add, CostResult, CostsExt}; +use grovedb_merk::CryptoHash; +use grovedb_path::{SubtreePath, SubtreePathBuilder}; + +use crate::{ + element::{Delta, MaxReferenceHop}, + merk_cache::{MerkCache, MerkHandle}, + operations::insert::InsertOptions, + reference_path::ReferencePathType, + util::{self, ResolvedReference}, + Element, ElementFlags, Error, +}; + +const META_BACKWARD_REFERENCES_PREFIX: &'static [u8] = b"refs"; + +pub type SlotIdx = usize; + +/// Flag to indicate whether the bidirectional reference should be deleted when +/// the pointing-to item no longer exists or becomes incompatible. +pub type CascadeOnUpdate = bool; + +#[derive(Clone, Encode, Decode, PartialEq, Eq, Hash)] +#[cfg_attr(not(any(feature = "full", feature = "visualize")), derive(Debug))] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub struct BidirectionalReference { + pub forward_reference_path: ReferencePathType, + pub backward_reference_slot: SlotIdx, + pub cascade_on_update: CascadeOnUpdate, + pub max_hop: MaxReferenceHop, + pub flags: Option, +} + +impl BidirectionalReference { + /// Given current path removes backward reference from the merk and key + /// where this bidirectional references points to + fn remove_backward_reference<'db, 'b, 'c, B: AsRef<[u8]>>( + self, + merk_cache: &'c MerkCache<'db, 'b, B>, + current_path: SubtreePathBuilder<'b, B>, + current_key: &[u8], + ) -> CostResult<(), Error> { + let mut cost = Default::default(); + + let ResolvedReference { + mut target_merk, + target_key, + .. + } = cost_return_on_error!( + &mut cost, + util::follow_reference_once( + merk_cache, + current_path, + current_key, + self.forward_reference_path + ) + ); + + cost_return_on_error!( + &mut cost, + Self::remove_backward_reference_resolved( + &mut target_merk, + &target_key, + self.backward_reference_slot + ) + ); + + Ok(()).wrap_with_cost(cost) + } + + fn remove_backward_reference_resolved<'db, 'c>( + target_merk: &mut MerkHandle<'db, 'c>, + target_key: &[u8], + slot_idx: SlotIdx, + ) -> CostResult<(), Error> { + let mut cost = Default::default(); + + let (prefix, mut bits) = cost_return_on_error!( + &mut cost, + get_backward_references_bitvec(target_merk, &target_key) + ); + + bits.set(slot_idx, false); + + cost_return_on_error!( + &mut cost, + target_merk + .for_merk( + |m| m.put_meta(prefix.clone(), bits.into_inner()[0].to_be_bytes().to_vec()) + ) + .map_err(Error::MerkError) + ); + + let mut indexed_prefix = prefix; + write!(&mut indexed_prefix, "{}", slot_idx).expect("no io involved"); + + cost_return_on_error!( + &mut cost, + target_merk + .for_merk(|m| m.delete_meta(&indexed_prefix)) + .map_err(Error::MerkError) + ); + + Ok(()).wrap_with_cost(cost) + } +} + +/// Insert bidirectional reference at specified location performing required +/// checks and updates +pub(crate) fn process_bidirectional_reference_insertion<'db, 'b, 'k, B: AsRef<[u8]>>( + merk_cache: &MerkCache<'db, 'b, B>, + path: SubtreePath<'b, B>, + key: &[u8], + mut reference: BidirectionalReference, + options: Option, +) -> CostResult<(), Error> { + let mut cost = Default::default(); + + // Since we limit what kind of elements a bidirectional reference can target, a + // check goes first: + let ResolvedReference { + mut target_merk, + target_key, + target_element, + target_node_value_hash, + .. + } = cost_return_on_error!( + &mut cost, + util::follow_reference_once( + merk_cache, + path.derive_owned(), + key, + reference.forward_reference_path.clone(), + ) + ); + + if !matches!( + target_element, + Element::ItemWithBackwardsReferences(..) + | Element::SumItemWithBackwardsReferences(..) + | Element::BidirectionalReference(..) + ) { + return Err(Error::BidirectionalReferenceRule( + "Bidirectional references can only point variants with backward references support" + .to_owned(), + )) + .wrap_with_cost(cost); + } + + // If target item is a bidirectional reference itself, we limit the number of + // backward references it supports by 1: + if let Element::BidirectionalReference(..) = target_element { + let (_, bitvec) = cost_return_on_error!( + &mut cost, + get_backward_references_bitvec(&mut target_merk, &target_key) + ); + + if !bitvec.not_any() { + return Err(Error::BidirectionalReferenceRule( + "Number of backward references for a single bidirectional references is limited \ + to 1" + .to_owned(), + )) + .wrap_with_cost(cost); + } + } + + // If the target element passes the first check, attempt to add backward + // reference: + let inverted_reference = cost_return_on_error_no_add!( + cost, + reference + .forward_reference_path + .invert(path.clone(), key) + .ok_or_else(|| Error::BidirectionalReferenceRule( + "unable to get an inverted reference".to_owned() + )) + ); + let slot = cost_return_on_error!( + &mut cost, + add_backward_reference( + &mut target_merk, + &target_key, + BackwardReference { + inverted_reference, + cascade_on_update: reference.cascade_on_update, + }, + ) + ); + + // Update the reference we insert with a backward reference slot used + reference.backward_reference_slot = slot; + + // Proceed with bidirectional reference insertion as regular reference + let mut merk = cost_return_on_error!(&mut cost, merk_cache.get_merk(path.derive_owned())); + + let previous_value = cost_return_on_error!( + &mut cost, + merk.for_merk(|m| Element::get_optional(m, key, true, merk_cache.version)) + ); + + if let Some(Element::BidirectionalReference(ref old_ref)) = previous_value { + if old_ref == &reference { + // Short-circuit if nothing was changed + return Ok(()).wrap_with_cost(cost); + } + } + + cost_return_on_error!( + &mut cost, + merk.for_merk(|m| { + Element::BidirectionalReference(reference).insert_reference( + m, + key, + target_node_value_hash, + options.map(|o| o.as_merk_options()), + &merk_cache.version, + ) + }) + ); + + match previous_value { + // If previous value was another bidirectional reference, backward reference of + // an older one shall be removed from target merks' meta + Some(Element::BidirectionalReference(reference)) => { + cost_return_on_error!( + &mut cost, + reference.remove_backward_reference(merk_cache, path.derive_owned(), key) + ); + + // This also requires propagation, because new target means new value hash and + // backward references' chain shall be notified: + cost_return_on_error!( + &mut cost, + propagate_backward_references( + merk_cache, + merk, + path.derive_owned(), + key.to_vec(), + target_node_value_hash + ) + ); + } + // If overwriting items with backward references it is an error since they can have many + // backward references when inserted bidirectional reference can have only one + Some( + Element::ItemWithBackwardsReferences(..) | Element::SumItemWithBackwardsReferences(..), + ) => { + return Err(Error::BidirectionalReferenceRule( + "insertion of bidirectional reference cannot override elements with backward \ + references (item/sum item) since only one backward reference is supported for \ + bidirectional reference and those may have up to 32" + .to_owned(), + )) + .wrap_with_cost(cost) + } + // Insertion into new place shall allocate empty bitvec of backward references + None => { + let prefix = make_meta_prefix(key); + cost_return_on_error!( + &mut cost, + merk.for_merk(|m| m.put_meta(prefix, 0u32.to_be_bytes().to_vec())) + .map_err(Error::MerkError) + ); + } + // If regular item/sum item was overwritten then no actions needed + _ => {} + } + + Ok(()).wrap_with_cost(cost) +} + +/// Post-processing of possible backward references relationships after +/// insertion of anything but bidirectional reference +pub(crate) fn process_update_element_with_backward_references<'db, 'b, 'c, B: AsRef<[u8]>>( + merk_cache: &'c MerkCache<'db, 'b, B>, + merk: MerkHandle<'db, 'c>, + path: SubtreePath<'b, B>, + key: &[u8], + delta: Delta, +) -> CostResult<(), Error> { + let mut cost = Default::default(); + + // On no changes no propagations shall happen: + if !delta.has_changed() { + return Ok(()).wrap_with_cost(cost); + } + + // If there was no overwrite we short-circuit as well: + let Some(old) = delta.old else { + return Ok(()).wrap_with_cost(cost); + }; + + match (old, delta.new) { + ( + Element::ItemWithBackwardsReferences(..) | Element::SumItemWithBackwardsReferences(..), + Element::ItemWithBackwardsReferences(..) | Element::SumItemWithBackwardsReferences(..), + ) => { + // Update with another backward references-compatible element variant, that + // means value hash propagation across backward references' chains: + cost_return_on_error!( + &mut cost, + propagate_backward_references( + merk_cache, + merk, + path.derive_owned(), + key.to_vec(), + cost_return_on_error!(&mut cost, delta.new.value_hash(&merk_cache.version)) + ) + ); + } + ( + Element::ItemWithBackwardsReferences(..) | Element::SumItemWithBackwardsReferences(..), + _, + ) => { + // Update with non backward references-compatible element, equals to cascade + // deletion of references' chains: + cost_return_on_error!( + &mut cost, + delete_backward_references_recursively( + merk_cache, + merk, + path.derive_owned(), + key.to_vec() + ) + ); + } + + ( + Element::BidirectionalReference(reference), + Element::ItemWithBackwardsReferences(..) | Element::SumItemWithBackwardsReferences(..), + ) => { + // Overwrite of bidirectional reference with backward references-compatible + // elements triggers propagation and removes one backward reference because of + // removal of old bidi ref + + cost_return_on_error!( + &mut cost, + propagate_backward_references( + merk_cache, + merk, + path.derive_owned(), + key.to_vec(), + cost_return_on_error!(&mut cost, delta.new.value_hash(&merk_cache.version)) + ) + ); + + cost_return_on_error!( + &mut cost, + reference.remove_backward_reference(merk_cache, path.derive_owned(), key) + ); + } + (Element::BidirectionalReference(reference), _) => { + // Overwrite of bidirectional reference with non backward + // references-compatible element shall trigger recursive deletion + // and removal of backward refrence from the element where the bidi + // ref in question used to point to + + // Since we're overwriting with backward references-incompatible element we + // issue a recursive deletion of backward references chains: + cost_return_on_error!( + &mut cost, + delete_backward_references_recursively( + merk_cache, + merk, + path.derive_owned(), + key.to_vec() + ) + ); + + cost_return_on_error!( + &mut cost, + reference.remove_backward_reference(merk_cache, path.derive_owned(), key) + ); + } + _ => { + // All other overwrites don't require special attention + } + } + + Ok(()).wrap_with_cost(cost) +} + +/// Recursively deletes all backward references' chains of a key if all of them +/// allow cascade deletion. +fn delete_backward_references_recursively<'db, 'b, 'c, B: AsRef<[u8]>>( + merk_cache: &'c MerkCache<'db, 'b, B>, + merk: MerkHandle<'db, 'c>, + path: SubtreePathBuilder<'b, B>, + key: Vec, +) -> CostResult<(), Error> { + let mut cost = Default::default(); + let mut queue = VecDeque::new(); + + queue.push_back((merk, path, key)); + + // Just like with propagation we follow all references chains... + while let Some((mut current_merk, current_path, current_key)) = queue.pop_front() { + let backward_references = cost_return_on_error!( + &mut cost, + get_backward_references(&mut current_merk, ¤t_key) + ); + for (idx, backward_ref) in backward_references.into_iter() { + if !backward_ref.cascade_on_update { + return Err(Error::BidirectionalReferenceRule( + "deletion of backward references through deletion of an element requires \ + `cascade_on_update` setting" + .to_owned(), + )) + .wrap_with_cost(cost); + } + + let ResolvedReference { + target_merk: origin_bidi_merk, + target_path: origin_bidi_path, + target_key: origin_bidi_key, + .. + } = cost_return_on_error!( + &mut cost, + util::follow_reference_once( + merk_cache, + current_path.clone(), + ¤t_key, + backward_ref.inverted_reference + ) + ); + + // ... except removing backward references from meta... + cost_return_on_error!( + &mut cost, + BidirectionalReference::remove_backward_reference_resolved( + &mut current_merk, + ¤t_key, + idx + ) + ); + + queue.push_back((origin_bidi_merk, origin_bidi_path, origin_bidi_key)); + } + + // ... and the element altogether + cost_return_on_error!( + &mut cost, + current_merk.for_merk(|m| Element::delete( + m, + current_key, + None, + false, + false, + &merk_cache.version + )) + ); + } + + Ok(()).wrap_with_cost(cost) +} + +/// Recursively updates all backward references' chains of a key. +fn propagate_backward_references<'db, 'b, 'c, B: AsRef<[u8]>>( + merk_cache: &'c MerkCache<'db, 'b, B>, + merk: MerkHandle<'db, 'c>, + path: SubtreePathBuilder<'b, B>, + key: Vec, + referenced_element_value_hash: CryptoHash, +) -> CostResult<(), Error> { + let mut cost = Default::default(); + let mut queue = VecDeque::new(); + + queue.push_back((merk, path, key)); + + while let Some((mut current_merk, current_path, current_key)) = queue.pop_front() { + let backward_references = cost_return_on_error!( + &mut cost, + get_backward_references(&mut current_merk, ¤t_key) + ); + for (_, backward_ref) in backward_references.into_iter() { + let ResolvedReference { + target_merk: mut origin_bidi_merk, + target_path: origin_bidi_path, + target_key: origin_bidi_key, + target_element: origin_bidi_ref, + .. + } = cost_return_on_error!( + &mut cost, + util::follow_reference_once( + merk_cache, + current_path.clone(), + ¤t_key, + backward_ref.inverted_reference + ) + ); + + cost_return_on_error!( + &mut cost, + origin_bidi_merk.for_merk(|m| { + origin_bidi_ref.insert_reference( + m, + &origin_bidi_key, + referenced_element_value_hash, + None, + merk_cache.version, + ) + }) + ); + + queue.push_back((origin_bidi_merk, origin_bidi_path, origin_bidi_key)); + } + } + + Ok(()).wrap_with_cost(cost) +} + +#[derive(Debug, Encode, Decode, PartialEq)] +pub(crate) struct BackwardReference { + pub(crate) inverted_reference: ReferencePathType, + pub(crate) cascade_on_update: bool, +} + +impl BackwardReference { + fn serialize(&self) -> Result, Error> { + let config = config::standard().with_big_endian().with_no_limit(); + bincode::encode_to_vec(self, config).map_err(|e| { + Error::CorruptedData(format!("unable to serialize backward reference {}", e)) + }) + } + + fn deserialize(bytes: &[u8]) -> Result { + let config = config::standard().with_big_endian().with_no_limit(); + Ok(bincode::decode_from_slice(bytes, config) + .map_err(|e| Error::CorruptedData(format!("unable to deserialize element {}", e)))? + .0) + } +} + +type Prefix = Vec; + +fn make_meta_prefix(key: &[u8]) -> Vec { + let mut backrefs_for_key = META_BACKWARD_REFERENCES_PREFIX.to_vec(); + backrefs_for_key.extend_from_slice(&key.len().to_be_bytes()); + backrefs_for_key.extend_from_slice(key); + + backrefs_for_key +} + +/// Get bitvec of backward references' slots for a key of a subtree. +/// Prefix for a Merk's meta storage is made of constant keyword, lenght of the +/// key and the key itself. Under the prefix GroveDB stores bitvec, and slots +/// for backward references are integers appended to the prefix. +fn get_backward_references_bitvec<'db, 'c>( + merk: &mut MerkHandle<'db, 'c>, + key: &[u8], +) -> CostResult<(Prefix, BitArray<[u32; 1], Lsb0>), Error> { + let mut cost = Default::default(); + + let backrefs_for_key = make_meta_prefix(key); + + let stored_bytes = cost_return_on_error!( + &mut cost, + merk.for_merk(|m| m + .get_meta(backrefs_for_key.clone()) + .map_ok(|opt_v| opt_v.map(|v| v.to_vec()))) + .map_err(Error::MerkError) + ); + + let bits: BitArray<[u32; 1], Lsb0> = if let Some(bytes) = stored_bytes { + cost_return_on_error_no_add!( + cost, + bytes + .try_into() + .map(|b| BitArray::new([u32::from_be_bytes(b)])) + .map_err(|_| Error::InternalError( + "backward references' bitvec is expected to be 4 bytes".to_owned() + )) + ) + } else { + Default::default() + }; + + Ok((backrefs_for_key, bits)).wrap_with_cost(cost) +} + +/// Adds backward reference to meta storage of a subtree. +/// +/// Only up to 32 backward references are allowed, for that reason we use a +/// bitvec to locate them. That way the bits is stored under +/// [META_BACKWARD_REFERENCES_PREFIX] with key and references themselves are +/// located under this prefix with index (0-31) appended. +fn add_backward_reference<'db, 'c>( + target_merk: &mut MerkHandle<'db, 'c>, + key: &[u8], + backward_reference: BackwardReference, +) -> CostResult { + let mut cost = Default::default(); + + let (prefix, mut bits) = + cost_return_on_error!(&mut cost, get_backward_references_bitvec(target_merk, key)); + + if let Some(free_index) = bits.first_zero() { + let mut idx_prefix = prefix.clone(); + write!(&mut idx_prefix, "{free_index}").expect("no io involved"); + + let serialized_ref = cost_return_on_error_no_add!(cost, backward_reference.serialize()); + cost_return_on_error!( + &mut cost, + target_merk + .for_merk(|m| m.put_meta(idx_prefix, serialized_ref)) + .map_err(Error::MerkError) + ); + + bits.set(free_index, true); + + cost_return_on_error!( + &mut cost, + target_merk + .for_merk(|m| m.put_meta(prefix, bits.into_inner()[0].to_be_bytes().to_vec())) + .map_err(Error::MerkError) + ); + + Ok(free_index) + } else { + Err(Error::BidirectionalReferenceRule( + "only up to 32 backward references for an item are supported".to_owned(), + )) + } + .wrap_with_cost(cost) +} + +/// Return a vector of backward references to the item +fn get_backward_references<'db, 'c>( + merk: &mut MerkHandle<'db, 'c>, + key: &[u8], +) -> CostResult, Error> { + let mut cost = Default::default(); + + let (prefix, bits) = + cost_return_on_error!(&mut cost, get_backward_references_bitvec(merk, key)); + + let mut backward_references = Vec::new(); + + for idx in bits.iter_ones() { + let mut indexed_prefix = prefix.clone(); + write!(&mut indexed_prefix, "{idx}").expect("no io involved"); + + let bytes_opt = cost_return_on_error!( + &mut cost, + merk.for_merk(|m| m + .get_meta(indexed_prefix) + .map_ok(|opt_v| opt_v.map(|v| v.to_vec()))) + .map_err(Error::MerkError) + ); + + let bytes = cost_return_on_error_no_add!( + cost, + bytes_opt.ok_or_else(|| { + Error::InternalError( + "backward references bitvec and slot are out of sync".to_owned(), + ) + }) + ); + + backward_references.push(( + idx, + cost_return_on_error_no_add!(cost, BackwardReference::deserialize(&bytes)), + )); + } + + Ok(backward_references).wrap_with_cost(cost) +} + +#[cfg(test)] +mod tests { + use grovedb_path::{SubtreePath, SubtreePathBuilder}; + use grovedb_version::version::GroveVersion; + use pretty_assertions::{assert_eq, assert_ne}; + + use super::*; + use crate::{ + merk_cache::MerkCache, + tests::{make_deep_tree, make_test_grovedb, ANOTHER_TEST_LEAF, TEST_LEAF}, + }; + + #[test] + fn add_multiple_backward_references() { + let version = GroveVersion::latest(); + let db = make_test_grovedb(&version); + let tx = db.start_transaction(); + let cache = MerkCache::new(&db, &tx, &version); + + let mut merk = cache.get_merk(SubtreePathBuilder::new()).unwrap().unwrap(); + + let slot0 = add_backward_reference( + &mut merk, + TEST_LEAF, + BackwardReference { + inverted_reference: ReferencePathType::AbsolutePathReference(vec![ + b"dummy1".to_vec() + ]), + cascade_on_update: false, + }, + ) + .unwrap() + .unwrap(); + + let slot1 = add_backward_reference( + &mut merk, + TEST_LEAF, + BackwardReference { + inverted_reference: ReferencePathType::AbsolutePathReference(vec![ + b"dummy2".to_vec() + ]), + cascade_on_update: false, + }, + ) + .unwrap() + .unwrap(); + + let slot2 = add_backward_reference( + &mut merk, + TEST_LEAF, + BackwardReference { + inverted_reference: ReferencePathType::AbsolutePathReference(vec![ + b"dummy3".to_vec() + ]), + cascade_on_update: false, + }, + ) + .unwrap() + .unwrap(); + + assert_eq!(slot0, 0); + assert_eq!(slot1, 1); + assert_eq!(slot2, 2); + + assert_eq!( + get_backward_references(&mut merk, TEST_LEAF) + .unwrap() + .unwrap() + .into_iter() + .map( + |( + _, + BackwardReference { + inverted_reference, .. + }, + )| inverted_reference + ) + .collect::>(), + vec![ + ReferencePathType::AbsolutePathReference(vec![b"dummy1".to_vec()]), + ReferencePathType::AbsolutePathReference(vec![b"dummy2".to_vec()]), + ReferencePathType::AbsolutePathReference(vec![b"dummy3".to_vec()]), + ] + ); + } + + #[test] + fn using_free_slots() { + let version = GroveVersion::latest(); + let db = make_test_grovedb(&version); + let tx = db.start_transaction(); + let cache = MerkCache::new(&db, &tx, &version); + + let mut merk = cache.get_merk(SubtreePathBuilder::new()).unwrap().unwrap(); + + let slot0 = add_backward_reference( + &mut merk, + TEST_LEAF, + BackwardReference { + inverted_reference: ReferencePathType::AbsolutePathReference(vec![ + b"dummy1".to_vec() + ]), + cascade_on_update: false, + }, + ) + .unwrap() + .unwrap(); + + let slot1 = add_backward_reference( + &mut merk, + TEST_LEAF, + BackwardReference { + inverted_reference: ReferencePathType::AbsolutePathReference(vec![ + b"dummy2".to_vec() + ]), + cascade_on_update: false, + }, + ) + .unwrap() + .unwrap(); + + let slot2 = add_backward_reference( + &mut merk, + TEST_LEAF, + BackwardReference { + inverted_reference: ReferencePathType::AbsolutePathReference(vec![ + b"dummy3".to_vec() + ]), + cascade_on_update: false, + }, + ) + .unwrap() + .unwrap(); + + assert_eq!(slot0, 0); + assert_eq!(slot1, 1); + assert_eq!(slot2, 2); + + BidirectionalReference::remove_backward_reference_resolved(&mut merk, TEST_LEAF, 1) + .unwrap() + .unwrap(); + + assert_eq!( + add_backward_reference( + &mut merk, + TEST_LEAF, + BackwardReference { + inverted_reference: ReferencePathType::AbsolutePathReference(vec![ + b"dummy4".to_vec() + ]), + cascade_on_update: false, + }, + ) + .unwrap() + .unwrap(), + 1 + ); + } + + #[test] + fn overflow() { + let version = GroveVersion::latest(); + let db = make_test_grovedb(&version); + let tx = db.start_transaction(); + let cache = MerkCache::new(&db, &tx, &version); + + let mut merk = cache.get_merk(SubtreePathBuilder::new()).unwrap().unwrap(); + + (0..32).for_each(|_| { + add_backward_reference( + &mut merk, + TEST_LEAF, + BackwardReference { + inverted_reference: ReferencePathType::AbsolutePathReference(vec![ + b"dummy1".to_vec() + ]), + cascade_on_update: false, + }, + ) + .unwrap() + .unwrap(); + }); + + assert!(add_backward_reference( + &mut merk, + TEST_LEAF, + BackwardReference { + inverted_reference: ReferencePathType::AbsolutePathReference(vec![ + b"dummy1".to_vec() + ]), + cascade_on_update: false, + }, + ) + .unwrap() + .is_err()); + } + + #[test] + fn overflow_for_bidi_reference() { + let version = GroveVersion::latest(); + let db = make_test_grovedb(&version); + + // Create an item that support backward references + db.insert( + SubtreePath::from(&[TEST_LEAF]), + b"key1", + Element::new_item_allowing_bidirectional_references(b"value".to_vec()), + None, + None, + version, + ) + .unwrap() + .unwrap(); + + let tx = db.start_transaction(); + + let mut cache = MerkCache::new(&db, &tx, &version); + + // Add a bidirectional reference to the item + process_bidirectional_reference_insertion( + &mut cache, + SubtreePath::from(&[ANOTHER_TEST_LEAF]), + b"key2", + BidirectionalReference { + forward_reference_path: ReferencePathType::AbsolutePathReference(vec![ + TEST_LEAF.to_vec(), + b"key1".to_vec(), + ]), + backward_reference_slot: 0, + cascade_on_update: false, + max_hop: None, + flags: None, + }, + None, + ) + .unwrap() + .unwrap(); + + // Add a bidirectional reference that points to the bidirectional reference + // above + assert!(matches!( + process_bidirectional_reference_insertion( + &mut cache, + SubtreePath::from(&[ANOTHER_TEST_LEAF]), + b"key3", + BidirectionalReference { + forward_reference_path: ReferencePathType::AbsolutePathReference(vec![ + ANOTHER_TEST_LEAF.to_vec(), + b"key2".to_vec(), + ]), + backward_reference_slot: 0, + cascade_on_update: false, + max_hop: None, + flags: None, + }, + None, + ) + .unwrap(), + Ok(()) + )); + + // Try to add another one, this should fail as only one bidirectional reference + // can point to single bidirectional reference + assert!(matches!( + process_bidirectional_reference_insertion( + &mut cache, + SubtreePath::from(&[ANOTHER_TEST_LEAF]), + b"key4", + BidirectionalReference { + forward_reference_path: ReferencePathType::AbsolutePathReference(vec![ + ANOTHER_TEST_LEAF.to_vec(), + b"key2".to_vec(), + ]), + backward_reference_slot: 0, + cascade_on_update: false, + max_hop: None, + flags: None, + }, + None, + ) + .unwrap(), + Err(Error::BidirectionalReferenceRule(..)) + )); + } + + // Following tests will cover all insertion scenarios. + // Roughly speaking there are three types of elements we care about: + // 1. Bidirectional reference + // 2. Items that support backward references + // 3. The rest + + #[test] + fn bidi_reference_insertion_no_overwrite() { + // Expecting to add backward reference to the pointed to key + + let version = GroveVersion::latest(); + let db = make_deep_tree(&version); + let tx = db.start_transaction(); + let merk_cache = MerkCache::new(&db, &tx, &version); + + let target_path = SubtreePath::from(&[ANOTHER_TEST_LEAF, b"innertree2"]); + let mut target_merk = merk_cache + .get_merk(target_path.derive_owned()) + .unwrap() + .unwrap(); + let target_key = b"item_key"; + + let inserted_item = + Element::new_item_allowing_bidirectional_references(b"item_value".to_vec()); + + target_merk + .for_merk(|m| inserted_item.insert(m, target_key, None, &version)) + .unwrap() + .unwrap(); + + process_update_element_with_backward_references( + &merk_cache, + target_merk, + target_path.clone(), + target_key, + Delta { + new: &inserted_item, + old: None, + }, + ) + .unwrap() + .unwrap(); + + let ref_path = SubtreePath::from(&[TEST_LEAF, b"innertree"]); + let ref_key = b"ref_key"; + + let ref_path2 = SubtreePath::from(&[TEST_LEAF, b"innertree4"]); + let ref_key2 = b"ref_key2"; + + process_bidirectional_reference_insertion( + &merk_cache, + ref_path.clone(), + ref_key, + BidirectionalReference { + forward_reference_path: ReferencePathType::AbsolutePathReference( + target_path.derive_owned_with_child(target_key).to_vec(), + ), + backward_reference_slot: 0, + cascade_on_update: false, + max_hop: None, + flags: None, + }, + None, + ) + .unwrap() + .unwrap(); + + process_bidirectional_reference_insertion( + &merk_cache, + ref_path2.clone(), + ref_key2, + BidirectionalReference { + forward_reference_path: ReferencePathType::AbsolutePathReference( + target_path.derive_owned_with_child(target_key).to_vec(), + ), + backward_reference_slot: 0, + cascade_on_update: false, + max_hop: None, + flags: None, + }, + None, + ) + .unwrap() + .unwrap(); + + let backward_reference1 = &get_backward_references( + &mut merk_cache + .get_merk(target_path.derive_owned()) + .unwrap() + .unwrap(), + target_key, + ) + .unwrap() + .unwrap()[0] + .1; + + let backward_reference2 = &get_backward_references( + &mut merk_cache + .get_merk(target_path.derive_owned()) + .unwrap() + .unwrap(), + target_key, + ) + .unwrap() + .unwrap()[1] + .1; + + // Backward references on target item points to bidirectional references that + // were inserted + assert_eq!( + backward_reference1 + .inverted_reference + .clone() + .absolute_qualified_path(target_path.derive_owned(), target_key) + .unwrap(), + ref_path.derive_owned_with_child(ref_key) + ); + + assert_eq!( + backward_reference2 + .inverted_reference + .clone() + .absolute_qualified_path(target_path.derive_owned(), target_key) + .unwrap(), + ref_path2.derive_owned_with_child(ref_key2) + ); + } + + #[test] + fn overwriting_bidi_reference_with_another_bidi_reference() { + // Expecting to add new backward reference, remove the old one and value hash + // propagation as well since it will target different item with different hash + + let version = GroveVersion::latest(); + let db = make_deep_tree(&version); + let tx = db.start_transaction(); + let merk_cache = MerkCache::new(&db, &tx, &version); + + let target_path = SubtreePath::from(&[ANOTHER_TEST_LEAF, b"innertree2"]); + let mut target_merk = merk_cache + .get_merk(target_path.derive_owned()) + .unwrap() + .unwrap(); + let target_key = b"item_key"; + + let inserted_item = + Element::new_item_allowing_bidirectional_references(b"item_value".to_vec()); + + target_merk + .for_merk(|m| inserted_item.insert(m, target_key, None, &version)) + .unwrap() + .unwrap(); + + let target_path2 = SubtreePath::from(&[ANOTHER_TEST_LEAF, b"innertree3"]); + let mut target_merk2 = merk_cache + .get_merk(target_path2.derive_owned()) + .unwrap() + .unwrap(); + let target_key2 = b"item_key2"; + + let inserted_item2 = + Element::new_item_allowing_bidirectional_references(b"item_value2".to_vec()); + + target_merk2 + .for_merk(|m| inserted_item2.insert(m, target_key2, None, &version)) + .unwrap() + .unwrap(); + + process_update_element_with_backward_references( + &merk_cache, + target_merk, + target_path.clone(), + target_key, + Delta { + new: &inserted_item, + old: None, + }, + ) + .unwrap() + .unwrap(); + + process_update_element_with_backward_references( + &merk_cache, + target_merk2, + target_path2.clone(), + target_key2, + Delta { + new: &inserted_item2, + old: None, + }, + ) + .unwrap() + .unwrap(); + + let ref_path = SubtreePath::from(&[TEST_LEAF, b"innertree"]); + let ref_key = b"ref_key"; + + process_bidirectional_reference_insertion( + &merk_cache, + ref_path.clone(), + ref_key, + BidirectionalReference { + forward_reference_path: ReferencePathType::AbsolutePathReference( + target_path.derive_owned_with_child(target_key).to_vec(), + ), + backward_reference_slot: 0, + cascade_on_update: false, + max_hop: None, + flags: None, + }, + None, + ) + .unwrap() + .unwrap(); + + let backward_reference = &get_backward_references( + &mut merk_cache + .get_merk(target_path.derive_owned()) + .unwrap() + .unwrap(), + target_key, + ) + .unwrap() + .unwrap()[0] + .1; + + assert_eq!( + backward_reference + .inverted_reference + .clone() + .absolute_qualified_path(target_path.derive_owned(), target_key) + .unwrap(), + ref_path.derive_owned_with_child(ref_key) + ); + + let last_ref_path = SubtreePath::from(&[TEST_LEAF, b"innertree4"]); + let last_ref_key = b"last_ref_key"; + + // Adding yet another bidirectional reference that points to the first one. + // Keeping its value hash we can check if a change of the upstream reference + // will affect the value hash of one in question + process_bidirectional_reference_insertion( + &merk_cache, + last_ref_path.clone(), + last_ref_key, + BidirectionalReference { + forward_reference_path: ReferencePathType::AbsolutePathReference( + ref_path.derive_owned_with_child(ref_key).to_vec(), + ), + backward_reference_slot: 0, + cascade_on_update: false, + max_hop: None, + flags: None, + }, + None, + ) + .unwrap() + .unwrap(); + + let mut last_ref_merk = merk_cache + .get_merk(last_ref_path.derive_owned()) + .unwrap() + .unwrap(); + + let prev_value_hash = last_ref_merk + .for_merk(|m| Element::get_value_hash(m, last_ref_key, true, &version)) + .unwrap() + .unwrap(); + + // Overwrite bidirectional reference with another bidirectional reference: + process_bidirectional_reference_insertion( + &merk_cache, + ref_path.clone(), + ref_key, + BidirectionalReference { + forward_reference_path: ReferencePathType::AbsolutePathReference( + target_path2.derive_owned_with_child(target_key2).to_vec(), + ), + backward_reference_slot: 0, + cascade_on_update: false, + max_hop: None, + flags: None, + }, + None, + ) + .unwrap() + .unwrap(); + + let post_value_hash = last_ref_merk + .for_merk(|m| Element::get_value_hash(m, last_ref_key, true, &version)) + .unwrap() + .unwrap(); + + assert!(&get_backward_references( + &mut merk_cache + .get_merk(target_path.derive_owned()) + .unwrap() + .unwrap(), + target_key, + ) + .unwrap() + .unwrap() + .is_empty()); + + let backward_reference2 = &get_backward_references( + &mut merk_cache + .get_merk(target_path2.derive_owned()) + .unwrap() + .unwrap(), + target_key2, + ) + .unwrap() + .unwrap()[0] + .1; + + assert_eq!( + backward_reference2 + .inverted_reference + .clone() + .absolute_qualified_path(target_path2.derive_owned(), target_key2) + .unwrap(), + ref_path.derive_owned_with_child(ref_key) + ); + + assert_ne!(prev_value_hash, post_value_hash); + } + + #[test] + fn overwriting_bidi_reference_with_backward_reference_compatible_item() { + let version = GroveVersion::latest(); + let db = make_deep_tree(&version); + let tx = db.start_transaction(); + let merk_cache = MerkCache::new(&db, &tx, &version); + + let target_path = SubtreePath::from(&[ANOTHER_TEST_LEAF, b"innertree2"]); + let mut target_merk = merk_cache + .get_merk(target_path.derive_owned()) + .unwrap() + .unwrap(); + let target_key = b"item_key"; + + let inserted_item = + Element::new_item_allowing_bidirectional_references(b"item_value".to_vec()); + + target_merk + .for_merk(|m| inserted_item.insert(m, target_key, None, &version)) + .unwrap() + .unwrap(); + + process_update_element_with_backward_references( + &merk_cache, + target_merk, + target_path.clone(), + target_key, + Delta { + new: &inserted_item, + old: None, + }, + ) + .unwrap() + .unwrap(); + + let ref_path = SubtreePath::from(&[TEST_LEAF, b"innertree"]); + let ref_key = b"ref_key"; + + let last_ref_path = SubtreePath::from(&[TEST_LEAF, b"innertree4"]); + let last_ref_key = b"last_ref_key"; + + process_bidirectional_reference_insertion( + &merk_cache, + ref_path.clone(), + ref_key, + BidirectionalReference { + forward_reference_path: ReferencePathType::AbsolutePathReference( + target_path.derive_owned_with_child(target_key).to_vec(), + ), + backward_reference_slot: 0, + cascade_on_update: false, + max_hop: None, + flags: None, + }, + None, + ) + .unwrap() + .unwrap(); + + // Insert yet another bidirectional reference that points to the previous one + // and forms that way a chain of references + process_bidirectional_reference_insertion( + &merk_cache, + last_ref_path.clone(), + last_ref_key, + BidirectionalReference { + forward_reference_path: ReferencePathType::AbsolutePathReference( + ref_path.derive_owned_with_child(ref_key).to_vec(), + ), + backward_reference_slot: 0, + cascade_on_update: false, + max_hop: None, + flags: None, + }, + None, + ) + .unwrap() + .unwrap(); + + let mut last_ref_merk = merk_cache + .get_merk(last_ref_path.derive_owned()) + .unwrap() + .unwrap(); + + let prev_value_hash = last_ref_merk + .for_merk(|m| Element::get_value_hash(m, last_ref_key, true, &version)) + .unwrap() + .unwrap(); + + // Overwrite first reference with item that supports backward references + let mut ref_merk = merk_cache + .get_merk(ref_path.derive_owned()) + .unwrap() + .unwrap(); + + let ref_overwrite_item = + Element::new_item_allowing_bidirectional_references(b"newvalue".to_vec()); + let delta = ref_merk + .for_merk(|m| ref_overwrite_item.insert_if_changed_value(m, ref_key, None, &version)) + .unwrap() + .unwrap(); + + process_update_element_with_backward_references( + &merk_cache, + ref_merk, + ref_path, + ref_key, + delta, + ) + .unwrap() + .unwrap(); + + let post_value_hash = last_ref_merk + .for_merk(|m| Element::get_value_hash(m, last_ref_key, true, &version)) + .unwrap() + .unwrap(); + + // Latest reference in chain has its value hash updated + assert_ne!(prev_value_hash, post_value_hash); + + // Old target has no backward references + assert!(get_backward_references( + &mut merk_cache + .get_merk(target_path.derive_owned()) + .unwrap() + .unwrap(), + target_key, + ) + .unwrap() + .unwrap() + .is_empty()); + } + + #[test] + fn overwriting_bidi_reference_with_backward_reference_incompatible_item() { + let version = GroveVersion::latest(); + let db = make_deep_tree(&version); + let tx = db.start_transaction(); + let merk_cache = MerkCache::new(&db, &tx, &version); + + let target_path = SubtreePath::from(&[ANOTHER_TEST_LEAF, b"innertree2"]); + let mut target_merk = merk_cache + .get_merk(target_path.derive_owned()) + .unwrap() + .unwrap(); + let target_key = b"item_key"; + + let inserted_item = + Element::new_item_allowing_bidirectional_references(b"item_value".to_vec()); + + target_merk + .for_merk(|m| inserted_item.insert(m, target_key, None, &version)) + .unwrap() + .unwrap(); + + process_update_element_with_backward_references( + &merk_cache, + target_merk, + target_path.clone(), + target_key, + Delta { + new: &inserted_item, + old: None, + }, + ) + .unwrap() + .unwrap(); + + let ref_path = SubtreePath::from(&[TEST_LEAF, b"innertree"]); + let ref_key = b"ref_key"; + + let last_ref_path = SubtreePath::from(&[TEST_LEAF, b"innertree4"]); + let last_ref_key = b"last_ref_key"; + + process_bidirectional_reference_insertion( + &merk_cache, + ref_path.clone(), + ref_key, + BidirectionalReference { + forward_reference_path: ReferencePathType::AbsolutePathReference( + target_path.derive_owned_with_child(target_key).to_vec(), + ), + backward_reference_slot: 0, + cascade_on_update: false, + max_hop: None, + flags: None, + }, + None, + ) + .unwrap() + .unwrap(); + + // Insert yet another bidirectional reference that points to the previous one + // and forms that way a chain of references + process_bidirectional_reference_insertion( + &merk_cache, + last_ref_path.clone(), + last_ref_key, + BidirectionalReference { + forward_reference_path: ReferencePathType::AbsolutePathReference( + ref_path.derive_owned_with_child(ref_key).to_vec(), + ), + backward_reference_slot: 0, + // Note that it is set to true + cascade_on_update: true, + max_hop: None, + flags: None, + }, + None, + ) + .unwrap() + .unwrap(); + + let mut last_ref_merk = merk_cache + .get_merk(last_ref_path.derive_owned()) + .unwrap() + .unwrap(); + + // Overwrite first reference with item that doesn't support backward references + let mut ref_merk = merk_cache + .get_merk(ref_path.derive_owned()) + .unwrap() + .unwrap(); + + let ref_overwrite_item = Element::new_item(b"newvalue".to_vec()); + let delta = ref_merk + .for_merk(|m| ref_overwrite_item.insert_if_changed_value(m, ref_key, None, &version)) + .unwrap() + .unwrap(); + + process_update_element_with_backward_references( + &merk_cache, + ref_merk, + ref_path, + ref_key, + delta, + ) + .unwrap() + .unwrap(); + + let latest_ref_element = last_ref_merk + .for_merk(|m| Element::get_optional(m, last_ref_key, true, &version)) + .unwrap() + .unwrap(); + + // Latest reference in chain has gone + assert!(latest_ref_element.is_none()); + + // Old target has no backward references + assert!(get_backward_references( + &mut merk_cache + .get_merk(target_path.derive_owned()) + .unwrap() + .unwrap(), + target_key, + ) + .unwrap() + .unwrap() + .is_empty()); + } + + #[test] + fn overwriting_backward_reference_compatible_item_with_bidi_reference() { + let version = GroveVersion::latest(); + let db = make_deep_tree(&version); + let tx = db.start_transaction(); + let merk_cache = MerkCache::new(&db, &tx, &version); + + let target_path = SubtreePath::from(&[ANOTHER_TEST_LEAF, b"innertree2"]); + let mut target_merk = merk_cache + .get_merk(target_path.derive_owned()) + .unwrap() + .unwrap(); + let target_key = b"item_key"; + + let inserted_item = + Element::new_item_allowing_bidirectional_references(b"item_value".to_vec()); + + target_merk + .for_merk(|m| inserted_item.insert(m, target_key, None, &version)) + .unwrap() + .unwrap(); + + process_update_element_with_backward_references( + &merk_cache, + target_merk, + target_path.clone(), + target_key, + Delta { + new: &inserted_item, + old: None, + }, + ) + .unwrap() + .unwrap(); + + // Attempt to overwrite item with bidi ref shall fail because it can't hold all + // possible backward references + assert!(process_bidirectional_reference_insertion( + &merk_cache, + target_path.clone(), + target_key, + BidirectionalReference { + forward_reference_path: ReferencePathType::AbsolutePathReference(vec![ + b"literally".to_vec(), + b"anything".to_vec() + ]), + backward_reference_slot: 0, + cascade_on_update: false, + max_hop: None, + flags: None, + }, + None, + ) + .unwrap() + .is_err()); + } + + #[test] + fn overwriting_backward_reference_compatible_item_with_backward_reference_compatible_item() { + let version = GroveVersion::latest(); + let db = make_deep_tree(&version); + let tx = db.start_transaction(); + let merk_cache = MerkCache::new(&db, &tx, &version); + + let target_path = SubtreePath::from(&[ANOTHER_TEST_LEAF, b"innertree2"]); + let mut target_merk = merk_cache + .get_merk(target_path.derive_owned()) + .unwrap() + .unwrap(); + let target_key = b"item_key"; + + let inserted_item = + Element::new_item_allowing_bidirectional_references(b"item_value".to_vec()); + + target_merk + .for_merk(|m| inserted_item.insert(m, target_key, None, &version)) + .unwrap() + .unwrap(); + + process_update_element_with_backward_references( + &merk_cache, + target_merk, + target_path.clone(), + target_key, + Delta { + new: &inserted_item, + old: None, + }, + ) + .unwrap() + .unwrap(); + + let ref_path = SubtreePath::from(&[TEST_LEAF, b"innertree"]); + let ref_key = b"ref_key"; + + let next_ref_path = SubtreePath::from(&[ANOTHER_TEST_LEAF, b"innertree3"]); + let next_ref_key = b"next_ref_key"; + + let last_ref_path = SubtreePath::from(&[TEST_LEAF, b"innertree4"]); + let last_ref_key = b"last_ref_key"; + + // Create bidi ref chain to the item to check propagation + process_bidirectional_reference_insertion( + &merk_cache, + ref_path.clone(), + ref_key, + BidirectionalReference { + forward_reference_path: ReferencePathType::AbsolutePathReference( + target_path.derive_owned_with_child(target_key).to_vec(), + ), + backward_reference_slot: 0, + cascade_on_update: false, + max_hop: None, + flags: None, + }, + None, + ) + .unwrap() + .unwrap(); + + process_bidirectional_reference_insertion( + &merk_cache, + next_ref_path.clone(), + next_ref_key, + BidirectionalReference { + forward_reference_path: ReferencePathType::AbsolutePathReference( + ref_path.derive_owned_with_child(ref_key).to_vec(), + ), + backward_reference_slot: 0, + cascade_on_update: false, + max_hop: None, + flags: None, + }, + None, + ) + .unwrap() + .unwrap(); + + process_bidirectional_reference_insertion( + &merk_cache, + last_ref_path.clone(), + last_ref_key, + BidirectionalReference { + forward_reference_path: ReferencePathType::AbsolutePathReference( + next_ref_path.derive_owned_with_child(next_ref_key).to_vec(), + ), + backward_reference_slot: 0, + cascade_on_update: false, + max_hop: None, + flags: None, + }, + None, + ) + .unwrap() + .unwrap(); + + let mut last_ref_merk = merk_cache + .get_merk(last_ref_path.derive_owned()) + .unwrap() + .unwrap(); + + let prev_value_hash = last_ref_merk + .for_merk(|m| Element::get_value_hash(m, last_ref_key, true, &version)) + .unwrap() + .unwrap(); + + // Update target item + + let inserted_item_new = + Element::new_item_allowing_bidirectional_references(b"item_value2".to_vec()); + + let mut target_merk = merk_cache + .get_merk(target_path.derive_owned()) + .unwrap() + .unwrap(); + let delta = target_merk + .for_merk(|m| inserted_item_new.insert_if_changed_value(m, target_key, None, &version)) + .unwrap() + .unwrap(); + + process_update_element_with_backward_references( + &merk_cache, + target_merk, + target_path.clone(), + target_key, + delta, + ) + .unwrap() + .unwrap(); + + let post_value_hash = last_ref_merk + .for_merk(|m| Element::get_value_hash(m, last_ref_key, true, &version)) + .unwrap() + .unwrap(); + + assert_ne!(prev_value_hash, post_value_hash); + } + + #[test] + fn overwriting_backward_reference_compatible_item_with_backward_reference_incompatible_item() { + let version = GroveVersion::latest(); + let db = make_deep_tree(&version); + let tx = db.start_transaction(); + let merk_cache = MerkCache::new(&db, &tx, &version); + + let target_path = SubtreePath::from(&[ANOTHER_TEST_LEAF, b"innertree2"]); + let mut target_merk = merk_cache + .get_merk(target_path.derive_owned()) + .unwrap() + .unwrap(); + let target_key = b"item_key"; + + let inserted_item = + Element::new_item_allowing_bidirectional_references(b"item_value".to_vec()); + + target_merk + .for_merk(|m| inserted_item.insert(m, target_key, None, &version)) + .unwrap() + .unwrap(); + + process_update_element_with_backward_references( + &merk_cache, + target_merk, + target_path.clone(), + target_key, + Delta { + new: &inserted_item, + old: None, + }, + ) + .unwrap() + .unwrap(); + + let ref_path = SubtreePath::from(&[TEST_LEAF, b"innertree"]); + let ref_key = b"ref_key"; + + let next_ref_path = SubtreePath::from(&[ANOTHER_TEST_LEAF, b"innertree3"]); + let next_ref_key = b"next_ref_key"; + + let last_ref_path = SubtreePath::from(&[TEST_LEAF, b"innertree4"]); + let last_ref_key = b"last_ref_key"; + + // Create bidi ref chain to the item to check propagation + process_bidirectional_reference_insertion( + &merk_cache, + ref_path.clone(), + ref_key, + BidirectionalReference { + forward_reference_path: ReferencePathType::AbsolutePathReference( + target_path.derive_owned_with_child(target_key).to_vec(), + ), + backward_reference_slot: 0, + // Must be true to allow cascade deletion + cascade_on_update: true, + max_hop: None, + flags: None, + }, + None, + ) + .unwrap() + .unwrap(); + + process_bidirectional_reference_insertion( + &merk_cache, + next_ref_path.clone(), + next_ref_key, + BidirectionalReference { + forward_reference_path: ReferencePathType::AbsolutePathReference( + ref_path.derive_owned_with_child(ref_key).to_vec(), + ), + backward_reference_slot: 0, + // Must be true to allow cascade deletion + cascade_on_update: true, + max_hop: None, + flags: None, + }, + None, + ) + .unwrap() + .unwrap(); + + process_bidirectional_reference_insertion( + &merk_cache, + last_ref_path.clone(), + last_ref_key, + BidirectionalReference { + forward_reference_path: ReferencePathType::AbsolutePathReference( + next_ref_path.derive_owned_with_child(next_ref_key).to_vec(), + ), + backward_reference_slot: 0, + cascade_on_update: true, + max_hop: None, + flags: None, + }, + None, + ) + .unwrap() + .unwrap(); + + let mut last_ref_merk = merk_cache + .get_merk(last_ref_path.derive_owned()) + .unwrap() + .unwrap(); + + // Update target item + + let inserted_item_new = Element::new_item(b"item_value2".to_vec()); + + let mut target_merk = merk_cache + .get_merk(target_path.derive_owned()) + .unwrap() + .unwrap(); + let delta = target_merk + .for_merk(|m| inserted_item_new.insert_if_changed_value(m, target_key, None, &version)) + .unwrap() + .unwrap(); + + process_update_element_with_backward_references( + &merk_cache, + target_merk, + target_path.clone(), + target_key, + delta, + ) + .unwrap() + .unwrap(); + + assert!(last_ref_merk + .for_merk(|m| Element::get_optional(m, last_ref_key, true, &version)) + .unwrap() + .unwrap() + .is_none()); + + assert!(merk_cache + .get_merk(ref_path.derive_owned()) + .unwrap() + .unwrap() + .for_merk(|m| Element::get_optional(m, ref_key, true, &version)) + .unwrap() + .unwrap() + .is_none()); + } +} diff --git a/grovedb/src/debugger.rs b/grovedb/src/debugger.rs index 7475dbbf..3a1dbaef 100644 --- a/grovedb/src/debugger.rs +++ b/grovedb/src/debugger.rs @@ -32,6 +32,7 @@ use tokio_util::sync::CancellationToken; use tower_http::services::ServeDir; use crate::{ + bidirectional_references::BidirectionalReference, operations::proof::{GroveDBProof, LayerProof, ProveOptions}, query_result_type::{QueryResultElement, QueryResultElements, QueryResultType}, reference_path::ReferencePathType, @@ -519,10 +520,13 @@ fn query_item_to_grovedb(item: QueryItem) -> crate::QueryItem { fn element_to_grovedbg(element: crate::Element) -> grovedbg_types::Element { match element { - crate::Element::Item(value, element_flags) => grovedbg_types::Element::Item { - value, - element_flags, - }, + crate::Element::Item(value, element_flags) + | crate::Element::ItemWithBackwardsReferences(value, element_flags) => { + grovedbg_types::Element::Item { + value, + element_flags, + } + } crate::Element::Tree(root_key, element_flags) => grovedbg_types::Element::Subtree { root_key, element_flags, @@ -531,15 +535,28 @@ fn element_to_grovedbg(element: crate::Element) -> grovedbg_types::Element { ReferencePathType::AbsolutePathReference(path), _, element_flags, - ) => grovedbg_types::Element::Reference(grovedbg_types::Reference::AbsolutePathReference { - path, - element_flags, - }), + ) + | crate::Element::BidirectionalReference(BidirectionalReference { + forward_reference_path: ReferencePathType::AbsolutePathReference(path), + flags: element_flags, + .. + }) => { + grovedbg_types::Element::Reference(grovedbg_types::Reference::AbsolutePathReference { + path, + element_flags, + }) + } crate::Element::Reference( ReferencePathType::UpstreamRootHeightReference(n_keep, path_append), _, element_flags, - ) => grovedbg_types::Element::Reference( + ) + | crate::Element::BidirectionalReference(BidirectionalReference { + forward_reference_path: + ReferencePathType::UpstreamRootHeightReference(n_keep, path_append), + flags: element_flags, + .. + }) => grovedbg_types::Element::Reference( grovedbg_types::Reference::UpstreamRootHeightReference { n_keep: n_keep.into(), path_append, @@ -553,7 +570,16 @@ fn element_to_grovedbg(element: crate::Element) -> grovedbg_types::Element { ), _, element_flags, - ) => grovedbg_types::Element::Reference( + ) + | crate::Element::BidirectionalReference(BidirectionalReference { + forward_reference_path: + ReferencePathType::UpstreamRootHeightWithParentPathAdditionReference( + n_keep, + path_append, + ), + flags: element_flags, + .. + }) => grovedbg_types::Element::Reference( grovedbg_types::Reference::UpstreamRootHeightWithParentPathAdditionReference { n_keep: n_keep.into(), path_append, @@ -564,7 +590,13 @@ fn element_to_grovedbg(element: crate::Element) -> grovedbg_types::Element { ReferencePathType::UpstreamFromElementHeightReference(n_remove, path_append), _, element_flags, - ) => grovedbg_types::Element::Reference( + ) + | crate::Element::BidirectionalReference(BidirectionalReference { + forward_reference_path: + ReferencePathType::UpstreamFromElementHeightReference(n_remove, path_append), + flags: element_flags, + .. + }) => grovedbg_types::Element::Reference( grovedbg_types::Reference::UpstreamFromElementHeightReference { n_remove: n_remove.into(), path_append, @@ -575,7 +607,12 @@ fn element_to_grovedbg(element: crate::Element) -> grovedbg_types::Element { ReferencePathType::CousinReference(swap_parent), _, element_flags, - ) => grovedbg_types::Element::Reference(grovedbg_types::Reference::CousinReference { + ) + | crate::Element::BidirectionalReference(BidirectionalReference { + forward_reference_path: ReferencePathType::CousinReference(swap_parent), + flags: element_flags, + .. + }) => grovedbg_types::Element::Reference(grovedbg_types::Reference::CousinReference { swap_parent, element_flags, }), @@ -583,7 +620,12 @@ fn element_to_grovedbg(element: crate::Element) -> grovedbg_types::Element { ReferencePathType::RemovedCousinReference(swap_parent), _, element_flags, - ) => { + ) + | crate::Element::BidirectionalReference(BidirectionalReference { + forward_reference_path: ReferencePathType::RemovedCousinReference(swap_parent), + flags: element_flags, + .. + }) => { grovedbg_types::Element::Reference(grovedbg_types::Reference::RemovedCousinReference { swap_parent, element_flags, @@ -593,14 +635,22 @@ fn element_to_grovedbg(element: crate::Element) -> grovedbg_types::Element { ReferencePathType::SiblingReference(sibling_key), _, element_flags, - ) => grovedbg_types::Element::Reference(grovedbg_types::Reference::SiblingReference { + ) + | crate::Element::BidirectionalReference(BidirectionalReference { + forward_reference_path: ReferencePathType::SiblingReference(sibling_key), + flags: element_flags, + .. + }) => grovedbg_types::Element::Reference(grovedbg_types::Reference::SiblingReference { sibling_key, element_flags, }), - crate::Element::SumItem(value, element_flags) => grovedbg_types::Element::SumItem { - value, - element_flags, - }, + crate::Element::SumItem(value, element_flags) + | crate::Element::SumItemWithBackwardsReferences(value, element_flags) => { + grovedbg_types::Element::SumItem { + value, + element_flags, + } + } crate::Element::SumTree(root_key, sum, element_flags) => grovedbg_types::Element::Sumtree { root_key, sum, diff --git a/grovedb/src/element/constructor.rs b/grovedb/src/element/constructor.rs index 91143ec8..e573911e 100644 --- a/grovedb/src/element/constructor.rs +++ b/grovedb/src/element/constructor.rs @@ -40,24 +40,57 @@ impl Element { Element::Item(item_value, None) } + #[cfg(feature = "full")] + /// Set element to an item without flags that allows for a backwards + /// reference + pub fn new_item_allowing_bidirectional_references(item_value: Vec) -> Self { + Element::ItemWithBackwardsReferences(item_value, None) + } + #[cfg(feature = "full")] /// Set element to an item with flags pub fn new_item_with_flags(item_value: Vec, flags: Option) -> Self { Element::Item(item_value, flags) } + #[cfg(feature = "full")] + /// Set element to an item with flags and that allows for a backwards + /// reference + pub fn new_item_allowing_bidirectional_references_with_flags( + item_value: Vec, + flags: Option, + ) -> Self { + Element::ItemWithBackwardsReferences(item_value, flags) + } + #[cfg(feature = "full")] /// Set element to a sum item without flags pub fn new_sum_item(value: i64) -> Self { Element::SumItem(value, None) } + #[cfg(feature = "full")] + /// Set element to a sum item hat allows for backwards references without + /// flags + pub fn new_sum_item_allowing_bidirectional_references(value: i64) -> Self { + Element::SumItemWithBackwardsReferences(value, None) + } + #[cfg(feature = "full")] /// Set element to a sum item with flags pub fn new_sum_item_with_flags(value: i64, flags: Option) -> Self { Element::SumItem(value, flags) } + #[cfg(feature = "full")] + /// Set element to a sum item hat allows for backwards references with flags + pub fn new_sum_item_allowing_bidirectional_references_with_flags( + value: i64, + flags: Option, + ) -> Self { + Element::SumItemWithBackwardsReferences(value, flags) + } + #[cfg(feature = "full")] /// Set element to a reference without flags pub fn new_reference(reference_path: ReferencePathType) -> Self { diff --git a/grovedb/src/element/get.rs b/grovedb/src/element/get.rs index bd904fb7..ba999490 100644 --- a/grovedb/src/element/get.rs +++ b/grovedb/src/element/get.rs @@ -149,7 +149,10 @@ impl Element { .transpose() ); match &element { - Some(Element::Item(..)) | Some(Element::Reference(..)) => { + Some(Element::Item(..)) + | Some(Element::Reference(..)) + | Some(Element::ItemWithBackwardsReferences(..)) + | Some(Element::BidirectionalReference(..)) => { // while the loaded item might be a sum item, it is given for free // as it would be very hard to know in advance cost.storage_loaded_bytes = KV::value_byte_cost_size_for_key_and_value_lengths( @@ -158,7 +161,8 @@ impl Element { false, ) as u64 } - Some(Element::SumItem(_, flags)) => { + Some(Element::SumItem(_, flags)) + | Some(Element::SumItemWithBackwardsReferences(_, flags)) => { let cost_size = SUM_ITEM_COST_SIZE; let flags_len = flags.as_ref().map_or(0, |flags| { let flags_len = flags.len() as u32; diff --git a/grovedb/src/element/helpers.rs b/grovedb/src/element/helpers.rs index 2ca87cd0..25baeef7 100644 --- a/grovedb/src/element/helpers.rs +++ b/grovedb/src/element/helpers.rs @@ -17,6 +17,8 @@ use grovedb_version::{check_grovedb_v0, version::GroveVersion}; #[cfg(feature = "full")] use integer_encoding::VarInt; +#[cfg(any(feature = "full"))] +use crate::bidirectional_references::BidirectionalReference; #[cfg(feature = "full")] use crate::reference_path::path_from_reference_path_type; #[cfg(any(feature = "full", feature = "verify"))] @@ -162,7 +164,10 @@ impl Element { | Element::Item(_, flags) | Element::Reference(_, _, flags) | Element::SumTree(.., flags) - | Element::SumItem(_, flags) => flags, + | Element::SumItem(_, flags) + | Element::ItemWithBackwardsReferences(_, flags) + | Element::SumItemWithBackwardsReferences(_, flags) + | Element::BidirectionalReference(BidirectionalReference { flags, .. }) => flags, } } @@ -174,7 +179,10 @@ impl Element { | Element::Item(_, flags) | Element::Reference(_, _, flags) | Element::SumTree(.., flags) - | Element::SumItem(_, flags) => flags, + | Element::SumItem(_, flags) + | Element::ItemWithBackwardsReferences(_, flags) + | Element::SumItemWithBackwardsReferences(_, flags) + | Element::BidirectionalReference(BidirectionalReference { flags, .. }) => flags, } } @@ -186,7 +194,10 @@ impl Element { | Element::Item(_, flags) | Element::Reference(_, _, flags) | Element::SumTree(.., flags) - | Element::SumItem(_, flags) => flags, + | Element::SumItem(_, flags) + | Element::ItemWithBackwardsReferences(_, flags) + | Element::SumItemWithBackwardsReferences(_, flags) + | Element::BidirectionalReference(BidirectionalReference { flags, .. }) => flags, } } @@ -198,7 +209,12 @@ impl Element { | Element::Item(_, flags) | Element::Reference(_, _, flags) | Element::SumTree(.., flags) - | Element::SumItem(_, flags) => *flags = new_flags, + | Element::SumItem(_, flags) + | Element::ItemWithBackwardsReferences(_, flags) + | Element::SumItemWithBackwardsReferences(_, flags) + | Element::BidirectionalReference(BidirectionalReference { flags, .. }) => { + *flags = new_flags + } } } diff --git a/grovedb/src/element/mod.rs b/grovedb/src/element/mod.rs index 7594fbb2..2cd7a49e 100644 --- a/grovedb/src/element/mod.rs +++ b/grovedb/src/element/mod.rs @@ -30,12 +30,15 @@ use grovedb_merk::estimated_costs::SUM_VALUE_EXTRA_COST; use grovedb_merk::estimated_costs::{LAYER_COST_SIZE, SUM_LAYER_COST_SIZE}; #[cfg(feature = "full")] use grovedb_visualize::visualize_to_vec; +pub(crate) use insert::Delta; -use crate::operations::proof::util::hex_to_ascii; #[cfg(any(feature = "full", feature = "verify"))] use crate::reference_path::ReferencePathType; #[cfg(feature = "full")] use crate::OperationCost; +use crate::{ + bidirectional_references::BidirectionalReference, operations::proof::util::hex_to_ascii, +}; #[cfg(any(feature = "full", feature = "verify"))] /// Optional meta-data to be stored per element @@ -84,6 +87,13 @@ pub enum Element { /// Same as Element::Tree but underlying Merk sums value of it's summable /// nodes SumTree(Option>, SumValue, Option), + /// A reference to an object by its path + BidirectionalReference(BidirectionalReference), + /// An ordinary value that has a backwards reference + ItemWithBackwardsReferences(Vec, Option), + /// Signed integer value that can be totaled in a sum tree that has a + /// backwards reference + SumItemWithBackwardsReferences(SumValue, Option), } impl fmt::Display for Element { @@ -110,6 +120,24 @@ impl fmt::Display for Element { .map_or(String::new(), |f| format!(", flags: {:?}", f)) ) } + Element::BidirectionalReference(BidirectionalReference { + forward_reference_path, + cascade_on_update, + max_hop, + flags, + .. + }) => { + // TODO: print something on backward_references + write!( + f, + "BidirectionalReference({forward_reference_path}, max_hop: {}{}, cascade: \ + {cascade_on_update})", + max_hop.map_or("None".to_string(), |h| h.to_string()), + flags + .as_ref() + .map_or(String::new(), |f| format!(", flags: {:?}", f)) + ) + } Element::Tree(root_key, flags) => { write!( f, @@ -141,6 +169,22 @@ impl fmt::Display for Element { .map_or(String::new(), |f| format!(", flags: {:?}", f)) ) } + Element::ItemWithBackwardsReferences(data, flags) => write!( + f, + "ItemWithBackwardReferences({}{})", + hex_to_ascii(data), + flags + .as_ref() + .map_or(String::new(), |f| format!(", flags: {:?}", f)) + ), + Element::SumItemWithBackwardsReferences(sum_value, flags) => write!( + f, + "SumItemWithBackwardReferences({}{})", + sum_value, + flags + .as_ref() + .map_or(String::new(), |f| format!(", flags: {:?}", f)) + ), } } } @@ -153,6 +197,9 @@ impl Element { Element::Tree(..) => "tree", Element::SumItem(..) => "sum item", Element::SumTree(..) => "sum tree", + Element::BidirectionalReference(..) => "bidirectional reference", + Element::ItemWithBackwardsReferences(..) => "item with backwards references", + Element::SumItemWithBackwardsReferences(..) => "sum item with backwards references", } } diff --git a/grovedb/src/estimated_costs/average_case_costs.rs b/grovedb/src/estimated_costs/average_case_costs.rs index 1eeea0b7..30831bf6 100644 --- a/grovedb/src/estimated_costs/average_case_costs.rs +++ b/grovedb/src/estimated_costs/average_case_costs.rs @@ -16,7 +16,9 @@ use grovedb_merk::{ HASH_LENGTH, }; use grovedb_storage::{worst_case_costs::WorstKeyLength, Storage}; -use grovedb_version::{check_grovedb_v0, check_grovedb_v0_with_cost, version::GroveVersion}; +use grovedb_version::{ + check_grovedb_v0, check_grovedb_v0_with_cost, error::GroveVersionError, version::GroveVersion, +}; use integer_encoding::VarInt; use crate::{ diff --git a/grovedb/src/estimated_costs/worst_case_costs.rs b/grovedb/src/estimated_costs/worst_case_costs.rs index 304deaeb..a6aacb4e 100644 --- a/grovedb/src/estimated_costs/worst_case_costs.rs +++ b/grovedb/src/estimated_costs/worst_case_costs.rs @@ -18,7 +18,9 @@ use grovedb_merk::{ HASH_LENGTH, }; use grovedb_storage::{worst_case_costs::WorstKeyLength, Storage}; -use grovedb_version::{check_grovedb_v0, check_grovedb_v0_with_cost, version::GroveVersion}; +use grovedb_version::{ + check_grovedb_v0, check_grovedb_v0_with_cost, error::GroveVersionError, version::GroveVersion, +}; use integer_encoding::VarInt; use crate::{ diff --git a/grovedb/src/lib.rs b/grovedb/src/lib.rs index b7741acf..1602e5c0 100644 --- a/grovedb/src/lib.rs +++ b/grovedb/src/lib.rs @@ -127,6 +127,8 @@ #[cfg(feature = "full")] pub mod batch; +#[cfg(feature = "full")] +mod bidirectional_references; #[cfg(feature = "grovedbg")] pub mod debugger; #[cfg(any(feature = "full", feature = "verify"))] @@ -159,6 +161,7 @@ use std::sync::Arc; #[cfg(feature = "full")] use std::{collections::HashMap, option::Option::None, path::Path}; +use bidirectional_references::BidirectionalReference; #[cfg(feature = "grovedbg")] use debugger::start_visualizer; #[cfg(any(feature = "full", feature = "verify"))] @@ -792,7 +795,10 @@ impl GroveDb { grove_version, )?); } - Element::Item(..) | Element::SumItem(..) => { + Element::Item(..) + | Element::SumItem(..) + | Element::ItemWithBackwardsReferences(..) + | Element::SumItemWithBackwardsReferences(..) => { let (kv_value, element_value_hash) = merk .get_value_and_value_hash( &key, @@ -815,7 +821,14 @@ impl GroveDb { ); } } - Element::Reference(ref reference_path, ..) => { + Element::Reference(ref reference_path, ..) + | Element::BidirectionalReference( + BidirectionalReference { + forward_reference_path: ref reference_path, + .. + }, + .., + ) => { // Skip this whole check if we don't `verify_references` if !verify_references { continue; diff --git a/grovedb/src/operations/get/query.rs b/grovedb/src/operations/get/query.rs index 6f725808..0959562a 100644 --- a/grovedb/src/operations/get/query.rs +++ b/grovedb/src/operations/get/query.rs @@ -12,8 +12,9 @@ use integer_encoding::VarInt; #[cfg(feature = "full")] use crate::element::SumValue; use crate::{ - element::QueryOptions, operations::proof::ProveOptions, - query_result_type::PathKeyOptionalElementTrio, util::TxRef, Transaction, + bidirectional_references::BidirectionalReference, element::QueryOptions, + operations::proof::ProveOptions, query_result_type::PathKeyOptionalElementTrio, util::TxRef, + Transaction, }; #[cfg(feature = "full")] use crate::{ @@ -193,7 +194,14 @@ where { .follow_element ); match element { - Element::Reference(reference_path, ..) => { + Element::Reference(reference_path, ..) + | Element::BidirectionalReference( + BidirectionalReference { + forward_reference_path: reference_path, + .. + }, + .., + ) => { match reference_path { ReferencePathType::AbsolutePathReference(absolute_path) => { // While `map` on iterator is lazy, we should accumulate costs @@ -221,7 +229,11 @@ where { )), } } - Element::Item(..) | Element::SumItem(..) | Element::SumTree(..) => Ok(element), + Element::Item(..) + | Element::SumItem(..) + | Element::SumTree(..) + | Element::ItemWithBackwardsReferences(..) + | Element::SumItemWithBackwardsReferences(..) => Ok(element), Element::Tree(..) => Err(Error::InvalidQuery("path_queries can not refer to trees")), } } @@ -310,7 +322,14 @@ where { .map(|result_item| match result_item { QueryResultElement::ElementResultItem(element) => { match element { - Element::Reference(reference_path, ..) => { + Element::Reference(reference_path, ..) + | Element::BidirectionalReference( + BidirectionalReference { + forward_reference_path: reference_path, + .. + }, + .., + ) => { match reference_path { ReferencePathType::AbsolutePathReference(absolute_path) => { // While `map` on iterator is lazy, we should accumulate costs @@ -340,8 +359,13 @@ where { )), } } - Element::Item(item, _) => Ok(item), - Element::SumItem(item, _) => Ok(item.encode_var_vec()), + Element::Item(item, _) | Element::ItemWithBackwardsReferences(item, ..) => { + Ok(item) + } + Element::SumItem(item, _) + | Element::SumItemWithBackwardsReferences(item, ..) => { + Ok(item.encode_var_vec()) + } Element::Tree(..) | Element::SumTree(..) => Err(Error::InvalidQuery( "path_queries can only refer to items and references", )), @@ -397,7 +421,14 @@ where { .map(|result_item| match result_item { QueryResultElement::ElementResultItem(element) => { match element { - Element::Reference(reference_path, ..) => { + Element::Reference(reference_path, ..) + | Element::BidirectionalReference( + BidirectionalReference { + forward_reference_path: reference_path, + .. + }, + .., + ) => { match reference_path { ReferencePathType::AbsolutePathReference(absolute_path) => { // While `map` on iterator is lazy, we should accumulate costs @@ -434,8 +465,11 @@ where { )), } } - Element::Item(item, _) => Ok(QueryItemOrSumReturnType::ItemData(item)), - Element::SumItem(sum_value, _) => { + Element::Item(item, _) | Element::ItemWithBackwardsReferences(item, ..) => { + Ok(QueryItemOrSumReturnType::ItemData(item)) + } + Element::SumItem(sum_value, _) + | Element::SumItemWithBackwardsReferences(sum_value, ..) => { Ok(QueryItemOrSumReturnType::SumValue(sum_value)) } Element::SumTree(_, sum_value, _) => { @@ -492,7 +526,14 @@ where { .map(|result_item| match result_item { QueryResultElement::ElementResultItem(element) => { match element { - Element::Reference(reference_path, ..) => { + Element::Reference(reference_path, ..) + | Element::BidirectionalReference( + BidirectionalReference { + forward_reference_path: reference_path, + .. + }, + .., + ) => { match reference_path { ReferencePathType::AbsolutePathReference(absolute_path) => { // While `map` on iterator is lazy, we should accumulate costs @@ -522,13 +563,15 @@ where { )), } } - Element::SumItem(item, _) => Ok(item), - Element::Tree(..) | Element::SumTree(..) | Element::Item(..) => { - Err(Error::InvalidQuery( - "path_queries over sum items can only refer to sum items and \ - references", - )) - } + Element::SumItem(item, _) + | Element::SumItemWithBackwardsReferences(item, ..) => Ok(item), + Element::Tree(..) + | Element::SumTree(..) + | Element::Item(..) + | Element::ItemWithBackwardsReferences(..) => Err(Error::InvalidQuery( + "path_queries over sum items can only refer to sum items and \ + references", + )), } } _ => Err(Error::CorruptedCodeExecution( diff --git a/grovedb/src/operations/insert/mod.rs b/grovedb/src/operations/insert/mod.rs index 0c5f03e6..8acc9068 100644 --- a/grovedb/src/operations/insert/mod.rs +++ b/grovedb/src/operations/insert/mod.rs @@ -11,6 +11,9 @@ use grovedb_storage::Storage; use grovedb_version::{check_grovedb_v0_with_cost, version::GroveVersion}; use crate::{ + bidirectional_references::{ + process_bidirectional_reference_insertion, process_update_element_with_backward_references, + }, merk_cache::{MerkCache, MerkHandle}, util::{self, TxRef}, Element, Error, GroveDb, TransactionArg, @@ -172,9 +175,10 @@ impl GroveDb { reference_path.clone() ) ); + let referenced_item: Element = resolved_reference.target_element; if matches!( - resolved_reference.target_element, + referenced_item, Element::Tree(_, _) | Element::SumTree(_, _, _) ) { return Err(Error::NotSupported( @@ -183,9 +187,9 @@ impl GroveDb { .wrap_with_cost(cost); } - cost_return_on_error!( + let delta = cost_return_on_error!( &mut cost, - subtree_to_insert_into.for_merk(|m| element.insert_reference( + subtree_to_insert_into.for_merk(|m| element.insert_reference_if_changed_value( m, key, resolved_reference.target_node_value_hash, @@ -193,6 +197,17 @@ impl GroveDb { grove_version, )) ); + + cost_return_on_error!( + &mut cost, + process_update_element_with_backward_references( + merk_cache, + subtree_to_insert_into.clone(), + path, + key, + delta + ) + ); } Element::Tree(ref value, _) | Element::SumTree(ref value, ..) => { @@ -202,9 +217,9 @@ impl GroveDb { )) .wrap_with_cost(cost); } else { - cost_return_on_error!( + let delta = cost_return_on_error!( &mut cost, - subtree_to_insert_into.for_merk(|m| element.insert_subtree( + subtree_to_insert_into.for_merk(|m| element.insert_subtree_if_changed( m, key, NULL_HASH, @@ -212,18 +227,51 @@ impl GroveDb { grove_version )) ); + + cost_return_on_error!( + &mut cost, + process_update_element_with_backward_references( + merk_cache, + subtree_to_insert_into.clone(), + path, + key, + delta + ) + ); } } - _ => { + Element::BidirectionalReference(reference) => { cost_return_on_error!( &mut cost, - subtree_to_insert_into.for_merk(|m| element.insert( + process_bidirectional_reference_insertion( + merk_cache, + path, + key, + reference, + Some(options) + ) + ); + } + _ => { + let delta = cost_return_on_error!( + &mut cost, + subtree_to_insert_into.for_merk(|m| element.insert_if_changed_value( m, key, Some(options.as_merk_options()), grove_version )) ); + cost_return_on_error!( + &mut cost, + process_update_element_with_backward_references( + merk_cache, + subtree_to_insert_into.clone(), + path, + key, + delta + ) + ); } } @@ -1820,13 +1868,13 @@ mod tests { assert_eq!( cost, OperationCost { - seek_count: 9, // todo: verify this + seek_count: 10, // todo: verify this storage_cost: StorageCost { added_bytes: 0, replaced_bytes: 409, // todo: verify this removed_bytes: NoStorageRemoval }, - storage_loaded_bytes: 487, // todo verify this + storage_loaded_bytes: 558, // todo verify this hash_node_calls: 11, } ); diff --git a/grovedb/src/operations/proof/verify.rs b/grovedb/src/operations/proof/verify.rs index c6a34c1f..810606f7 100644 --- a/grovedb/src/operations/proof/verify.rs +++ b/grovedb/src/operations/proof/verify.rs @@ -337,8 +337,11 @@ impl GroveDb { Element::Tree(None, _) | Element::SumTree(None, ..) | Element::SumItem(..) + | Element::SumItemWithBackwardsReferences(..) | Element::Item(..) - | Element::Reference(..) => { + | Element::ItemWithBackwardsReferences(..) + | Element::Reference(..) + | Element::BidirectionalReference(..) => { return Err(Error::InvalidProof( "Proof has lower layer for a non Tree".into(), )); diff --git a/grovedb/src/util.rs b/grovedb/src/util.rs index 70db057c..6809742e 100644 --- a/grovedb/src/util.rs +++ b/grovedb/src/util.rs @@ -13,6 +13,7 @@ use grovedb_version::{check_grovedb_v0_with_cost, version::GroveVersion}; use grovedb_visualize::DebugByteVectors; use crate::{ + bidirectional_references::BidirectionalReference, merk_cache::{MerkCache, MerkHandle}, operations::MAX_REFERENCE_HOPS, reference_path::ReferencePathType, @@ -216,7 +217,14 @@ pub(crate) fn follow_reference<'db, 'b, 'c, B: AsRef<[u8]>>( ); match element { - Element::Reference(ref_path, ..) => { + Element::Reference(ref_path, ..) + | Element::BidirectionalReference( + BidirectionalReference { + forward_reference_path: ref_path, + .. + }, + .., + ) => { current_path = referred_path; current_key = referred_key; current_ref = ref_path; diff --git a/grovedb/src/visualize.rs b/grovedb/src/visualize.rs index 8b531cc0..6ae691c9 100644 --- a/grovedb/src/visualize.rs +++ b/grovedb/src/visualize.rs @@ -57,6 +57,16 @@ impl Visualize for Element { } } } + Element::ItemWithBackwardsReferences(value, flags) => { + drawer.write(b"item_with_backwards_references :")?; + drawer = value.visualize(drawer)?; + + if let Some(f) = flags { + if !f.is_empty() { + drawer = f.visualize(drawer)?; + } + } + } Element::SumItem(value, flags) => { drawer.write(format!("sum_item: {value}").as_bytes())?; @@ -66,6 +76,15 @@ impl Visualize for Element { } } } + Element::SumItemWithBackwardsReferences(value, flags) => { + drawer.write(format!("sum_item_with_backwards_references: {value}",).as_bytes())?; + + if let Some(f) = flags { + if !f.is_empty() { + drawer = f.visualize(drawer)?; + } + } + } Element::Reference(_ref, ..) => { drawer.write(b"ref")?; // drawer.write(b"ref: [path: ")?; @@ -100,6 +119,9 @@ impl Visualize for Element { } } } + Element::BidirectionalReference(..) => { + drawer.write(b"bidi ref")?; + } } Ok(drawer) }