From 0d92cebd434410bce284f52c0840c3bd4fba7c02 Mon Sep 17 00:00:00 2001 From: Evgeny Fomin Date: Fri, 8 Nov 2024 15:30:02 +0100 Subject: [PATCH] wip --- grovedb/src/element/insert.rs | 15 ++- grovedb/src/element/mod.rs | 13 ++ grovedb/src/merk_cache.rs | 244 ++++++++++++++++++++++++++++++++-- 3 files changed, 257 insertions(+), 15 deletions(-) diff --git a/grovedb/src/element/insert.rs b/grovedb/src/element/insert.rs index ae231766..1d418d56 100644 --- a/grovedb/src/element/insert.rs +++ b/grovedb/src/element/insert.rs @@ -349,6 +349,7 @@ impl Element { /// If transaction is not passed, the batch will be written immediately. /// If transaction is passed, the operation will be committed on the /// transaction commit. + /// Returns `bool` that indicates if a reference propagation is required. pub fn insert_reference<'db, K: AsRef<[u8]>, S: StorageContext<'db>>( self, merk: &mut Merk, @@ -356,7 +357,7 @@ impl Element { referenced_value: Hash, options: Option, grove_version: &GroveVersion, - ) -> CostResult<(), Error> { + ) -> CostResult { check_grovedb_v0_with_cost!( "insert_reference", grove_version.grovedb_versions.element.insert_reference @@ -364,13 +365,16 @@ impl Element { let mut cost = Default::default(); - let to_insert = if let Some(mut prev) = cost_return_on_error!( + let (ref_updated, to_insert) = if let Some(mut prev) = cost_return_on_error!( &mut cost, Self::get_optional_from_storage(&merk.storage, key.as_ref(), grove_version) ) { - cost_return_on_error_no_add!(cost, self.promote_to_referenced_variant(&mut prev)) + ( + !prev.eq_no_backreferences(&self) && prev.has_backward_references(), + cost_return_on_error_no_add!(cost, self.promote_to_referenced_variant(&mut prev)), + ) } else { - self + (false, self) }; let serialized = match to_insert.serialize(grove_version) { @@ -402,6 +406,7 @@ impl Element { grove_version, ) .map_err(|e| Error::CorruptedData(e.to_string())) + .map_ok(|_| ref_updated) } #[cfg(feature = "full")] @@ -540,7 +545,7 @@ impl Element { } /// Adds info on reference that points to this element. - fn referenced_from( + pub(crate) fn referenced_from( mut self, reference: ReferencePathType, cascade_on_update: CascadeOnUpdate, diff --git a/grovedb/src/element/mod.rs b/grovedb/src/element/mod.rs index 4c65da51..d607d35e 100644 --- a/grovedb/src/element/mod.rs +++ b/grovedb/src/element/mod.rs @@ -365,6 +365,19 @@ impl Element { _ => false, } } + + /// Downgrades `Element` variants with backward references to regular + /// variants. + pub(crate) fn cut_backreferences(self) -> Self { + match self { + Element::ItemWithBackwardsReferences(value, _, flags) => Element::Item(value, flags), + Element::SumItemWithBackwardsReferences(sum, _, flags) => Element::SumItem(sum, flags), + Element::BidirectionalReference(ref_path, _, max_hops, flags) => { + Element::Reference(ref_path, max_hops, flags) + } + x => x, + } + } } #[cfg(any(feature = "full", feature = "visualize"))] diff --git a/grovedb/src/merk_cache.rs b/grovedb/src/merk_cache.rs index 70be0ea0..bdd6bc05 100644 --- a/grovedb/src/merk_cache.rs +++ b/grovedb/src/merk_cache.rs @@ -2,20 +2,19 @@ //! after usage automatically. use std::{ - borrow::Cow, cell::RefCell, collections::{btree_map::Entry, BTreeMap, HashSet}, mem::{self, MaybeUninit}, ops::Deref, }; -use grovedb_costs::{cost_return_on_error, CostResult, CostsExt}; -use grovedb_merk::{Merk, MerkOptions}; +use grovedb_costs::{cost_return_on_error, cost_return_on_error_no_add, CostResult, CostsExt}; +use grovedb_merk::{tree::NULL_HASH, CryptoHash, Merk, MerkOptions}; use grovedb_path::SubtreePath; use grovedb_storage::{rocksdb_storage::PrefixedRocksDbTransactionContext, StorageBatch}; use grovedb_version::version::GroveVersion; -use crate::{Element, Error, GroveDb, Transaction}; +use crate::{reference_path::ReferencePathType, Element, Error, GroveDb, Transaction}; type TxMerk<'db> = Merk>; @@ -192,6 +191,87 @@ impl<'db, 'b, B: AsRef<[u8]>> MerkCache<'db, 'b, B> { propagation_result.map_ok(|_| result_batch) } + /// Inserts a reference into a cached Merk. + /// The reason why this has to be a whole `MerkCache` method is that + /// references involve opening and modifying several Merks at once, this + /// makes it out of scope for a single `MerkHandle`. + pub(crate) fn insert_reference<'c>( + &'c mut self, + path: SubtreePath<'b, B>, + key: &[u8], + ref_element: ReferenceElement, + cascade_on_update: bool, + options: Option, + ) -> CostResult<(), Error> { + let mut cost = Default::default(); + + let follow_reference_result = cost_return_on_error!( + &mut cost, + self.follow_reference(path.clone(), key, ref_element.get_ref_path()) + ); + + let value_hash = cost_return_on_error!( + &mut cost, + follow_reference_result + .last_element + .value_hash(&self.version) + ); + + // The insertion of a reference requires that its hash value be equal to the + // hash value of the item in the end of the reference's chain: + let [mut merk] = cost_return_on_error!(&mut cost, self.get_multi_mut([path])); + cost_return_on_error!( + &mut cost, + merk.insert_internal(key, ref_element.0.clone(), options, Some(value_hash)) + ); + + let version = self.version.clone(); // TODO + + // The closest referenced item's backward references list of the chain shall be + // updated with a new entry: + let [mut closest_merk] = cost_return_on_error!( + &mut cost, + self.get_multi_mut([follow_reference_result.first_path]) + ); + let mut closest_element = cost_return_on_error!( + &mut cost, + Element::get( + closest_merk.deref(), + &follow_reference_result.first_key, + true, + &version + ) + ); + // Updated backward references information: + closest_element = cost_return_on_error_no_add!( + cost, + closest_element.referenced_from(ref_element.to_ref_path(), cascade_on_update) + ); + // And write it back: + cost_return_on_error!( + &mut cost, + closest_merk.insert_internal( + &follow_reference_result.first_key, + closest_element, + None, + Some(value_hash), + ) + ); + + todo!() + } + + /// Follows a reference returning the first link in the references chain and + /// also the referenced item at the end of it. + fn follow_reference( + &mut self, + self_path: SubtreePath<'b, B>, + self_key: &[u8], + reference_path: &ReferencePathType, + ) -> CostResult, Error> { + todo!() + } + /// Perform propagation of references' chains marked as changed. fn propagate_updated_references(&mut self) -> CostResult<(), Error> { todo!() @@ -242,6 +322,62 @@ impl<'db, 'b, B: AsRef<[u8]>> MerkCache<'db, 'b, B> { } } +struct FollowReferenceResult<'b, B> { + first_path: SubtreePath<'b, B>, + first_key: Vec, + last_element: Element, +} + +/// A wrapper type to ensure `Element` is a reference wherever it is required. +pub(crate) struct ReferenceElement(Element); + +impl ReferenceElement { + fn get_ref_path(&self) -> &ReferencePathType { + match &self.0 { + Element::Reference(ref_path, ..) | Element::BidirectionalReference(ref_path, ..) => { + ref_path + } + _ => unreachable!(), + } + } + + fn to_ref_path(self) -> ReferencePathType { + match self.0 { + Element::Reference(ref_path, ..) | Element::BidirectionalReference(ref_path, ..) => { + ref_path + } + _ => unreachable!(), + } + } +} + +impl TryFrom for ReferenceElement { + type Error = (); + + fn try_from(value: Element) -> Result { + match value { + element @ Element::Reference(..) | element @ Element::BidirectionalReference(..) => { + Ok(Self(element)) + } + _ => Err(()), + } + } +} + +/// A wrapper type to ensure `Element` is not a reference. +pub(crate) struct NonReferenceElement(Element); + +impl TryFrom for NonReferenceElement { + type Error = (); + + fn try_from(value: Element) -> Result { + match value { + Element::Reference(..) | Element::BidirectionalReference(..) => Err(()), + element => Ok(Self(element)), + } + } +} + /// Handle to a cached Merk. pub(crate) struct MerkHandle<'db, 'c, 'b, B> { merk: &'c mut TxMerk<'db>, @@ -276,10 +412,20 @@ impl<'db, 'c, 'b, B> Deref for MerkHandle<'db, 'c, 'b, B> { impl<'db, 'c, 'b, B: AsRef<[u8]>> MerkHandle<'db, 'c, 'b, B> { pub(crate) fn insert( + &mut self, + key: &[u8], + NonReferenceElement(element): NonReferenceElement, + options: Option, + ) -> CostResult<(), Error> { + self.insert_internal(key, element, options, None) + } + + fn insert_internal( &mut self, key: &[u8], element: Element, options: Option, + referenced_value_hash: Option, ) -> CostResult<(), Error> { let mut costs = Default::default(); @@ -287,13 +433,39 @@ impl<'db, 'c, 'b, B: AsRef<[u8]>> MerkHandle<'db, 'c, 'b, B> { // references should be propagated after if cost_return_on_error!( &mut costs, - element.insert_if_changed_value(self.merk, key, options, self.version) - ) - .0 - { + match element { + Element::Item(..) + | Element::SumItem(..) + | Element::ItemWithBackwardsReferences(..) + | Element::SumItemWithBackwardsReferences(..) => element + .insert_if_changed_value(self.merk, key, options, self.version) + .map_ok(|r| r.0), + Element::Reference(..) | Element::BidirectionalReference(..) => element + .insert_reference( + self.merk, + key, + referenced_value_hash.expect("todo"), + options, + self.version + ), + Element::Tree(ref value, ..) | Element::SumTree(ref value, ..) => + if value.is_some() { + Err(Error::InvalidCodeExecution( + "a tree should be empty at the moment of insertion when not using \ + batches", + )) + .wrap_with_cost(Default::default()) + } else { + element + .insert_subtree(self.merk, key, NULL_HASH, options, self.version) + .map_ok(|_| false) + }, + } + ) { self.updated_reference_handle .mark_updated_reference(key.to_vec()); } + *self.to_propagate = true; Ok(()).wrap_with_cost(costs) @@ -322,6 +494,7 @@ mod tests { use super::MerkCache; use crate::{ + reference_path::ReferencePathType, tests::{make_deep_tree, ANOTHER_TEST_LEAF, TEST_LEAF}, Element, }; @@ -397,7 +570,11 @@ mod tests { .unwrap() .unwrap(); subtree - .insert(b"ayy", Element::new_item(b"lmao".to_vec()), None) + .insert( + b"ayy", + Element::new_item(b"lmao".to_vec()).try_into().unwrap(), + None, + ) .unwrap() .unwrap(); @@ -419,7 +596,11 @@ mod tests { .unwrap() .unwrap(); subtree - .insert(b"ayy", Element::new_item(b"lmao".to_vec()), None) + .insert( + b"ayy", + Element::new_item(b"lmao".to_vec()).try_into().unwrap(), + None, + ) .unwrap() .unwrap(); @@ -435,4 +616,47 @@ mod tests { db.root_hash(Some(&tx), &version).unwrap().unwrap() ) } + + #[test] + fn changes_to_referenced_values_are_marked_uncommitted() { + let version = GroveVersion::latest(); + let db = make_deep_tree(&version); + let tx = db.start_transaction(); + + let mut cache = MerkCache::new(&db, &tx, version); + cache + .insert_reference( + SubtreePath::from(&[TEST_LEAF, b"innertree"]), + b"ayy", + Element::new_reference(ReferencePathType::AbsolutePathReference(vec![ + ANOTHER_TEST_LEAF.to_vec(), + b"innertree2".to_vec(), + b"k3".to_vec(), + ])) + .try_into() + .unwrap(), + false, + None, + ) + .unwrap() + .unwrap(); + + assert!(cache.updated_references.borrow().is_empty()); + + let [mut subtree] = cache + .get_multi_mut([SubtreePath::from(&[ANOTHER_TEST_LEAF, b"innertree2"])]) + .unwrap() + .unwrap(); + + subtree + .insert( + b"k3", + Element::new_item(b"huh".to_vec()).try_into().unwrap(), + None, + ) + .unwrap() + .unwrap(); + + assert!(!cache.updated_references.borrow().is_empty()); + } }