Skip to content

Commit

Permalink
feat: Improve verify grovedb + better debug statements (#269)
Browse files Browse the repository at this point in the history 
* Improve verify grovedb + better debug statements

* clean up

* return error rather than panic for verify_grovedb

* fix compile errors
  • Loading branch information
@iammadab
iammadab authored Sep 25, 2023
1 parent 8380363 commit 75b4df1
Show file tree
Hide file tree
Showing 2 changed files with 152 additions and 40 deletions.
55 changes: 33 additions & 22 deletions grovedb/src/batch/mod.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -371,32 +371,42 @@ impl fmt::Debug for GroveDbOp {

let op_dbg = match &self.op {
Op::Insert { element } => match element {
Element::Item(..) => "Insert Item",
Element::Reference(..) => "Insert Ref",
Element::Tree(..) => "Insert Tree",
Element::SumTree(..) => "Insert Sum Tree",
Element::SumItem(..) => "Insert Sum Item",
Element::Item(..) => "Insert Item".to_string(),
Element::Reference(..) => "Insert Ref".to_string(),
Element::Tree(..) => "Insert Tree".to_string(),
Element::SumTree(..) => "Insert Sum Tree".to_string(),
Element::SumItem(..) => "Insert Sum Item".to_string(),
},
Op::Replace { element } => match element {
Element::Item(..) => "Replace Item",
Element::Reference(..) => "Replace Ref",
Element::Tree(..) => "Replace Tree",
Element::SumTree(..) => "Replace Sum Tree",
Element::SumItem(..) => "Replace Sum Item",
Element::Item(..) => "Replace Item".to_string(),
Element::Reference(..) => "Replace Ref".to_string(),
Element::Tree(..) => "Replace Tree".to_string(),
Element::SumTree(..) => "Replace Sum Tree".to_string(),
Element::SumItem(..) => "Replace Sum Item".to_string(),
},
Op::Patch { element, .. } => match element {
Element::Item(..) => "Patch Item",
Element::Reference(..) => "Patch Ref",
Element::Tree(..) => "Patch Tree",
Element::SumTree(..) => "Patch Sum Tree",
Element::SumItem(..) => "Patch Sum Item",
Element::Item(..) => "Patch Item".to_string(),
Element::Reference(..) => "Patch Ref".to_string(),
Element::Tree(..) => "Patch Tree".to_string(),
Element::SumTree(..) => "Patch Sum Tree".to_string(),
Element::SumItem(..) => "Patch Sum Item".to_string(),
},
Op::RefreshReference { .. } => "Refresh Reference",
Op::Delete => "Delete",
Op::DeleteTree => "Delete Tree",
Op::DeleteSumTree => "Delete Sum Tree",
Op::ReplaceTreeRootKey { .. } => "Replace Tree Hash and Root Key",
Op::InsertTreeWithRootHash { .. } => "Insert Tree Hash and Root Key",
Op::RefreshReference {
reference_path_type,
max_reference_hop,
trust_refresh_reference,
..
} => {
format!(
"Refresh Reference: path {:?}, max_hop {:?}, trust_reference {} ",
reference_path_type, max_reference_hop, trust_refresh_reference
)
}
Op::Delete => "Delete".to_string(),
Op::DeleteTree => "Delete Tree".to_string(),
Op::DeleteSumTree => "Delete Sum Tree".to_string(),
Op::ReplaceTreeRootKey { .. } => "Replace Tree Hash and Root Key".to_string(),
Op::InsertTreeWithRootHash { .. } => "Insert Tree Hash and Root Key".to_string(),
};

f.debug_struct("GroveDbOp")
Expand Down Expand Up @@ -1150,7 +1160,8 @@ where
let Element::Reference(path_reference, max_reference_hop, _) = &element else {
return Err(Error::InvalidInput(
"trying to refresh a an element that is not a reference",
)).wrap_with_cost(cost)
))
.wrap_with_cost(cost);
};

let merk_feature_type = if is_sum_tree {
Expand Down
137 changes: 119 additions & 18 deletions grovedb/src/lib.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -224,6 +224,7 @@ pub use crate::error::Error;
use crate::helpers::raw_decode;
#[cfg(feature = "full")]
use crate::util::{root_merk_optional_tx, storage_context_optional_tx};
use crate::Error::MerkError;

#[cfg(feature = "full")]
type Hash = [u8; 32];
Expand Down Expand Up @@ -783,8 +784,11 @@ impl GroveDb {
}

/// Method to visualize hash mismatch after verification
pub fn visualize_verify_grovedb(&self) -> HashMap<String, (String, String, String)> {
self.verify_grovedb()
pub fn visualize_verify_grovedb(
&self,
) -> Result<HashMap<String, (String, String, String)>, Error> {
Ok(self
.verify_grovedb(None)?
.iter()
.map(|(path, (root_hash, expected, actual))| {
(
Expand All @@ -799,27 +803,41 @@ impl GroveDb {
),
)
})
.collect()
.collect())
}

/// Method to check that the value_hash of Element::Tree nodes are computed
/// correctly.
pub fn verify_grovedb(&self) -> HashMap<Vec<Vec<u8>>, (CryptoHash, CryptoHash, CryptoHash)> {
let root_merk = self
.open_non_transactional_merk_at_path(SubtreePath::empty(), None)
.unwrap()
.expect("should exist");
self.verify_merk_and_submerks(root_merk, &SubtreePath::empty(), None)
pub fn verify_grovedb(
&self,
transaction: TransactionArg,
) -> Result<HashMap<Vec<Vec<u8>>, (CryptoHash, CryptoHash, CryptoHash)>, Error> {
if let Some(transaction) = transaction {
let root_merk = self
.open_transactional_merk_at_path(SubtreePath::empty(), transaction, None)
.unwrap()?;
self.verify_merk_and_submerks_in_transaction(
root_merk,
&SubtreePath::empty(),
None,
transaction,
)
} else {
let root_merk = self
.open_non_transactional_merk_at_path(SubtreePath::empty(), None)
.unwrap()?;
self.verify_merk_and_submerks(root_merk, &SubtreePath::empty(), None)
}
}

/// Verifies that the root hash of the given merk and all submerks match
/// those of the merk and submerks at the given path. Returns any issues.
fn verify_merk_and_submerks<'db, B: AsRef<[u8]>>(
fn verify_merk_and_submerks<'db, B: AsRef<[u8]>, S: StorageContext<'db>>(
&'db self,
merk: Merk<PrefixedRocksDbStorageContext>,
merk: Merk<S>,
path: &SubtreePath<B>,
batch: Option<&'db StorageBatch>,
) -> HashMap<Vec<Vec<u8>>, (CryptoHash, CryptoHash, CryptoHash)> {
) -> Result<HashMap<Vec<Vec<u8>>, (CryptoHash, CryptoHash, CryptoHash)>, Error> {
let mut all_query = Query::new();
all_query.insert_all();

Expand All @@ -828,20 +846,83 @@ impl GroveDb {
let mut element_iterator = KVIterator::new(merk.storage.raw_iter(), &all_query).unwrap();

while let Some((key, element_value)) = element_iterator.next_kv().unwrap() {
let element = raw_decode(&element_value).unwrap();
let element = raw_decode(&element_value)?;
if element.is_tree() {
let (kv_value, element_value_hash) = merk
.get_value_and_value_hash(&key, true)
.unwrap()
.unwrap()
.unwrap();
.map_err(MerkError)?
.ok_or(Error::CorruptedData(
"expected merk to contain value at key".to_string(),
))?;
let new_path = path.derive_owned_with_child(key);
let new_path_ref = SubtreePath::from(&new_path);

let inner_merk = self
.open_non_transactional_merk_at_path(new_path_ref.clone(), batch)
.unwrap()?;
let root_hash = inner_merk.root_hash().unwrap();

let actual_value_hash = value_hash(&kv_value).unwrap();
let combined_value_hash = combine_hash(&actual_value_hash, &root_hash).unwrap();

if combined_value_hash != element_value_hash {
issues.insert(
new_path.to_vec(),
(root_hash, combined_value_hash, element_value_hash),
);
}
issues.extend(self.verify_merk_and_submerks(inner_merk, &new_path_ref, batch)?);
} else if element.is_item() {
let (kv_value, element_value_hash) = merk
.get_value_and_value_hash(&key, true)
.unwrap()
.map_err(MerkError)?
.ok_or(Error::CorruptedData(
"expected merk to contain value at key".to_string(),
))?;
let actual_value_hash = value_hash(&kv_value).unwrap();
if actual_value_hash != element_value_hash {
issues.insert(
path.derive_owned_with_child(key).to_vec(),
(actual_value_hash, element_value_hash, actual_value_hash),
);
}
}
}
Ok(issues)
}

fn verify_merk_and_submerks_in_transaction<'db, B: AsRef<[u8]>, S: StorageContext<'db>>(
&'db self,
merk: Merk<S>,
path: &SubtreePath<B>,
batch: Option<&'db StorageBatch>,
transaction: &Transaction,
) -> Result<HashMap<Vec<Vec<u8>>, (CryptoHash, CryptoHash, CryptoHash)>, Error> {
let mut all_query = Query::new();
all_query.insert_all();

let _in_sum_tree = merk.is_sum_tree;
let mut issues = HashMap::new();
let mut element_iterator = KVIterator::new(merk.storage.raw_iter(), &all_query).unwrap();

while let Some((key, element_value)) = element_iterator.next_kv().unwrap() {
let element = raw_decode(&element_value)?;
if element.is_tree() {
let (kv_value, element_value_hash) = merk
.get_value_and_value_hash(&key, true)
.unwrap()
.expect("should exist");
.map_err(MerkError)?
.ok_or(Error::CorruptedData(
"expected merk to contain value at key".to_string(),
))?;
let new_path = path.derive_owned_with_child(key);
let new_path_ref = SubtreePath::from(&new_path);

let inner_merk = self
.open_transactional_merk_at_path(new_path_ref.clone(), transaction, batch)
.unwrap()?;
let root_hash = inner_merk.root_hash().unwrap();

let actual_value_hash = value_hash(&kv_value).unwrap();
Expand All @@ -853,9 +934,29 @@ impl GroveDb {
(root_hash, combined_value_hash, element_value_hash),
);
}
issues.extend(self.verify_merk_and_submerks(inner_merk, &new_path_ref, batch));
issues.extend(self.verify_merk_and_submerks_in_transaction(
inner_merk,
&new_path_ref,
batch,
transaction,
)?);
} else if element.is_item() {
let (kv_value, element_value_hash) = merk
.get_value_and_value_hash(&key, true)
.unwrap()
.map_err(MerkError)?
.ok_or(Error::CorruptedData(
"expected merk to contain value at key".to_string(),
))?;
let actual_value_hash = value_hash(&kv_value).unwrap();
if actual_value_hash != element_value_hash {
issues.insert(
path.derive_owned_with_child(key).to_vec(),
(actual_value_hash, element_value_hash, actual_value_hash),
);
}
}
}
issues
Ok(issues)
}
}

0 comments on commit 75b4df1

Please sign in to comment.