Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: state sync optimization #346

Open
wants to merge 7 commits into
base: develop
Choose a base branch
from
+1,172 −613
Open

feat: state sync optimization #346

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
91ea7df
feat: optimization refactor
ogabrielides Dec 20, 2024
e18a10f
fix: display non-utf8 path
ogabrielides Dec 23, 2024
f937923
remove check on height
ogabrielides Dec 23, 2024
70460eb
refactor: various refactoring and comments
ogabrielides Dec 26, 2024
995aec7
refactor: suggestions and richer doc
ogabrielides Dec 30, 2024
dfcef7a
refactor: better type alias
ogabrielides Dec 30, 2024
5098940
fix: build for verify feature
ogabrielides Dec 30, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
148 changes: 127 additions & 21 deletions grovedb/src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -240,6 +240,25 @@
#[cfg(feature = "full")]
pub type TransactionArg<'db, 'a> = Option<&'a Transaction<'db>>;

/// Type alias for the return type of the `verify_merk_and_submerks` and
/// `verify_grovedb` functions. It represents a mapping of paths (as vectors of
/// vectors of bytes) to a tuple of three cryptographic hashes: the root hash,
/// the combined value hash, and the expected value hash.
#[cfg(feature = "full")]
type VerificationIssues = HashMap<Vec<Vec<u8>>, (CryptoHash, CryptoHash, CryptoHash)>;

/// Type alias for the return type of the `open_merk_for_replication` function.
/// It represents a tuple containing:
/// - A `Merk` instance with a prefixed RocksDB immediate storage context.
/// - An optional `root_key`, represented as a vector of bytes.
/// - A boolean indicating whether the Merk is a sum tree.
#[cfg(feature = "full")]
type OpenedMerkForReplication<'tx> = (
Merk<PrefixedRocksDbImmediateStorageContext<'tx>>,
Option<Vec<u8>>,
bool,
);

#[cfg(feature = "full")]
impl GroveDb {
/// Opens a given path
Expand Down Expand Up @@ -330,14 +349,54 @@
}
}

fn open_transactional_merk_by_prefix<'db>(
&'db self,
prefix: SubtreePrefix,
root_key: Option<Vec<u8>>,
is_sum_tree: bool,
tx: &'db Transaction,
batch: Option<&'db StorageBatch>,
grove_version: &GroveVersion,
) -> CostResult<Merk<PrefixedRocksDbTransactionContext<'db>>, Error> {
let mut cost = OperationCost::default();
let storage = self
.db
.get_transactional_storage_context_by_subtree_prefix(prefix, batch, tx)
.unwrap_add_cost(&mut cost);
if root_key.is_some() {
Merk::open_layered_with_root_key(
storage,
root_key,
is_sum_tree,
Some(&Element::value_defined_cost_for_serialized_value),
grove_version,
)
.map_err(|_| {
Error::CorruptedData(
"cannot open a subtree by prefix with given root key".to_owned(),
)
})
.add_cost(cost)
} else {
Merk::open_base(
storage,
false,
Some(&Element::value_defined_cost_for_serialized_value),
grove_version,
)
.map_err(|_| Error::CorruptedData("cannot open a root subtree by prefix".to_owned()))
.add_cost(cost)
}
}

/// Opens a Merk at given path for with direct write access. Intended for
/// replication purposes.
fn open_merk_for_replication<'tx, 'db: 'tx, 'b, B>(
&'db self,
path: SubtreePath<'b, B>,
tx: &'tx Transaction<'db>,
grove_version: &GroveVersion,
) -> Result<Merk<PrefixedRocksDbImmediateStorageContext<'tx>>, Error>
) -> Result<OpenedMerkForReplication<'tx>, Error>
where
B: AsRef<[u8]> + 'b,
{
Expand All @@ -364,31 +423,39 @@
.unwrap()?;
let is_sum_tree = element.is_sum_tree();
if let Element::Tree(root_key, _) | Element::SumTree(root_key, ..) = element {
Merk::open_layered_with_root_key(
storage,
Ok((
Merk::open_layered_with_root_key(
storage,
root_key.clone(),
is_sum_tree,
Some(&Element::value_defined_cost_for_serialized_value),
grove_version,
)
.map_err(|_| {
Error::CorruptedData("cannot open a subtree with given root key".to_owned())
})
.unwrap()?,
root_key,
is_sum_tree,
Some(&Element::value_defined_cost_for_serialized_value),
grove_version,
)
.map_err(|_| {
Error::CorruptedData("cannot open a subtree with given root key".to_owned())
})
.unwrap()
))
} else {
Err(Error::CorruptedPath(
"cannot open a subtree as parent exists but is not a tree".to_string(),
))
}
} else {
Merk::open_base(
storage,
Ok((
Merk::open_base(
storage,
false,
None::<&fn(&[u8], &GroveVersion) -> Option<ValueDefinedCostType>>,
grove_version,
)
.map_err(|_| Error::CorruptedData("cannot open a the root subtree".to_owned()))
.unwrap()?,
None,
false,
None::<&fn(&[u8], &GroveVersion) -> Option<ValueDefinedCostType>>,
grove_version,
)
.map_err(|_| Error::CorruptedData("cannot open a the root subtree".to_owned()))
.unwrap()
))
}
}

Expand All @@ -398,7 +465,7 @@
path: SubtreePath<'b, B>,
batch: Option<&'db StorageBatch>,
grove_version: &GroveVersion,
) -> CostResult<Merk<PrefixedRocksDbStorageContext>, Error>
) -> CostResult<Merk<PrefixedRocksDbStorageContext<'db>>, Error>
where
B: AsRef<[u8]> + 'b,
{
Expand Down Expand Up @@ -458,6 +525,45 @@
}
}

fn open_non_transactional_merk_by_prefix<'db>(
&'db self,
prefix: SubtreePrefix,
root_key: Option<Vec<u8>>,
is_sum_tree: bool,
batch: Option<&'db StorageBatch>,
grove_version: &GroveVersion,
) -> CostResult<Merk<PrefixedRocksDbStorageContext<'db>>, Error> {
let mut cost = OperationCost::default();
let storage = self
.db
.get_storage_context_by_subtree_prefix(prefix, batch)
.unwrap_add_cost(&mut cost);
if root_key.is_some() {
Merk::open_layered_with_root_key(
storage,
root_key,
is_sum_tree,
Some(&Element::value_defined_cost_for_serialized_value),
grove_version,
)
.map_err(|_| {
Error::CorruptedData(
"cannot open a subtree by prefix with given root key".to_owned(),
)
})
.add_cost(cost)
} else {
Merk::open_base(
storage,
false,
Some(&Element::value_defined_cost_for_serialized_value),
grove_version,
)
.map_err(|_| Error::CorruptedData("cannot open a root subtree by prefix".to_owned()))
.add_cost(cost)
}
}

/// Creates a checkpoint
pub fn create_checkpoint<P: AsRef<Path>>(&self, path: P) -> Result<(), Error> {
self.db.create_checkpoint(path).map_err(|e| e.into())
Expand Down Expand Up @@ -935,7 +1041,7 @@
verify_references: bool,
allow_cache: bool,
grove_version: &GroveVersion,
) -> Result<HashMap<Vec<Vec<u8>>, (CryptoHash, CryptoHash, CryptoHash)>, Error> {
) -> Result<VerificationIssues, Error> {
if let Some(transaction) = transaction {
let root_merk = self
.open_transactional_merk_at_path(
Expand Down Expand Up @@ -979,7 +1085,7 @@
verify_references: bool,
allow_cache: bool,
grove_version: &GroveVersion,
) -> Result<HashMap<Vec<Vec<u8>>, (CryptoHash, CryptoHash, CryptoHash)>, Error> {
) -> Result<VerificationIssues, Error> {
let mut all_query = Query::new();
all_query.insert_all();

Expand Down Expand Up @@ -1114,16 +1220,16 @@
Ok(issues)
}

fn verify_merk_and_submerks_in_transaction<'db, B: AsRef<[u8]>, S: StorageContext<'db>>(
&'db self,
merk: Merk<S>,
path: &SubtreePath<B>,
batch: Option<&'db StorageBatch>,
transaction: &Transaction,
verify_references: bool,
allow_cache: bool,
grove_version: &GroveVersion,
) -> Result<HashMap<Vec<Vec<u8>>, (CryptoHash, CryptoHash, CryptoHash)>, Error> {
) -> Result<VerificationIssues, Error> {

Check warning on line 1232 in grovedb/src/lib.rs

View workflow job for this annotation

GitHub Actions / clippy

this function has too many arguments (8/7) 

warning: this function has too many arguments (8/7)
    --> grovedb/src/lib.rs:1223:5
     |
1223 | /     fn verify_merk_and_submerks_in_transaction<'db, B: AsRef<[u8]>, S: StorageContext<'db>>(
1224 | |         &'db self,
1225 | |         merk: Merk<S>,
1226 | |         path: &SubtreePath<B>,
...    |
1231 | |         grove_version: &GroveVersion,
1232 | |     ) -> Result<VerificationIssues, Error> {
     | |__________________________________________^
     |
     = help: for further information visit https://rust-lang.github.io/rust-clippy/master/index.html#too_many_arguments
let mut all_query = Query::new();
all_query.insert_all();

Expand Down
Loading
Loading