provide raw store component base on RocksDB (#454)

Cargo.toml

@@ -9,6 +9,7 @@ members = [
  "moveos/moveos-stdlib-builder",
  "moveos/moveos",
  "moveos/moveos-common",
+ "moveos/raw-store",
  "crates/rooch-key",
  "crates/rooch-types",
  "crates/rooch-framework",
@@ -61,6 +62,7 @@ moveos = { path = "moveos/moveos" }
 moveos-cli = { path = "moveos/moveos-cli" }
 moveos-common = { path = "moveos/moveos-common" }
 moveos-verifier = { path = "moveos/moveos-verifier" }
+raw-store = { path = "moveos/raw-store" }
 
 # crates for Rooch
 rooch = { path = "crates/rooch" }
@@ -160,13 +162,23 @@ versions = "4.1.0"
 pretty_assertions = "1.2.0"
 syn = { version = "1.0.104", features = ["full", "extra-traits"] }
 quote = "1.0"
-proc-macro2 = "1.0"
+proc-macro2 = "1.0.47"
 derive-syn-parse = "0.1.5"
 unescape = "0.1.0"
 tempfile = "3.2.0"
 regex = "1.8.4"
 walkdir = "2.3.3"
 
+rocksdb = { version = "0.21.0", features = ["snappy", "lz4", "zstd", "zlib", "multi-threaded-cf"], default-features = false }
+bincode = "1.3.3"
+collectable = "0.0.2"
+fdlimit = "0.2.1"
+tap = "1.0.1"
+num_cpus = "1.14.0"
+prometheus = "0.13.3"
+hdrhistogram = "7.5.1"
+ouroboros = "0.15.5"
+rstest = "0.16.0"
 
 # Note: the BEGIN and END comments below are required for external tooling. Do not remove.
 # BEGIN MOVE DEPENDENCIES

moveos/moveos-store/Cargo.toml

@@ -26,4 +26,5 @@ move-core-types = { workspace = true }
 move-resource-viewer = { workspace = true }
 
 moveos-types = { workspace = true }
-moveos-stdlib = { workspace = true }
+moveos-stdlib = { workspace = true }
+raw-store = { workspace = true }

moveos/raw-store/Cargo.toml

@@ -0,0 +1,77 @@
+[package]
+name = "raw-store"
+version = "0.1.0"
+
+# Workspace inherited keys
+authors = { workspace = true }
+edition = { workspace = true }
+homepage = { workspace = true }
+license = { workspace = true }
+publish = { workspace = true }
+repository = { workspace = true }
+rust-version = { workspace = true }
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+#[dependencies]
+#bcs = "0.1.4"
+#bincode = "1.3.3"
+#collectable = "0.0.2"
+#eyre = "0.6.8"
+#fdlimit = "0.2.1"
+#once_cell = "1.15.0"
+#tap = "1.0.1"
+#num_cpus = "1.14.0"
+#prometheus = "0.13.3"
+#hdrhistogram = "7.5.1"
+## deactivation of bzip2 due to https://github.com/rust-rocksdb/rust-rocksdb/issues/609
+#rocksdb = { version = "0.20.1", features = ["snappy", "lz4", "zstd", "zlib", "multi-threaded-cf"], default-features = false }
+#serde = { version = "1.0.140", features = ["derive"] }
+#thiserror = "1.0.37"
+#tokio = { workspace = true, features = ["full", "test-util"] }
+#tracing = "0.1.37"
+#ouroboros = "0.15.5"
+#rand = "0.8.5"
+#async-trait = "0.1.57"
+#itertools = "0.10.5"
+
+#sui-macros = { path = "../sui-macros" }
+#workspace-hack = { version = "0.1", path = "../workspace-hack" }
+
+[dependencies]
+anyhow = { workspace = true }
+bcs = { workspace = true }
+#smt = { workspace = true }
+serde = { workspace = true }
+serde_bytes = { workspace = true }
+hex = { workspace = true }
+parking_lot = { workspace = true }
+
+rocksdb = { workspace = true }
+prometheus = { workspace = true }
+tokio = { workspace = true }
+bincode = { workspace = true }
+collectable = { workspace = true }
+once_cell = { workspace = true }
+eyre = { workspace = true }
+fdlimit = { workspace = true }
+tap = { workspace = true }
+num_cpus = { workspace = true }
+hdrhistogram = { workspace = true }
+ouroboros = { workspace = true }
+rand = { workspace = true }
+async-trait = { workspace = true }
+itertools = { workspace = true }
+thiserror = { workspace = true }
+tracing = { workspace = true }
+futures = { workspace = true }
+rstest = { workspace = true }
+tempfile = { workspace = true }
+
+move-core-types = { workspace = true }
+move-resource-viewer = { workspace = true }
+
+# Most packages should depend on sui-simulator instead of directly on msim, but for typed-store
+# that creates a circular dependency.
+#[target.'cfg(msim)'.dependencies]
+#msim = { git = "https://github.com/MystenLabs/mysten-sim.git", rev = "e9011f96b84615b63cd8b5835e606a2fc218a1bd", package = "msim" }

moveos/raw-store/src/lib.rs

@@ -0,0 +1,22 @@
+// Copyright (c) RoochNetwork
+// SPDX-License-Identifier: Apache-2.0
+
+// Copyright (c) 2021, Facebook, Inc. and its affiliates
+// Copyright (c) Mysten Labs, Inc.
+// SPDX-License-Identifier: Apache-2.0
+#![warn(
+ future_incompatible,
+ nonstandard_style,
+ rust_2018_idioms,
+ rust_2021_compatibility
+)]
+
+pub mod traits;
+pub use traits::Map;
+pub mod rocks;
+pub use rocks::RawStoreError;
+pub mod macros;
+pub mod metrics;
+pub mod test_db;
+
+pub type StoreError = rocks::RawStoreError;

moveos/raw-store/src/macros.rs

@@ -0,0 +1,138 @@
+// Copyright (c) RoochNetwork
+// SPDX-License-Identifier: Apache-2.0
+
+// Copyright (c) Mysten Labs, Inc.
+// SPDX-License-Identifier: Apache-2.0
+
+use futures::future::BoxFuture;
+use std::collections::HashMap;
+use std::future::Future;
+use std::sync::Arc;
+
+/// Simply evaluates expr.
+#[macro_export]
+macro_rules! nondeterministic {
+ ($expr: expr) => {
+ $expr
+ };
+}
+
+type FpCallback = dyn Fn() -> Option<BoxFuture<'static, ()>> + Send + Sync + 'static;
+type FpMap = HashMap<&'static str, Arc<FpCallback>>;
+
+fn with_fp_map<T>(func: impl FnOnce(&mut FpMap) -> T) -> T {
+ use once_cell::sync::Lazy;
+ use std::sync::Mutex;
+
+ static MAP: Lazy<Mutex<FpMap>> = Lazy::new(Default::default);
+ let mut map = MAP.lock().unwrap();
+ func(&mut map)
+}
+
+fn get_callback(identifier: &'static str) -> Option<Arc<FpCallback>> {
+ with_fp_map(|map| map.get(identifier).cloned())
+}
+
+pub fn handle_fail_point(identifier: &'static str) {
+ if let Some(callback) = get_callback(identifier) {
+ tracing::error!("hit failpoint {}", identifier);
+ assert!(
+ callback().is_none(),
+ "sync failpoint must not return future"
+ );
+ }
+}
+
+pub async fn handle_fail_point_async(identifier: &'static str) {
+ if let Some(callback) = get_callback(identifier) {
+ tracing::error!("hit async failpoint {}", identifier);
+ let fut = callback().expect("async callback must return future");
+ fut.await;
+ }
+}
+
+fn register_fail_point_impl(
+ identifier: &'static str,
+ callback: Arc<dyn Fn() -> Option<BoxFuture<'static, ()>> + Sync + Send + 'static>,
+) {
+ with_fp_map(move |map| {
+ assert!(
+ map.insert(identifier, callback).is_none(),
+ "duplicate fail point registration"
+ );
+ })
+}
+
+pub fn register_fail_point(identifier: &'static str, callback: impl Fn() + Sync + Send + 'static) {
+ register_fail_point_impl(
+ identifier,
+ Arc::new(move || {
+ callback();
+ None
+ }),
+ );
+}
+
+pub fn register_fail_point_async<F>(
+ identifier: &'static str,
+ callback: impl Fn() -> F + Sync + Send + 'static,
+) where
+ F: Future<Output = ()> + Sync + Send + 'static,
+{
+ register_fail_point_impl(identifier, Arc::new(move || Some(Box::pin(callback()))));
+}
+
+pub fn register_fail_points(
+ identifiers: &[&'static str],
+ callback: impl Fn() + Sync + Send + 'static,
+) {
+ let cb = Arc::new(move || {
+ callback();
+ None
+ });
+ for id in identifiers {
+ register_fail_point_impl(id, cb.clone());
+ }
+}
+
+#[cfg(not(any(fail_points)))]
+#[macro_export]
+macro_rules! fail_point {
+ ($tag: expr) => {};
+}
+
+#[cfg(not(any(fail_points)))]
+#[macro_export]
+macro_rules! fail_point_async {
+ ($tag: expr) => {};
+}
+
+// These tests need to be run in release mode, since debug mode does overflow checks by default!
+#[cfg(test)]
+mod test {
+ // use super::*;
+
+ // Uncomment to test error messages
+ // #[with_checked_arithmetic]
+ // struct TestStruct;
+
+ macro_rules! pass_through {
+ ($($tt:tt)*) => {
+ $($tt)*
+ }
+ }
+
+ #[test]
+ fn test_skip_checked_arithmetic() {
+ // comment out this attr to test the error message
+ pass_through! {
+ fn unchecked_add(a: i32, b: i32) -> i32 {
+ a + b
+ }
+ }
+
+ // this will not panic even if we pass in (i32::MAX, 1), because we skipped processing
+ // the item macro, so we also need to make sure it doesn't panic in debug mode.
+ unchecked_add(1, 2);
+ }
+}