Fix rocksdb config (#1386)

crates/sovereign-sdk/full-node/db/sov-db/Cargo.toml

@@ -26,6 +26,7 @@ bincode = { workspace = true }
 borsh = { workspace = true, default-features = true, features = ["bytes", "rc"] }
 byteorder = { workspace = true, default-features = true }
 hex = { workspace = true }
+num_cpus = { workspace = true }
 proptest = { workspace = true, optional = true, default-features = true }
 proptest-derive = { workspace = true, optional = true }
 rlimit = { workspace = true }

crates/sovereign-sdk/full-node/db/sov-db/src/ledger_db/mod.rs

@@ -105,11 +105,12 @@ impl LedgerDB {
     #[instrument(level = "trace", skip_all, err)]
     pub fn with_config(cfg: &RocksdbConfig) -> Result<Self, anyhow::Error> {
         let path = cfg.path.join(LEDGER_DB_PATH_SUFFIX);
+        let raw_options = cfg.as_raw_options(false);
         let inner = DB::open(
             path,
             "ledger-db",
             LEDGER_TABLES.iter().copied(),
-            &cfg.as_rocksdb_options(false),
+            &raw_options,
         )?;
 
         let next_item_numbers = ItemNumbers {

crates/sovereign-sdk/full-node/db/sov-db/src/native_db.rs

@@ -32,12 +32,13 @@ impl<Q> NativeDB<Q> {
 
     /// Initialize [`sov_schema_db::DB`] that matches tables and columns for NativeDB
     pub fn setup_schema_db(cfg: &RocksdbConfig) -> anyhow::Result<sov_schema_db::DB> {
+        let raw_options = cfg.as_raw_options(false);
         let path = cfg.path.join(Self::DB_PATH_SUFFIX);
         sov_schema_db::DB::open(
             path,
             Self::DB_NAME,
             NATIVE_TABLES.iter().copied(),
-            &cfg.as_rocksdb_options(false),
+            &raw_options,
         )
     }
     /// Convert it to [`ReadOnlyDbSnapshot`] which cannot be edited anymore

crates/sovereign-sdk/full-node/db/sov-db/src/rocks_db_config.rs

@@ -4,6 +4,7 @@ use std::path::Path;
 
 use rlimit::{getrlimit, Resource};
 use rocksdb::{BlockBasedOptions, Cache, Options};
+use sov_schema_db::RawRocksdbOptions;
 use tracing::warn;
 
 /// Port selected RocksDB options for tuning underlying rocksdb instance of our state db.
@@ -41,11 +42,11 @@ impl<'a> RocksdbConfig<'a> {
         }
     }
 
-    /// Build [`rocksdb::Options`] from [`RocksdbConfig`]
-    pub fn as_rocksdb_options(&self, readonly: bool) -> Options {
-        let mut db_opts = Options::default();
+    /// Build [`RawRocksdbOptions`] from [`RocksdbConfig`]
+    pub fn as_raw_options(&self, readonly: bool) -> RawRocksdbOptions {
+        let mut db_options = Options::default();
 
-        let mut block_based_options = BlockBasedOptions::default();
+        let mut block_options = BlockBasedOptions::default();
         /*
          * The following settings are recommended in:
          * https://github.com/facebook/rocksdb/wiki/memory-usage-in-rocksdb
@@ -57,32 +58,33 @@ impl<'a> RocksdbConfig<'a> {
         // could still have OOM errors when trying to allocate more for the cache even if the capacity
         // is reached.
         let cache = Cache::new_lru_cache(100 * 1024 * 1024); // 100 MB
-        block_based_options.set_block_cache(&cache);
+        block_options.set_block_cache(&cache);
         // jemalloc friendly bloom filter sizing
-        block_based_options.set_optimize_filters_for_memory(true);
+        block_options.set_optimize_filters_for_memory(true);
         // By default our block size is 4KB, we set this to 32KB.
         // Increasing the size of the block decreases the number of blocks,
-        // therefore, less memory consumption for indicies.
-        block_based_options.set_block_size(32 * 1024);
+        // therefore, less memory consumption for indices.
+        block_options.set_block_size(32 * 1024);
         // Default is Snappy but Lz4 is recommend
         // https://github.com/facebook/rocksdb/wiki/Compression
-        db_opts.set_compression_type(rocksdb::DBCompressionType::Lz4);
-        db_opts.set_compression_options_parallel_threads(4);
+        db_options.set_compression_type(rocksdb::DBCompressionType::Lz4);
 
-        db_opts.set_max_open_files(self.max_open_files);
-        db_opts.set_max_total_wal_size(self.max_total_wal_size);
-        db_opts.set_max_background_jobs(self.max_background_jobs);
-        if !readonly {
-            // Increase write buffer size to reduce allocations.
-            db_opts.set_write_buffer_size(30 * 1024 * 1024); // 30 MB
-            db_opts.set_block_based_table_factory(&block_based_options);
+        let allowed_cores = std::cmp::max(1, num_cpus::get() / 2) as i32;
+        db_options.set_compression_options_parallel_threads(allowed_cores);
+        db_options.increase_parallelism(allowed_cores);
 
-            db_opts.create_if_missing(true);
-            db_opts.create_missing_column_families(true);
-            db_opts.set_atomic_flush(true);
+        db_options.set_max_open_files(self.max_open_files);
+        db_options.set_max_total_wal_size(self.max_total_wal_size);
+        db_options.set_max_background_jobs(self.max_background_jobs);
+        if !readonly {
+            db_options.create_if_missing(true);
+            db_options.create_missing_column_families(true);
         }
 
-        db_opts
+        RawRocksdbOptions {
+            db_options,
+            block_options,
+        }
     }
 }
 

crates/sovereign-sdk/full-node/db/sov-db/src/state_db.rs

@@ -40,12 +40,13 @@ impl<Q> StateDB<Q> {
 
     /// Initialize [`sov_schema_db::DB`] that should be used by snapshots.
     pub fn setup_schema_db(cfg: &RocksdbConfig) -> anyhow::Result<sov_schema_db::DB> {
+        let raw_options = cfg.as_raw_options(false);
         let state_db_path = cfg.path.join(Self::DB_PATH_SUFFIX);
         sov_schema_db::DB::open(
             state_db_path,
             Self::DB_NAME,
             STATE_TABLES.iter().copied(),
-            &cfg.as_rocksdb_options(false),
+            &raw_options,
         )
     }
 

crates/sovereign-sdk/full-node/db/sov-schema-db/src/lib.rs

@@ -56,15 +56,16 @@ impl DB {
         path: impl AsRef<Path>,
         name: &'static str,
         column_families: impl IntoIterator<Item = impl Into<String>>,
-        db_opts: &rocksdb::Options,
+        options: &RawRocksdbOptions,
     ) -> anyhow::Result<Self> {
         let db = DB::open_with_cfds(
-            db_opts,
+            &options.db_options,
             path,
             name,
             column_families.into_iter().map(|cf_name| {
                 let mut cf_opts = rocksdb::Options::default();
                 cf_opts.set_compression_type(rocksdb::DBCompressionType::Lz4);
+                cf_opts.set_block_based_table_factory(&options.block_options);
                 rocksdb::ColumnFamilyDescriptor::new(cf_name, cf_opts)
             }),
         )?;
@@ -328,6 +329,15 @@ impl DB {
     }
 }
 
+/// Raw rocksdb config wrapper. Useful to convert user provided config into
+/// the actual rocksdb config with all defaults set.
+pub struct RawRocksdbOptions {
+    /// Global db options
+    pub db_options: rocksdb::Options,
+    /// Per column-family options
+    pub block_options: rocksdb::BlockBasedOptions,
+}
+
 /// Readability alias for a key in the DB.
 pub type SchemaKey = Vec<u8>;
 /// Readability alias for a value in the DB.
@@ -386,8 +396,17 @@ mod tests {
         db_opts.create_if_missing(true);
         db_opts.create_missing_column_families(true);
 
-        let db = DB::open(tmpdir.path(), "test_db_debug", column_families, &db_opts)
-            .expect("Failed to open DB.");
+        let block_opts = rocksdb::BlockBasedOptions::default();
+        let db = DB::open(
+            tmpdir.path(),
+            "test_db_debug",
+            column_families,
+            &RawRocksdbOptions {
+                db_options: db_opts,
+                block_options: block_opts,
+            },
+        )
+        .expect("Failed to open DB.");
 
         let db_debug = format!("{:?}", db);
         assert!(db_debug.contains("test_db_debug"));

crates/sovereign-sdk/full-node/db/sov-schema-db/tests/db_test.rs

@@ -6,7 +6,7 @@ use std::path::Path;
 use rocksdb::DEFAULT_COLUMN_FAMILY_NAME;
 use sov_schema_db::schema::{ColumnFamilyName, Result};
 use sov_schema_db::test::TestField;
-use sov_schema_db::{define_schema, Schema, SchemaBatch, DB};
+use sov_schema_db::{define_schema, RawRocksdbOptions, Schema, SchemaBatch, DB};
 use tempfile::TempDir;
 
 // Creating two schemas that share exactly the same structure but are stored in different column
@@ -28,7 +28,17 @@ fn open_db(dir: impl AsRef<Path>) -> DB {
     let mut db_opts = rocksdb::Options::default();
     db_opts.create_if_missing(true);
     db_opts.create_missing_column_families(true);
-    DB::open(dir, "test", get_column_families(), &db_opts).expect("Failed to open DB.")
+    let block_opts = rocksdb::BlockBasedOptions::default();
+    DB::open(
+        dir,
+        "test",
+        get_column_families(),
+        &RawRocksdbOptions {
+            db_options: db_opts,
+            block_options: block_opts,
+        },
+    )
+    .expect("Failed to open DB.")
 }
 
 fn open_db_read_only(dir: &TempDir) -> DB {

crates/sovereign-sdk/full-node/db/sov-schema-db/tests/iterator_test.rs

@@ -8,7 +8,8 @@ use sov_schema_db::schema::{KeyDecoder, KeyEncoder, ValueCodec};
 use sov_schema_db::snapshot::{DbSnapshot, ReadOnlyLock, SingleSnapshotQueryManager};
 use sov_schema_db::test::{KeyPrefix1, KeyPrefix2, TestCompositeField, TestField};
 use sov_schema_db::{
-    define_schema, Operation, Schema, SchemaBatch, SchemaIterator, SeekKeyEncoder, DB,
+    define_schema, Operation, RawRocksdbOptions, Schema, SchemaBatch, SchemaIterator,
+    SeekKeyEncoder, DB,
 };
 use tempfile::TempDir;
 
@@ -44,7 +45,17 @@ impl TestDB {
         let mut db_opts = rocksdb::Options::default();
         db_opts.create_if_missing(true);
         db_opts.create_missing_column_families(true);
-        let db = DB::open(tmpdir.path(), "test", column_families, &db_opts).unwrap();
+        let block_opts = rocksdb::BlockBasedOptions::default();
+        let db = DB::open(
+            tmpdir.path(),
+            "test",
+            column_families,
+            &RawRocksdbOptions {
+                db_options: db_opts,
+                block_options: block_opts,
+            },
+        )
+        .unwrap();
 
         db.put::<S>(&TestCompositeField(1, 0, 0), &TestField(100))
             .unwrap();

crates/sovereign-sdk/full-node/sov-prover-storage-manager/src/snapshot_manager.rs

@@ -309,13 +309,8 @@ mod tests {
 
     fn create_test_db(path: &std::path::Path) -> sov_schema_db::DB {
         let tables = vec![DUMMY_STATE_CF.to_string()];
-        sov_schema_db::DB::open(
-            path,
-            "test_db",
-            tables,
-            &RocksdbConfig::new(path, None).as_rocksdb_options(false),
-        )
-        .unwrap()
+        let raw_options = RocksdbConfig::new(path, None).as_raw_options(false);
+        sov_schema_db::DB::open(path, "test_db", tables, &raw_options).unwrap()
     }
 
     #[test]