From c020608417f7295cb56b0821744017930b8106c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ahmet=20Yaz=C4=B1c=C4=B1?= <75089142+yaziciahmet@users.noreply.github.com> Date: Wed, 30 Oct 2024 14:31:27 +0100 Subject: [PATCH] Fix rocksdb config (#1386) --- Cargo.lock | 1 + .../full-node/db/sov-db/Cargo.toml | 1 + .../full-node/db/sov-db/src/ledger_db/mod.rs | 3 +- .../full-node/db/sov-db/src/native_db.rs | 3 +- .../db/sov-db/src/rocks_db_config.rs | 44 ++++++++++--------- .../full-node/db/sov-db/src/state_db.rs | 3 +- .../full-node/db/sov-schema-db/src/lib.rs | 27 ++++++++++-- .../db/sov-schema-db/tests/db_test.rs | 14 +++++- .../db/sov-schema-db/tests/iterator_test.rs | 15 ++++++- .../src/snapshot_manager.rs | 9 +--- 10 files changed, 81 insertions(+), 39 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1ae71d6cc..3cae4de96 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7508,6 +7508,7 @@ dependencies = [ "criterion", "hex", "jmt", + "num_cpus", "proptest", "proptest-derive", "rand 0.8.5", diff --git a/crates/sovereign-sdk/full-node/db/sov-db/Cargo.toml b/crates/sovereign-sdk/full-node/db/sov-db/Cargo.toml index 17493fed0..176db14e4 100644 --- a/crates/sovereign-sdk/full-node/db/sov-db/Cargo.toml +++ b/crates/sovereign-sdk/full-node/db/sov-db/Cargo.toml @@ -26,6 +26,7 @@ bincode = { workspace = true } borsh = { workspace = true, default-features = true, features = ["bytes", "rc"] } byteorder = { workspace = true, default-features = true } hex = { workspace = true } +num_cpus = { workspace = true } proptest = { workspace = true, optional = true, default-features = true } proptest-derive = { workspace = true, optional = true } rlimit = { workspace = true } diff --git a/crates/sovereign-sdk/full-node/db/sov-db/src/ledger_db/mod.rs b/crates/sovereign-sdk/full-node/db/sov-db/src/ledger_db/mod.rs index e07a8f2d1..816b33c39 100644 --- a/crates/sovereign-sdk/full-node/db/sov-db/src/ledger_db/mod.rs +++ b/crates/sovereign-sdk/full-node/db/sov-db/src/ledger_db/mod.rs @@ -105,11 +105,12 @@ impl LedgerDB { #[instrument(level = "trace", skip_all, err)] pub fn with_config(cfg: &RocksdbConfig) -> Result { let path = cfg.path.join(LEDGER_DB_PATH_SUFFIX); + let raw_options = cfg.as_raw_options(false); let inner = DB::open( path, "ledger-db", LEDGER_TABLES.iter().copied(), - &cfg.as_rocksdb_options(false), + &raw_options, )?; let next_item_numbers = ItemNumbers { diff --git a/crates/sovereign-sdk/full-node/db/sov-db/src/native_db.rs b/crates/sovereign-sdk/full-node/db/sov-db/src/native_db.rs index 946c5084a..d409e9404 100644 --- a/crates/sovereign-sdk/full-node/db/sov-db/src/native_db.rs +++ b/crates/sovereign-sdk/full-node/db/sov-db/src/native_db.rs @@ -32,12 +32,13 @@ impl NativeDB { /// Initialize [`sov_schema_db::DB`] that matches tables and columns for NativeDB pub fn setup_schema_db(cfg: &RocksdbConfig) -> anyhow::Result { + let raw_options = cfg.as_raw_options(false); let path = cfg.path.join(Self::DB_PATH_SUFFIX); sov_schema_db::DB::open( path, Self::DB_NAME, NATIVE_TABLES.iter().copied(), - &cfg.as_rocksdb_options(false), + &raw_options, ) } /// Convert it to [`ReadOnlyDbSnapshot`] which cannot be edited anymore diff --git a/crates/sovereign-sdk/full-node/db/sov-db/src/rocks_db_config.rs b/crates/sovereign-sdk/full-node/db/sov-db/src/rocks_db_config.rs index acfb5d4de..f18eb1659 100644 --- a/crates/sovereign-sdk/full-node/db/sov-db/src/rocks_db_config.rs +++ b/crates/sovereign-sdk/full-node/db/sov-db/src/rocks_db_config.rs @@ -4,6 +4,7 @@ use std::path::Path; use rlimit::{getrlimit, Resource}; use rocksdb::{BlockBasedOptions, Cache, Options}; +use sov_schema_db::RawRocksdbOptions; use tracing::warn; /// Port selected RocksDB options for tuning underlying rocksdb instance of our state db. @@ -41,11 +42,11 @@ impl<'a> RocksdbConfig<'a> { } } - /// Build [`rocksdb::Options`] from [`RocksdbConfig`] - pub fn as_rocksdb_options(&self, readonly: bool) -> Options { - let mut db_opts = Options::default(); + /// Build [`RawRocksdbOptions`] from [`RocksdbConfig`] + pub fn as_raw_options(&self, readonly: bool) -> RawRocksdbOptions { + let mut db_options = Options::default(); - let mut block_based_options = BlockBasedOptions::default(); + let mut block_options = BlockBasedOptions::default(); /* * The following settings are recommended in: * https://github.com/facebook/rocksdb/wiki/memory-usage-in-rocksdb @@ -57,32 +58,33 @@ impl<'a> RocksdbConfig<'a> { // could still have OOM errors when trying to allocate more for the cache even if the capacity // is reached. let cache = Cache::new_lru_cache(100 * 1024 * 1024); // 100 MB - block_based_options.set_block_cache(&cache); + block_options.set_block_cache(&cache); // jemalloc friendly bloom filter sizing - block_based_options.set_optimize_filters_for_memory(true); + block_options.set_optimize_filters_for_memory(true); // By default our block size is 4KB, we set this to 32KB. // Increasing the size of the block decreases the number of blocks, - // therefore, less memory consumption for indicies. - block_based_options.set_block_size(32 * 1024); + // therefore, less memory consumption for indices. + block_options.set_block_size(32 * 1024); // Default is Snappy but Lz4 is recommend // https://github.com/facebook/rocksdb/wiki/Compression - db_opts.set_compression_type(rocksdb::DBCompressionType::Lz4); - db_opts.set_compression_options_parallel_threads(4); + db_options.set_compression_type(rocksdb::DBCompressionType::Lz4); - db_opts.set_max_open_files(self.max_open_files); - db_opts.set_max_total_wal_size(self.max_total_wal_size); - db_opts.set_max_background_jobs(self.max_background_jobs); - if !readonly { - // Increase write buffer size to reduce allocations. - db_opts.set_write_buffer_size(30 * 1024 * 1024); // 30 MB - db_opts.set_block_based_table_factory(&block_based_options); + let allowed_cores = std::cmp::max(1, num_cpus::get() / 2) as i32; + db_options.set_compression_options_parallel_threads(allowed_cores); + db_options.increase_parallelism(allowed_cores); - db_opts.create_if_missing(true); - db_opts.create_missing_column_families(true); - db_opts.set_atomic_flush(true); + db_options.set_max_open_files(self.max_open_files); + db_options.set_max_total_wal_size(self.max_total_wal_size); + db_options.set_max_background_jobs(self.max_background_jobs); + if !readonly { + db_options.create_if_missing(true); + db_options.create_missing_column_families(true); } - db_opts + RawRocksdbOptions { + db_options, + block_options, + } } } diff --git a/crates/sovereign-sdk/full-node/db/sov-db/src/state_db.rs b/crates/sovereign-sdk/full-node/db/sov-db/src/state_db.rs index 442b03963..5e0883cb0 100644 --- a/crates/sovereign-sdk/full-node/db/sov-db/src/state_db.rs +++ b/crates/sovereign-sdk/full-node/db/sov-db/src/state_db.rs @@ -40,12 +40,13 @@ impl StateDB { /// Initialize [`sov_schema_db::DB`] that should be used by snapshots. pub fn setup_schema_db(cfg: &RocksdbConfig) -> anyhow::Result { + let raw_options = cfg.as_raw_options(false); let state_db_path = cfg.path.join(Self::DB_PATH_SUFFIX); sov_schema_db::DB::open( state_db_path, Self::DB_NAME, STATE_TABLES.iter().copied(), - &cfg.as_rocksdb_options(false), + &raw_options, ) } diff --git a/crates/sovereign-sdk/full-node/db/sov-schema-db/src/lib.rs b/crates/sovereign-sdk/full-node/db/sov-schema-db/src/lib.rs index 295ca25c1..9347be18e 100644 --- a/crates/sovereign-sdk/full-node/db/sov-schema-db/src/lib.rs +++ b/crates/sovereign-sdk/full-node/db/sov-schema-db/src/lib.rs @@ -56,15 +56,16 @@ impl DB { path: impl AsRef, name: &'static str, column_families: impl IntoIterator>, - db_opts: &rocksdb::Options, + options: &RawRocksdbOptions, ) -> anyhow::Result { let db = DB::open_with_cfds( - db_opts, + &options.db_options, path, name, column_families.into_iter().map(|cf_name| { let mut cf_opts = rocksdb::Options::default(); cf_opts.set_compression_type(rocksdb::DBCompressionType::Lz4); + cf_opts.set_block_based_table_factory(&options.block_options); rocksdb::ColumnFamilyDescriptor::new(cf_name, cf_opts) }), )?; @@ -328,6 +329,15 @@ impl DB { } } +/// Raw rocksdb config wrapper. Useful to convert user provided config into +/// the actual rocksdb config with all defaults set. +pub struct RawRocksdbOptions { + /// Global db options + pub db_options: rocksdb::Options, + /// Per column-family options + pub block_options: rocksdb::BlockBasedOptions, +} + /// Readability alias for a key in the DB. pub type SchemaKey = Vec; /// Readability alias for a value in the DB. @@ -386,8 +396,17 @@ mod tests { db_opts.create_if_missing(true); db_opts.create_missing_column_families(true); - let db = DB::open(tmpdir.path(), "test_db_debug", column_families, &db_opts) - .expect("Failed to open DB."); + let block_opts = rocksdb::BlockBasedOptions::default(); + let db = DB::open( + tmpdir.path(), + "test_db_debug", + column_families, + &RawRocksdbOptions { + db_options: db_opts, + block_options: block_opts, + }, + ) + .expect("Failed to open DB."); let db_debug = format!("{:?}", db); assert!(db_debug.contains("test_db_debug")); diff --git a/crates/sovereign-sdk/full-node/db/sov-schema-db/tests/db_test.rs b/crates/sovereign-sdk/full-node/db/sov-schema-db/tests/db_test.rs index 5a9b0e0c7..26d38fe14 100644 --- a/crates/sovereign-sdk/full-node/db/sov-schema-db/tests/db_test.rs +++ b/crates/sovereign-sdk/full-node/db/sov-schema-db/tests/db_test.rs @@ -6,7 +6,7 @@ use std::path::Path; use rocksdb::DEFAULT_COLUMN_FAMILY_NAME; use sov_schema_db::schema::{ColumnFamilyName, Result}; use sov_schema_db::test::TestField; -use sov_schema_db::{define_schema, Schema, SchemaBatch, DB}; +use sov_schema_db::{define_schema, RawRocksdbOptions, Schema, SchemaBatch, DB}; use tempfile::TempDir; // Creating two schemas that share exactly the same structure but are stored in different column @@ -28,7 +28,17 @@ fn open_db(dir: impl AsRef) -> DB { let mut db_opts = rocksdb::Options::default(); db_opts.create_if_missing(true); db_opts.create_missing_column_families(true); - DB::open(dir, "test", get_column_families(), &db_opts).expect("Failed to open DB.") + let block_opts = rocksdb::BlockBasedOptions::default(); + DB::open( + dir, + "test", + get_column_families(), + &RawRocksdbOptions { + db_options: db_opts, + block_options: block_opts, + }, + ) + .expect("Failed to open DB.") } fn open_db_read_only(dir: &TempDir) -> DB { diff --git a/crates/sovereign-sdk/full-node/db/sov-schema-db/tests/iterator_test.rs b/crates/sovereign-sdk/full-node/db/sov-schema-db/tests/iterator_test.rs index a90282e50..aeac215c4 100644 --- a/crates/sovereign-sdk/full-node/db/sov-schema-db/tests/iterator_test.rs +++ b/crates/sovereign-sdk/full-node/db/sov-schema-db/tests/iterator_test.rs @@ -8,7 +8,8 @@ use sov_schema_db::schema::{KeyDecoder, KeyEncoder, ValueCodec}; use sov_schema_db::snapshot::{DbSnapshot, ReadOnlyLock, SingleSnapshotQueryManager}; use sov_schema_db::test::{KeyPrefix1, KeyPrefix2, TestCompositeField, TestField}; use sov_schema_db::{ - define_schema, Operation, Schema, SchemaBatch, SchemaIterator, SeekKeyEncoder, DB, + define_schema, Operation, RawRocksdbOptions, Schema, SchemaBatch, SchemaIterator, + SeekKeyEncoder, DB, }; use tempfile::TempDir; @@ -44,7 +45,17 @@ impl TestDB { let mut db_opts = rocksdb::Options::default(); db_opts.create_if_missing(true); db_opts.create_missing_column_families(true); - let db = DB::open(tmpdir.path(), "test", column_families, &db_opts).unwrap(); + let block_opts = rocksdb::BlockBasedOptions::default(); + let db = DB::open( + tmpdir.path(), + "test", + column_families, + &RawRocksdbOptions { + db_options: db_opts, + block_options: block_opts, + }, + ) + .unwrap(); db.put::(&TestCompositeField(1, 0, 0), &TestField(100)) .unwrap(); diff --git a/crates/sovereign-sdk/full-node/sov-prover-storage-manager/src/snapshot_manager.rs b/crates/sovereign-sdk/full-node/sov-prover-storage-manager/src/snapshot_manager.rs index 6617fccc2..e8b70a17a 100644 --- a/crates/sovereign-sdk/full-node/sov-prover-storage-manager/src/snapshot_manager.rs +++ b/crates/sovereign-sdk/full-node/sov-prover-storage-manager/src/snapshot_manager.rs @@ -309,13 +309,8 @@ mod tests { fn create_test_db(path: &std::path::Path) -> sov_schema_db::DB { let tables = vec![DUMMY_STATE_CF.to_string()]; - sov_schema_db::DB::open( - path, - "test_db", - tables, - &RocksdbConfig::new(path, None).as_rocksdb_options(false), - ) - .unwrap() + let raw_options = RocksdbConfig::new(path, None).as_raw_options(false); + sov_schema_db::DB::open(path, "test_db", tables, &raw_options).unwrap() } #[test]