From a2f11f6d7e51858b32b193fd05eec3b2d4a136ee Mon Sep 17 00:00:00 2001 From: Evgeniy Scherbina Date: Fri, 19 Jul 2024 13:31:29 -0400 Subject: [PATCH] Allow individual database configuration (#8) * Allow individual database configuration * Update README.md --- README.md | 27 +++++++++++- opendb_rocksdb.go | 95 ++++++++++++++++++++++++++++-------------- opendb_rocksdb_test.go | 22 ++++++++++ 3 files changed, 111 insertions(+), 33 deletions(-) diff --git a/README.md b/README.md index d29d360..9973402 100644 --- a/README.md +++ b/README.md @@ -35,6 +35,31 @@ There are 2 main scenarios: - opendb loads stored rocksdb configuration and starts with it - opendb overrides options which explicitly specified in appOpts (app.toml) +#### Individual database configuration + +`app.toml` example: +```toml +[rocksdb] +enable-metrics = true +report-metrics-interval-secs = 15 +max-open-files = 16384 +... + +[rocksdb.blockstore] +max-open-files = -1 +block_size = 4096 + +[rocksdb.tx_index] +max-open-files = -1 +block_size = 4096 +``` + +`[rocksdb]` section contains `default`/`fallback` database configuration, but you can override it with `database-specific` configuration + +it's very convenient approach, because rocksdb configuration is pretty extensive, contains around 30 parameters, but usually we override only few of them + +so we can define standard configuration in `[rocksdb]` section, and then override only few params in `database-specific` configurations + ### List of databases: | Name | Subsystem | IAVL V1 size as of 10.5 millions blocks | IAVL V1 number of SST files as of 10.5 millions blocks | @@ -45,7 +70,7 @@ There are 2 main scenarios: | state.db | Ethermint | 282 GB | 5.5k | | tx_index.db | Ethermint | 504 GB | 11.6k | | evidence.db | Ethermint | 28 MB | 270 | -| evm_indexer.db | Ethermint | 2.2 GB | 230 | +| evmindexer.db | Ethermint | 2.2 GB | 230 | ### List of reported rocksdb metrics: diff --git a/opendb_rocksdb.go b/opendb_rocksdb.go index 5e37e07..a75c963 100644 --- a/opendb_rocksdb.go +++ b/opendb_rocksdb.go @@ -42,39 +42,39 @@ const ( DefaultColumnFamilyName = "default" - enableMetricsOptName = "rocksdb.enable-metrics" - reportMetricsIntervalSecsOptName = "rocksdb.report-metrics-interval-secs" + enableMetricsOptName = "enable-metrics" + reportMetricsIntervalSecsOptName = "report-metrics-interval-secs" defaultReportMetricsIntervalSecs = 15 - maxOpenFilesDBOptName = "rocksdb.max-open-files" - maxFileOpeningThreadsDBOptName = "rocksdb.max-file-opening-threads" - tableCacheNumshardbitsDBOptName = "rocksdb.table_cache_numshardbits" - allowMMAPWritesDBOptName = "rocksdb.allow_mmap_writes" - allowMMAPReadsDBOptName = "rocksdb.allow_mmap_reads" - useFsyncDBOptName = "rocksdb.use_fsync" - useAdaptiveMutexDBOptName = "rocksdb.use_adaptive_mutex" - bytesPerSyncDBOptName = "rocksdb.bytes_per_sync" - maxBackgroundJobsDBOptName = "rocksdb.max-background-jobs" - - writeBufferSizeCFOptName = "rocksdb.write-buffer-size" - numLevelsCFOptName = "rocksdb.num-levels" - maxWriteBufferNumberCFOptName = "rocksdb.max_write_buffer_number" - minWriteBufferNumberToMergeCFOptName = "rocksdb.min_write_buffer_number_to_merge" - maxBytesForLevelBaseCFOptName = "rocksdb.max_bytes_for_level_base" - maxBytesForLevelMultiplierCFOptName = "rocksdb.max_bytes_for_level_multiplier" - targetFileSizeBaseCFOptName = "rocksdb.target_file_size_base" - targetFileSizeMultiplierCFOptName = "rocksdb.target_file_size_multiplier" - level0FileNumCompactionTriggerCFOptName = "rocksdb.level0_file_num_compaction_trigger" - level0SlowdownWritesTriggerCFOptName = "rocksdb.level0_slowdown_writes_trigger" - - blockCacheSizeBBTOOptName = "rocksdb.block_cache_size" - bitsPerKeyBBTOOptName = "rocksdb.bits_per_key" - blockSizeBBTOOptName = "rocksdb.block_size" - cacheIndexAndFilterBlocksBBTOOptName = "rocksdb.cache_index_and_filter_blocks" - pinL0FilterAndIndexBlocksInCacheBBTOOptName = "rocksdb.pin_l0_filter_and_index_blocks_in_cache" - formatVersionBBTOOptName = "rocksdb.format_version" - - asyncIOReadOptName = "rocksdb.read-async-io" + maxOpenFilesDBOptName = "max-open-files" + maxFileOpeningThreadsDBOptName = "max-file-opening-threads" + tableCacheNumshardbitsDBOptName = "table_cache_numshardbits" + allowMMAPWritesDBOptName = "allow_mmap_writes" + allowMMAPReadsDBOptName = "allow_mmap_reads" + useFsyncDBOptName = "use_fsync" + useAdaptiveMutexDBOptName = "use_adaptive_mutex" + bytesPerSyncDBOptName = "bytes_per_sync" + maxBackgroundJobsDBOptName = "max-background-jobs" + + writeBufferSizeCFOptName = "write-buffer-size" + numLevelsCFOptName = "num-levels" + maxWriteBufferNumberCFOptName = "max_write_buffer_number" + minWriteBufferNumberToMergeCFOptName = "min_write_buffer_number_to_merge" + maxBytesForLevelBaseCFOptName = "max_bytes_for_level_base" + maxBytesForLevelMultiplierCFOptName = "max_bytes_for_level_multiplier" + targetFileSizeBaseCFOptName = "target_file_size_base" + targetFileSizeMultiplierCFOptName = "target_file_size_multiplier" + level0FileNumCompactionTriggerCFOptName = "level0_file_num_compaction_trigger" + level0SlowdownWritesTriggerCFOptName = "level0_slowdown_writes_trigger" + + blockCacheSizeBBTOOptName = "block_cache_size" + bitsPerKeyBBTOOptName = "bits_per_key" + blockSizeBBTOOptName = "block_size" + cacheIndexAndFilterBlocksBBTOOptName = "cache_index_and_filter_blocks" + pinL0FilterAndIndexBlocksInCacheBBTOOptName = "pin_l0_filter_and_index_blocks_in_cache" + formatVersionBBTOOptName = "format_version" + + asyncIOReadOptName = "read-async-io" ) // AppOptions is the same interface as provided by cosmos-sdk, see for details: @@ -84,9 +84,40 @@ type AppOptions interface { Get(string) interface{} } +// rocksDBOptions implements AppOptions interface. +// It does it by wrapping another AppOptions, but also takes into account dbName. +type rocksDBOptions struct { + appOpts AppOptions + dbName string +} + +func newRocksDBOptions(appOpts AppOptions, dbName string) *rocksDBOptions { + return &rocksDBOptions{ + appOpts: appOpts, + dbName: dbName, + } +} + +// Get constructs database-specific and fallback keys and use them to get value from underlying AppOptions. +// Database-specific key takes precedence over fallback key. +func (opts *rocksDBOptions) Get(key string) interface{} { + // get value using database-specific key + dbSpecificKey := fmt.Sprintf("rocksdb.%v.%v", opts.dbName, key) + if opts.appOpts.Get(dbSpecificKey) != nil { + return opts.appOpts.Get(dbSpecificKey) + } + + // get value using fallback key + fallbackKey := fmt.Sprintf("rocksdb.%v", key) + return opts.appOpts.Get(fallbackKey) +} + func OpenDB(appOpts AppOptions, dataDir string, dbName string, backendType dbm.BackendType) (dbm.DB, error) { + // wrap AppOptions with rocksDBOptions to make sure dbName is considered when applying configuration + // it allows individual database configuration + rocksDBOpts := newRocksDBOptions(appOpts, dbName) if backendType == dbm.RocksDBBackend { - return openRocksdb(dataDir, dbName, appOpts) + return openRocksdb(dataDir, dbName, rocksDBOpts) } return dbm.NewDB(dbName, backendType, dataDir) diff --git a/opendb_rocksdb_test.go b/opendb_rocksdb_test.go index c7ec03e..2307e00 100644 --- a/opendb_rocksdb_test.go +++ b/opendb_rocksdb_test.go @@ -28,6 +28,28 @@ func (m *mockAppOptions) Get(key string) interface{} { return m.opts[key] } +func TestRocksDBOptions(t *testing.T) { + mockAppOptions := newMockAppOptions(map[string]interface{}{ + // fallback configuration + "rocksdb.max-open-files": 16_384, + "rocksdb.block_size": 16_384, + + // database-specific configuration + "rocksdb.tx_index.max-open-files": 4096, + "rocksdb.tx_index.block_size": 4096, + }) + + // there isn't database-specific configuration for application database, so fallback to fallback configuration + appDBOpts := newRocksDBOptions(mockAppOptions, "application") + require.Equal(t, 16_384, appDBOpts.Get("max-open-files")) + require.Equal(t, 16_384, appDBOpts.Get("block_size")) + + // there is database-specific configuration for tx_index database, so use it instead of fallback configuration + txIndexDBOpts := newRocksDBOptions(mockAppOptions, "tx_index") + require.Equal(t, 4096, txIndexDBOpts.Get("max-open-files")) + require.Equal(t, 4096, txIndexDBOpts.Get("block_size")) +} + func TestOpenRocksdb(t *testing.T) { t.Run("db already exists", func(t *testing.T) { defaultOpts := newDefaultOptions()