Skip to content

Commit

Permalink
feat: ELECTRS_ROCKSDB_WRITE_BUFFER_SIZE
Browse files Browse the repository at this point in the history
In testing setups large write buffer sizes lead to large
db files that potentially waste resources / interfere
with CIs limits.

This change adds an ability to overwrite the default behavior.
  • Loading branch information
dpc committed Dec 14, 2023
1 parent adedee1 commit a6169da
Showing 1 changed file with 16 additions and 1 deletion.
17 changes: 16 additions & 1 deletion src/new_index/db.rs
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
use rocksdb;

use std::path::Path;
use std::str::FromStr;

use crate::config::Config;
use crate::util::Bytes;
Expand Down Expand Up @@ -80,6 +81,20 @@ pub enum DBFlush {
Enable,
}

fn get_write_buffer_size() -> usize {
const VAR_NAME: &str = "ELECTRS_ROCKSDB_WRITE_BUFFER_SIZE";

if let Ok(var) = std::env::var(VAR_NAME) {
let size = FromStr::from_str(&var).expect("Could not parse {VAR_NAME}");

info!("Using custom write buffer size: {size}");

size
} else {
256 << 20
}
}

impl DB {
pub fn open(path: &Path, config: &Config) -> DB {
debug!("opening DB at {:?}", path);
Expand All @@ -89,7 +104,7 @@ impl DB {
db_opts.set_compaction_style(rocksdb::DBCompactionStyle::Level);
db_opts.set_compression_type(rocksdb::DBCompressionType::Snappy);
db_opts.set_target_file_size_base(1_073_741_824);
db_opts.set_write_buffer_size(256 << 20);
db_opts.set_write_buffer_size(get_write_buffer_size());
db_opts.set_disable_auto_compactions(true); // for initial bulk load

// db_opts.set_advise_random_on_open(???);
Expand Down

0 comments on commit a6169da

Please sign in to comment.