From c94924baefd166d71bb9d7d1218f324401d84cce Mon Sep 17 00:00:00 2001 From: Abner Zheng Date: Wed, 29 May 2024 13:58:35 +0800 Subject: [PATCH] fix --- .../src/bin/compaction-simulator.rs | 6 +- mini-lsm-starter/src/compact/leveled.rs | 4 +- mini-lsm-starter/src/lsm_storage.rs | 8 ++- mini-lsm-starter/src/tests.rs | 1 + mini-lsm-starter/src/tests/harness.rs | 28 +++++----- mini-lsm-starter/src/tests/week2_day5.rs | 3 + mini-lsm-starter/src/tests/week3_day1.rs | 56 +++++++++++++++++++ 7 files changed, 86 insertions(+), 20 deletions(-) create mode 100644 mini-lsm-starter/src/tests/week3_day1.rs diff --git a/mini-lsm-starter/src/bin/compaction-simulator.rs b/mini-lsm-starter/src/bin/compaction-simulator.rs index e63f8b2..8c6c1e8 100644 --- a/mini-lsm-starter/src/bin/compaction-simulator.rs +++ b/mini-lsm-starter/src/bin/compaction-simulator.rs @@ -14,8 +14,9 @@ use mini_lsm_wrapper::key::KeyBytes; use mini_lsm_wrapper::lsm_storage::LsmStorageState; use mini_lsm_wrapper::mem_table::MemTable; use mini_lsm_wrapper::table::SsTable; -use rand::SeedableRng; -use rand_chacha::{ChaCha12Rng, ChaCha8Rng}; +use rand::Rng; +use rand_chacha::rand_core::SeedableRng; +use rand_chacha::ChaCha12Rng; #[derive(Parser, Debug)] #[command(author, version, about, long_about = None)] @@ -188,7 +189,6 @@ impl MockStorage { } fn generate_random_key_range() -> (KeyBytes, KeyBytes) { - use rand::Rng; let mut rng = ChaCha12Rng::seed_from_u64(10086); let begin: usize = rng.gen_range(0..(1 << 31)); let end: usize = begin + rng.gen_range((1 << 10)..(1 << 31)); diff --git a/mini-lsm-starter/src/compact/leveled.rs b/mini-lsm-starter/src/compact/leveled.rs index 235edd3..daa0688 100644 --- a/mini-lsm-starter/src/compact/leveled.rs +++ b/mini-lsm-starter/src/compact/leveled.rs @@ -209,11 +209,11 @@ impl LeveledCompactionController { } res }; - // find the right place to insert compacted SsTable - let first_key = snapshot.sstables[&output[0]].first_key(); if lower_level_kept.is_empty() { snapshot.levels[*lower_level] = (*level_id, output.to_vec()); } else { + // find the right place to insert compacted SsTable + let first_key = snapshot.sstables[&output[0]].first_key(); let insert_idx = lower_level_kept .iter() .position(|sst_id| snapshot.sstables[sst_id].last_key() > first_key); diff --git a/mini-lsm-starter/src/lsm_storage.rs b/mini-lsm-starter/src/lsm_storage.rs index a9b127b..25aa336 100644 --- a/mini-lsm-starter/src/lsm_storage.rs +++ b/mini-lsm-starter/src/lsm_storage.rs @@ -346,7 +346,13 @@ impl LsmStorageInner { for record in records { match record { ManifestRecord::Flush(sst_id) => { - state.l0_sstables.insert(0, sst_id); + if compaction_controller.flush_to_l0() { + println!("flush memtable{sst_id} to l0"); + state.l0_sstables.insert(0, sst_id); + } else { + println!("flush memtable{sst_id} to new tier"); + state.levels.insert(0, (sst_id, vec![sst_id])); + } let option = memtable_to_recover.pop(); assert_eq!(option, Some(sst_id), "failed to flush memtable"); sst_need_load.insert(sst_id); diff --git a/mini-lsm-starter/src/tests.rs b/mini-lsm-starter/src/tests.rs index 0a30b32..a5acad1 100644 --- a/mini-lsm-starter/src/tests.rs +++ b/mini-lsm-starter/src/tests.rs @@ -15,3 +15,4 @@ mod week2_day3; mod week2_day4; mod week2_day5; mod week2_day6; +mod week3_day1; diff --git a/mini-lsm-starter/src/tests/harness.rs b/mini-lsm-starter/src/tests/harness.rs index 64df423..d2208b0 100644 --- a/mini-lsm-starter/src/tests/harness.rs +++ b/mini-lsm-starter/src/tests/harness.rs @@ -104,20 +104,20 @@ where // as_bytes(iter.key().for_testing_key_ref()) // ); assert!(iter.is_valid()); - // assert_eq!( - // k, - // iter.key().for_testing_key_ref(), - // "expected key: {:?}, actual key: {:?}", - // k, - // as_bytes(iter.key().for_testing_key_ref()), - // ); - // assert_eq!( - // v, - // iter.value(), - // "expected value: {:?}, actual value: {:?}", - // v, - // as_bytes(iter.value()), - // ); + assert_eq!( + k, + iter.key().for_testing_key_ref(), + "expected key: {:?}, actual key: {:?}", + k, + as_bytes(iter.key().for_testing_key_ref()), + ); + assert_eq!( + v, + iter.value(), + "expected value: {:?}, actual value: {:?}", + v, + as_bytes(iter.value()), + ); iter.next().unwrap(); } assert!(!iter.is_valid()); diff --git a/mini-lsm-starter/src/tests/week2_day5.rs b/mini-lsm-starter/src/tests/week2_day5.rs index 9ceedb3..398b2b2 100644 --- a/mini-lsm-starter/src/tests/week2_day5.rs +++ b/mini-lsm-starter/src/tests/week2_day5.rs @@ -1,3 +1,5 @@ +use std::thread::sleep; +use std::time::Duration; use tempfile::tempdir; use crate::{ @@ -62,6 +64,7 @@ fn test_integration(compaction_options: CompactionOptions) { .force_freeze_memtable(&storage.inner.state_lock.lock()) .unwrap(); } + sleep(Duration::from_secs(1)); storage.close().unwrap(); // ensure all SSTs are flushed assert!(storage.inner.state.read().memtable.is_empty()); diff --git a/mini-lsm-starter/src/tests/week3_day1.rs b/mini-lsm-starter/src/tests/week3_day1.rs new file mode 100644 index 0000000..319c7f3 --- /dev/null +++ b/mini-lsm-starter/src/tests/week3_day1.rs @@ -0,0 +1,56 @@ +use std::sync::Arc; + +use bytes::Bytes; +use tempfile::tempdir; + +use crate::key::KeySlice; +use crate::table::{FileObject, SsTable, SsTableBuilder, SsTableIterator}; + +use super::harness::{check_iter_result_by_key_and_ts, generate_sst_with_ts}; + +#[test] +#[ignore] +fn test_sst_build_multi_version_simple() { + let mut builder = SsTableBuilder::new(16); + builder.add( + KeySlice::for_testing_from_slice_with_ts(b"233", 233), + b"233333", + ); + builder.add( + KeySlice::for_testing_from_slice_with_ts(b"233", 0), + b"2333333", + ); + let dir = tempdir().unwrap(); + builder.build_for_test(dir.path().join("1.sst")).unwrap(); +} + +fn generate_test_data() -> Vec<((Bytes, u64), Bytes)> { + (0..100) + .map(|id| { + ( + (Bytes::from(format!("key{:05}", id / 5)), 5 - (id % 5)), + Bytes::from(format!("value{:05}", id)), + ) + }) + .collect() +} + +#[test] +#[ignore] +fn test_sst_build_multi_version_hard() { + let dir = tempdir().unwrap(); + let data = generate_test_data(); + generate_sst_with_ts(1, dir.path().join("1.sst"), data.clone(), None); + let sst = Arc::new( + SsTable::open( + 1, + None, + FileObject::open(&dir.path().join("1.sst")).unwrap(), + ) + .unwrap(), + ); + check_iter_result_by_key_and_ts( + &mut SsTableIterator::create_and_seek_to_first(sst).unwrap(), + data, + ); +}