From 2a0a6190103ac1b6b547b762622b66467e10893f Mon Sep 17 00:00:00 2001
From: Luiz Irber <contact+github@luizirber.org>
Date: Mon, 9 Oct 2023 16:15:23 -0700
Subject: [PATCH] save version in DB

---
 src/core/src/index/revindex/disk_revindex.rs | 12 +++++++++++-
 src/core/src/index/revindex/mod.rs           |  9 +++++++--
 2 files changed, 18 insertions(+), 3 deletions(-)

diff --git a/src/core/src/index/revindex/disk_revindex.rs b/src/core/src/index/revindex/disk_revindex.rs
index 5ab4a5c321..05efad9ecb 100644
--- a/src/core/src/index/revindex/disk_revindex.rs
+++ b/src/core/src/index/revindex/disk_revindex.rs
@@ -12,7 +12,7 @@ use crate::collection::{Collection, CollectionSet};
 use crate::encodings::{Color, Idx};
 use crate::index::revindex::{
     self as module, prepare_query, stats_for_cf, Datasets, DbStats, HashToColor, QueryColors,
-    RevIndexOps, DB, HASHES, MANIFEST, METADATA, STORAGE_SPEC,
+    RevIndexOps, DB, HASHES, MANIFEST, METADATA, STORAGE_SPEC, VERSION,
 };
 use crate::index::{GatherResult, SigCounter};
 use crate::manifest::Manifest;
@@ -23,6 +23,8 @@ use crate::sketch::Sketch;
 use crate::storage::{InnerStorage, Storage};
 use crate::Result;
 
+const DB_VERSION: u8 = 1;
+
 fn compute_color(idxs: &Datasets) -> Color {
     let s = BuildHasherDefault::<twox_hash::Xxh3Hash128>::default();
     let mut hasher = s.build_hasher();
@@ -126,6 +128,9 @@ impl RevIndex {
     fn load_collection_from_rocksdb(db: Arc<DB>) -> Result<CollectionSet> {
         let cf_metadata = db.cf_handle(METADATA).unwrap();
 
+        let rdr = db.get_cf(&cf_metadata, VERSION)?.unwrap();
+        assert_eq!(rdr[0], DB_VERSION);
+
         let rdr = db.get_cf(&cf_metadata, MANIFEST)?.unwrap();
         let manifest = Manifest::from_reader(&rdr[..])?;
 
@@ -144,6 +149,11 @@ impl RevIndex {
     fn save_collection(&self) -> Result<()> {
         let cf_metadata = self.db.cf_handle(METADATA).unwrap();
 
+        // save DB version
+        // TODO: probably should go together with a more general
+        //       saving procedure used in create/update
+        self.db.put_cf(&cf_metadata, VERSION, &[DB_VERSION])?;
+
         // write manifest
         let mut wtr = vec![];
         {
diff --git a/src/core/src/index/revindex/mod.rs b/src/core/src/index/revindex/mod.rs
index 36245c604d..0765ee71d9 100644
--- a/src/core/src/index/revindex/mod.rs
+++ b/src/core/src/index/revindex/mod.rs
@@ -30,11 +30,15 @@ type HashToColorT = HashMap<HashIntoType, Color, BuildNoHashHasher<HashIntoType>
 #[derive(Serialize, Deserialize)]
 pub struct HashToColor(HashToColorT);
 
+// Column families
 const HASHES: &str = "hashes";
 const COLORS: &str = "colors";
 const METADATA: &str = "metadata";
+
+// DB metadata saved in the METADATA column family
 const MANIFEST: &str = "manifest";
 const STORAGE_SPEC: &str = "storage_spec";
+const VERSION: &str = "version";
 
 #[enum_dispatch(RevIndexOps)]
 pub enum RevIndex {
@@ -205,11 +209,12 @@ impl RevIndex {
         block_opts.set_block_size(16 * 1024);
         block_opts.set_cache_index_and_filter_blocks(true);
         block_opts.set_pin_l0_filter_and_index_blocks_in_cache(true);
-        block_opts.set_format_version(5);
+        block_opts.set_format_version(6);
         opts.set_block_based_table_factory(&block_opts);
         // End of updated defaults
 
-        opts.increase_parallelism(8);
+        opts.increase_parallelism(rayon::current_num_threads() as i32);
+        //opts.max_background_jobs = 6;
         // opts.optimize_level_style_compaction();
         // opts.optimize_universal_style_compaction();