handle both old and new meta page formats

Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "jammdb"
 description = "An embedded single-file database for Rust"
-version = "0.10.0"
+version = "0.11.0"
 authors = ["PJ Tatlow <[email protected]>"]
 edition = "2021"
 license = "MIT OR Apache-2.0"
@@ -10,13 +10,7 @@ readme = "README.md"
 keywords = ["db", "database", "embedded-database", "memory-map"]
 categories = ["database", "database-implementations"]
 
-exclude = [
-    ".*.yml",
-    ".github/*",
-    "ci/*",
-    "tests/*",
-    "makefile",
-]
+exclude = [".*.yml", ".github/*", "ci/*", "tests/*", "makefile"]
 
 [dependencies]
 libc = "0.2.149"
@@ -26,6 +20,7 @@ fs4 = "0.7.0"
 bytes = "1.5.0"
 bumpalo = "3.14.0"
 fnv = "1.0.7"
+sha3 = "0.10.8"
 
 [dev-dependencies]
 bytes = { version = "1", features = ["serde"] }

src/bucket.rs

@@ -1029,6 +1029,7 @@ mod tests {
                 let tx = db.tx(true).unwrap();
                 let b = tx.create_bucket("abc").unwrap();
                 tx.delete_bucket("abc").unwrap();
+                #[allow(clippy::redundant_closure_call)]
                 $value(&b);
     		}
     	)*
@@ -1089,6 +1090,7 @@ mod tests {
                 }
                 let tx = db.tx($rw)?;
                 let b = tx.get_bucket("abc")?;
+                #[allow(clippy::redundant_closure_call)]
                 $value(&b);
                 Ok(())
     		}

src/db.rs

@@ -283,56 +283,65 @@ impl DBInner {
 
     pub(crate) fn meta(&self) -> Result<Meta> {
         let data = self.data.lock()?;
-        let meta1 = Page::from_buf(&data, 0, self.pagesize).meta();
-
-        // Double check that we have the right pagesize before we read the second page.
-        if meta1.valid() && meta1.pagesize != self.pagesize {
-            assert_eq!(
-                meta1.pagesize, self.pagesize,
-                "Invalid pagesize from meta1 {}. Expected {}.",
-                meta1.pagesize, self.pagesize
-            );
-        }
 
-        let meta2 = Page::from_buf(&data, 1, self.pagesize).meta();
-        let meta = match (meta1.valid(), meta2.valid()) {
-            (true, true) => {
-                assert_eq!(
-                    meta1.pagesize, self.pagesize,
-                    "Invalid pagesize from meta1 {}. Expected {}.",
-                    meta1.pagesize, self.pagesize
-                );
-                assert_eq!(
-                    meta2.pagesize, self.pagesize,
-                    "Invalid pagesize from meta2 {}. Expected {}.",
-                    meta2.pagesize, self.pagesize
-                );
-                if meta1.tx_id > meta2.tx_id {
-                    meta1
-                } else {
-                    meta2
+        macro_rules! check_meta {
+            ($func:ident) => {{
+                let meta1 = Page::from_buf(&data, 0, self.pagesize).$func();
+                // Double check that we have the right pagesize before we read the second page.
+                if meta1.valid() && meta1.pagesize != self.pagesize {
+                    assert_eq!(
+                        meta1.pagesize, self.pagesize,
+                        "Invalid pagesize from meta1 {}. Expected {}.",
+                        meta1.pagesize, self.pagesize
+                    );
                 }
-            }
-            (true, false) => {
-                assert_eq!(
-                    meta1.pagesize, self.pagesize,
-                    "Invalid pagesize from meta1 {}. Expected {}.",
-                    meta1.pagesize, self.pagesize
-                );
-                meta1
-            }
-            (false, true) => {
-                assert_eq!(
-                    meta2.pagesize, self.pagesize,
-                    "Invalid pagesize from meta2 {}. Expected {}.",
-                    meta2.pagesize, self.pagesize
-                );
-                meta2
-            }
-            (false, false) => panic!("NO VALID META PAGES"),
-        };
+                let meta2 = Page::from_buf(&data, 1, self.pagesize).$func();
+                match (meta1.valid(), meta2.valid()) {
+                    (true, true) => {
+                        assert_eq!(
+                            meta1.pagesize, self.pagesize,
+                            "Invalid pagesize from meta1 {}. Expected {}.",
+                            meta1.pagesize, self.pagesize
+                        );
+                        assert_eq!(
+                            meta2.pagesize, self.pagesize,
+                            "Invalid pagesize from meta2 {}. Expected {}.",
+                            meta2.pagesize, self.pagesize
+                        );
+                        if meta1.tx_id > meta2.tx_id {
+                            Some(meta1)
+                        } else {
+                            Some(meta2)
+                        }
+                    }
+                    (true, false) => {
+                        assert_eq!(
+                            meta1.pagesize, self.pagesize,
+                            "Invalid pagesize from meta1 {}. Expected {}.",
+                            meta1.pagesize, self.pagesize
+                        );
+                        Some(meta1)
+                    }
+                    (false, true) => {
+                        assert_eq!(
+                            meta2.pagesize, self.pagesize,
+                            "Invalid pagesize from meta2 {}. Expected {}.",
+                            meta2.pagesize, self.pagesize
+                        );
+                        Some(meta2)
+                    }
+                    (false, false) => None,
+                }
+            }};
+        }
 
-        Ok(meta.clone())
+        if let Some(meta) = check_meta!(meta) {
+            Ok(meta.clone())
+        } else if let Some(old_meta) = check_meta!(old_meta) {
+            Ok(old_meta.into())
+        } else {
+            panic!("NO VALID META PAGES");
+        }
     }
 }
 

src/meta.rs

@@ -1,5 +1,7 @@
-use fnv::FnvHasher;
 use std::hash::Hasher;
+
+use fnv::FnvHasher;
+
 use crate::{bucket::BucketMeta, page::PageID};
 
 #[repr(C)]
@@ -38,7 +40,6 @@ impl Meta {
     }
 }
 
-
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -71,3 +72,76 @@ mod tests {
         assert_eq!(meta.hash, meta.hash_self());
     }
 }
+
+// OldMeta is the metadata format for versions <= 0.10
+// For now we check all databases for either metadata version,
+// but always write the new format.
+use std::io::Write;
+
+use bytes::BufMut;
+use sha3::{Digest, Sha3_256};
+
+#[repr(C)]
+#[derive(Debug, Clone)]
+pub(crate) struct OldMeta {
+    pub(crate) meta_page: u32,
+    pub(crate) magic: u32,
+    pub(crate) version: u32,
+    pub(crate) pagesize: u64,
+    pub(crate) root: BucketMeta,
+    pub(crate) num_pages: PageID,
+    pub(crate) freelist_page: PageID,
+    pub(crate) tx_id: u64,
+    pub(crate) hash: [u8; 32],
+}
+
+impl OldMeta {
+    pub(crate) fn valid(&self) -> bool {
+        self.hash == self.hash_self()
+    }
+
+    pub(crate) fn hash_self(&self) -> [u8; 32] {
+        let mut hash_result: [u8; 32] = [0; 32];
+        let mut hasher = Sha3_256::new();
+        hasher.update(self.bytes());
+        let hash = hasher.finalize();
+        assert_eq!(hash.len(), 32);
+        hash_result.copy_from_slice(&hash[..]);
+        hash_result
+    }
+
+    fn bytes(&self) -> bytes::Bytes {
+        let buf = bytes::BytesMut::new();
+        let mut w = buf.writer();
+        let _ = w.write(&self.meta_page.to_be_bytes());
+        let _ = w.write(&self.magic.to_be_bytes());
+        let _ = w.write(&self.version.to_be_bytes());
+        let _ = w.write(&self.pagesize.to_be_bytes());
+        let _ = w.write(&self.root.root_page.to_be_bytes());
+        let _ = w.write(&self.root.next_int.to_be_bytes());
+        let _ = w.write(&self.num_pages.to_be_bytes());
+        let _ = w.write(&self.freelist_page.to_be_bytes());
+        let _ = w.write(&self.tx_id.to_be_bytes());
+
+        w.into_inner().freeze()
+    }
+}
+
+impl From<&OldMeta> for Meta {
+    fn from(val: &OldMeta) -> Self {
+        let mut m = Meta {
+            meta_page: val.meta_page,
+            magic: val.magic,
+            version: val.version,
+            pagesize: val.pagesize,
+            root: val.root,
+            num_pages: val.num_pages,
+            freelist_page: val.freelist_page,
+            tx_id: val.tx_id,
+            hash: 0,
+        };
+
+        m.hash = m.hash_self();
+        m
+    }
+}

src/page.rs

@@ -9,7 +9,7 @@ use memmap2::Mmap;
 
 use crate::{
     errors::Result,
-    meta::Meta,
+    meta::{Meta, OldMeta},
     node::{Node, NodeData, NodeType},
 };
 
@@ -66,57 +66,107 @@ impl Page {
     }
 
     pub(crate) fn meta(&self) -> &Meta {
-        assert_eq!(self.page_type, Page::TYPE_META);
+        assert_eq!(
+            self.page_type,
+            Page::TYPE_META,
+            "Did not find meta page, found {}",
+            self.page_type
+        );
         unsafe { &*(&self.ptr as *const u64 as *const Meta) }
     }
 
+    pub(crate) fn old_meta(&self) -> &OldMeta {
+        assert_eq!(
+            self.page_type,
+            Page::TYPE_META,
+            "Did not find meta page, found {}",
+            self.page_type
+        );
+        unsafe { &*(&self.ptr as *const u64 as *const OldMeta) }
+    }
+
     pub(crate) fn meta_mut(&mut self) -> &mut Meta {
-        assert_eq!(self.page_type, Page::TYPE_META);
+        assert_eq!(
+            self.page_type,
+            Page::TYPE_META,
+            "Did not find meta page, found {}",
+            self.page_type
+        );
         unsafe { &mut *(&mut self.ptr as *mut u64 as *mut Meta) }
     }
 
     pub(crate) fn freelist(&self) -> &[PageID] {
-        assert_eq!(self.page_type, Page::TYPE_FREELIST);
+        assert_eq!(
+            self.page_type,
+            Page::TYPE_FREELIST,
+            "Did not find freelist page, found {}",
+            self.page_type
+        );
         unsafe {
             let start = &self.ptr as *const u64 as *const PageID;
             from_raw_parts(start, self.count as usize)
         }
     }
 
     pub(crate) fn freelist_mut(&mut self) -> &mut [PageID] {
-        assert_eq!(self.page_type, Page::TYPE_FREELIST);
+        assert_eq!(
+            self.page_type,
+            Page::TYPE_FREELIST,
+            "Did not find freelist page, found {}",
+            self.page_type
+        );
         unsafe {
             let start = &self.ptr as *const u64 as *mut PageID;
             from_raw_parts_mut(start, self.count as usize)
         }
     }
 
     pub(crate) fn leaf_elements(&self) -> &[LeafElement] {
-        assert_eq!(self.page_type, Page::TYPE_LEAF);
+        assert_eq!(
+            self.page_type,
+            Page::TYPE_LEAF,
+            "Did not find leaf page, found {}",
+            self.page_type
+        );
         unsafe {
             let start = &self.ptr as *const u64 as *const LeafElement;
             from_raw_parts(start, self.count as usize)
         }
     }
 
     pub(crate) fn branch_elements(&self) -> &[BranchElement] {
-        assert_eq!(self.page_type, Page::TYPE_BRANCH);
+        assert_eq!(
+            self.page_type,
+            Page::TYPE_BRANCH,
+            "Did not find branch page, found {}",
+            self.page_type
+        );
         unsafe {
             let start = &self.ptr as *const u64 as *const BranchElement;
             from_raw_parts(start, self.count as usize)
         }
     }
 
     pub(crate) fn leaf_elements_mut(&mut self) -> &mut [LeafElement] {
-        assert_eq!(self.page_type, Page::TYPE_LEAF);
+        assert_eq!(
+            self.page_type,
+            Page::TYPE_LEAF,
+            "Did not find leaf page, found {}",
+            self.page_type
+        );
         unsafe {
             let start = &self.ptr as *const u64 as *const LeafElement as *mut LeafElement;
             from_raw_parts_mut(start, self.count as usize)
         }
     }
 
     pub(crate) fn branch_elements_mut(&mut self) -> &mut [BranchElement] {
-        assert_eq!(self.page_type, Page::TYPE_BRANCH);
+        assert_eq!(
+            self.page_type,
+            Page::TYPE_BRANCH,
+            "Did not find branch page, found {}",
+            self.page_type
+        );
         unsafe {
             let start = &self.ptr as *const u64 as *const BranchElement as *mut BranchElement;
             from_raw_parts_mut(start, self.count as usize)

tests/common/record.rs

@@ -416,12 +416,7 @@ pub fn log_playback(name: &str) -> Result<(), Error> {
     Ok(())
 }
 
-fn mutate_buckets<'tx, F>(
-    tx: &Tx<'tx>,
-    root: &mut FakeNode,
-    path: &Vec<Bytes>,
-    f: F,
-) -> Result<(), Error>
+fn mutate_buckets<F>(tx: &Tx, root: &mut FakeNode, path: &Vec<Bytes>, f: F) -> Result<(), Error>
 where
     F: Fn(&Bucket, &mut BTreeMap<Bytes, FakeNode>) -> Result<(), Error>,
 {