sparse file support

src/fs.rs

@@ -547,16 +547,21 @@ impl FileSystem {
 #[cfg(test)]
 mod tests {
 
+    use ic_stable_structures::memory_manager::{MemoryId, MemoryManager};
     use ic_stable_structures::{Memory, VectorMemory};
 
+    use crate::test_utils::write_text_at_offset;
     use crate::{
         error::Error,
         fs::{DstBuf, FdFlags, SrcBuf},
         runtime::{
             structure_helpers::find_node,
             types::{FdStat, OpenFlags},
         },
-        storage::types::{FileSize, FileType},
+        storage::{
+            stable::StableStorage,
+            types::{FileSize, FileType},
+        },
         test_utils::{
             new_vector_memory, read_text_file, test_fs, test_fs_setups, test_fs_transient,
             write_text_fd, write_text_file,
@@ -1353,6 +1358,32 @@ mod tests {
         println!("{:?}", buf);
     }
 
+    #[test]
+    fn reading_mounted_memory_after_upgrade() {
+        let memory_manager = MemoryManager::init(new_vector_memory());
+        let memory = memory_manager.get(MemoryId::new(1));
+
+        let storage = StableStorage::new_with_memory_manager(&memory_manager, 200..210);
+        let mut fs = FileSystem::new(Box::new(storage)).unwrap();
+        fs.mount_memory_file("test.txt", Box::new(memory.clone()))
+            .unwrap();
+
+        let root_fd = fs.root_fd();
+        let content = "ABCDEFG123";
+        write_text_file(&mut fs, root_fd, "test.txt", content, 2).unwrap();
+
+        // imitate canister upgrade (we keep the memory manager but recreate the file system with the same virtual memories)
+        let storage = StableStorage::new_with_memory_manager(&memory_manager, 200..210);
+        let mut fs = FileSystem::new(Box::new(storage)).unwrap();
+        fs.mount_memory_file("test.txt", Box::new(memory.clone()))
+            .unwrap();
+        let root_fd = fs.root_fd();
+
+        let content = read_text_file(&mut fs, root_fd, "test.txt", 0, 100);
+
+        assert_eq!(content, "ABCDEFG123ABCDEFG123");
+    }
+
     #[test]
     fn deleting_mounted_file_fails() {
         let memory: VectorMemory = new_vector_memory();
@@ -1367,7 +1398,7 @@ mod tests {
         let res = fs.remove_file(root_fd, "test.txt");
 
         assert!(
-            res.is_err(),
+            res == Err(Error::CannotRemoveMountedMemoryFile),
             "Deleting a mounted file should not be allowed!"
         );
 
@@ -1478,4 +1509,113 @@ mod tests {
             assert!(res.is_err());
         }
     }
+
+    fn create_file_with_size(filename: &str, size: FileSize, fs: &mut FileSystem) -> Fd {
+        let fd = fs
+            .open_or_create(
+                fs.root_fd,
+                filename,
+                FdStat::default(),
+                OpenFlags::CREATE,
+                12,
+            )
+            .unwrap();
+        let mut meta = fs.metadata(fd).unwrap();
+        meta.size = size;
+
+        fs.set_metadata(fd, meta).unwrap();
+
+        fd
+    }
+
+    // test sparse files
+    #[test]
+    fn set_size_for_an_empty_file() {
+        let filename = "test.txt";
+
+        for mut fs in test_fs_setups(filename) {
+            let root_fd = fs.root_fd();
+            let fd = create_file_with_size(filename, 15, &mut fs);
+
+            let content = read_text_file(&mut fs, root_fd, filename, 0, 100);
+
+            assert_eq!(content, "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0");
+            println!("{:?}", content);
+
+            write_text_at_offset(&mut fs, fd, "abc", 3, 3).unwrap();
+
+            let content = read_text_file(&mut fs, root_fd, filename, 1, 100);
+
+            assert_eq!(content, "\0\0abcabcabc\0\0\0");
+            println!("{:?}", content);
+        }
+    }
+
+    #[test]
+    fn create_file_missing_chunk_in_the_middle() {
+        let filename = "test.txt";
+
+        for mut fs in test_fs_setups(filename) {
+            let chunk_size = fs.storage.chunk_size();
+
+            let root_fd = fs.root_fd();
+            let fd = create_file_with_size(filename, chunk_size as FileSize * 2 + 500, &mut fs);
+
+            let content = read_text_file(&mut fs, root_fd, filename, 0, chunk_size * 10);
+
+            let vec = vec![0; chunk_size * 2 + 500];
+
+            let expected = String::from_utf8(vec).unwrap();
+
+            assert_eq!(content, expected);
+
+            write_text_at_offset(&mut fs, fd, "abc", 33, 3).unwrap();
+            write_text_at_offset(&mut fs, fd, "abc", 33, chunk_size as FileSize * 2 + 100).unwrap();
+
+            let content = read_text_file(&mut fs, root_fd, filename, 0, chunk_size * 10);
+
+            let mut expected = vec![0u8; chunk_size * 2 + 500];
+
+            let pattern = b"abc".repeat(33);
+            expected[3..3 + 99].copy_from_slice(&pattern[..]);
+            expected[chunk_size * 2 + 100..chunk_size * 2 + 100 + 99].copy_from_slice(&pattern[..]);
+
+            let expected = String::from_utf8(expected).unwrap();
+
+            assert_eq!(content, expected);
+        }
+    }
+
+    #[test]
+    fn iterate_file_only_middle_chunk_is_present() {
+        let filename = "test.txt";
+
+        for mut fs in test_fs_setups(filename) {
+            let chunk_size = fs.storage.chunk_size();
+
+            let root_fd = fs.root_fd();
+            let fd = create_file_with_size(filename, chunk_size as FileSize * 2 + 500, &mut fs);
+
+            let content = read_text_file(&mut fs, root_fd, filename, 0, chunk_size * 10);
+
+            let vec = vec![0; chunk_size * 2 + 500];
+
+            let expected = String::from_utf8(vec).unwrap();
+
+            assert_eq!(content, expected);
+
+            write_text_at_offset(&mut fs, fd, "abc", 33, chunk_size as FileSize + 100).unwrap();
+
+            let content = read_text_file(&mut fs, root_fd, filename, 0, chunk_size * 10);
+
+            let mut expected = vec![0u8; chunk_size * 2 + 500];
+
+            let pattern = b"abc".repeat(33);
+            expected[chunk_size + 100..chunk_size + 100 + 99].copy_from_slice(&pattern[..]);
+
+            let expected = String::from_utf8(expected).unwrap();
+
+            assert_eq!(content, expected);
+        }
+    }
 }

src/runtime/structure_helpers.rs

@@ -118,6 +118,13 @@ pub fn create_dir_entry(
     }
 
     let node = storage.new_node();
+
+    let chunk_type = if entry_type == FileType::RegularFile {
+        Some(storage.chunk_type())
+    } else {
+        None
+    };
+
     storage.put_metadata(
         node,
         Metadata {
@@ -132,6 +139,7 @@ pub fn create_dir_entry(
             },
             first_dir_entry: None,
             last_dir_entry: None,
+            chunk_type,
         },
     );
 

src/runtime/types.rs

@@ -1,4 +1,5 @@
 use bitflags::bitflags;
+use serde::{Deserialize, Serialize};
 
 #[derive(Copy, Clone, Debug)]
 pub struct FdStat {
@@ -43,10 +44,10 @@ impl ChunkSize {
     ];
 }
 
-#[derive(Clone, Copy, Debug, PartialEq)]
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
 pub enum ChunkType {
-    V1,
-    V2,
+    V1 = 1,
+    V2 = 2,
 }
 
 bitflags! {

src/storage.rs

@@ -9,6 +9,7 @@ use crate::{
 
 mod allocator;
 pub mod dummy;
+mod iterator;
 mod journal;
 pub mod stable;
 pub mod transient;

src/storage/dummy.rs

@@ -137,6 +137,7 @@ mod tests {
                 times: Times::default(),
                 first_dir_entry: Some(42),
                 last_dir_entry: Some(24),
+                chunk_type: None,
             },
         )
     }