more tests for sparse files

src/fs.rs

@@ -550,6 +550,7 @@ mod tests {
     use ic_stable_structures::memory_manager::{MemoryId, MemoryManager};
     use ic_stable_structures::{Memory, VectorMemory};
 
+    use crate::test_utils::write_text_at_offset;
     use crate::{
         error::Error,
         fs::{DstBuf, FdFlags, SrcBuf},
@@ -1509,5 +1510,112 @@ mod tests {
         }
     }
 
+    fn create_file_with_size(filename: &str, size: FileSize, fs: &mut FileSystem) -> Fd {
+        let fd = fs
+            .open_or_create(
+                fs.root_fd,
+                filename,
+                FdStat::default(),
+                OpenFlags::CREATE,
+                12,
+            )
+            .unwrap();
+        let mut meta = fs.metadata(fd).unwrap();
+        meta.size = size;
+
+        fs.set_metadata(fd, meta).unwrap();
+
+        fd
+    }
+
     // test sparse files
+    #[test]
+    fn set_size_for_an_empty_file() {
+        let filename = "test.txt";
+
+        for mut fs in test_fs_setups(filename) {
+            let root_fd = fs.root_fd();
+            let fd = create_file_with_size(filename, 15, &mut fs);
+
+            let content = read_text_file(&mut fs, root_fd, filename, 0, 100);
+
+            assert_eq!(content, "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0");
+            println!("{:?}", content);
+
+            write_text_at_offset(&mut fs, fd, "abc", 3, 3).unwrap();
+
+            let content = read_text_file(&mut fs, root_fd, filename, 1, 100);
+
+            assert_eq!(content, "\0\0abcabcabc\0\0\0");
+            println!("{:?}", content);
+        }
+    }
+
+    #[test]
+    fn create_file_missing_chunk_in_the_middle() {
+        let filename = "test.txt";
+
+        for mut fs in test_fs_setups(filename) {
+            let chunk_size = fs.storage.chunk_size();
+
+            let root_fd = fs.root_fd();
+            let fd = create_file_with_size(filename, chunk_size as FileSize * 2 + 500, &mut fs);
+
+            let content = read_text_file(&mut fs, root_fd, filename, 0, chunk_size * 10);
+
+            let vec = vec![0; chunk_size * 2 + 500];
+
+            let expected = String::from_utf8(vec).unwrap();
+
+            assert_eq!(content, expected);
+
+            write_text_at_offset(&mut fs, fd, "abc", 33, 3).unwrap();
+            write_text_at_offset(&mut fs, fd, "abc", 33, chunk_size as FileSize * 2 + 100).unwrap();
+
+            let content = read_text_file(&mut fs, root_fd, filename, 0, chunk_size * 10);
+
+            let mut expected = vec![0u8; chunk_size * 2 + 500];
+
+            let pattern = b"abc".repeat(33);
+            expected[3..3 + 99].copy_from_slice(&pattern[..]);
+            expected[chunk_size * 2 + 100..chunk_size * 2 + 100 + 99].copy_from_slice(&pattern[..]);
+
+            let expected = String::from_utf8(expected).unwrap();
+
+            assert_eq!(content, expected);
+        }
+    }
+
+    #[test]
+    fn iterate_file_only_middle_chunk_is_present() {
+        let filename = "test.txt";
+
+        for mut fs in test_fs_setups(filename) {
+            let chunk_size = fs.storage.chunk_size();
+
+            let root_fd = fs.root_fd();
+            let fd = create_file_with_size(filename, chunk_size as FileSize * 2 + 500, &mut fs);
+
+            let content = read_text_file(&mut fs, root_fd, filename, 0, chunk_size * 10);
+
+            let vec = vec![0; chunk_size * 2 + 500];
+
+            let expected = String::from_utf8(vec).unwrap();
+
+            assert_eq!(content, expected);
+
+            write_text_at_offset(&mut fs, fd, "abc", 33, chunk_size as FileSize * 1 + 100).unwrap();
+
+            let content = read_text_file(&mut fs, root_fd, filename, 0, chunk_size * 10);
+
+            let mut expected = vec![0u8; chunk_size * 2 + 500];
+
+            let pattern = b"abc".repeat(33);
+            expected[chunk_size * 1 + 100..chunk_size * 1 + 100 + 99].copy_from_slice(&pattern[..]);
+
+            let expected = String::from_utf8(expected).unwrap();
+
+            assert_eq!(content, expected);
+        }
+    }
 }

src/storage/iterator.rs

@@ -124,7 +124,7 @@ mod tests {
 
         let buf = vec![142u8; write_size];
 
-        storage.write(node, 0, &*buf).unwrap();
+        storage.write(node, 0, &buf).unwrap();
 
         let meta = storage.get_metadata(node).unwrap();
         let file_size = meta.size;

src/storage/stable.rs

@@ -540,6 +540,9 @@ impl<M: Memory> Storage for StableStorage<M> {
             let remainder = file_size - offset;
             let to_read = remainder.min(buf.len() as FileSize);
 
+            // grow memory also for reading
+            grow_memory(memory.as_ref(), offset + to_read);
+
             memory.read(offset, &mut buf[..to_read as usize]);
             to_read
         } else {
@@ -785,6 +788,9 @@ impl<M: Memory> Storage for StableStorage<M> {
         while remainder > 0 {
             let to_read = remainder.min(buf.len() as FileSize);
 
+            // grow memory also for reading
+            grow_memory(memory.as_ref(), offset + to_read);
+
             memory.read(offset, &mut buf[..to_read as usize]);
 
             self.write(node, offset, &buf[..to_read as usize])?;

src/storage/transient.rs

@@ -150,15 +150,21 @@ impl Storage for TransientStorage {
             return Ok(0);
         }
 
-        let size_read = if let Some(memory) = self.get_mounted_memory(node) {
+        let size_read = if let Some(memory) = self.active_mounts.get(&node) {
             let remainder = file_size - offset;
             let to_read = remainder.min(buf.len() as FileSize);
 
+            // grow memory also for reading
+            grow_memory(memory.as_ref(), offset + to_read);
+
             memory.read(offset, &mut buf[..to_read as usize]);
             to_read
         } else {
             let start_index = (offset / FILE_CHUNK_SIZE_V1 as FileSize) as FileChunkIndex;
 
+            let end_index = ((offset + buf.len() as FileSize) / FILE_CHUNK_SIZE_V1 as FileSize + 1)
+                as FileChunkIndex;
+
             let mut chunk_offset =
                 offset - start_index as FileSize * FILE_CHUNK_SIZE_V1 as FileSize;
 
@@ -167,28 +173,48 @@ impl Storage for TransientStorage {
             let mut size_read: FileSize = 0;
             let mut remainder = file_size - offset;
 
-            for ((nd, _idx), value) in self.filechunk.range(range) {
-                assert!(*nd == node);
-
-                // finished reading, buffer full
-                if size_read == buf.len() as FileSize {
-                    break;
-                }
+            let mut iter = self.filechunk.range(range);
+            let mut cur_fetched = None;
 
+            for cur_index in start_index..end_index {
                 let chunk_space = FILE_CHUNK_SIZE_V1 as FileSize - chunk_offset;
 
                 let to_read = remainder
                     .min(chunk_space)
                     .min(buf.len() as FileSize - size_read);
 
-                let write_buf = &mut buf[size_read as usize..size_read as usize + to_read as usize];
+                // finished reading, buffer full
+                if size_read == buf.len() as FileSize {
+                    break;
+                }
 
-                write_buf.copy_from_slice(
-                    &value.bytes[chunk_offset as usize..chunk_offset as usize + to_read as usize],
-                );
+                if cur_fetched.is_none() {
+                    cur_fetched = iter.next();
+                }
 
-                chunk_offset = 0;
+                let read_buf = &mut buf[size_read as usize..size_read as usize + to_read as usize];
+
+                if let Some(((nd, idx), value)) = cur_fetched {
+                    if *idx == cur_index {
+                        assert!(*nd == node);
+
+                        read_buf.copy_from_slice(
+                            &value.bytes
+                                [chunk_offset as usize..chunk_offset as usize + to_read as usize],
+                        );
+
+                        // consume token
+                        cur_fetched = None;
+                    } else {
+                        // fill up with zeroes
+                        read_buf.iter_mut().for_each(|m| *m = 0)
+                    }
+                } else {
+                    // fill up with zeroes
+                    read_buf.iter_mut().for_each(|m| *m = 0)
+                }
 
+                chunk_offset = 0;
                 size_read += to_read;
                 remainder -= to_read;
             }

src/test_utils.rs

@@ -114,6 +114,32 @@ pub fn write_text_fd(
     Ok(())
 }
 
+use crate::fs::{FileSize, SrcBuf};
+
+#[cfg(test)]
+pub fn write_text_at_offset(
+    fs: &mut FileSystem,
+    file_fd: u32,
+    content: &str,
+    times: usize,
+    offset: FileSize,
+) -> Result<(), Error> {
+    let mut str = "".to_string();
+
+    for _ in 0..times {
+        str.push_str(content)
+    }
+
+    let src = SrcBuf {
+        buf: str.as_ptr(),
+        len: str.len(),
+    };
+
+    fs.write_vec_with_offset(file_fd, &[src], offset)?;
+
+    Ok(())
+}
+
 #[cfg(test)]
 pub fn read_text_file(
     fs: &mut FileSystem,