From 59f1c0128b464400c5e96b0efb6af02003939e22 Mon Sep 17 00:00:00 2001 From: Gabriel Viganotti Date: Wed, 13 Mar 2024 10:46:10 -0300 Subject: [PATCH 1/4] refactor(cli): breaking up acc-packet logic within its own mod --- sn_cli/src/subcommands/acc_packet.rs | 154 ++--------------- .../subcommands/acc_packet/change_tracking.rs | 156 ++++++++++++++++++ 2 files changed, 168 insertions(+), 142 deletions(-) create mode 100644 sn_cli/src/subcommands/acc_packet/change_tracking.rs diff --git a/sn_cli/src/subcommands/acc_packet.rs b/sn_cli/src/subcommands/acc_packet.rs index b72260d9f0..84064fc3df 100644 --- a/sn_cli/src/subcommands/acc_packet.rs +++ b/sn_cli/src/subcommands/acc_packet.rs @@ -6,18 +6,24 @@ // KIND, either express or implied. Please review the Licences for the specific language governing // permissions and limitations relating to use of the SAFE Network Software. -use super::files::ChunkManager; +mod change_tracking; + +use change_tracking::*; -use serde::{Deserialize, Serialize}; -use sn_client::protocol::storage::{Chunk, RegisterAddress, RetryStrategy}; -use sn_client::registers::EntryHash; -use sn_client::transfers::HotWallet; -use sn_client::{Client, FilesApi, FolderEntry, FoldersApi, Metadata, WalletClient}; +use super::files::ChunkManager; use crate::subcommands::files::{ self, download::download_file, iterative_uploader::IterativeUploader, upload::FilesUploadOptions, }; + +use sn_client::{ + protocol::storage::{Chunk, RegisterAddress, RetryStrategy}, + registers::EntryHash, + transfers::HotWallet, + Client, FilesApi, FolderEntry, FoldersApi, Metadata, WalletClient, +}; + use color_eyre::{ eyre::{bail, eyre}, Result, @@ -28,7 +34,6 @@ use std::{ BTreeMap, }, ffi::OsString, - fmt, fs::{create_dir_all, remove_dir_all, remove_file, File}, io::Write, path::{Path, PathBuf}, @@ -37,87 +42,6 @@ use tokio::task::JoinSet; use walkdir::{DirEntry, WalkDir}; use xor_name::XorName; -// Name of hidden folder where tracking information and metadata is locally kept. -const SAFE_TRACKING_CHANGES_DIR: &str = ".safe"; - -// Subfolder where files metadata will be cached -const METADATA_CACHE_DIR: &str = "metadata"; - -// Name of the file where metadata about root folder is locally kept. -const ROOT_FOLDER_METADATA_FILENAME: &str = "root_folder.addr"; - -// Container to keep track in memory what changes are detected in local Folders hierarchy and files. -type Folders = BTreeMap; - -// Type of local changes detected to a Folder -#[derive(Clone, Debug, PartialEq)] -enum FolderChange { - NoChange, - NewFolder, - NewEntries, -} - -impl FolderChange { - /// Returns true if it's currently set to NewFolder. - pub fn is_new_folder(&self) -> bool { - self == &Self::NewFolder - } - - /// If it's currently set to NoChange then switch it to NewEntries. - /// Otherwise we don't need to change it as the entire Folder will need to be uploaded. - pub fn has_new_entries(&mut self) { - if self == &Self::NoChange { - *self = Self::NewEntries; - } - } -} - -// Changes detected locally which eventually can be applied and upload to network. -#[derive(Default)] -struct ChangesToApply { - folders: Folders, - mutations: Vec, -} - -// Type of mutation detected locally. -#[derive(Debug)] -enum Mutation { - NewFile(MetadataTrackingInfo), - FileRemoved((PathBuf, XorName)), - FileContentChanged((XorName, MetadataTrackingInfo)), - NewFolder(MetadataTrackingInfo), - FolderRemoved((PathBuf, XorName)), -} - -impl fmt::Display for Mutation { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - match self { - Self::NewFile(tracking_info) => { - write!(f, "New file: {:?}", tracking_info.file_path) - } - Self::FileRemoved((path, _)) => write!(f, "File removed: {path:?}"), - Self::FileContentChanged((_, tracking_info)) => { - write!(f, "File content changed: {:?}", tracking_info.file_path) - } - Self::NewFolder(tracking_info) => { - write!(f, "New folder: {:?}", tracking_info.file_path) - } - Self::FolderRemoved((path, _)) => write!(f, "Folder removed: {path:?}"), - } - } -} - -// Information stored locally to keep track of local changes to files/folders. -// TODO: to make file changes discovery more efficient, and prevent chunking for -// such purposes, add more info like file size and last modified timestamp. -#[derive(Debug, Serialize, Deserialize, PartialEq)] -struct MetadataTrackingInfo { - file_path: PathBuf, - meta_xorname: XorName, - metadata: Metadata, - entry_hash: EntryHash, -} - /// Object which allows a user to store and manage files, wallets, etc., with the ability /// and tools necessary to keep a local instance in sync with its remote version stored on the network. /// TODO: currently only files and folders are supported, wallets, keys, etc., to be added later. @@ -847,60 +771,6 @@ impl AccountPacket { } } -// Build absolute paths for the different dirs to be used for locally tracking changes -fn build_tracking_info_paths(path: &Path) -> Result<(PathBuf, PathBuf, PathBuf)> { - let files_dir = path.to_path_buf().canonicalize()?; - let tracking_info_dir = files_dir.join(SAFE_TRACKING_CHANGES_DIR); - let meta_dir = tracking_info_dir.join(METADATA_CACHE_DIR); - create_dir_all(&meta_dir) - .map_err(|err| eyre!("The path provided needs to be a directory: {err}"))?; - - Ok((files_dir, tracking_info_dir, meta_dir)) -} - -fn read_tracking_info_from_disk( - meta_dir: &Path, -) -> Result> { - let mut curr_tracking_info = BTreeMap::new(); - for entry in WalkDir::new(meta_dir) - .into_iter() - .flatten() - .filter(|e| e.file_type().is_file() && e.file_name() != ROOT_FOLDER_METADATA_FILENAME) - { - let path = entry.path(); - let bytes = std::fs::read(path) - .map_err(|err| eyre!("Error while reading the tracking info from {path:?}: {err}"))?; - let tracking_info: MetadataTrackingInfo = rmp_serde::from_slice(&bytes) - .map_err(|err| eyre!("Error while deserializing tracking info from {path:?}: {err}"))?; - - curr_tracking_info.insert(tracking_info.file_path.clone(), tracking_info); - } - - Ok(curr_tracking_info) -} - -// Store tracking info about the root folder in a file to keep track of any changes made -fn store_root_folder_tracking_info( - meta_dir: &Path, - root_folder_addr: RegisterAddress, - created: bool, -) -> Result<()> { - let path = meta_dir.join(ROOT_FOLDER_METADATA_FILENAME); - let mut meta_file = File::create(path)?; - meta_file.write_all(&rmp_serde::to_vec(&(root_folder_addr, created))?)?; - - Ok(()) -} - -// Read the tracking info about the root folder -fn read_root_folder_addr(meta_dir: &Path) -> Result<(RegisterAddress, bool)> { - let path = meta_dir.join(ROOT_FOLDER_METADATA_FILENAME); - let bytes = std::fs::read(&path) - .map_err(|err| eyre!("Error while reading the tracking info from {path:?}: {err:?}"))?; - - Ok(rmp_serde::from_slice(&bytes)?) -} - // Given an absolute path, find the Folder containing such item, and remove it from its entries. fn remove_from_parent(folders: &mut Folders, path: &Path, entry_hash: EntryHash) -> Result<()> { if let Some((parent_folder, folder_change)) = path.parent().and_then(|p| folders.get_mut(p)) { diff --git a/sn_cli/src/subcommands/acc_packet/change_tracking.rs b/sn_cli/src/subcommands/acc_packet/change_tracking.rs new file mode 100644 index 0000000000..8cb2b89c39 --- /dev/null +++ b/sn_cli/src/subcommands/acc_packet/change_tracking.rs @@ -0,0 +1,156 @@ +// Copyright 2024 MaidSafe.net limited. +// +// This SAFE Network Software is licensed to you under The General Public License (GPL), version 3. +// Unless required by applicable law or agreed to in writing, the SAFE Network Software distributed +// under the GPL Licence is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. Please review the Licences for the specific language governing +// permissions and limitations relating to use of the SAFE Network Software. + +use sn_client::{protocol::storage::RegisterAddress, registers::EntryHash, FoldersApi, Metadata}; + +use color_eyre::{eyre::eyre, Result}; +use serde::{Deserialize, Serialize}; +use std::{ + collections::BTreeMap, + fmt, + fs::{create_dir_all, File}, + io::Write, + path::{Path, PathBuf}, +}; +use walkdir::WalkDir; +use xor_name::XorName; + +// Name of hidden folder where tracking information and metadata is locally kept. +pub(super) const SAFE_TRACKING_CHANGES_DIR: &str = ".safe"; + +// Subfolder where files metadata will be cached +pub(super) const METADATA_CACHE_DIR: &str = "metadata"; + +// Name of the file where metadata about root folder is locally kept. +pub(super) const ROOT_FOLDER_METADATA_FILENAME: &str = "root_folder.addr"; + +// Container to keep track in memory what changes are detected in local Folders hierarchy and files. +pub(super) type Folders = BTreeMap; + +// Type of local changes detected to a Folder +#[derive(Clone, Debug, PartialEq)] +pub(super) enum FolderChange { + NoChange, + NewFolder, + NewEntries, +} + +impl FolderChange { + /// Returns true if it's currently set to NewFolder. + pub fn is_new_folder(&self) -> bool { + self == &Self::NewFolder + } + + /// If it's currently set to NoChange then switch it to NewEntries. + /// Otherwise we don't need to change it as the entire Folder will need to be uploaded. + pub fn has_new_entries(&mut self) { + if self == &Self::NoChange { + *self = Self::NewEntries; + } + } +} + +// Changes detected locally which eventually can be applied and upload to network. +#[derive(Default)] +pub(super) struct ChangesToApply { + pub folders: Folders, + pub mutations: Vec, +} + +// Type of mutation detected locally. +#[derive(Debug)] +pub(super) enum Mutation { + NewFile(MetadataTrackingInfo), + FileRemoved((PathBuf, XorName)), + FileContentChanged((XorName, MetadataTrackingInfo)), + NewFolder(MetadataTrackingInfo), + FolderRemoved((PathBuf, XorName)), +} + +impl fmt::Display for Mutation { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match self { + Self::NewFile(tracking_info) => { + write!(f, "New file: {:?}", tracking_info.file_path) + } + Self::FileRemoved((path, _)) => write!(f, "File removed: {path:?}"), + Self::FileContentChanged((_, tracking_info)) => { + write!(f, "File content changed: {:?}", tracking_info.file_path) + } + Self::NewFolder(tracking_info) => { + write!(f, "New folder: {:?}", tracking_info.file_path) + } + Self::FolderRemoved((path, _)) => write!(f, "Folder removed: {path:?}"), + } + } +} + +// Information stored locally to keep track of local changes to files/folders. +// TODO: to make file changes discovery more efficient, and prevent chunking for +// such purposes, add more info like file size and last modified timestamp. +#[derive(Debug, Serialize, Deserialize, PartialEq)] +pub(super) struct MetadataTrackingInfo { + pub file_path: PathBuf, + pub meta_xorname: XorName, + pub metadata: Metadata, + pub entry_hash: EntryHash, +} + +// Build absolute paths for the different dirs to be used for locally tracking changes +pub(super) fn build_tracking_info_paths(path: &Path) -> Result<(PathBuf, PathBuf, PathBuf)> { + let files_dir = path.to_path_buf().canonicalize()?; + let tracking_info_dir = files_dir.join(SAFE_TRACKING_CHANGES_DIR); + let meta_dir = tracking_info_dir.join(METADATA_CACHE_DIR); + create_dir_all(&meta_dir) + .map_err(|err| eyre!("The path provided needs to be a directory: {err}"))?; + + Ok((files_dir, tracking_info_dir, meta_dir)) +} + +pub(super) fn read_tracking_info_from_disk( + meta_dir: &Path, +) -> Result> { + let mut curr_tracking_info = BTreeMap::new(); + for entry in WalkDir::new(meta_dir) + .into_iter() + .flatten() + .filter(|e| e.file_type().is_file() && e.file_name() != ROOT_FOLDER_METADATA_FILENAME) + { + let path = entry.path(); + let bytes = std::fs::read(path) + .map_err(|err| eyre!("Error while reading the tracking info from {path:?}: {err}"))?; + let tracking_info: MetadataTrackingInfo = rmp_serde::from_slice(&bytes) + .map_err(|err| eyre!("Error while deserializing tracking info from {path:?}: {err}"))?; + + curr_tracking_info.insert(tracking_info.file_path.clone(), tracking_info); + } + + Ok(curr_tracking_info) +} + +// Store tracking info about the root folder in a file to keep track of any changes made +pub(super) fn store_root_folder_tracking_info( + meta_dir: &Path, + root_folder_addr: RegisterAddress, + created: bool, +) -> Result<()> { + let path = meta_dir.join(ROOT_FOLDER_METADATA_FILENAME); + let mut meta_file = File::create(path)?; + meta_file.write_all(&rmp_serde::to_vec(&(root_folder_addr, created))?)?; + + Ok(()) +} + +// Read the tracking info about the root folder +pub(super) fn read_root_folder_addr(meta_dir: &Path) -> Result<(RegisterAddress, bool)> { + let path = meta_dir.join(ROOT_FOLDER_METADATA_FILENAME); + let bytes = std::fs::read(&path) + .map_err(|err| eyre!("Error while reading the tracking info from {path:?}: {err:?}"))?; + + Ok(rmp_serde::from_slice(&bytes)?) +} From ef594e82d7e5cad5c27cd04031cd0478672d66ea Mon Sep 17 00:00:00 2001 From: Gabriel Viganotti Date: Wed, 13 Mar 2024 17:03:20 -0300 Subject: [PATCH 2/4] test(acc-packet): adding unit test to private methods/helpers --- sn_cli/src/subcommands/acc_packet.rs | 168 +++++++++++++++++++-------- 1 file changed, 121 insertions(+), 47 deletions(-) diff --git a/sn_cli/src/subcommands/acc_packet.rs b/sn_cli/src/subcommands/acc_packet.rs index 84064fc3df..c361f560cc 100644 --- a/sn_cli/src/subcommands/acc_packet.rs +++ b/sn_cli/src/subcommands/acc_packet.rs @@ -805,13 +805,17 @@ fn find_by_name_in_parent_folder(name: &str, path: &Path, folders: &Folders) -> mod tests { // All tests require a network running so Clients can be instantiated. - use super::{read_root_folder_addr, read_tracking_info_from_disk, AccountPacket}; + use super::{ + read_root_folder_addr, read_tracking_info_from_disk, AccountPacket, Metadata, + MetadataTrackingInfo, + }; use crate::subcommands::files::upload::FilesUploadOptions; + use bytes::Bytes; use sn_client::{ protocol::storage::{Chunk, RetryStrategy}, - registers::RegisterAddress, + registers::{EntryHash, RegisterAddress}, test_utils::{get_funded_wallet, get_new_client, random_file_chunk}, - FolderEntry, BATCH_SIZE, + Client, FolderEntry, BATCH_SIZE, }; use bls::SecretKey; @@ -832,12 +836,68 @@ mod tests { }; #[tokio::test] - async fn test_acc_packet_from_empty_dir() -> Result<()> { + async fn test_acc_packet_private_helpers() -> Result<()> { + let (client, root_folder_addr) = get_client_and_folder_addr().await?; + + let tmp_dir = tempfile::tempdir()?; + let wallet_dir = tmp_dir.path(); + let files_path = tmp_dir.path().join("myfiles"); + create_dir_all(&files_path)?; + + let acc_packet = AccountPacket::init( + client.clone(), + wallet_dir, + &files_path, + Some(root_folder_addr), + )?; + + assert_eq!(acc_packet.root_folder_addr(), root_folder_addr); + + let mut test_files = create_test_files_on_disk(&files_path)?; let mut rng = rand::thread_rng(); - let owner_sk = SecretKey::random(); - let owner_pk = owner_sk.public_key(); - let root_folder_addr = RegisterAddress::new(XorName::random(&mut rng), owner_pk); - let client = get_new_client(owner_sk).await?; + let dummy_metadata = Metadata { + name: "dummy".to_string(), + content: FolderEntry::File(Chunk::new(Bytes::new())), + }; + for (relative_path, _) in test_files.iter() { + let abs_path = files_path.join(relative_path); + + // test helper which calculates relative paths based on root files dir of acc packet + assert!( + matches!(acc_packet.get_relative_path(&abs_path), Ok(p) if &p == relative_path), + "AccountPacket::get_relative_path helper returned invalid path" + ); + + // let's test helper to store tracking info + // use just dummy/invalid metadata and meta-xorname since we won't verify it + let meta_xorname = XorName::random(&mut rng); + acc_packet.store_tracking_info(MetadataTrackingInfo { + file_path: abs_path, + meta_xorname, + metadata: dummy_metadata.clone(), + entry_hash: EntryHash::default(), + })?; + assert!(acc_packet.meta_dir.join(hex::encode(meta_xorname)).exists()); + } + + // let's test helpers to read and remove tracking info + let tracking_info = read_tracking_info_from_disk(&acc_packet.meta_dir)?; + assert_eq!(tracking_info.len(), test_files.len()); + for (abs_path, info) in tracking_info.iter() { + assert!(test_files.remove(abs_path).is_some()); + acc_packet.remove_tracking_info(info.meta_xorname); + assert!(!acc_packet + .meta_dir + .join(hex::encode(info.meta_xorname)) + .exists()); + } + + Ok(()) + } + + #[tokio::test] + async fn test_acc_packet_from_empty_dir() -> Result<()> { + let (client, root_folder_addr) = get_client_and_folder_addr().await?; let tmp_dir = tempfile::tempdir()?; let wallet_dir = tmp_dir.path(); @@ -876,11 +936,7 @@ mod tests { #[tokio::test] async fn test_acc_packet_upload_download() -> Result<()> { - let mut rng = rand::thread_rng(); - let owner_sk = SecretKey::random(); - let owner_pk = owner_sk.public_key(); - let root_folder_addr = RegisterAddress::new(XorName::random(&mut rng), owner_pk); - let client = get_new_client(owner_sk).await?; + let (client, root_folder_addr) = get_client_and_folder_addr().await?; let tmp_dir = tempfile::tempdir()?; let wallet_dir = tmp_dir.path(); @@ -918,11 +974,7 @@ mod tests { #[tokio::test] async fn test_acc_packet_sync_mutations() -> Result<()> { - let mut rng = rand::thread_rng(); - let owner_sk = SecretKey::random(); - let owner_pk = owner_sk.public_key(); - let root_folder_addr = RegisterAddress::new(XorName::random(&mut rng), owner_pk); - let client = get_new_client(owner_sk).await?; + let (client, root_folder_addr) = get_client_and_folder_addr().await?; let tmp_dir = tempfile::tempdir()?; let wallet_dir = tmp_dir.path(); @@ -952,35 +1004,7 @@ mod tests { .await?; // let's make mutations to the clone: - // - modify the content of a file - let new_chunk = random_file_chunk(); - let file2modify = Path::new("file0.txt"); - OpenOptions::new() - .write(true) - .open(clone_files_path.join(file2modify))? - .write_all(new_chunk.value())?; - expected_files.insert(file2modify.to_path_buf(), Some(new_chunk)); - // - remove one of the files - let file2remove = Path::new("dir1").join("file1.txt"); - remove_file(clone_files_path.join(&file2remove))?; - expected_files.remove(&file2remove); - // we need to keep the empty dir within the list of expected files though - expected_files.insert(Path::new("dir1").to_path_buf(), None); - // - remove one of the dirs - let dir2remove = Path::new("dir2"); - remove_dir_all(clone_files_path.join(dir2remove))?; - expected_files.remove(dir2remove); - // - create new dir - let dir2create = Path::new("dir2").join("dir2_1"); - create_dir_all(clone_files_path.join(&dir2create))?; - expected_files.insert(dir2create.to_path_buf(), None); - // - create new file - create_dir_all(clone_files_path.join("dir3").join("dir3_1"))?; - let file2create = Path::new("dir3").join("dir3_1").join("file3.txt"); - let mut file = File::create(clone_files_path.join(&file2create))?; - let new_chunk = random_file_chunk(); - file.write_all(new_chunk.value())?; - expected_files.insert(file2create, Some(new_chunk)); + mutate_test_files_on_disk(&clone_files_path, &mut expected_files)?; // and finally, sync the clone up with the network cloned_acc_packet.sync(SYNC_OPTS).await?; @@ -996,6 +1020,18 @@ mod tests { Ok(()) } + // Helpers functions to generate and verify test data + + // Build a new Client and a random address for a Folder (Register) + async fn get_client_and_folder_addr() -> Result<(Client, RegisterAddress)> { + let mut rng = rand::thread_rng(); + let owner_sk = SecretKey::random(); + let owner_pk = owner_sk.public_key(); + let root_folder_addr = RegisterAddress::new(XorName::random(&mut rng), owner_pk); + let client = get_new_client(owner_sk).await?; + Ok((client, root_folder_addr)) + } + // Create a hard-coded set of test files and dirs on disk fn create_test_files_on_disk(base_path: &Path) -> Result>> { // let's create a hierarchy with dirs and files with random content @@ -1025,6 +1061,44 @@ mod tests { Ok(files) } + // Apply a hard-coded set of mutations to test files and dirs on disk + fn mutate_test_files_on_disk( + path: &Path, + test_files: &mut BTreeMap>, + ) -> Result<()> { + // - modify the content of a file + let new_chunk = random_file_chunk(); + let file2modify = Path::new("file0.txt"); + OpenOptions::new() + .write(true) + .open(path.join(file2modify))? + .write_all(new_chunk.value())?; + test_files.insert(file2modify.to_path_buf(), Some(new_chunk)); + // - remove one of the files + let file2remove = Path::new("dir1").join("file1.txt"); + remove_file(path.join(&file2remove))?; + test_files.remove(&file2remove); + // we need to keep the empty dir within the list of expected files though + test_files.insert(Path::new("dir1").to_path_buf(), None); + // - remove one of the dirs + let dir2remove = Path::new("dir2"); + remove_dir_all(path.join(dir2remove))?; + test_files.remove(dir2remove); + // - create new dir + let dir2create = Path::new("dir2").join("dir2_1"); + create_dir_all(path.join(&dir2create))?; + test_files.insert(dir2create.to_path_buf(), None); + // - create new file + create_dir_all(path.join("dir3").join("dir3_1"))?; + let file2create = Path::new("dir3").join("dir3_1").join("file3.txt"); + let mut file = File::create(path.join(&file2create))?; + let new_chunk = random_file_chunk(); + file.write_all(new_chunk.value())?; + test_files.insert(file2create, Some(new_chunk)); + + Ok(()) + } + // Helper to check if a dir is empty fn is_empty_dir(path: &Path) -> bool { path.read_dir() From e7a3187be8c4e97188ea651151e4fecff4851c11 Mon Sep 17 00:00:00 2001 From: Gabriel Viganotti Date: Thu, 14 Mar 2024 09:37:16 -0300 Subject: [PATCH 3/4] test(acc-packet): adding unit test for acc-packet changes scanning logic --- sn_cli/src/subcommands/acc_packet.rs | 63 +++++++++++++++++++++++++--- 1 file changed, 57 insertions(+), 6 deletions(-) diff --git a/sn_cli/src/subcommands/acc_packet.rs b/sn_cli/src/subcommands/acc_packet.rs index c361f560cc..fcf5efa78a 100644 --- a/sn_cli/src/subcommands/acc_packet.rs +++ b/sn_cli/src/subcommands/acc_packet.rs @@ -809,7 +809,7 @@ mod tests { read_root_folder_addr, read_tracking_info_from_disk, AccountPacket, Metadata, MetadataTrackingInfo, }; - use crate::subcommands::files::upload::FilesUploadOptions; + use crate::subcommands::{acc_packet::Mutation, files::upload::FilesUploadOptions}; use bytes::Bytes; use sn_client::{ protocol::storage::{Chunk, RetryStrategy}, @@ -972,6 +972,57 @@ mod tests { Ok(()) } + #[tokio::test] + async fn test_acc_packet_scan_files_and_folders_changes() -> Result<()> { + let (client, root_folder_addr) = get_client_and_folder_addr().await?; + + let tmp_dir = tempfile::tempdir()?; + let wallet_dir = tmp_dir.path(); + let _ = get_funded_wallet(&client, wallet_dir).await?; + + let files_path = tmp_dir.path().join("myaccpacket-to-scan"); + let mut test_files = create_test_files_on_disk(&files_path)?; + let files_path = files_path.canonicalize()?; + + let mut acc_packet = AccountPacket::init( + client.clone(), + wallet_dir, + &files_path, + Some(root_folder_addr), + )?; + + let changes = acc_packet.scan_files_and_folders_for_changes(false)?; + // verify changes detected + assert_eq!(changes.mutations.len(), 4); + assert!(changes.mutations.iter().all(|mutation| { + matches!(mutation, Mutation::NewFile(i) if i.file_path == files_path.join("file0.txt")) + || matches!(mutation, Mutation::NewFile(i) if i.file_path == files_path.join("dir1").join("file1.txt")) + || matches!(mutation, Mutation::NewFolder(i) if i.file_path == files_path.join("dir1")) + || matches!(mutation, Mutation::NewFolder(i) if i.file_path == files_path.join("dir2")) + }), "at least one of the mutations detected was unexpected/incorrect"); + + acc_packet.sync(SYNC_OPTS).await?; + + // let's make some mutations/changes + mutate_test_files_on_disk(&files_path, &mut test_files)?; + + let changes = acc_packet.scan_files_and_folders_for_changes(false)?; + // verify new changes detected + assert_eq!(changes.mutations.len(), 8); + assert!(changes.mutations.iter().all(|mutation| { + matches!(mutation, Mutation::FileContentChanged((_,i)) if i.file_path == files_path.join("file0.txt")) + || matches!(mutation, Mutation::FileRemoved((p, _)) if p == &files_path.join("dir1").join("file1.txt")) + || matches!(mutation, Mutation::FolderRemoved((p,_)) if p == &files_path.join("dir2")) + || matches!(mutation, Mutation::NewFolder(i) if i.file_path == files_path.join("dir3")) + || matches!(mutation, Mutation::NewFolder(i) if i.file_path == files_path.join("dir3").join("dir3_1")) + || matches!(mutation, Mutation::NewFile(i) if i.file_path == files_path.join("dir3").join("dir3_1").join("file3.txt")) + || matches!(mutation, Mutation::NewFolder(i) if i.file_path == files_path.join("dir4")) + || matches!(mutation, Mutation::NewFolder(i) if i.file_path == files_path.join("dir4").join("dir4_1")) + }), "at least one of the mutations detected was unexpected/incorrect"); + + Ok(()) + } + #[tokio::test] async fn test_acc_packet_sync_mutations() -> Result<()> { let (client, root_folder_addr) = get_client_and_folder_addr().await?; @@ -1084,17 +1135,17 @@ mod tests { let dir2remove = Path::new("dir2"); remove_dir_all(path.join(dir2remove))?; test_files.remove(dir2remove); - // - create new dir - let dir2create = Path::new("dir2").join("dir2_1"); - create_dir_all(path.join(&dir2create))?; - test_files.insert(dir2create.to_path_buf(), None); - // - create new file + // - create new file within subdirs create_dir_all(path.join("dir3").join("dir3_1"))?; let file2create = Path::new("dir3").join("dir3_1").join("file3.txt"); let mut file = File::create(path.join(&file2create))?; let new_chunk = random_file_chunk(); file.write_all(new_chunk.value())?; test_files.insert(file2create, Some(new_chunk)); + // - create new subdirs + let dir2create = Path::new("dir4").join("dir4_1"); + create_dir_all(path.join(&dir2create))?; + test_files.insert(dir2create.to_path_buf(), None); Ok(()) } From b22cfe55d2ecb4da0f47c2f9efb1b91b4909d441 Mon Sep 17 00:00:00 2001 From: Gabriel Viganotti Date: Thu, 14 Mar 2024 12:33:30 -0300 Subject: [PATCH 4/4] test(acc-packet): adding test for acc-packet moved to a different location on disk --- sn_cli/src/subcommands/acc_packet.rs | 53 ++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/sn_cli/src/subcommands/acc_packet.rs b/sn_cli/src/subcommands/acc_packet.rs index fcf5efa78a..bf24ed3913 100644 --- a/sn_cli/src/subcommands/acc_packet.rs +++ b/sn_cli/src/subcommands/acc_packet.rs @@ -1071,6 +1071,59 @@ mod tests { Ok(()) } + // Acc-packets can be moved to different locations on local disk without affecting their tracking info. + // We disable this test for Windows since in CI the use of std::fs::rename gives a permissions issue. + #[cfg(any(target_os = "linux", target_os = "linux"))] + #[tokio::test] + async fn test_acc_packet_moved_folder() -> Result<()> { + let (client, root_folder_addr) = get_client_and_folder_addr().await?; + + let tmp_dir = tempfile::tempdir()?; + let wallet_dir = tmp_dir.path(); + let _ = get_funded_wallet(&client, wallet_dir).await?; + + let src_files_path = tmp_dir.path().join("myaccpacket-to-move"); + let mut test_files = create_test_files_on_disk(&src_files_path)?; + + let mut acc_packet = AccountPacket::init( + client.clone(), + wallet_dir, + &src_files_path, + Some(root_folder_addr), + )?; + + acc_packet.sync(SYNC_OPTS).await?; + + // let's make just one mutation before moving the dir to another disk location + let new_chunk = random_file_chunk(); + let file2modify = Path::new("dir1").join("file1.txt"); + OpenOptions::new() + .write(true) + .open(src_files_path.join(&file2modify))? + .write_all(new_chunk.value())?; + test_files.insert(file2modify, Some(new_chunk)); + + // let's now move it to another disk location + let moved_files_path = tmp_dir.path().join("myaccpacket-moved"); + create_dir_all(&moved_files_path)?; + std::fs::rename(src_files_path, &moved_files_path)?; + let moved_files_path = moved_files_path.canonicalize()?; + + let moved_acc_packet = + AccountPacket::from_path(client.clone(), wallet_dir, &moved_files_path)?; + + // verify only one change is detected still after moved to another location on disk + let changes = moved_acc_packet.scan_files_and_folders_for_changes(false)?; + assert_eq!(changes.mutations.len(), 1); + assert_eq!(changes.mutations.first().map(|mutation| { + matches!(mutation, Mutation::FileContentChanged((_,i)) if i.file_path == moved_files_path.join("dir1").join("file1.txt")) + }), Some(true)); + + check_tracking_info_match(&moved_acc_packet, &moved_acc_packet, test_files)?; + + Ok(()) + } + // Helpers functions to generate and verify test data // Build a new Client and a random address for a Folder (Register)