From 5fa06a91acf98f2f51ccca6914fc2eff9561c267 Mon Sep 17 00:00:00 2001 From: Gabriel Viganotti Date: Mon, 18 Mar 2024 10:35:08 -0300 Subject: [PATCH 1/2] feat(folders): folders and acc-packet APIs to accept an encryption key for metadata chunks --- sn_cli/src/subcommands/acc_packet.rs | 47 +++++++++++++++------- sn_cli/src/subcommands/folders.rs | 5 +-- sn_client/src/error.rs | 3 ++ sn_client/src/folders.rs | 60 +++++++++++++++++++++++----- sn_client/tests/folders_api.rs | 31 +++++++------- 5 files changed, 103 insertions(+), 43 deletions(-) diff --git a/sn_cli/src/subcommands/acc_packet.rs b/sn_cli/src/subcommands/acc_packet.rs index bf24ed3913..1c95464a3d 100644 --- a/sn_cli/src/subcommands/acc_packet.rs +++ b/sn_cli/src/subcommands/acc_packet.rs @@ -8,6 +8,7 @@ mod change_tracking; +use bls::PublicKey; use change_tracking::*; use super::files::ChunkManager; @@ -58,6 +59,8 @@ pub struct AccountPacket { impl AccountPacket { /// Initialise directory as a fresh new packet with the given/random address for its root Folder. + /// The client's key will be used for encrypting the files/folders metadata chunks. Currently, + /// when the user requests to encrypt data, the provided Client's signer key is used. pub fn init( client: Client, wallet_dir: &Path, @@ -180,7 +183,7 @@ impl AccountPacket { /// both pushing and pulling changes to/form the network. pub async fn sync(&mut self, options: FilesUploadOptions) -> Result<()> { let ChangesToApply { folders, mutations } = - self.scan_files_and_folders_for_changes(false)?; + self.scan_files_and_folders_for_changes(options.make_data_public)?; if mutations.is_empty() { println!("No local changes made to files/folders to be pushed to network."); @@ -368,16 +371,20 @@ impl AccountPacket { // Scan existing files and folders on disk, generating a report of all the detected // changes based on the tracking info kept locally. - // TODO: encrypt the data-map and metadata if make_data_public is false. - fn scan_files_and_folders_for_changes( - &self, - _make_data_public: bool, - ) -> Result { + // If make_data_public is false the metadata chunks are encrypted. + fn scan_files_and_folders_for_changes(&self, make_data_public: bool) -> Result { // we don't use the local cache in order to realise of any changes made to files content. let mut chunk_manager = ChunkManager::new(&self.tracking_info_dir); chunk_manager.chunk_with_iter(self.iter_only_files(), false, false)?; - let mut changes = self.read_folders_hierarchy_from_disk()?; + let encryption_pk = if make_data_public { + None + } else { + // we pass down the key to encrypt the metadata chunk of any new content detected. + Some(self.client.signer_pk()) + }; + + let mut changes = self.read_folders_hierarchy_from_disk(encryption_pk)?; // add chunked files to the corresponding Folders let folders = &mut changes.folders; @@ -392,13 +399,14 @@ impl AccountPacket { Ok(Some(tracking_info)) => match &tracking_info.metadata.content { FolderEntry::File(chunk) => { if chunk.address() != &chunked_file.head_chunk_address { - // TODO: we need to encrypt the data-map and metadata if make_data_public is false. let (entry_hash, meta_xorname, metadata) = replace_item_in_folder( &mut parent_folder, tracking_info.entry_hash, chunked_file.file_name.clone(), chunked_file.data_map.clone(), + encryption_pk, )?; + changes.mutations.push(Mutation::FileContentChanged(( tracking_info.meta_xorname, MetadataTrackingInfo { @@ -412,12 +420,12 @@ impl AccountPacket { } FolderEntry::Folder(_) => { // New file found where there used to be a folder - // TODO: we need to encrypt the data-map and metadata if make_data_public is false. let (entry_hash, meta_xorname, metadata) = replace_item_in_folder( &mut parent_folder, tracking_info.entry_hash, chunked_file.file_name.clone(), chunked_file.data_map.clone(), + encryption_pk, )?; changes .mutations @@ -430,11 +438,11 @@ impl AccountPacket { } }, Ok(None) => { - // TODO: we need to encrypt the data-map and metadata if make_data_public is false. let (entry_hash, meta_xorname, metadata) = parent_folder.get_mut().0.add_file( chunked_file.file_name.clone(), chunked_file.data_map.clone(), + encryption_pk, )?; parent_folder.get_mut().1.has_new_entries(); @@ -485,8 +493,12 @@ impl AccountPacket { Ok(changes) } - // Build Folders hierarchy from the set files dir - fn read_folders_hierarchy_from_disk(&self) -> Result { + // Build Folders hierarchy from the set files dir. The metadata chunk of every new folder + // will be encrpyted if an encrpytion key has been provided. + fn read_folders_hierarchy_from_disk( + &self, + encryption_pk: Option, + ) -> Result { let mut changes = ChangesToApply::default(); for (dir_path, depth, parent, dir_name) in self.iter_only_dirs().filter_map(|entry| { entry.path().parent().map(|parent| { @@ -513,7 +525,7 @@ impl AccountPacket { if folder_change.is_new_folder() { let (entry_hash, meta_xorname, metadata) = - parent_folder.add_folder(dir_name, curr_folder_addr)?; + parent_folder.add_folder(dir_name, curr_folder_addr, encryption_pk)?; parent_folder_change.has_new_entries(); changes @@ -781,15 +793,22 @@ fn remove_from_parent(folders: &mut Folders, path: &Path, entry_hash: EntryHash) } // Replace a file/folder item from a given Folder (passed in as a container's OccupiedEntry'). +// The metadata chunk of the new item (folder/file) will be encrpyted if a key has been provided. fn replace_item_in_folder( folder: &mut OccupiedEntry<'_, PathBuf, (FoldersApi, FolderChange)>, entry_hash: EntryHash, file_name: OsString, data_map: Chunk, + encryption_pk: Option, ) -> Result<(EntryHash, XorName, Metadata)> { let (ref mut folders_api, ref mut folder_change) = folder.get_mut(); folder_change.has_new_entries(); - let res = folders_api.replace_file(entry_hash, file_name.clone(), data_map.clone())?; + let res = folders_api.replace_file( + entry_hash, + file_name.clone(), + data_map.clone(), + encryption_pk, + )?; Ok(res) } diff --git a/sn_cli/src/subcommands/folders.rs b/sn_cli/src/subcommands/folders.rs index 4d0b3a0045..408bec4d0c 100644 --- a/sn_cli/src/subcommands/folders.rs +++ b/sn_cli/src/subcommands/folders.rs @@ -102,7 +102,7 @@ pub(crate) async fn folders_cmds( ) -> Result<()> { match cmds { FoldersCmds::Init { path, folder_addr } => { - // init path as a fresh new folder + // initialise path as a fresh new Folder with a random network address if none was provided let root_folder_addr = folder_addr.and_then(|hex_str| RegisterAddress::from_hex(&hex_str).ok()); let acc_packet = @@ -115,7 +115,7 @@ pub(crate) async fn folders_cmds( make_data_public, retry_strategy, } => { - // init path as a fresh new folder + // initialise path as a fresh new Folder with a random network address let mut acc_packet = AccountPacket::init(client.clone(), root_dir, &path, None)?; let options = FilesUploadOptions { @@ -163,7 +163,6 @@ pub(crate) async fn folders_cmds( } FoldersCmds::Status { path } => { let acc_packet = AccountPacket::from_path(client.clone(), root_dir, &path)?; - acc_packet.status()?; } FoldersCmds::Sync { diff --git a/sn_client/src/error.rs b/sn_client/src/error.rs index 33e46b6c47..2bd4059d75 100644 --- a/sn_client/src/error.rs +++ b/sn_client/src/error.rs @@ -41,6 +41,9 @@ pub enum Error { #[error("Chunks error {0}.")] Chunks(#[from] super::chunks::Error), + #[error("Decrypting a Folder's item failed.")] + FolderEntryDecryption, + #[error("SelfEncryption Error {0}.")] SelfEncryptionIO(#[from] self_encryption::Error), diff --git a/sn_client/src/folders.rs b/sn_client/src/folders.rs index a683aab75e..6b621ed174 100644 --- a/sn_client/src/folders.rs +++ b/sn_client/src/folders.rs @@ -6,10 +6,11 @@ // KIND, either express or implied. Please review the Licences for the specific language governing // permissions and limitations relating to use of the SAFE Network Software. -use crate::FilesApi; +use crate::{Error, FilesApi}; use super::{error::Result, Client, ClientRegister, WalletClient}; +use bls::{Ciphertext, PublicKey}; use self_encryption::MAX_CHUNK_SIZE; use sn_protocol::{ storage::{Chunk, ChunkAddress, RegisterAddress, RetryStrategy}, @@ -41,6 +42,9 @@ pub struct Metadata { pub content: FolderEntry, } +// This is the entry value used in Folders to mark a removed file/folder. +const REMOVED_ENTRY_MARK: XorName = XorName([0; XOR_NAME_LEN]); + /// Folders APIs. #[derive(Clone)] pub struct FoldersApi { @@ -102,10 +106,12 @@ impl FoldersApi { } /// Add provided file as entry of this Folder (locally). + /// The new file's metadata chunk will be encrypted if a key has been provided. pub fn add_file( &mut self, file_name: OsString, data_map_chunk: Chunk, + encryption_pk: Option, ) -> Result<(EntryHash, XorName, Metadata)> { // create metadata Chunk for this entry let metadata = Metadata { @@ -113,14 +119,16 @@ impl FoldersApi { content: FolderEntry::File(data_map_chunk), }; - self.add_entry(metadata, &BTreeSet::default()) + self.add_entry(metadata, &BTreeSet::default(), encryption_pk) } /// Add subfolder as entry of this Folder (locally). + /// The new folder's metadata chunk will be encrypted if a key has been provided. pub fn add_folder( &mut self, folder_name: OsString, address: RegisterAddress, + encryption_pk: Option, ) -> Result<(EntryHash, XorName, Metadata)> { // create metadata Chunk for this entry let metadata = Metadata { @@ -128,15 +136,17 @@ impl FoldersApi { content: FolderEntry::Folder(address), }; - self.add_entry(metadata, &BTreeSet::default()) + self.add_entry(metadata, &BTreeSet::default(), encryption_pk) } /// Replace an existing file with the provided one (locally). + /// The new file's metadata chunk will be encrypted if a key has been provided. pub fn replace_file( &mut self, existing_entry: EntryHash, file_name: OsString, data_map_chunk: Chunk, + encryption_pk: Option, ) -> Result<(EntryHash, XorName, Metadata)> { // create metadata Chunk for this entry let metadata = Metadata { @@ -144,13 +154,17 @@ impl FoldersApi { content: FolderEntry::File(data_map_chunk), }; - self.add_entry(metadata, &vec![existing_entry].into_iter().collect()) + self.add_entry( + metadata, + &vec![existing_entry].into_iter().collect(), + encryption_pk, + ) } /// Remove a file/folder item from this Folder (locally). pub fn remove_item(&mut self, existing_entry: EntryHash) -> Result<()> { let _ = self.register.write_atop( - &XorName::default(), + &REMOVED_ENTRY_MARK, &vec![existing_entry].into_iter().collect(), )?; Ok(()) @@ -207,7 +221,7 @@ impl FoldersApi { .register .read() .iter() - .map(|(_, meta_xorname)| xorname_from_entry(meta_xorname)) + .map(|(_, meta_xorname_entry)| xorname_from_entry(meta_xorname_entry)) .collect(); self.metadata @@ -227,8 +241,7 @@ impl FoldersApi { let mut entries = BTreeMap::new(); for (entry_hash, entry) in self.register.read() { let meta_xorname = xorname_from_entry(&entry); - if meta_xorname == XorName::default() { - // this is the mark signaling a removed file/folder, so we skip it + if meta_xorname == REMOVED_ENTRY_MARK { continue; } @@ -240,7 +253,21 @@ impl FoldersApi { .client .get_chunk(ChunkAddress::new(meta_xorname), false, None) .await?; - let metadata: Metadata = rmp_serde::from_slice(chunk.value())?; + + // let's first assume it's unencrypted + let metadata: Metadata = match rmp_serde::from_slice(chunk.value()) { + Ok(metadata) => metadata, + Err(err) => { + // let's try to decrypt it then + let cipher = Ciphertext::from_bytes(chunk.value()).map_err(|_| err)?; + let data = self + .client + .signer() + .decrypt(&cipher) + .ok_or(Error::FolderEntryDecryption)?; + rmp_serde::from_slice(&data)? + } + }; self.metadata.insert(meta_xorname, (metadata.clone(), None)); metadata } @@ -265,14 +292,25 @@ impl FoldersApi { }) } - // Add the given entry to the underlying Register as well as creating the metadata Chunk + // Add the given entry to the underlying Register as well as creating the metadata Chunk. + // If an encryption key is given, the metadata chunk will be encrpyted with it. fn add_entry( &mut self, metadata: Metadata, children: &BTreeSet, + encryption_pk: Option, ) -> Result<(EntryHash, XorName, Metadata)> { let mut bytes = BytesMut::with_capacity(MAX_CHUNK_SIZE); - bytes.put(rmp_serde::to_vec(&metadata)?.as_slice()); + let serialised_metadata = rmp_serde::to_vec(&metadata)?; + if let Some(pk) = encryption_pk { + bytes.put( + pk.encrypt(serialised_metadata.as_slice()) + .to_bytes() + .as_slice(), + ); + } else { + bytes.put(serialised_metadata.as_slice()); + } let meta_chunk = Chunk::new(bytes.freeze()); let meta_xorname = *meta_chunk.name(); diff --git a/sn_client/tests/folders_api.rs b/sn_client/tests/folders_api.rs index 467cb04c89..5350510e6a 100644 --- a/sn_client/tests/folders_api.rs +++ b/sn_client/tests/folders_api.rs @@ -35,7 +35,7 @@ async fn test_folder_basics() -> Result<()> { let file_chunk = random_file_chunk(); let (file_entry_hash, file_meta_xorname, file_metadata) = - folders_api.add_file("file.txt".into(), file_chunk.clone())?; + folders_api.add_file("file.txt".into(), file_chunk.clone(), None)?; assert_eq!( file_metadata, Metadata { @@ -45,7 +45,7 @@ async fn test_folder_basics() -> Result<()> { ); let (subdir_entry_hash, subdir_meta_xorname, subdir_metadata) = - folders_api.add_folder("subdir".into(), address_subdir)?; + folders_api.add_folder("subdir".into(), address_subdir, None)?; assert_eq!( subdir_metadata, Metadata { @@ -109,9 +109,10 @@ async fn test_folder_remove_replace_entries() -> Result<()> { let file3_chunk = random_file_chunk(); let file4_chunk = random_file_chunk(); - let (file1_entry_hash, _, _) = folders_api.add_file("file1.txt".into(), file1_chunk.clone())?; + let (file1_entry_hash, _, _) = + folders_api.add_file("file1.txt".into(), file1_chunk.clone(), None)?; let (file2_entry_hash, file2_meta_xorname, file2_metadata) = - folders_api.add_file("file2.txt".into(), file2_chunk.clone())?; + folders_api.add_file("file2.txt".into(), file2_chunk.clone(), None)?; assert_eq!(folders_api.entries().await?.len(), 2); assert!(folders_api.contains(&file1_entry_hash)); @@ -137,7 +138,7 @@ async fn test_folder_remove_replace_entries() -> Result<()> { // now we test replacing file2.txt with file3.txt let (file3_entry_hash, file3_meta_xorname, file3_metadata) = - folders_api.replace_file(file2_entry_hash, "file3.txt".into(), file3_chunk)?; + folders_api.replace_file(file2_entry_hash, "file3.txt".into(), file3_chunk, None)?; assert!(!folders_api.contains(&file2_entry_hash)); assert!(folders_api.contains(&file3_entry_hash)); assert!(folders_api.find_by_name("file1.txt").is_none()); @@ -158,7 +159,7 @@ async fn test_folder_remove_replace_entries() -> Result<()> { // let's add file4.txt, and check that final state is correct let (file4_entry_hash, file4_meta_xorname, file4_metadata) = - folders_api.add_file("file4.txt".into(), file4_chunk)?; + folders_api.add_file("file4.txt".into(), file4_chunk, None)?; assert!(!folders_api.contains(&file1_entry_hash)); assert!(!folders_api.contains(&file2_entry_hash)); @@ -203,13 +204,13 @@ async fn test_folder_retrieve() -> Result<()> { let file1_chunk = random_file_chunk(); let (file1_entry_hash, file1_meta_xorname, file1_metadata) = - folder.add_file("file1.txt".into(), file1_chunk.clone())?; + folder.add_file("file1.txt".into(), file1_chunk.clone(), None)?; let (subfolder_entry_hash, subfolder_meta_xorname, subfolder_metadata) = - folder.add_folder("subfolder".into(), *subfolder.address())?; + folder.add_folder("subfolder".into(), *subfolder.address(), None)?; let file2_chunk = random_file_chunk(); let (file2_entry_hash, file2_meta_xorname, file2_metadata) = - subfolder.add_file("file2.txt".into(), file2_chunk.clone())?; + subfolder.add_file("file2.txt".into(), file2_chunk.clone(), None)?; // let's pay for storage let mut addrs2pay = vec![folder.as_net_addr(), subfolder.as_net_addr()]; @@ -285,11 +286,11 @@ async fn test_folder_merge_changes() -> Result<()> { let file_a2_chunk = random_file_chunk(); let (file_a1_entry_hash, file_a1_meta_xorname, file_a1_metadata) = - folder_a.add_file("fileA1.txt".into(), file_a1_chunk.clone())?; + folder_a.add_file("fileA1.txt".into(), file_a1_chunk.clone(), None)?; let (subfolder_a_entry_hash, subfolder_a_meta_xorname, subfolder_a_metadata) = - folder_a.add_folder("subfolderA".into(), *subfolder_a.address())?; + folder_a.add_folder("subfolderA".into(), *subfolder_a.address(), None)?; let (file_a2_entry_hash, file_a2_meta_xorname, file_a2_metadata) = - subfolder_a.add_file("fileA2.txt".into(), file_a2_chunk.clone())?; + subfolder_a.add_file("fileA2.txt".into(), file_a2_chunk.clone(), None)?; let mut folder_b = FoldersApi::new(client.clone(), wallet_dir, Some(folder_addr))?; let mut subfolder_b = FoldersApi::new(client.clone(), wallet_dir, Some(subfolder_addr))?; @@ -297,11 +298,11 @@ async fn test_folder_merge_changes() -> Result<()> { let file_b2_chunk = random_file_chunk(); let (file_b1_entry_hash, file_b1_meta_xorname, file_b1_metadata) = - folder_b.add_file("fileB1.txt".into(), file_b1_chunk.clone())?; + folder_b.add_file("fileB1.txt".into(), file_b1_chunk.clone(), None)?; let (subfolder_b_entry_hash, subfolder_b_meta_xorname, subfolder_b_metadata) = - folder_b.add_folder("subfolderB".into(), *subfolder_b.address())?; + folder_b.add_folder("subfolderB".into(), *subfolder_b.address(), None)?; let (file_b2_entry_hash, file_b2_meta_xorname, file_b2_metadata) = - subfolder_b.add_file("fileB2.txt".into(), file_b2_chunk.clone())?; + subfolder_b.add_file("fileB2.txt".into(), file_b2_chunk.clone(), None)?; // let's pay for storage let mut addrs2pay = vec![folder_a.as_net_addr(), subfolder_a.as_net_addr()]; From 25773714dd43595370ce4049a6162c106b75cb50 Mon Sep 17 00:00:00 2001 From: Gabriel Viganotti Date: Mon, 18 Mar 2024 11:46:05 -0300 Subject: [PATCH 2/2] docs(cli): adding some high-level design documentation to the acc-packet codebase --- sn_cli/src/subcommands/acc_packet.rs | 51 ++++++++++++++++++++++++++-- 1 file changed, 49 insertions(+), 2 deletions(-) diff --git a/sn_cli/src/subcommands/acc_packet.rs b/sn_cli/src/subcommands/acc_packet.rs index 1c95464a3d..d428035bb9 100644 --- a/sn_cli/src/subcommands/acc_packet.rs +++ b/sn_cli/src/subcommands/acc_packet.rs @@ -43,9 +43,56 @@ use tokio::task::JoinSet; use walkdir::{DirEntry, WalkDir}; use xor_name::XorName; -/// Object which allows a user to store and manage files, wallets, etc., with the ability -/// and tools necessary to keep a local instance in sync with its remote version stored on the network. +/// An `AccountPacket` object allows users to store and manage files, wallets, etc., with the ability +/// and tools necessary to keep an instance tracking a local storage path, as well as keeping it in sync +/// with its remote version stored on the network. +/// A `Client` and a the location for a funded local hot-wallet are required by this object in order to be able to connect +/// to the network, paying for data storage, and upload/retrieve information to/from the network. +/// /// TODO: currently only files and folders are supported, wallets, keys, etc., to be added later. +/// TODO: currently the provided Client's key is used both for encrypting data and signing network operations. +/// Be able to provide specific keys, and/or a way to derive keys for these operations is still pending. +/// +/// The `AccountPacket` keeps a reference to the network address of the root Folder holding the user's +/// files/folder hierarchy. All tracking information is kept under the `.safe` directory on disk, whose +/// content is not uploaded to the network, but only kept locally in order to realise which files/dirs +/// the user has made changes on compared to their last version retrieved from the network. +/// +/// A subdirectory called `metadata` is kept under `.safe` directory with the following files: +/// - A file named `root_folder.addr` which contains the network address where the root Folder is stored, +/// which is the one holding the entire hierarchy of user's files/dirs to be kept in sync with local changes +/// made by the user. +/// - For each of the user's files/dirs, a serialised `MetadataTrackingInfo` instance is stored on using the +/// file/dir metadata chunk xorname as filename. The information stored in these files are used to realise +/// if changes were locally made by the user in comparison with the last version of such files/dirs retrieved +/// from the network. +/// Example of files generated within an account-packet to keep track of changes makde to user's files/dirs: +/// +/// ./my-acc-packet +/// ├── my-dir-1 +/// ├── my-file.txt +/// ├── my-subdir-1 +/// │ ├── other-dir +/// │ └── my-other-file.txt +/// └── .safe +/// ├── chunk_artifacts +/// │ ├── ... +/// │ ... +/// ├── metadata +/// │ ├── 082cc90c900fa08d36067246a1e6136a828f1aae4926268c4349c200d56e34b9 +/// │ ├── 102c5536a10682bc3cdd4a1915fe2ad5e839cb94d0d3f124d0c18aee1d49ce50 +/// │ ├── 31824937c47a979df64af591f2e43f76190e65af835c4b338cbe7a7ba3f7d3cb +/// │ ├── 36778e471083140bc111677e2a86e49f4c0c20bc14ff2ad610e22615b72260b8 +/// │ ├── 3edd953cc320449e09b69b7b1b909a53874ee477f602f1a807dfd8057378367e +/// │ └── root_folder.addr +/// └── uploaded_files +/// ├── ... +/// ... +/// +/// There are other files which are stored under `.safe/chunk_artifacts` and `.safe/uploaded_files` directories +/// which are managed by the `ChunkManager` in order to locally cache chunked files, and a list of files +/// already uploaded to the network, to prevent from chunking and/or uploading the same files again. For more +/// details about these files, please refer to the `ChunkManager` module. pub struct AccountPacket { client: Client, wallet_dir: PathBuf,