feat(folders): Folders and AccPacket to accept an encryption key for metadata chunks

sn_cli/src/subcommands/acc_packet.rs

@@ -8,6 +8,7 @@
 
 mod change_tracking;
 
+use bls::PublicKey;
 use change_tracking::*;
 
 use super::files::ChunkManager;
@@ -42,9 +43,56 @@
 use walkdir::{DirEntry, WalkDir};
 use xor_name::XorName;
 
-/// Object which allows a user to store and manage files, wallets, etc., with the ability
-/// and tools necessary to keep a local instance in sync with its remote version stored on the network.
+/// An `AccountPacket` object allows users to store and manage files, wallets, etc., with the ability
+/// and tools necessary to keep an instance tracking a local storage path, as well as keeping it in sync
+/// with its remote version stored on the network.
+/// A `Client` and a the location for a funded local hot-wallet are required by this object in order to be able to connect
+/// to the network, paying for data storage, and upload/retrieve information to/from the network.  
+///
 /// TODO: currently only files and folders are supported, wallets, keys, etc., to be added later.
+/// TODO: currently the provided Client's key is used both for encrypting data and signing network operations.
+/// Be able to provide specific keys, and/or a way to derive keys for these operations is still pending.
+///
+/// The `AccountPacket` keeps a reference to the network address of the root Folder holding the user's
+/// files/folder hierarchy. All tracking information is kept under the `.safe` directory on disk, whose
+/// content is not uploaded to the network, but only kept locally in order to realise which files/dirs
+/// the user has made changes on compared to their last version retrieved from the network.
+///
+/// A subdirectory called `metadata` is kept under `.safe` directory with the following files:
+/// - A file named `root_folder.addr` which contains the network address where the root Folder is stored,
+/// which is the one holding the entire hierarchy of user's files/dirs to be kept in sync with local changes
+/// made by the user.
+/// - For each of the user's files/dirs, a serialised `MetadataTrackingInfo` instance is stored on using the
+/// file/dir metadata chunk xorname as filename. The information stored in these files are used to realise
+/// if changes were locally made by the user in comparison with the last version of such files/dirs retrieved
+/// from the network.
+/// Example of files generated within an account-packet to keep track of changes makde to user's files/dirs:
+///
+/// ./my-acc-packet
+/// ├── my-dir-1
+/// ├── my-file.txt
+/// ├── my-subdir-1
+/// │   ├── other-dir
+/// │   └── my-other-file.txt
+/// └── .safe
+///     ├── chunk_artifacts
+///     │   ├── ...
+///     │   ...
+///     ├── metadata
+///     │   ├── 082cc90c900fa08d36067246a1e6136a828f1aae4926268c4349c200d56e34b9
+///     │   ├── 102c5536a10682bc3cdd4a1915fe2ad5e839cb94d0d3f124d0c18aee1d49ce50
+///     │   ├── 31824937c47a979df64af591f2e43f76190e65af835c4b338cbe7a7ba3f7d3cb
+///     │   ├── 36778e471083140bc111677e2a86e49f4c0c20bc14ff2ad610e22615b72260b8
+///     │   ├── 3edd953cc320449e09b69b7b1b909a53874ee477f602f1a807dfd8057378367e
+///     │   └── root_folder.addr
+///     └── uploaded_files
+///         ├── ...
+///         ...
+///
+/// There are other files which are stored under `.safe/chunk_artifacts` and `.safe/uploaded_files` directories
+/// which are managed by the `ChunkManager` in order to locally cache chunked files, and a list of files
+/// already uploaded to the network, to prevent from chunking and/or uploading the same files again. For more
+/// details about these files, please refer to the `ChunkManager` module.
 pub struct AccountPacket {
     client: Client,
     wallet_dir: PathBuf,
@@ -58,6 +106,8 @@
 
 impl AccountPacket {
     /// Initialise directory as a fresh new packet with the given/random address for its root Folder.
+    /// The client's key will be used for encrypting the files/folders metadata chunks. Currently,
+    /// when the user requests to encrypt data, the provided Client's signer key is used.
     pub fn init(
         client: Client,
         wallet_dir: &Path,
@@ -180,7 +230,7 @@
     /// both pushing and pulling changes to/form the network.
     pub async fn sync(&mut self, options: FilesUploadOptions) -> Result<()> {
         let ChangesToApply { folders, mutations } =
-            self.scan_files_and_folders_for_changes(false)?;
+            self.scan_files_and_folders_for_changes(options.make_data_public)?;
 
         if mutations.is_empty() {
             println!("No local changes made to files/folders to be pushed to network.");
@@ -368,16 +418,20 @@
 
     // Scan existing files and folders on disk, generating a report of all the detected
     // changes based on the tracking info kept locally.
-    // TODO: encrypt the data-map and metadata if make_data_public is false.
-    fn scan_files_and_folders_for_changes(
-        &self,
-        _make_data_public: bool,
-    ) -> Result<ChangesToApply> {
+    // If make_data_public is false the metadata chunks are encrypted.
+    fn scan_files_and_folders_for_changes(&self, make_data_public: bool) -> Result<ChangesToApply> {
         // we don't use the local cache in order to realise of any changes made to files content.
         let mut chunk_manager = ChunkManager::new(&self.tracking_info_dir);
         chunk_manager.chunk_with_iter(self.iter_only_files(), false, false)?;
 
-        let mut changes = self.read_folders_hierarchy_from_disk()?;
+        let encryption_pk = if make_data_public {
+            None
+        } else {
+            // we pass down the key to encrypt the metadata chunk of any new content detected.
+            Some(self.client.signer_pk())
+        };
+
+        let mut changes = self.read_folders_hierarchy_from_disk(encryption_pk)?;
 
         // add chunked files to the corresponding Folders
         let folders = &mut changes.folders;
@@ -392,13 +446,14 @@
                     Ok(Some(tracking_info)) => match &tracking_info.metadata.content {
                         FolderEntry::File(chunk) => {
                             if chunk.address() != &chunked_file.head_chunk_address {
-                                // TODO: we need to encrypt the data-map and metadata if make_data_public is false.
                                 let (entry_hash, meta_xorname, metadata) = replace_item_in_folder(
                                     &mut parent_folder,
                                     tracking_info.entry_hash,
                                     chunked_file.file_name.clone(),
                                     chunked_file.data_map.clone(),
+                                    encryption_pk,
                                 )?;
+
                                 changes.mutations.push(Mutation::FileContentChanged((
                                     tracking_info.meta_xorname,
                                     MetadataTrackingInfo {
@@ -412,12 +467,12 @@
                         }
                         FolderEntry::Folder(_) => {
                             // New file found where there used to be a folder
-                            // TODO: we need to encrypt the data-map and metadata if make_data_public is false.
                             let (entry_hash, meta_xorname, metadata) = replace_item_in_folder(
                                 &mut parent_folder,
                                 tracking_info.entry_hash,
                                 chunked_file.file_name.clone(),
                                 chunked_file.data_map.clone(),
+                                encryption_pk,
                             )?;
                             changes
                                 .mutations
@@ -430,11 +485,11 @@
                         }
                     },
                     Ok(None) => {
-                        // TODO: we need to encrypt the data-map and metadata if make_data_public is false.
                         let (entry_hash, meta_xorname, metadata) =
                             parent_folder.get_mut().0.add_file(
                                 chunked_file.file_name.clone(),
                                 chunked_file.data_map.clone(),
+                                encryption_pk,
                             )?;
                         parent_folder.get_mut().1.has_new_entries();
 
@@ -485,8 +540,12 @@
         Ok(changes)
     }
 
-    // Build Folders hierarchy from the set files dir
-    fn read_folders_hierarchy_from_disk(&self) -> Result<ChangesToApply> {
+    // Build Folders hierarchy from the set files dir. The metadata chunk of every new folder
+    // will be encrpyted if an encrpytion key has been provided.
+    fn read_folders_hierarchy_from_disk(
+        &self,
+        encryption_pk: Option<PublicKey>,
+    ) -> Result<ChangesToApply> {
         let mut changes = ChangesToApply::default();
         for (dir_path, depth, parent, dir_name) in self.iter_only_dirs().filter_map(|entry| {
             entry.path().parent().map(|parent| {
@@ -513,7 +572,7 @@
 
                 if folder_change.is_new_folder() {
                     let (entry_hash, meta_xorname, metadata) =
-                        parent_folder.add_folder(dir_name, curr_folder_addr)?;
+                        parent_folder.add_folder(dir_name, curr_folder_addr, encryption_pk)?;
                     parent_folder_change.has_new_entries();
 
                     changes
@@ -781,15 +840,22 @@
 }
 
 // Replace a file/folder item from a given Folder (passed in as a container's OccupiedEntry').
+// The metadata chunk of the new item (folder/file) will be encrpyted if a key has been provided.
 fn replace_item_in_folder(
     folder: &mut OccupiedEntry<'_, PathBuf, (FoldersApi, FolderChange)>,
     entry_hash: EntryHash,
     file_name: OsString,
     data_map: Chunk,
+    encryption_pk: Option<PublicKey>,
 ) -> Result<(EntryHash, XorName, Metadata)> {
     let (ref mut folders_api, ref mut folder_change) = folder.get_mut();
     folder_change.has_new_entries();
-    let res = folders_api.replace_file(entry_hash, file_name.clone(), data_map.clone())?;
+    let res = folders_api.replace_file(
+        entry_hash,
+        file_name.clone(),
+        data_map.clone(),
+        encryption_pk,
+    )?;
     Ok(res)
 }
 

sn_cli/src/subcommands/folders.rs

@@ -102,7 +102,7 @@ pub(crate) async fn folders_cmds(
 ) -> Result<()> {
     match cmds {
         FoldersCmds::Init { path, folder_addr } => {
-            // init path as a fresh new folder
+            // initialise path as a fresh new Folder with a random network address if none was provided
             let root_folder_addr =
                 folder_addr.and_then(|hex_str| RegisterAddress::from_hex(&hex_str).ok());
             let acc_packet =
@@ -115,7 +115,7 @@ pub(crate) async fn folders_cmds(
             make_data_public,
             retry_strategy,
         } => {
-            // init path as a fresh new folder
+            // initialise path as a fresh new Folder with a random network address
             let mut acc_packet = AccountPacket::init(client.clone(), root_dir, &path, None)?;
 
             let options = FilesUploadOptions {
@@ -163,7 +163,6 @@ pub(crate) async fn folders_cmds(
         }
         FoldersCmds::Status { path } => {
             let acc_packet = AccountPacket::from_path(client.clone(), root_dir, &path)?;
-
             acc_packet.status()?;
         }
         FoldersCmds::Sync {

sn_client/src/error.rs

@@ -41,6 +41,9 @@ pub enum Error {
     #[error("Chunks error {0}.")]
     Chunks(#[from] super::chunks::Error),
 
+    #[error("Decrypting a Folder's item failed.")]
+    FolderEntryDecryption,
+
     #[error("SelfEncryption Error {0}.")]
     SelfEncryptionIO(#[from] self_encryption::Error),