diff --git a/autonomi-cli/src/actions/download.rs b/autonomi-cli/src/actions/download.rs deleted file mode 100644 index ba004930e3..0000000000 --- a/autonomi-cli/src/actions/download.rs +++ /dev/null @@ -1,52 +0,0 @@ -// Copyright 2024 MaidSafe.net limited. -// -// This SAFE Network Software is licensed to you under The General Public License (GPL), version 3. -// Unless required by applicable law or agreed to in writing, the SAFE Network Software distributed -// under the GPL Licence is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. Please review the Licences for the specific language governing -// permissions and limitations relating to use of the SAFE Network Software. - -use super::get_progress_bar; -use autonomi::{client::address::str_to_addr, Client}; -use color_eyre::eyre::{eyre, Context, Result}; -use std::path::PathBuf; - -pub async fn download(addr: &str, dest_path: &str, client: &mut Client) -> Result<()> { - let address = str_to_addr(addr).wrap_err("Failed to parse data address")?; - let archive = client - .archive_get(address) - .await - .wrap_err("Failed to fetch data from address")?; - - let progress_bar = get_progress_bar(archive.map.len() as u64)?; - let mut all_errs = vec![]; - for (path, addr) in archive.map { - progress_bar.println(format!("Fetching file: {path:?}...")); - let bytes = match client.data_get(addr).await { - Ok(bytes) => bytes, - Err(e) => { - let err = format!("Failed to fetch file {path:?}: {e}"); - all_errs.push(err); - continue; - } - }; - - let path = PathBuf::from(dest_path).join(path); - let here = PathBuf::from("."); - let parent = path.parent().unwrap_or_else(|| &here); - std::fs::create_dir_all(parent)?; - std::fs::write(path, bytes)?; - progress_bar.clone().inc(1); - } - progress_bar.finish_and_clear(); - - if all_errs.is_empty() { - println!("Successfully downloaded data at: {addr}"); - Ok(()) - } else { - let err_no = all_errs.len(); - eprintln!("{err_no} errors while downloading data at: {addr}"); - eprintln!("{all_errs:#?}"); - Err(eyre!("Errors while downloading data")) - } -} diff --git a/autonomi-cli/src/actions/mod.rs b/autonomi-cli/src/actions/mod.rs index 8b4662c3d9..98ef491064 100644 --- a/autonomi-cli/src/actions/mod.rs +++ b/autonomi-cli/src/actions/mod.rs @@ -7,10 +7,6 @@ // permissions and limitations relating to use of the SAFE Network Software. mod connect; -mod download; mod progress_bar; pub use connect::connect_to_network; -pub use download::download; - -pub use progress_bar::get_progress_bar; diff --git a/autonomi-cli/src/actions/progress_bar.rs b/autonomi-cli/src/actions/progress_bar.rs index 2fcfe0ba20..5e2c6c914e 100644 --- a/autonomi-cli/src/actions/progress_bar.rs +++ b/autonomi-cli/src/actions/progress_bar.rs @@ -10,6 +10,7 @@ use color_eyre::eyre::Result; use indicatif::{ProgressBar, ProgressStyle}; use std::time::Duration; +#[allow(dead_code)] pub fn get_progress_bar(length: u64) -> Result { let progress_bar = ProgressBar::new(length); progress_bar.set_style( diff --git a/autonomi-cli/src/commands.rs b/autonomi-cli/src/commands.rs index bb718df43a..4c2067aa87 100644 --- a/autonomi-cli/src/commands.rs +++ b/autonomi-cli/src/commands.rs @@ -9,6 +9,7 @@ mod file; mod register; mod vault; +use std::path::PathBuf; use clap::Subcommand; use color_eyre::Result; @@ -55,7 +56,7 @@ pub enum FileCmd { /// The address of the file to download. addr: String, /// The destination file path. - dest_file: String, + dest_file: PathBuf, }, /// List previous uploads diff --git a/autonomi-cli/src/commands/file.rs b/autonomi-cli/src/commands/file.rs index d99a848214..bfa4719460 100644 --- a/autonomi-cli/src/commands/file.rs +++ b/autonomi-cli/src/commands/file.rs @@ -8,9 +8,11 @@ use crate::utils::collect_upload_summary; use autonomi::client::address::addr_to_str; +use autonomi::client::address::str_to_addr; use autonomi::Multiaddr; use color_eyre::eyre::Context; use color_eyre::eyre::Result; +use std::path::Path; use std::path::PathBuf; pub async fn cost(file: &str, peers: Vec) -> Result<()> { @@ -26,22 +28,35 @@ pub async fn cost(file: &str, peers: Vec) -> Result<()> { println!("Total cost: {cost}"); Ok(()) } -pub async fn upload(file: &str, peers: Vec) -> Result<()> { +pub async fn upload(path: &str, peers: Vec) -> Result<()> { let wallet = crate::keys::load_evm_wallet()?; let mut client = crate::actions::connect_to_network(peers).await?; let event_receiver = client.enable_client_events(); let (upload_summary_thread, upload_completed_tx) = collect_upload_summary(event_receiver); - println!("Uploading data to network..."); + let path = PathBuf::from(path); + + let xor_name = if path.is_dir() { + println!("Uploading directory: {path:?}"); + info!("Uploading directory: {path:?}"); + client + .dir_upload(&path, &wallet) + .await + .wrap_err("Failed to upload directory")? + } else { + println!("Uploading file: {path:?}"); + info!("Uploading file: {path:?}"); + client + .file_upload(&path, &wallet) + .await + .wrap_err("Failed to upload file")? + }; - let xor_name = client - .dir_upload(PathBuf::from(file), &wallet) - .await - .wrap_err("Failed to upload file")?; let addr = addr_to_str(xor_name); - println!("Successfully uploaded: {file}"); + println!("Successfully uploaded: {path:?}"); println!("At address: {addr}"); + info!("Successfully uploaded: {path:?} at address: {addr}"); if let Ok(()) = upload_completed_tx.send(()) { let summary = upload_summary_thread.await?; if summary.record_count == 0 { @@ -50,13 +65,18 @@ pub async fn upload(file: &str, peers: Vec) -> Result<()> { println!("Number of chunks uploaded: {}", summary.record_count); println!("Total cost: {} AttoTokens", summary.tokens_spent); } + info!("Summary for upload of data {path:?} at {addr:?}: {summary:?}"); } Ok(()) } -pub async fn download(addr: &str, dest_path: &str, peers: Vec) -> Result<()> { - let mut client = crate::actions::connect_to_network(peers).await?; - crate::actions::download(addr, dest_path, &mut client).await +pub async fn download(addr: &str, dest_path: &Path, peers: Vec) -> Result<()> { + let client = crate::actions::connect_to_network(peers).await?; + let address = str_to_addr(addr).wrap_err("Failed to parse data address")?; + + client.download_file_or_dir(address, dest_path).await?; + + Ok(()) } pub fn list(_peers: Vec) -> Result<()> { diff --git a/autonomi-cli/src/main.rs b/autonomi-cli/src/main.rs index 2cf4e9cd61..de4cdcf4c4 100644 --- a/autonomi-cli/src/main.rs +++ b/autonomi-cli/src/main.rs @@ -49,14 +49,16 @@ async fn main() -> Result<()> { fn init_logging_and_metrics(opt: &Opt) -> Result<(ReloadHandle, Option)> { let logging_targets = vec![ + ("autonomi-cli".to_string(), Level::TRACE), + ("autonomi".to_string(), Level::TRACE), + ("evmlib".to_string(), Level::TRACE), + ("sn_evm".to_string(), Level::TRACE), ("sn_networking".to_string(), Level::INFO), ("sn_build_info".to_string(), Level::TRACE), - ("autonomi-cli".to_string(), Level::TRACE), ("sn_logging".to_string(), Level::TRACE), ("sn_peers_acquisition".to_string(), Level::TRACE), ("sn_protocol".to_string(), Level::TRACE), ("sn_registers".to_string(), Level::TRACE), - ("sn_evm".to_string(), Level::TRACE), ]; let mut log_builder = LogBuilder::new(logging_targets); log_builder.output_dest(opt.log_output_dest.clone()); diff --git a/autonomi/src/client/archive.rs b/autonomi/src/client/archive.rs index d3cf9714ec..f38ca24cbc 100644 --- a/autonomi/src/client/archive.rs +++ b/autonomi/src/client/archive.rs @@ -30,7 +30,7 @@ pub struct Archive { impl Archive { /// Deserialize from bytes. - pub fn from_bytes(data: Bytes) -> Result { + pub fn from_bytes(data: &Bytes) -> Result { let root: Archive = rmp_serde::from_slice(&data[..])?; Ok(root) @@ -49,7 +49,7 @@ impl Client { /// Fetch an archive from the network pub async fn archive_get(&self, addr: ArchiveAddr) -> Result { let data = self.data_get(addr).await?; - Ok(Archive::from_bytes(data)?) + Ok(Archive::from_bytes(&data)?) } /// Upload an archive to the network diff --git a/autonomi/src/client/fs.rs b/autonomi/src/client/fs.rs index 8fff06324c..674e03fc2b 100644 --- a/autonomi/src/client/fs.rs +++ b/autonomi/src/client/fs.rs @@ -10,7 +10,7 @@ use crate::client::Client; use bytes::Bytes; use sn_evm::EvmWallet; use std::collections::HashMap; -use std::path::PathBuf; +use std::path::Path; use super::archive::{Archive, ArchiveAddr}; use super::data::{DataAddr, GetError, PutError}; @@ -48,7 +48,7 @@ impl Client { pub async fn file_download( &self, data_addr: DataAddr, - to_dest: PathBuf, + to_dest: &Path, ) -> Result<(), DownloadError> { let data = self.data_get(data_addr).await?; if let Some(parent) = to_dest.parent() { @@ -62,20 +62,52 @@ impl Client { pub async fn dir_download( &self, archive_addr: ArchiveAddr, - to_dest: PathBuf, + to_dest: &Path, ) -> Result<(), DownloadError> { let archive = self.archive_get(archive_addr).await?; for (path, addr) in archive.map { - self.file_download(addr, to_dest.join(path)).await?; + self.file_download(addr, &to_dest.join(path)).await?; } Ok(()) } + /// Download either a file or a directory depending on the data present at the provided address. + pub async fn download_file_or_dir( + &self, + address: DataAddr, + to_dest: &Path, + ) -> Result<(), DownloadError> { + let data = self.data_get(address).await?; + + if let Ok(archive) = Archive::from_bytes(&data) { + info!("Got an Archive from bytes, unpacking directory to {to_dest:?}"); + for (path, addr) in archive.map { + let dest = to_dest.join(path); + + #[cfg(feature = "loud")] + println!("Downloading file: {addr:?} to {dest:?}"); + + debug!("Downloading archived file: {addr:?} to {dest:?}"); + self.file_download(addr, &dest).await?; + } + } else { + info!("The downloaded data is not an Archive, saving it as a file."); + #[cfg(feature = "loud")] + println!("Downloading file: {address:?} to {to_dest:?}"); + if let Some(parent) = to_dest.parent() { + tokio::fs::create_dir_all(parent).await?; + } + tokio::fs::write(to_dest, data).await?; + } + + Ok(()) + } + /// Upload a directory to the network. The directory is recursively walked. /// Reads all files, splits into chunks, uploads chunks, uploads datamaps, uploads archive, returns ArchiveAddr (pointing to the archive) pub async fn dir_upload( &self, - dir_path: PathBuf, + dir_path: &Path, wallet: &EvmWallet, ) -> Result { let mut map = HashMap::new(); @@ -87,13 +119,13 @@ impl Client { continue; } - let path = entry.path().to_path_buf(); + let path = entry.path(); tracing::info!("Uploading file: {path:?}"); #[cfg(feature = "loud")] println!("Uploading file: {path:?}"); - let file = self.file_upload(path.clone(), wallet).await?; + let file = self.file_upload(path, wallet).await?; - map.insert(path, file); + map.insert(path.to_path_buf(), file); } let archive = Archive { map }; @@ -106,9 +138,9 @@ impl Client { /// Upload a file to the network. /// Reads file, splits into chunks, uploads chunks, uploads datamap, returns DataAddr (pointing to the datamap) - async fn file_upload( + pub async fn file_upload( &self, - path: PathBuf, + path: &Path, wallet: &EvmWallet, ) -> Result { let data = tokio::fs::read(path).await?; @@ -119,7 +151,7 @@ impl Client { /// Get the cost to upload a file/dir to the network. /// quick and dirty implementation, please refactor once files are cleanly implemented - pub async fn file_cost(&self, path: &PathBuf) -> Result { + pub async fn file_cost(&self, path: &Path) -> Result { let mut map = HashMap::new(); let mut total_cost = sn_evm::Amount::ZERO; diff --git a/autonomi/src/client/mod.rs b/autonomi/src/client/mod.rs index f19216fe84..68dfe0d50a 100644 --- a/autonomi/src/client/mod.rs +++ b/autonomi/src/client/mod.rs @@ -202,6 +202,7 @@ pub enum ClientEvent { } /// Summary of an upload operation. +#[derive(Debug, Clone)] pub struct UploadSummary { pub record_count: usize, pub tokens_spent: Amount, diff --git a/autonomi/tests/fs.rs b/autonomi/tests/fs.rs index 5b1fce533b..4c286725aa 100644 --- a/autonomi/tests/fs.rs +++ b/autonomi/tests/fs.rs @@ -14,6 +14,7 @@ use sha2::{Digest, Sha256}; use sn_logging::LogBuilder; use std::fs::File; use std::io::{BufReader, Read}; +use std::path::PathBuf; use std::time::Duration; use test_utils::{evm::get_funded_wallet, peers_from_env}; use tokio::time::sleep; @@ -30,13 +31,13 @@ async fn dir_upload_download() -> Result<()> { let wallet = get_funded_wallet(); let addr = client - .dir_upload("tests/file/test_dir".into(), &wallet) + .dir_upload(&PathBuf::from("tests/file/test_dir"), &wallet) .await?; sleep(Duration::from_secs(10)).await; client - .dir_download(addr, "tests/file/test_dir_fetched".into()) + .dir_download(addr, &PathBuf::from("tests/file/test_dir_fetched")) .await?; // compare the two directories @@ -86,7 +87,7 @@ async fn file_into_vault() -> Result<()> { let client_sk = bls::SecretKey::random(); let addr = client - .dir_upload("tests/file/test_dir".into(), &wallet) + .dir_upload(&PathBuf::from("tests/file/test_dir"), &wallet) .await?; sleep(Duration::from_secs(2)).await; @@ -99,7 +100,7 @@ async fn file_into_vault() -> Result<()> { let new_client = Client::connect(&[]).await?; if let Some(ap) = new_client.fetch_and_decrypt_vault(&client_sk).await? { - let ap_archive_fetched = autonomi::client::archive::Archive::from_bytes(ap)?; + let ap_archive_fetched = autonomi::client::archive::Archive::from_bytes(&ap)?; assert_eq!( archive.map, ap_archive_fetched.map,