From 9b0dc52ca9c2192c3fc039b7381645a17c8d83d4 Mon Sep 17 00:00:00 2001 From: Jason Paul Date: Mon, 11 Mar 2024 22:47:00 +0000 Subject: [PATCH] feat: move ctu --- sn_cli/src/subcommands/acc_packet.rs | 51 ++------ sn_cli/src/subcommands/files.rs | 119 ++++++++++++------ .../subcommands/files/iterative_uploader.rs | 26 +--- sn_client/src/api.rs | 4 + 4 files changed, 95 insertions(+), 105 deletions(-) diff --git a/sn_cli/src/subcommands/acc_packet.rs b/sn_cli/src/subcommands/acc_packet.rs index 70eabc51a2..a2cb538578 100644 --- a/sn_cli/src/subcommands/acc_packet.rs +++ b/sn_cli/src/subcommands/acc_packet.rs @@ -14,6 +14,7 @@ use sn_client::registers::EntryHash; use sn_client::transfers::HotWallet; use sn_client::{Client, FilesApi, FolderEntry, FoldersApi, Metadata, WalletClient}; +use crate::subcommands::files; use crate::subcommands::files::{ download::download_file, iterative_uploader::IterativeUploader, upload::FilesUploadOptions, }; @@ -21,8 +22,6 @@ use color_eyre::{ eyre::{bail, eyre}, Result, }; -use rand::prelude::SliceRandom; -use rand::thread_rng; use std::{ collections::{ btree_map::{Entry, OccupiedEntry}, @@ -665,46 +664,14 @@ impl AccountPacket { let files_api = FilesApi::build(self.client.clone(), self.wallet_dir.clone())?; let mut chunk_manager = ChunkManager::new(&self.tracking_info_dir.clone()); - let chunks_to_upload = if chunk_manager.is_chunks_empty() { - let chunks = - chunk_manager.already_put_chunks(&self.files_dir, options.make_data_public)?; - println!( - "Files upload attempted previously, verifying {} chunks", - chunks.len() - ); - - let failed_chunks = files_api - .client() - .verify_uploaded_chunks(&chunks, options.batch_size) - .await?; - - chunk_manager.mark_completed( - chunks - .into_iter() - .filter(|c| !failed_chunks.contains(c)) - .map(|(xor, _)| xor), - )?; - - if failed_chunks.is_empty() { - // msg_files_already_uploaded_verified(); - // if !make_data_public { - // msg_not_public_by_default(); - // } - // msg_star_line(); - // if self.chunk_manager.completed_files().is_empty() { - // msg_chk_mgr_no_verified_file_nor_re_upload(); - // } - // msg_chunk_manager_upload_complete(self.chunk_manager); - bail!("") - } - // msg_unverified_chunks_reattempted(&failed_chunks.len()); - failed_chunks - } else { - let mut chunks = chunk_manager.get_chunks(); - let mut rng = thread_rng(); - chunks.shuffle(&mut rng); - chunks - }; + let chunks_to_upload = files::ctu( + &files_api, + &mut chunk_manager, + &self.files_dir.clone(), + options.batch_size, + options.make_data_public, + ) + .await?; IterativeUploader::new(chunk_manager, files_api) .iterate_upload(chunks_to_upload, self.files_dir.clone(), options.clone()) diff --git a/sn_cli/src/subcommands/files.rs b/sn_cli/src/subcommands/files.rs index 1ed1152fa5..3877f6ef20 100644 --- a/sn_cli/src/subcommands/files.rs +++ b/sn_cli/src/subcommands/files.rs @@ -136,45 +136,14 @@ pub(crate) async fn files_cmds( bail!("The provided file path is invalid. Please verify the path."); } } else { - let chunks_to_upload = if chunk_manager.is_chunks_empty() { - let chunks = chunk_manager.already_put_chunks(&file_path, make_data_public)?; - println!( - "Files upload attempted previously, verifying {} chunks", - chunks.len() - ); - - let failed_chunks = files_api - .client() - .verify_uploaded_chunks(&chunks, batch_size) - .await?; - - chunk_manager.mark_completed( - chunks - .into_iter() - .filter(|c| !failed_chunks.contains(c)) - .map(|(xor, _)| xor), - )?; - - if failed_chunks.is_empty() { - // msg_files_already_uploaded_verified(); - // if !make_data_public { - // msg_not_public_by_default(); - // } - // msg_star_line(); - // if self.chunk_manager.completed_files().is_empty() { - // msg_chk_mgr_no_verified_file_nor_re_upload(); - // } - // msg_chunk_manager_upload_complete(self.chunk_manager); - return Ok(()); - } - // msg_unverified_chunks_reattempted(&failed_chunks.len()); - failed_chunks - } else { - let mut chunks = chunk_manager.get_chunks(); - let mut rng = thread_rng(); - chunks.shuffle(&mut rng); - chunks - }; + let chunks_to_upload = ctu( + &files_api, + &mut chunk_manager, + &file_path, + batch_size, + make_data_public, + ) + .await?; IterativeUploader::new(chunk_manager, files_api) .iterate_upload( @@ -303,6 +272,51 @@ pub(crate) async fn files_cmds( Ok(()) } +pub async fn ctu( + files_api: &FilesApi, + chunk_manager: &mut ChunkManager, + file_path: &Path, + batch_size: usize, + make_data_public: bool, +) -> Result> { + let chunks_to_upload = if chunk_manager.is_chunks_empty() { + let chunks = chunk_manager.already_put_chunks(file_path, make_data_public)?; + + let failed_chunks = files_api + .client() + .verify_uploaded_chunks(&chunks, batch_size) + .await?; + + chunk_manager.mark_completed( + chunks + .into_iter() + .filter(|c| !failed_chunks.contains(c)) + .map(|(xor, _)| xor), + )?; + + if failed_chunks.is_empty() { + msg_files_already_uploaded_verified(); + if !make_data_public { + msg_not_public_by_default(); + } + msg_star_line(); + if chunk_manager.completed_files().is_empty() { + msg_chk_mgr_no_verified_file_nor_re_upload(); + } + iterative_uploader::msg_chunk_manager_upload_complete(chunk_manager.clone()); + bail!("failed chunks are empty") + } + msg_unverified_chunks_reattempted(&failed_chunks.len()); + failed_chunks + } else { + let mut chunks = chunk_manager.get_chunks(); + let mut rng = thread_rng(); + chunks.shuffle(&mut rng); + chunks + }; + Ok(chunks_to_upload) +} + pub fn get_progress_bar(length: u64) -> Result { let progress_bar = ProgressBar::new(length); progress_bar.set_style( @@ -313,3 +327,30 @@ pub fn get_progress_bar(length: u64) -> Result { progress_bar.enable_steady_tick(Duration::from_millis(100)); Ok(progress_bar) } + +fn msg_files_already_uploaded_verified() { + println!("All files were already uploaded and verified"); + println!("**************************************"); + println!("* Uploaded Files *"); +} + +fn msg_chk_mgr_no_verified_file_nor_re_upload() { + println!("chunk_manager doesn't have any verified_files, nor any failed_chunks to re-upload."); +} + +fn msg_not_public_by_default() { + println!("* *"); + println!("* These are not public by default. *"); + println!("* Reupload with `-p` option *"); + println!("* to publish the datamaps. *"); +} + +fn msg_star_line() { + println!("**************************************"); +} + +fn msg_unverified_chunks_reattempted(failed_amount: &usize) { + println!( + "{failed_amount} chunks were uploaded in the past but failed to verify. Will attempt to upload them again..." + ); +} diff --git a/sn_cli/src/subcommands/files/iterative_uploader.rs b/sn_cli/src/subcommands/files/iterative_uploader.rs index 0bf535bcac..be0abc8469 100644 --- a/sn_cli/src/subcommands/files/iterative_uploader.rs +++ b/sn_cli/src/subcommands/files/iterative_uploader.rs @@ -229,7 +229,8 @@ fn msg_final( msg_made_payment_info(files_upload.get_upload_storage_cost(), uploaded_chunks); } -fn msg_chunk_manager_upload_complete(chunk_manager: ChunkManager) { + +pub fn msg_chunk_manager_upload_complete(chunk_manager: ChunkManager) { for (file_name, addr) in chunk_manager.completed_files() { let hex_addr = addr.to_hex(); if let Some(file_name) = file_name.to_str() { @@ -278,28 +279,10 @@ fn msg_payment_details( println!("Made payment of {total_royalty_fees} for royalties fees"); println!("New wallet balance: {final_balance}"); } - -// fn msg_chk_mgr_no_verified_file_nor_re_upload() { -// println!("chunk_manager doesn't have any verified_files, nor any failed_chunks to re-upload."); -// } - fn msg_star_line() { println!("**************************************"); } -// fn msg_not_public_by_default() { -// println!("* *"); -// println!("* These are not public by default. *"); -// println!("* Reupload with `-p` option *"); -// println!("* to publish the datamaps. *"); -// } - -// fn msg_files_already_uploaded_verified() { -// println!("All files were already uploaded and verified"); -// println!("**************************************"); -// println!("* Uploaded Files *"); -// } - fn msg_uploading_chunks(chunks_to_upload_len: usize) { println!("Uploading {chunks_to_upload_len} chunks",); } @@ -315,8 +298,3 @@ fn msg_uploaded_files_banner() { println!("**************************************"); println!("* Uploaded Files *"); } -// fn msg_unverified_chunks_reattempted(failed_amount: &usize) { -// println!( -// "{failed_amount} chunks were uploaded in the past but failed to verify. Will attempt to upload them again..." -// ); -// } diff --git a/sn_client/src/api.rs b/sn_client/src/api.rs index 96cc3cdbaa..5c6caa9092 100644 --- a/sn_client/src/api.rs +++ b/sn_client/src/api.rs @@ -1107,6 +1107,10 @@ impl Client { } } + println!( + "Files upload attempted previously, verifying {} chunks", + failed_chunks.len() + ); Ok(failed_chunks) } }