Skip to content

Commit

Permalink
feat: move ctu
Browse files Browse the repository at this point in the history
  • Loading branch information
@JasonPaulGithub
JasonPaulGithub committed Mar 11, 2024
1 parent d3b1d49 commit 9b0dc52
Show file tree
Hide file tree
Showing 4 changed files with 95 additions and 105 deletions.
51 changes: 9 additions & 42 deletions sn_cli/src/subcommands/acc_packet.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,15 +14,14 @@ use sn_client::registers::EntryHash;
use sn_client::transfers::HotWallet;
use sn_client::{Client, FilesApi, FolderEntry, FoldersApi, Metadata, WalletClient};

use crate::subcommands::files;
use crate::subcommands::files::{
download::download_file, iterative_uploader::IterativeUploader, upload::FilesUploadOptions,
};
use color_eyre::{
eyre::{bail, eyre},
Result,
};
use rand::prelude::SliceRandom;
use rand::thread_rng;
use std::{
collections::{
btree_map::{Entry, OccupiedEntry},
Expand Down Expand Up @@ -665,46 +664,14 @@ impl AccountPacket {
let files_api = FilesApi::build(self.client.clone(), self.wallet_dir.clone())?;
let mut chunk_manager = ChunkManager::new(&self.tracking_info_dir.clone());

let chunks_to_upload = if chunk_manager.is_chunks_empty() {
let chunks =
chunk_manager.already_put_chunks(&self.files_dir, options.make_data_public)?;
println!(
"Files upload attempted previously, verifying {} chunks",
chunks.len()
);

let failed_chunks = files_api
.client()
.verify_uploaded_chunks(&chunks, options.batch_size)
.await?;

chunk_manager.mark_completed(
chunks
.into_iter()
.filter(|c| !failed_chunks.contains(c))
.map(|(xor, _)| xor),
)?;

if failed_chunks.is_empty() {
// msg_files_already_uploaded_verified();
// if !make_data_public {
// msg_not_public_by_default();
// }
// msg_star_line();
// if self.chunk_manager.completed_files().is_empty() {
// msg_chk_mgr_no_verified_file_nor_re_upload();
// }
// msg_chunk_manager_upload_complete(self.chunk_manager);
bail!("")
}
// msg_unverified_chunks_reattempted(&failed_chunks.len());
failed_chunks
} else {
let mut chunks = chunk_manager.get_chunks();
let mut rng = thread_rng();
chunks.shuffle(&mut rng);
chunks
};
let chunks_to_upload = files::ctu(
&files_api,
&mut chunk_manager,
&self.files_dir.clone(),
options.batch_size,
options.make_data_public,
)
.await?;

IterativeUploader::new(chunk_manager, files_api)
.iterate_upload(chunks_to_upload, self.files_dir.clone(), options.clone())
Expand Down
119 changes: 80 additions & 39 deletions sn_cli/src/subcommands/files.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -136,45 +136,14 @@ pub(crate) async fn files_cmds(
bail!("The provided file path is invalid. Please verify the path.");
}
} else {
let chunks_to_upload = if chunk_manager.is_chunks_empty() {
let chunks = chunk_manager.already_put_chunks(&file_path, make_data_public)?;
println!(
"Files upload attempted previously, verifying {} chunks",
chunks.len()
);

let failed_chunks = files_api
.client()
.verify_uploaded_chunks(&chunks, batch_size)
.await?;

chunk_manager.mark_completed(
chunks
.into_iter()
.filter(|c| !failed_chunks.contains(c))
.map(|(xor, _)| xor),
)?;

if failed_chunks.is_empty() {
// msg_files_already_uploaded_verified();
// if !make_data_public {
// msg_not_public_by_default();
// }
// msg_star_line();
// if self.chunk_manager.completed_files().is_empty() {
// msg_chk_mgr_no_verified_file_nor_re_upload();
// }
// msg_chunk_manager_upload_complete(self.chunk_manager);
return Ok(());
}
// msg_unverified_chunks_reattempted(&failed_chunks.len());
failed_chunks
} else {
let mut chunks = chunk_manager.get_chunks();
let mut rng = thread_rng();
chunks.shuffle(&mut rng);
chunks
};
let chunks_to_upload = ctu(
&files_api,
&mut chunk_manager,
&file_path,
batch_size,
make_data_public,
)
.await?;

IterativeUploader::new(chunk_manager, files_api)
.iterate_upload(
Expand Down Expand Up @@ -303,6 +272,51 @@ pub(crate) async fn files_cmds(
Ok(())
}

pub async fn ctu(
files_api: &FilesApi,
chunk_manager: &mut ChunkManager,
file_path: &Path,
batch_size: usize,
make_data_public: bool,
) -> Result<Vec<(XorName, PathBuf)>> {
let chunks_to_upload = if chunk_manager.is_chunks_empty() {
let chunks = chunk_manager.already_put_chunks(file_path, make_data_public)?;

let failed_chunks = files_api
.client()
.verify_uploaded_chunks(&chunks, batch_size)
.await?;

chunk_manager.mark_completed(
chunks
.into_iter()
.filter(|c| !failed_chunks.contains(c))
.map(|(xor, _)| xor),
)?;

if failed_chunks.is_empty() {
msg_files_already_uploaded_verified();
if !make_data_public {
msg_not_public_by_default();
}
msg_star_line();
if chunk_manager.completed_files().is_empty() {
msg_chk_mgr_no_verified_file_nor_re_upload();
}
iterative_uploader::msg_chunk_manager_upload_complete(chunk_manager.clone());
bail!("failed chunks are empty")
}
msg_unverified_chunks_reattempted(&failed_chunks.len());
failed_chunks
} else {
let mut chunks = chunk_manager.get_chunks();
let mut rng = thread_rng();
chunks.shuffle(&mut rng);
chunks
};
Ok(chunks_to_upload)
}

pub fn get_progress_bar(length: u64) -> Result<ProgressBar> {
let progress_bar = ProgressBar::new(length);
progress_bar.set_style(
Expand All @@ -313,3 +327,30 @@ pub fn get_progress_bar(length: u64) -> Result<ProgressBar> {
progress_bar.enable_steady_tick(Duration::from_millis(100));
Ok(progress_bar)
}

fn msg_files_already_uploaded_verified() {
println!("All files were already uploaded and verified");
println!("**************************************");
println!("* Uploaded Files *");
}

fn msg_chk_mgr_no_verified_file_nor_re_upload() {
println!("chunk_manager doesn't have any verified_files, nor any failed_chunks to re-upload.");
}

fn msg_not_public_by_default() {
println!("* *");
println!("* These are not public by default. *");
println!("* Reupload with `-p` option *");
println!("* to publish the datamaps. *");
}

fn msg_star_line() {
println!("**************************************");
}

fn msg_unverified_chunks_reattempted(failed_amount: &usize) {
println!(
"{failed_amount} chunks were uploaded in the past but failed to verify. Will attempt to upload them again..."
);
}
26 changes: 2 additions & 24 deletions sn_cli/src/subcommands/files/iterative_uploader.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -229,7 +229,8 @@ fn msg_final(

msg_made_payment_info(files_upload.get_upload_storage_cost(), uploaded_chunks);
}
fn msg_chunk_manager_upload_complete(chunk_manager: ChunkManager) {

pub fn msg_chunk_manager_upload_complete(chunk_manager: ChunkManager) {
for (file_name, addr) in chunk_manager.completed_files() {
let hex_addr = addr.to_hex();
if let Some(file_name) = file_name.to_str() {
Expand Down Expand Up @@ -278,28 +279,10 @@ fn msg_payment_details(
println!("Made payment of {total_royalty_fees} for royalties fees");
println!("New wallet balance: {final_balance}");
}

// fn msg_chk_mgr_no_verified_file_nor_re_upload() {
// println!("chunk_manager doesn't have any verified_files, nor any failed_chunks to re-upload.");
// }

fn msg_star_line() {
println!("**************************************");
}

// fn msg_not_public_by_default() {
// println!("* *");
// println!("* These are not public by default. *");
// println!("* Reupload with `-p` option *");
// println!("* to publish the datamaps. *");
// }

// fn msg_files_already_uploaded_verified() {
// println!("All files were already uploaded and verified");
// println!("**************************************");
// println!("* Uploaded Files *");
// }

fn msg_uploading_chunks(chunks_to_upload_len: usize) {
println!("Uploading {chunks_to_upload_len} chunks",);
}
Expand All @@ -315,8 +298,3 @@ fn msg_uploaded_files_banner() {
println!("**************************************");
println!("* Uploaded Files *");
}
// fn msg_unverified_chunks_reattempted(failed_amount: &usize) {
// println!(
// "{failed_amount} chunks were uploaded in the past but failed to verify. Will attempt to upload them again..."
// );
// }
4 changes: 4 additions & 0 deletions sn_client/src/api.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1107,6 +1107,10 @@ impl Client {
}
}

println!(
"Files upload attempted previously, verifying {} chunks",
failed_chunks.len()
);
Ok(failed_chunks)
}
}
Expand Down

0 comments on commit 9b0dc52

Please sign in to comment.