Skip to content

Commit

Permalink
refactor(cli): chunk-mgr to report files chunked/uploaded rather than…
Browse files Browse the repository at this point in the history
… bailing out
  • Loading branch information
bochaco authored and JasonPaulGithub committed Mar 11, 2024
1 parent 91b07b3 commit b5db038
Show file tree
Hide file tree
Showing 3 changed files with 31 additions and 33 deletions.
18 changes: 14 additions & 4 deletions sn_cli/src/subcommands/files.rs
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,10 @@ pub(crate) use chunk_manager::ChunkManager;

use crate::subcommands::files::iterative_uploader::IterativeUploader;
use clap::Parser;
use color_eyre::{eyre::eyre, Help, Result};
use color_eyre::{
eyre::{bail, eyre},
Help, Result,
};
use indicatif::{ProgressBar, ProgressStyle};
use sn_client::protocol::storage::{Chunk, ChunkAddress, RetryStrategy};
use sn_client::{Client, FilesApi, BATCH_SIZE};
Expand Down Expand Up @@ -115,18 +118,25 @@ pub(crate) async fn files_cmds(
retry_strategy,
make_data_public,
} => {
IterativeUploader::new(chunk_manager, files_api)
let total_files = IterativeUploader::new(chunk_manager, files_api)
.iterate_upload(
WalkDir::new(&file_path).into_iter().flatten(),
file_path,
file_path.clone(),
FilesUploadOptions {
make_data_public,
verify_store,
batch_size,
retry_strategy,
},
)
.await?
.await?;
if total_files == 0 {
if file_path.is_dir() {
bail!("The directory specified for upload is empty. Please verify the provided path.");
} else {
bail!("The provided file path is invalid. Please verify the path.");
}
}
}
FilesCmds::Download {
file_name,
Expand Down
32 changes: 8 additions & 24 deletions sn_cli/src/subcommands/files/chunk_manager.rs
Original file line number Diff line number Diff line change
Expand Up @@ -99,7 +99,7 @@ impl ChunkManager {
files_path: &Path,
read_cache: bool,
include_data_maps: bool,
) -> Result<()> {
) -> Result<usize> {
self.chunk_with_iter(
WalkDir::new(files_path).into_iter().flatten(),
read_cache,
Expand All @@ -115,7 +115,7 @@ impl ChunkManager {
entries_iter: impl Iterator<Item = DirEntry>,
read_cache: bool,
include_data_maps: bool,
) -> Result<()> {
) -> Result<usize> {
let now = Instant::now();
// clean up
self.files_to_chunk = Default::default();
Expand All @@ -125,14 +125,8 @@ impl ChunkManager {
self.resumed_files_count = 0;

// collect the files to chunk
let mut files_path_is_dir = false;
entries_iter.for_each(|entry| {
let is_file = entry.file_type().is_file();
if entry.depth() == 0 {
files_path_is_dir = !is_file;
}

if is_file {
if entry.file_type().is_file() {
let path_xor = PathXorName::new(entry.path());
info!(
"Added file {:?} with path_xor: {path_xor:?} to be chunked/resumed",
Expand All @@ -147,14 +141,8 @@ impl ChunkManager {
});
let total_files = self.files_to_chunk.len();
if total_files == 0 {
if files_path_is_dir {
bail!(
"The directory specified for upload is empty. Please verify the provided path."
);
} else {
bail!("The provided file path is invalid. Please verify the path.");
}
}
return Ok(0);
};

// resume the chunks from the artifacts dir
if read_cache {
Expand Down Expand Up @@ -199,7 +187,7 @@ impl ChunkManager {
"All files_to_chunk ({total_files:?}) were resumed. Returning the resumed chunks.",
);
debug!("It took {:?} to resume all the files", now.elapsed());
return Ok(());
return Ok(total_files);
}

let progress_bar = get_progress_bar(total_files as u64)?;
Expand Down Expand Up @@ -287,14 +275,10 @@ impl ChunkManager {
.collect::<Result<()>>()?;

progress_bar.finish_and_clear();
debug!(
"It took {:?} to chunk {} files",
now.elapsed(),
self.files_to_chunk.len()
);
debug!("It took {:?} to chunk {} files", now.elapsed(), total_files);
self.chunks.extend(chunked_files);

Ok(())
Ok(total_files)
}

// Try to resume the chunks
Expand Down
14 changes: 9 additions & 5 deletions sn_cli/src/subcommands/files/iterative_uploader.rs
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ impl IterativeUploader {
entries_iter: impl Iterator<Item = DirEntry>,
files_path: PathBuf,
options: FilesUploadOptions,
) -> Result<()> {
) -> Result<usize> {
let FilesUploadOptions {
make_data_public,
verify_store,
Expand All @@ -48,8 +48,12 @@ impl IterativeUploader {

msg_init(&files_path, &batch_size, &verify_store, make_data_public);

self.chunk_manager
.chunk_with_iter(entries_iter, true, make_data_public)?;
let total_files =
self.chunk_manager
.chunk_with_iter(entries_iter, true, make_data_public)?;
if total_files == 0 {
return Ok(0);
}

// Return early if we already uploaded them
let mut chunks_to_upload = if self.chunk_manager.is_chunks_empty() {
Expand Down Expand Up @@ -88,7 +92,7 @@ impl IterativeUploader {
msg_chk_mgr_no_verified_file_nor_re_upload();
}
msg_chunk_manager_upload_complete(self.chunk_manager);
return Ok(());
return Ok(total_files);
}
msg_unverified_chunks_reattempted(&failed_chunks.len());
failed_chunks
Expand Down Expand Up @@ -139,7 +143,7 @@ impl IterativeUploader {
files_upload,
);

Ok(())
Ok(total_files)
}

async fn upload_result(
Expand Down

0 comments on commit b5db038

Please sign in to comment.