Skip to content

Commit

Permalink
working
Browse files Browse the repository at this point in the history
  • Loading branch information
@joshuef
joshuef committed Dec 29, 2023
1 parent 855829e commit bb50a76
Show file tree
Hide file tree
Showing 4 changed files with 28 additions and 23 deletions.
39 changes: 23 additions & 16 deletions sn_cli/src/subcommands/files/chunk_manager.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,12 @@ impl ChunkManager {
/// Chunk all the files in the provided `files_path`
/// These are stored to the CHUNK_ARTIFACTS_DIR
/// if read_cache is true, will take cache from previous runs into account
pub(crate) fn chunk_path(&mut self, files_path: &Path, read_cache: bool) -> Result<()> {
pub(crate) fn chunk_path(
&mut self,
files_path: &Path,
read_cache: bool,
include_data_maps: bool,
) -> Result<()> {
println!("Starting to chunk {files_path:?} now.");
let now = Instant::now();
// clean up
Expand Down Expand Up @@ -175,7 +180,7 @@ impl ChunkManager {
}
};

match FilesApi::chunk_file(path, &file_chunks_dir, false) {
match FilesApi::chunk_file(path, &file_chunks_dir, include_data_maps) {
Ok((file_xor_addr, data_map, size, chunks)) => {
progress_bar.clone().inc(1);
debug!("Chunked {original_file_name:?} with {path_xor:?} into file's XorName: {file_xor_addr:?} of size {size}, and chunks len: {}", chunks.len());
Expand Down Expand Up @@ -311,7 +316,10 @@ impl ChunkManager {
let chunk_file = self.chunks.get(path_xor);

chunk_file.and_then(|chunk_file| {
warn!("CHUNK file has data_map {:?}", chunk_file.data_map.is_some());
warn!(
"CHUNK file has data_map {:?}",
chunk_file.data_map.is_some()
);
chunk_file
.data_map
.as_ref()
Expand All @@ -325,7 +333,6 @@ impl ChunkManager {

debug!("chunks len after: {:?}", chunks.len());


chunks
}

Expand Down Expand Up @@ -417,7 +424,7 @@ impl ChunkManager {
files_path: &Path,
make_files_public: bool,
) -> Result<Vec<(XorName, PathBuf)>> {
self.chunk_path(files_path, false)?;
self.chunk_path(files_path, false, make_files_public)?;
Ok(self.get_chunks(make_files_public))
}

Expand Down Expand Up @@ -544,7 +551,7 @@ mod tests {
let (_tmp_dir, mut manager, _, random_files_dir) = init_manager()?;
let artifacts_dir = manager.artifacts_dir.clone();
let _ = create_random_files(&random_files_dir, 1, 1)?;
manager.chunk_path(&random_files_dir, true)?;
manager.chunk_path(&random_files_dir, true, true)?;

let chunks = manager.get_chunks(true);
// 1. 1mb file produces 4 chunks
Expand Down Expand Up @@ -615,7 +622,7 @@ mod tests {
let (_tmp_dir, mut manager, _, random_files_dir) = init_manager()?;

let _ = create_random_files(&random_files_dir, 1, 1)?;
manager.chunk_path(&random_files_dir, true)?;
manager.chunk_path(&random_files_dir, true, true)?;

let path_xor = manager.chunks.keys().next().unwrap().clone();
let chunked_file = manager.chunks.values().next().unwrap().clone();
Expand Down Expand Up @@ -661,7 +668,7 @@ mod tests {
let (_tmp_dir, mut manager, _, random_files_dir) = init_manager()?;

let _ = create_random_files(&random_files_dir, 5, 5)?;
manager.chunk_path(&random_files_dir, true)?;
manager.chunk_path(&random_files_dir, true, true)?;
// cloned after chunking
let manager_clone = manager.clone();

Expand Down Expand Up @@ -703,10 +710,10 @@ mod tests {
let (_tmp_dir, mut manager, root_dir, random_files_dir) = init_manager()?;

let _ = create_random_files(&random_files_dir, 5, 5)?;
manager.chunk_path(&random_files_dir, true)?;
manager.chunk_path(&random_files_dir, true, true)?;

let mut new_manager = ChunkManager::new(&root_dir);
new_manager.chunk_path(&random_files_dir, true)?;
new_manager.chunk_path(&random_files_dir, true, true)?;

// 1. make sure the chunk counts match
let total_chunk_count = manager
Expand All @@ -730,7 +737,7 @@ mod tests {
let (_tmp_dir, mut manager, root_dir, random_files_dir) = init_manager()?;

let _ = create_random_files(&random_files_dir, 5, 5)?;
manager.chunk_path(&random_files_dir, true)?;
manager.chunk_path(&random_files_dir, true, true)?;

let total_chunks_count = manager
.chunks
Expand All @@ -751,7 +758,7 @@ mod tests {
.0;
manager.mark_completed([removed_chunk].into_iter());
let mut new_manager = ChunkManager::new(&root_dir);
new_manager.chunk_path(&random_files_dir, true)?;
new_manager.chunk_path(&random_files_dir, true, true)?;

// 1. we should have 1 completed chunk and (total_chunks_count-1) incomplete chunks
assert_eq!(manager.resumed_chunk_count, 0);
Expand All @@ -778,11 +785,11 @@ mod tests {
let (_tmp_dir, mut manager, root_dir, random_files_dir) = init_manager()?;

let _ = create_random_files(&random_files_dir, 5, 5)?;
manager.chunk_path(&random_files_dir, true)?;
manager.chunk_path(&random_files_dir, true, true)?;
manager.mark_completed_all();

let mut new_manager = ChunkManager::new(&root_dir);
new_manager.chunk_path(&random_files_dir, true)?;
new_manager.chunk_path(&random_files_dir, true, true)?;

// 1. we should have chunk entries, but 0 chunks inside them
assert_eq!(new_manager.chunks.len(), 5);
Expand Down Expand Up @@ -810,7 +817,7 @@ mod tests {

let mut random_files = create_random_files(&random_files_dir, 1, 1)?;
let random_file = random_files.remove(0);
manager.chunk_path(&random_files_dir, true)?;
manager.chunk_path(&random_files_dir, true, true)?;

let mut old_chunks_list = BTreeSet::new();
for entry in WalkDir::new(&manager.artifacts_dir).into_iter().flatten() {
Expand All @@ -832,7 +839,7 @@ mod tests {
fs::remove_file(&metadata_path)?;

// use the same manager to chunk the path
manager.chunk_path(&random_files_dir, true)?;
manager.chunk_path(&random_files_dir, true, true)?;
// nothing should be resumed
assert_eq!(manager.resumed_chunk_count, 0);
// but it should be re-chunked
Expand Down
2 changes: 1 addition & 1 deletion sn_cli/src/subcommands/files/mod.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -175,7 +175,7 @@ async fn upload_files(
bail!("The wallet is empty. Cannot upload any files! Please transfer some funds into the wallet");
}
let mut chunk_manager = ChunkManager::new(&root_dir);
chunk_manager.chunk_path(&files_path, true)?;
chunk_manager.chunk_path(&files_path, true, make_data_public)?;

// Return early if we already uploaded them
let mut chunks_to_upload = if chunk_manager.is_chunks_empty() {
Expand Down
9 changes: 4 additions & 5 deletions sn_client/src/files/api.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -170,13 +170,12 @@ impl FilesApi {
let data_map_path = chunk_dir.join(DATA_MAP_FILE);

// for chunk in additional_chunks.iter() {
// let file_path = output_dir.join(&hex::encode(chunk.name()));
// encrypted_chunks.push((*chunk.name(), file_path.to_path_buf()));
let mut output_file = File::create(file_path)?;
output_file.write_all(&data_map_chunk.value)?;
// let file_path = output_dir.join(&hex::encode(chunk.name()));
// encrypted_chunks.push((*chunk.name(), file_path.to_path_buf()));
let mut output_file = File::create(file_path)?;
output_file.write_all(&data_map_chunk.value)?;
// }


// let mut data_map_file = File::create(&data_map_path)?;
// data_map_file.write_all(&data_map_chunk.value)?;
if include_data_map_in_chunks {
Expand Down
1 change: 0 additions & 1 deletion sn_client/src/files/mod.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -170,7 +170,6 @@ impl Files {
/// Uploads the provided chunks to the network.
/// If you want to track the upload progress, use the `get_upload_events` method.
pub async fn upload_chunks(&mut self, chunks: Vec<(XorName, PathBuf)>) -> Result<()> {

// make sure we log that the event sender is absent atleast once
self.logged_event_sender_absence = false;

Expand Down

0 comments on commit bb50a76

Please sign in to comment.