Split layers according to size threshold-Push side #3
Conversation
derrickqi2003
changed the title
derrickqi2003
changed the title
.vscode/launch.json
Outdated
| @@ -0,0 +1,45 @@ | |||
| { | |||
There was a problem hiding this comment.
add .vscode to gitignore so we don't need to commit this file
src/planner.rs
Outdated
| } | ||
|
|
||
| impl ConexPlanner { | ||
| pub fn default() -> Self { | ||
| Self { | ||
| layer_to_files: Vec::new(), | ||
| //30 mb threshold | ||
| split_threshold: 30 * 1024 * 1024, |
There was a problem hiding this comment.
can this be passed in through CLI. this means it needs to be part of default() arguments, and then passed in through the Clap App in main.rs.
src/planner.rs
Outdated
| }); | ||
| } | ||
| } | ||
| //println!("ingested total size {}",total_size.to_string()); |
There was a problem hiding this comment.
| //println!("ingested total size {}",total_size.to_string()); |
src/planner.rs
Outdated
| let mut segment_idx = 0; | ||
| let mut remainder_size = file.size; | ||
| while remainder_size != 0 { | ||
| //println!("remainder size {}",remainder_size.to_string()); |
There was a problem hiding this comment.
| //println!("remainder size {}",remainder_size.to_string()); |
src/planner.rs
Outdated
| if frag.hard_link_to.is_some() { | ||
| //soft links don't count | ||
| current_layer_size += remainder_size; | ||
| } |
There was a problem hiding this comment.
let's test this. i don't think this will do it because we want any file that's a link to occupy 0 bytes of space of content. because both hard link and softlinks are pointers.
|
How do run end to end testing:
|
|
How to inspect local "blobs". The blobs are in zstd compressed tar format. The way to depress them is to tar has native support for zstd tar can let you list the directory without untarring them |
Showing the fixes to the threshold