Wp/ updated delete file to use pk as filename

src/backend.rs

@@ -1,2 +1 @@
 pub mod fs;
-// pub mod task;

src/backend/fs.rs

@@ -1,8 +1,8 @@
 use std::{
     fs::{self, OpenOptions},
-    io::{self, Read, Seek, Write},
-    path::{Path, PathBuf},
+    io::{Read, Seek, Write},
     pin::Pin,
+    str::FromStr,
     sync::Arc,
 };
 
@@ -134,6 +134,7 @@ pub fn write_segment(sk: &[u8], pk: &[u8], encoded: &Encoded) -> Result<BaoHash>
 
 pub fn write_catalog(file_hash: &Blake3Hash, segment_hashes: &[BaoHash]) -> Result<()> {
     debug!("Write catalog");
+
     let contents: Vec<u8> = segment_hashes
         .iter()
         .flat_map(|bao_hash| bao_hash.to_bytes())
@@ -255,73 +256,65 @@ pub fn read_catalog(file_hash: &Blake3Hash) -> Result<Vec<BaoHash>> {
     Ok(bao_hashes)
 }
 
-#[allow(unused_variables)]
-pub fn delete_file(pk: Secp256k1PubKey, file_bytes: &[u8]) -> Result<()> {
-    let pk_bytes = pk.to_bytes();
-    let (x_only_pk, _) = pk.into_inner().x_only_public_key();
-
-    let file_hash = Blake3Hash(blake3::keyed_hash(&x_only_pk.serialize(), file_bytes));
-    trace!(">>>>>file_hash::   {}", file_hash);
-
-    for vol in &SYS_CFG.volumes {
-        let seg_file = &vol.path.join(SEGMENT_DIR).join(file_hash.to_string());
-        let seg_dir = &vol.path.join(SEGMENT_DIR);
-        remove_dir_contents(seg_dir, seg_file.to_path_buf()).unwrap();
-    }
-
-    for vol in &SYS_CFG.volumes {
-        let cat_path = &vol.path.join(CATALOG_DIR).join(file_hash.to_string());
-        let cat = &vol.path.join(CATALOG_DIR);
-        remove_dir_catalogs(cat.to_path_buf(), cat_path.to_path_buf()).unwrap();
-    }
-    Ok(())
+pub async fn read_slices(
+    //hash: bao::Hash, // from Blake3
+    file_bytes: &[u8],
+    //slice_start: u64, //let slice_start = 65536;
+    //slice_len: u16,   // let slice_len = 8192;
+) -> Result<Vec<u8>> {
+    use std::io::prelude::*;
+
+    // Start by encoding some input.
+    let (encoded_input, hash) = bao::encode::encode(file_bytes);
+    debug!(">>> encoded INPUT LENGTH:: {:?}", &encoded_input.len());
+    debug!(">>> fs hash hash:: {:?}", &hash);
+
+    // Slice the encoding. These parameters are multiples of the chunk size, which avoids
+    // unnecessary overhead.
+    let slice_start = 65536;
+    let slice_len = 8192;
+    let encoded_cursor = std::io::Cursor::new(&encoded_input);
+    let mut extractor = bao::encode::SliceExtractor::new(encoded_cursor, slice_start, slice_len);
+    let mut slice = Vec::new();
+    extractor.read_to_end(&mut slice)?;
+
+    // Decode the slice. The result should be the same as the part of the input that the slice
+    // represents. Note that we're using the same hash that encoding produced, which is
+    // independent of the slice parameters. That's the whole point; if we just wanted to re-encode
+    // a portion of the input and wind up with a different hash, we wouldn't need slicing.
+    let mut decoded = Vec::new();
+    let mut decoder = bao::decode::SliceDecoder::new(&*slice, &hash, slice_start, slice_len);
+    decoder.read_to_end(&mut decoded)?;
+
+    debug!(
+        "usize vs length: {:?}",
+        &encoded_input[slice_start as usize..][..slice_len as usize].len()
+    );
+
+    // Like regular decoding, slice decoding will fail if the hash doesn't match.
+    // let mut bad_slice = slice.clone();
+    // debug!("BAD SLICE");
+    // let last_index = bad_slice.len() - 1;
+    // bad_slice[last_index] ^= 1;
+    // let mut decoder = bao::decode::SliceDecoder::new(&*bad_slice, &hash, slice_start, slice_len);
+    // let err = decoder.read_to_end(&mut Vec::new()).unwrap_err();
+    // assert_eq!(std::io::ErrorKind::InvalidData, err.kind());
+
+    Ok(decoded)
 }
 
-#[allow(unused_variables)]
-fn remove_dir_contents<P: AsRef<Path>>(path: P, seg_file: PathBuf) -> io::Result<()> {
-    trace!(">>> remove_Segment_contents");
-    for entry in fs::read_dir(path)? {
-        trace!("Delete Segment File at {:?}", entry);
-        fs::remove_file(entry?.path())?;
-    }
-    Ok(())
-}
-
-#[allow(unused_variables)]
-fn remove_dir_catalogs(path: PathBuf, file: PathBuf) -> io::Result<()> {
-    for entry in fs::read_dir(path)? {
-        trace!("Delete CATALOG File at {:?}", entry);
-        fs::remove_file(entry?.path())?;
+pub async fn delete_file(hash: &String) -> Result<()> {
+    let blake3_hash = &Blake3Hash(blake3::Hash::from_str(hash)?);
+    let catalog = read_catalog(blake3_hash)?;
+    for vol in &SYS_CFG.volumes {
+        for cat in &catalog {
+            let seg_file = &vol.path.join(SEGMENT_DIR).join(cat.to_string() + ".c15");
+            let is_removed = fs::remove_file(seg_file).is_ok();
+            trace!("SEG: {:?} REMOVED:  {:?}", &seg_file, is_removed);
+        }
+        let cat_path = &vol.path.join(CATALOG_DIR).join(hash);
+        let is_removed = fs::remove_file(cat_path).is_ok();
+        trace!("CAT: {:?}: REMOVED: {:?}", &cat_path, is_removed);
     }
     Ok(())
 }
-
-// fn remove_dir_segements<P: AsRef<Path>>(path: P, seg_file: PathBuf) -> io::Result<()> {
-//     trace!(">>> remove_Segment_contents");
-//     for entry in fs::read_dir(path)? {
-//         let entry = entry?;
-//         trace!("ENTRY Delete SEGMENT File at {:?}", entry);
-
-//         match &entry {
-//             seg_file => {
-//                 fs::remove_file(seg_file.path())?;
-//                 trace!("Delete Segment File at {:?}", seg_file);
-//             }
-//         }
-//     }
-//     Ok(())
-// }
-
-// fn remove_dir_catalogs(path: PathBuf, file: PathBuf) -> io::Result<()> {
-//     for entry in fs::read_dir(path)? {
-//         let entry = entry?;
-//         trace!("ENTRY Delete CATALOG File at {:?}", entry);
-//         match &entry {
-//             file => {
-//                 fs::remove_file(file.path())?;
-//                 trace!("FILE MATCH Delete CATALOG File at {:?}", file);
-//             }
-//         }
-//     }
-//     Ok(())
-// }

src/frontend/http.rs

@@ -70,11 +70,8 @@ async fn get_file(
 }
 
 #[axum_macros::debug_handler]
-async fn remove_file(
-    Path((pk, blake3_hash)): Path<(String, String)>,
-) -> Result<impl IntoResponse, AppError> {
-    let pk = Secp256k1PubKey::try_from(pk.as_str())?;
-    delete_file(pk, blake3_hash.as_bytes())?;
+async fn remove_file(Path(blake3_hash): Path<String>) -> Result<impl IntoResponse, AppError> {
+    delete_file(&blake3_hash).await?;
     Ok((StatusCode::OK, blake3_hash))
 }
 
@@ -89,9 +86,10 @@ async fn key(Path(pk): Path<String>) -> Result<impl IntoResponse, AppError> {
 
 pub async fn start() -> Result<()> {
     let app = Router::new()
-        .route("/remove/:pk/:blake3_hash", delete(remove_file))
+        .route("/remove/:blake3_hash", delete(remove_file))
         .route("/store/:pk", post(post_file))
         .route("/retrieve/:pk/:blake3_hash", get(get_file))
+        // .route("/slice/", get(get_slice))
         .route("/key/:pk", get(key))
         // .route("/catalog/:blake3_hash", get(get_catalog))
         // .route("/raw/:bao_hash", get(get_raw))

src/structs.rs

@@ -12,6 +12,16 @@ impl fmt::Display for Blake3Hash {
     }
 }
 
+impl TryFrom<&[u8]> for Blake3Hash {
+    type Error = Error;
+
+    fn try_from(value: &[u8]) -> Result<Self, Self::Error> {
+        let mut hash = [0_u8; 32];
+        hash.copy_from_slice(&value[0..32]);
+        Ok(Self(blake3::Hash::try_from(hash)?))
+    }
+}
+
 pub struct BaoHash(pub bao::Hash);
 
 impl BaoHash {

tests/file.rs

@@ -3,9 +3,9 @@ use std::fs;
 use anyhow::Result;
 use axum::body::Bytes;
 use carbonado_node::{
-    backend::fs::{read_file, write_file, FileStream},
-    config::node_shared_secret,
-    prelude::SEGMENT_SIZE,
+    backend::fs::{delete_file, read_file, write_file, FileStream},
+    config::{node_shared_secret, SYS_CFG},
+    prelude::{CATALOG_DIR, SEGMENT_DIR, SEGMENT_SIZE},
     structs::Secp256k1PubKey,
 };
 use futures_util::{stream, StreamExt};
@@ -63,100 +63,86 @@ async fn write_read() -> Result<()> {
         "Original keyed blake3 hash and streamed hash match",
     );
 
-    // TODO: This is not how you delete files
-    // let new_file_bytes = delete_file(Secp256k1PubKey(write_pk), &file_bytes).is_err();
-    // debug!("Write Delete:: deleted file:: {:?}", new_file_bytes);
+    // Delete file with hash across all volumes in config
+    let new_file_bytes = delete_file(&blake3_hash.to_string()).await.is_ok();
+    debug!("Write/Read Deleted file:: {:?}", new_file_bytes);
 
-    info!("Write/Delete test finished successfully!");
+    info!("Write/Read test finished successfully!");
 
     Ok(())
 }
 
 #[tokio::test]
-async fn check_catalog_exists() -> Result<()> {
+async fn read_write_delete_file() -> Result<()> {
     carbonado::utils::init_logging(RUST_LOG);
 
+    // write file to test delete
     let (_sk, pk) = generate_keypair(&mut thread_rng());
 
-    info!("Reading file bytes");
-    let file_bytes = fs::read("tests/samples/cat.gif")?;
-    debug!("{} bytes read", file_bytes.len());
-
-    let file_stream = stream::iter(file_bytes)
-        .chunks(1024 * 1024)
-        .map(|chunk| Ok(Bytes::from(chunk)))
-        .boxed();
-
-    info!("Writing file if not exists");
-    let is_ok = write_file(&Secp256k1PubKey(pk), file_stream).await.is_ok();
-    debug!("Skip writing file as File hash exists: {is_ok}");
-    assert!(is_ok);
-
-    let file_bytes = fs::read("tests/samples/cat.gif")?;
-    let file_stream = stream::iter(file_bytes)
-        .chunks(1024 * 1024)
-        .map(|chunk| Ok(Bytes::from(chunk)))
-        .boxed();
-
-    info!("Writing file if not exists");
-    let is_err = write_file(&Secp256k1PubKey(pk), file_stream).await.is_err();
-    debug!("Skip writing file as File hash exists: {is_err}");
-    assert!(is_err);
-
-    Ok(())
-}
-
-#[tokio::test]
-async fn read_write_delete_file() -> Result<()> {
-    carbonado::utils::init_logging(RUST_LOG);
-
-    info!("Write Delete:: Reading file bytes");
     let file_bytes = fs::read("tests/samples/cat.gif")?;
     debug!("{} Write Delete:: bytes read", file_bytes.len());
 
-    let file_stream = stream::iter(file_bytes.clone())
+    let file_stream = stream::iter(file_bytes)
         .chunks(1024 * 1024)
         .map(|chunk| Ok(Bytes::from(chunk)))
         .boxed();
 
-    // first file to write
-    let (_sk, pk) = generate_keypair(&mut thread_rng());
+    info!("Write Delete:: Writing file if not exists in order to test delete");
+    let final_blake3_hash = write_file(&Secp256k1PubKey(pk), file_stream).await?;
 
-    // info!("Write Delete:: Writing file if not exists in order to test delete");
-    let file_did_write = write_file(&Secp256k1PubKey(pk), file_stream).await.is_ok();
+    info!(
+        "File To Delete as blake3_hash:: {} ",
+        final_blake3_hash.to_string()
+    );
 
-    if file_did_write {
-        info!(
-            "Write File Group One to Delete File as blake3_hash:: {} ",
-            file_did_write.to_string()
+    // Check number of catalog files
+    let mut current_volume = -1; // start for volume 0
+    for vol in &SYS_CFG.volumes {
+        current_volume += 1;
+        let seg_file = &vol.path.join(CATALOG_DIR);
+
+        let paths = fs::read_dir(seg_file).unwrap();
+        let mut counter = -1;
+        for path in paths {
+            counter += 1;
+            debug!(
+                "Catalog count: {:?} :: path: {}",
+                counter,
+                path.unwrap().path().display()
+            )
+        }
+        debug!(
+            "For Volume: {}, total catalogs deleted: {:?}",
+            current_volume, counter
         );
     }
 
-    // second file to write
-    let (_sk, pk) = generate_keypair(&mut thread_rng());
-
-    let file_stream = stream::iter(file_bytes)
-        .chunks(1024 * 1024)
-        .map(|chunk| Ok(Bytes::from(chunk)))
-        .boxed();
-
-    info!("Write Delete:: Writing file if not exists in order to test delete");
-    let blake3_hash = write_file(&Secp256k1PubKey(pk), file_stream).await.is_ok();
-
-    if blake3_hash {
-        info!(
-            "Write File in Group Two to Delete File as blake3_hash:: {} ",
-            blake3_hash.to_string()
+    // Check number of segment files
+    let mut current_volume = -1; // start for volume 0
+    for vol in &SYS_CFG.volumes {
+        current_volume += 1;
+        let seg_file = &vol.path.join(SEGMENT_DIR);
+
+        let paths = fs::read_dir(seg_file).unwrap();
+        let mut counter = -1; // account for .DS_store in dir .../segments/.DS_Store
+        for path in paths {
+            counter += 1;
+            debug!(
+                "Segment count: {:?} :: path: {}",
+                counter,
+                path.unwrap().path().display()
+            )
+        }
+        debug!(
+            "For Volume: {}, total segments deleted: {:?}",
+            current_volume, counter
         );
     }
 
-    // TODO: This is not how you delete files
-    // let new_file_bytes = delete_file(Secp256k1PubKey(pk), &file_bytes).is_err();
-    // debug!("Write Delete:: deleted file:: {:?}", new_file_bytes);
-
-    // TODO: Check number of files
+    // Delete Files for blake3_hash
+    let new_file_bytes = delete_file(&final_blake3_hash.to_string()).await.is_ok();
+    debug!("Deleted files success:: {:?}", new_file_bytes);
 
-    debug!(" >>>> Public Key Generated :: {:?} :: {}", _sk, pk);
     info!("Write/Delete test finished successfully!");
 
     Ok(())