Skip to content

Commit

Permalink
fix tests
Browse files Browse the repository at this point in the history
  • Loading branch information
qiweiii committed Aug 10, 2024
1 parent 08bc394 commit b9a7b8c
Show file tree
Hide file tree
Showing 3 changed files with 9 additions and 87 deletions.
2 changes: 1 addition & 1 deletion Utils/Sources/Utils/ErasureCoding.swift
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,7 @@ public class SubShardDecoder {
subshard_decoder_free(decoder)
}

/// Decoded reconstruct result
public class Decoded {
private var result: UnsafeMutablePointer<ReconstructResult>

Expand All @@ -101,7 +102,6 @@ public class SubShardDecoder {
let numSegments = Int(result.pointee.num_segments)
let segmentTuplesPtr = result.pointee.segments

// Safely access the segments array
let bufferPtr = UnsafeMutableBufferPointer<SegmentTuple>(start: segmentTuplesPtr, count: numSegments)
segments = Array(bufferPtr)

Expand Down
87 changes: 5 additions & 82 deletions Utils/Sources/erasure-coding/src/ffi.rs
Original file line number Diff line number Diff line change
Expand Up @@ -87,18 +87,18 @@ pub extern "C" fn subshard_encoder_construct(

match encoder.construct_chunks(r_segments) {
Ok(result) => {
let total_chunks = result.len() * TOTAL_SHARDS;
let mut data: Vec<u8> = Vec::with_capacity(total_chunks);
let total_subshards = result.len() * TOTAL_SHARDS;
let mut data: Vec<u8> = Vec::with_capacity(total_subshards);

for boxed_array in result {
for chunk in boxed_array.iter() {
data.extend_from_slice(chunk);
for subshard in boxed_array.iter() {
data.extend_from_slice(subshard);
}
}

unsafe {
ptr::copy_nonoverlapping(data.as_ptr(), out_chunks, data.len());
*out_len = total_chunks;
*out_len = total_subshards;
}

std::mem::forget(data);
Expand Down Expand Up @@ -218,80 +218,3 @@ pub extern "C" fn subshard_decoder_reconstruct(
}
}
}

#[cfg(test)]
mod tests {
use std::fs;

use super::*;
use erasure_coding::{SubShard, SUBSHARD_SIZE};
use serde::{Deserialize, Serialize};
use serde_json;

#[derive(Serialize, Deserialize, Debug)]
struct JsonData {
data: String,
segment: SegmentData,
}

#[derive(Serialize, Deserialize, Debug)]
struct SegmentData {
segments: Vec<Segment>,
}

#[derive(Serialize, Deserialize, Debug)]
struct Segment {
segment_ec: Vec<String>,
}

#[test]
fn test_reconstruct_from_json() {
let file =
fs::File::open("../../Tests/UtilsTests/TestData/ec/erasure-coding-test-data.json")
.expect("file should open read only");
let json_data: JsonData =
serde_json::from_reader(file).expect("file should be proper JSON");

// Convert segment_ec data back to bytes and prepare subshards
let mut subshards: Vec<(u8, ChunkIndex, SubShard)> = Vec::new();
for (segment_idx, segment) in json_data.segment.segments.iter().enumerate() {
for (chunk_idx, chunk) in segment.segment_ec.iter().enumerate() {
let chunk_bytes: Vec<u8> = hex::decode(chunk).expect("Failed to decode hex string");
if chunk_idx >= 684 {
let mut subshard = [0u8; SUBSHARD_SIZE];
subshard[..chunk_bytes.len()].copy_from_slice(&chunk_bytes);
subshards.push((segment_idx as u8, ChunkIndex(chunk_idx as u16), subshard));
}
}
}

// Initialize decoder, call reconstruct!
let mut decoder = SubShardDecoder::new().unwrap();

let cloned_subshards: Vec<(u8, ChunkIndex, &[u8; 12])> =
subshards.iter().map(|t| (t.0, t.1, &t.2)).collect();

let (reconstructed_segments, _nb_decode) = decoder
.reconstruct(&mut cloned_subshards.iter().cloned())
.unwrap();

// Check the result
// println!("Reconstructed Segments: {:x?}", reconstructed_segments);
// println!("Number of Decodes: {}", nb_decode);

assert_eq!(reconstructed_segments.len(), 1);
let original_data_bytes =
hex::decode(&json_data.data).expect("Failed to decode hex string");
// Verify that the data attribute matches the first 342 bytes of the reconstructed data in the first segment
if let Some((_, first_segment)) = reconstructed_segments.get(0) {
assert_eq!(
&first_segment.data[..342],
&original_data_bytes[..342],
"The first 342 bytes of the reconstructed data do not match the original data."
);
println!("Reconstructed successfully! YAY");
} else {
panic!("No reconstructed segments found.");
}
}
}
7 changes: 3 additions & 4 deletions Utils/Tests/UtilsTests/ErasureCodeTest.swift
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ struct ErasureCodeTests {

if chunkIdx >= 684 {
var subshard: [UInt8] = Array(repeating: 0, count: Int(SUBSHARD_SIZE))
subshard[0 ..< chunkBytes.count].append(contentsOf: chunkBytes)
subshard[0 ..< chunkBytes.count] = [UInt8](chunkBytes)[...]
subshards.append(SubShardTuple(
seg_index: UInt8(segmentIdx),
chunk_index: ChunkIndex(chunkIdx),
Expand Down Expand Up @@ -85,12 +85,11 @@ struct ErasureCodeTests {
case let .success(decoded):
#expect(decoded.numDecoded == 1)
let segmentTuples = decoded.segments
#expect(segmentTuples.count == 1)
let segment = segmentTuples[0].segment

let originalDataBytes = Data(fromHexString: testCase.data)!
let segmentData = Data(UnsafeBufferPointer(start: segment.data, count: Int(SEGMENT_SIZE)))
// #expect(segmentData[0 ..< 342] == originalDataBytes[0 ..< 342])
#expect(segmentData[0 ..< 2] == originalDataBytes[0 ..< 2])
#expect(segmentData[0 ..< 342] == originalDataBytes[0 ..< 342])
case let .failure(error):
Issue.record("Expected success, got \(error)")
}
Expand Down

0 comments on commit b9a7b8c

Please sign in to comment.