Skip to content

Commit

Permalink
Test vectorized file write with multiple writes to the same file.
Browse files Browse the repository at this point in the history
  • Loading branch information
aliddell committed Dec 2, 2024
1 parent 97e93c1 commit fe238af
Show file tree
Hide file tree
Showing 3 changed files with 34 additions and 15 deletions.
8 changes: 4 additions & 4 deletions src/streaming/vectorized.file.writer.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -120,7 +120,7 @@ zarr::VectorizedFileWriter::~VectorizedFileWriter()
bool
zarr::VectorizedFileWriter::write_vectors(
const std::vector<std::vector<std::byte>>& buffers,
const std::vector<size_t>& offsets)
size_t offset)
{
std::lock_guard<std::mutex> lock(mutex_);

Expand Down Expand Up @@ -154,8 +154,8 @@ zarr::VectorizedFileWriter::write_vectors(
}

OVERLAPPED overlapped = { 0 };
overlapped.Offset = static_cast<DWORD>(offsets[0] & 0xFFFFFFFF);
overlapped.OffsetHigh = static_cast<DWORD>(offsets[0] >> 32);
overlapped.Offset = static_cast<DWORD>(offset & 0xFFFFFFFF);
overlapped.OffsetHigh = static_cast<DWORD>(offset >> 32);
overlapped.hEvent = CreateEvent(nullptr, TRUE, FALSE, nullptr);

DWORD bytes_written;
Expand Down Expand Up @@ -197,7 +197,7 @@ zarr::VectorizedFileWriter::write_vectors(
ssize_t bytes_written = pwritev(fd_,
iovecs.data(),
static_cast<int>(iovecs.size()),
static_cast<int>(offsets[0]));
static_cast<int>(offset));

if (bytes_written != total_bytes) {
auto error = get_last_error_as_string();
Expand Down
2 changes: 1 addition & 1 deletion src/streaming/vectorized.file.writer.hh
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ class VectorizedFileWriter
~VectorizedFileWriter();

bool write_vectors(const std::vector<std::vector<std::byte>>& buffers,
const std::vector<size_t>& offsets);
size_t offset);

std::mutex& mutex() { return mutex_; }

Expand Down
39 changes: 29 additions & 10 deletions tests/unit-tests/vectorized-file-write.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -14,20 +14,21 @@ write_to_file(const std::string& filename)
zarr::VectorizedFileWriter writer(filename);

std::vector<std::vector<std::byte>> data(10);
std::vector<size_t> offsets(10);
size_t offset = 0;
for (auto i = 0; i < data.size(); ++i) {
data[i].resize((i + 1) * 1024);
std::fill(data[i].begin(), data[i].end(), std::byte(i));

offsets[i] = offset;
offset += data[i].size();
file_size += data[i].size();
}
CHECK(writer.write_vectors(data, 0));

file_size = offsets.back() + data.back().size();
CHECK(writer.write_vectors(data, offsets));
// write more data
for (auto i = 0; i < 10; ++i) {
auto& vec = data[i];
std::fill(vec.begin(), vec.end(), std::byte(i + 10));
}
CHECK(writer.write_vectors(data, file_size));

return file_size;
return 2 * file_size;
}

void
Expand All @@ -41,10 +42,10 @@ verify_file_data(const std::string& filename, size_t file_size)

// Verify data pattern
size_t offset = 0;
for (size_t i = 0; i < 10; i++) {
for (size_t i = 0; i < 10; ++i) {
size_t size = (i + 1) * 1024;

for (size_t j = offset; j < offset + size; j++) {
for (size_t j = offset; j < offset + size; ++j) {
auto byte = (int)read_buffer[j];
EXPECT(byte == i,
"Data mismatch at offset ",
Expand All @@ -57,6 +58,23 @@ verify_file_data(const std::string& filename, size_t file_size)
}
offset += size;
}

for (size_t i = 0; i < 10; ++i) {
size_t size = (i + 1) * 1024;

for (size_t j = offset; j < offset + size; ++j) {
auto byte = (int)read_buffer[j];
EXPECT(byte == i + 10,
"Data mismatch at offset ",
j,
". Expected ",
i + 10,
" got ",
byte,
".");
}
offset += size;
}
}

int
Expand All @@ -74,6 +92,7 @@ main()
try {
const auto file_size = write_to_file(filename);
EXPECT(fs::exists(filename), "File not found: ", filename);
EXPECT(fs::file_size(filename) >= 112640); // sum(1:10) * 1024 * 2
verify_file_data(filename, file_size);

retval = 0;
Expand Down

0 comments on commit fe238af

Please sign in to comment.