Skip to content

Commit

Permalink
Addressing feedback from @davisp.
Browse files Browse the repository at this point in the history
  • Loading branch information
KiterLuc committed Sep 1, 2023
1 parent 89686b4 commit 8acbf07
Show file tree
Hide file tree
Showing 4 changed files with 80 additions and 39 deletions.
6 changes: 5 additions & 1 deletion tiledb/sm/query/readers/reader_base.cc
Original file line number Diff line number Diff line change
Expand Up @@ -718,7 +718,11 @@ ReaderBase::load_tile_chunk_data(
const FilterPipeline& filters = array_schema_.filters(name);
if (!var_size ||
!filters.skip_offsets_filtering(t_var->type(), array_schema_.version())) {
unfiltered_tile_size = t->load_chunk_data(tile_chunk_data, var_size);
if (var_size) {
unfiltered_tile_size = t->load_offsets_chunk_data(tile_chunk_data);
} else {
unfiltered_tile_size = t->load_chunk_data(tile_chunk_data);
}
}

if (var_size) {
Expand Down
3 changes: 0 additions & 3 deletions tiledb/sm/query/test/unit_query_condition.cc
Original file line number Diff line number Diff line change
Expand Up @@ -2675,9 +2675,6 @@ void test_apply_cells_sparse<char*>(
auto expected_iter = expected_cell_idx_vec.begin();
for (uint64_t cell_idx = 0; cell_idx < cells; ++cell_idx) {
if (result_bitmap[cell_idx]) {
if (*expected_iter != cell_idx) {
std::cout << "hello";
}
REQUIRE(*expected_iter == cell_idx);
++expected_iter;
}
Expand Down
76 changes: 47 additions & 29 deletions tiledb/sm/tile/tile.cc
Original file line number Diff line number Diff line change
Expand Up @@ -249,38 +249,17 @@ Status Tile::zip_coordinates() {
return Status::Ok();
}

uint64_t Tile::load_chunk_data(ChunkData& unfiltered_tile, bool is_offsets) {
assert(filtered());

Deserializer deserializer(filtered_data(), filtered_size());

// Make a pass over the tile to get the chunk information.
uint64_t num_chunks = deserializer.read<uint64_t>();

auto& filtered_chunks = unfiltered_tile.filtered_chunks_;
auto& chunk_offsets = unfiltered_tile.chunk_offsets_;
filtered_chunks.resize(num_chunks);
chunk_offsets.resize(num_chunks);
uint64_t total_orig_size = 0;
for (uint64_t i = 0; i < num_chunks; i++) {
auto& chunk = filtered_chunks[i];
chunk.unfiltered_data_size_ = deserializer.read<uint32_t>();
chunk.filtered_data_size_ = deserializer.read<uint32_t>();
chunk.filtered_metadata_size_ = deserializer.read<uint32_t>();
chunk.filtered_metadata_ = const_cast<void*>(
deserializer.get_ptr<void>(chunk.filtered_metadata_size_));
chunk.filtered_data_ = const_cast<void*>(
deserializer.get_ptr<void>(chunk.filtered_data_size_));

chunk_offsets[i] = total_orig_size;
total_orig_size += chunk.unfiltered_data_size_;
}
uint64_t Tile::load_chunk_data(ChunkData& chunk_data) {
return load_chunk_data(chunk_data, size());
}

if (total_orig_size + (is_offsets ? sizeof(uint64_t) : 0) != size()) {
throw TileStatusException("Incorrect unfiltered tile size allocated.");
uint64_t Tile::load_offsets_chunk_data(ChunkData& chunk_data) {
auto s = size();
if (s < 8) {
throw std::runtime_error("Offsets tile should at least be 8 bytes.");
}

return total_orig_size;
return load_chunk_data(chunk_data, s - 8);
}

void Tile::swap(Tile& tile) {
Expand Down Expand Up @@ -320,5 +299,44 @@ void WriterTile::swap(WriterTile& tile) {
std::swap(filtered_buffer_, tile.filtered_buffer_);
}

/* ********************************* */
/* PRIVATE FUNCTIONS */
/* ********************************* */

uint64_t Tile::load_chunk_data(
ChunkData& unfiltered_tile, uint64_t expected_original_size) {
assert(filtered());

Deserializer deserializer(filtered_data(), filtered_size());

// Make a pass over the tile to get the chunk information.
uint64_t num_chunks = deserializer.read<uint64_t>();

auto& filtered_chunks = unfiltered_tile.filtered_chunks_;
auto& chunk_offsets = unfiltered_tile.chunk_offsets_;
filtered_chunks.resize(num_chunks);
chunk_offsets.resize(num_chunks);
uint64_t total_orig_size = 0;
for (uint64_t i = 0; i < num_chunks; i++) {
auto& chunk = filtered_chunks[i];
chunk.unfiltered_data_size_ = deserializer.read<uint32_t>();
chunk.filtered_data_size_ = deserializer.read<uint32_t>();
chunk.filtered_metadata_size_ = deserializer.read<uint32_t>();
chunk.filtered_metadata_ = const_cast<void*>(
deserializer.get_ptr<void>(chunk.filtered_metadata_size_));
chunk.filtered_data_ = const_cast<void*>(
deserializer.get_ptr<void>(chunk.filtered_data_size_));

chunk_offsets[i] = total_orig_size;
total_orig_size += chunk.unfiltered_data_size_;
}

if (total_orig_size != expected_original_size) {
throw TileStatusException("Incorrect unfiltered tile size allocated.");
}

return total_orig_size;
}

} // namespace sm
} // namespace tiledb
34 changes: 28 additions & 6 deletions tiledb/sm/tile/tile.h
Original file line number Diff line number Diff line change
Expand Up @@ -267,6 +267,31 @@ class Tile : public TileBase {
*/
Status zip_coordinates();

/**
* Reads the chunk data of a tile buffer and populates a chunk data structure.
*
* @param chunk_data Tile chunk info, buffers and offsets.
* @return Original size.
*/
uint64_t load_chunk_data(ChunkData& chunk_data);

/**
* Reads the chunk data of a tile offsets buffer and populates a chunk data
* structure.
*
* @param chunk_data Tile chunk info, buffers and offsets.
* @return Original size.
*/
uint64_t load_offsets_chunk_data(ChunkData& chunk_data);

/** Swaps the contents (all field values) of this tile with the given tile. */
void swap(Tile& tile);

private:
/* ********************************* */
/* PRIVATE FUNCTIONS */
/* ********************************* */

/**
* Reads the chunk data of a tile buffer and populates a chunk data structure.
*
Expand All @@ -291,15 +316,12 @@ class Tile : public TileBase {
* chunkN_data (uint8_t[])
*
* @param chunk_data Tile chunk info, buffers and offsets.
* @param is_offsets Does the tile contains offsets?
* @param expected_original_size Expected size for the tile.
* @return Original size.
*/
uint64_t load_chunk_data(ChunkData& chunk_data, bool is_offsets = false);

/** Swaps the contents (all field values) of this tile with the given tile. */
void swap(Tile& tile);
uint64_t load_chunk_data(
ChunkData& chunk_data, uint64_t expected_original_size);

private:
/* ********************************* */
/* PRIVATE ATTRIBUTES */
/* ********************************* */
Expand Down

0 comments on commit 8acbf07

Please sign in to comment.