diff --git a/be/src/storage/metadata_util.cpp b/be/src/storage/metadata_util.cpp index 1ec368e67fa65..b9a63e9374179 100644 --- a/be/src/storage/metadata_util.cpp +++ b/be/src/storage/metadata_util.cpp @@ -146,6 +146,9 @@ static Status type_desc_to_pb(const std::vector& types, int* index, C // All struct fields all nullable now RETURN_IF_ERROR(type_desc_to_pb(types, index, field_pb)); field_pb->set_name(field.name); + if (field.__isset.id && field.id >= 0) { + field_pb->set_unique_id(field.id); + } } return Status::OK(); } diff --git a/be/src/storage/rowset/column_reader.cpp b/be/src/storage/rowset/column_reader.cpp index b56a4610669c1..8b2946d388d25 100644 --- a/be/src/storage/rowset/column_reader.cpp +++ b/be/src/storage/rowset/column_reader.cpp @@ -54,6 +54,7 @@ #include "storage/rowset/bitmap_index_reader.h" #include "storage/rowset/bloom_filter.h" #include "storage/rowset/bloom_filter_index_reader.h" +#include "storage/rowset/default_value_column_iterator.h" #include "storage/rowset/encoding_info.h" #include "storage/rowset/json_column_iterator.h" #include "storage/rowset/map_column_iterator.h" @@ -71,9 +72,10 @@ namespace starrocks { -StatusOr> ColumnReader::create(ColumnMetaPB* meta, Segment* segment) { +StatusOr> ColumnReader::create(ColumnMetaPB* meta, Segment* segment, + const TabletColumn* column) { auto r = std::make_unique(private_type(0), segment); - RETURN_IF_ERROR(r->_init(meta)); + RETURN_IF_ERROR(r->_init(meta, column)); return std::move(r); } @@ -110,11 +112,15 @@ ColumnReader::~ColumnReader() { MEM_TRACKER_SAFE_RELEASE(GlobalEnv::GetInstance()->column_metadata_mem_tracker(), sizeof(ColumnReader)); } -Status ColumnReader::_init(ColumnMetaPB* meta) { +Status ColumnReader::_init(ColumnMetaPB* meta, const TabletColumn* column) { _column_type = static_cast(meta->type()); _dict_page_pointer = PagePointer(meta->dict_page()); _total_mem_footprint = meta->total_mem_footprint(); - _name = meta->has_name() ? meta->name() : "None"; + if (column == nullptr) { + _name = meta->has_name() ? meta->name() : "None"; + } else { + _name = meta->has_name() ? meta->name() : column->name(); + } _column_unique_id = meta->unique_id(); if (meta->is_nullable()) _flags |= kIsNullableMask; @@ -189,7 +195,8 @@ Status ColumnReader::_init(ColumnMetaPB* meta) { if (_column_type == LogicalType::TYPE_JSON) { _sub_readers = std::make_unique(); for (int i = 0; i < meta->children_columns_size(); ++i) { - auto res = ColumnReader::create(meta->mutable_children_columns(i), _segment); + auto sub_column = (column != nullptr) ? column->subcolumn_ptr(i) : nullptr; + auto res = ColumnReader::create(meta->mutable_children_columns(i), _segment, sub_column); RETURN_IF_ERROR(res); _sub_readers->emplace_back(std::move(res).value()); } @@ -204,18 +211,19 @@ Status ColumnReader::_init(ColumnMetaPB* meta) { } _sub_readers->reserve(3); + auto sub_column = (column != nullptr) ? column->subcolumn_ptr(0) : nullptr; // elements - auto res = ColumnReader::create(meta->mutable_children_columns(0), _segment); + auto res = ColumnReader::create(meta->mutable_children_columns(0), _segment, sub_column); RETURN_IF_ERROR(res); _sub_readers->emplace_back(std::move(res).value()); // null flags - res = ColumnReader::create(meta->mutable_children_columns(1), _segment); + res = ColumnReader::create(meta->mutable_children_columns(1), _segment, nullptr); RETURN_IF_ERROR(res); _sub_readers->emplace_back(std::move(res).value()); // offsets - res = ColumnReader::create(meta->mutable_children_columns(2), _segment); + res = ColumnReader::create(meta->mutable_children_columns(2), _segment, nullptr); RETURN_IF_ERROR(res); _sub_readers->emplace_back(std::move(res).value()); } else { @@ -224,13 +232,14 @@ Status ColumnReader::_init(ColumnMetaPB* meta) { } _sub_readers->reserve(2); + auto sub_column = (column != nullptr) ? column->subcolumn_ptr(0) : nullptr; // elements - auto res = ColumnReader::create(meta->mutable_children_columns(0), _segment); + auto res = ColumnReader::create(meta->mutable_children_columns(0), _segment, sub_column); RETURN_IF_ERROR(res); _sub_readers->emplace_back(std::move(res).value()); // offsets - res = ColumnReader::create(meta->mutable_children_columns(1), _segment); + res = ColumnReader::create(meta->mutable_children_columns(1), _segment, nullptr); RETURN_IF_ERROR(res); _sub_readers->emplace_back(std::move(res).value()); } @@ -244,22 +253,22 @@ Status ColumnReader::_init(ColumnMetaPB* meta) { _sub_readers->reserve(4); // keys - auto res = ColumnReader::create(meta->mutable_children_columns(0), _segment); + auto res = ColumnReader::create(meta->mutable_children_columns(0), _segment, nullptr); RETURN_IF_ERROR(res); _sub_readers->emplace_back(std::move(res).value()); // values - res = ColumnReader::create(meta->mutable_children_columns(1), _segment); + res = ColumnReader::create(meta->mutable_children_columns(1), _segment, nullptr); RETURN_IF_ERROR(res); _sub_readers->emplace_back(std::move(res).value()); // null flags - res = ColumnReader::create(meta->mutable_children_columns(2), _segment); + res = ColumnReader::create(meta->mutable_children_columns(2), _segment, nullptr); RETURN_IF_ERROR(res); _sub_readers->emplace_back(std::move(res).value()); // offsets - res = ColumnReader::create(meta->mutable_children_columns(3), _segment); + res = ColumnReader::create(meta->mutable_children_columns(3), _segment, nullptr); RETURN_IF_ERROR(res); _sub_readers->emplace_back(std::move(res).value()); } else { @@ -269,17 +278,17 @@ Status ColumnReader::_init(ColumnMetaPB* meta) { _sub_readers->reserve(3); // keys - auto res = ColumnReader::create(meta->mutable_children_columns(0), _segment); + auto res = ColumnReader::create(meta->mutable_children_columns(0), _segment, nullptr); RETURN_IF_ERROR(res); _sub_readers->emplace_back(std::move(res).value()); // values - res = ColumnReader::create(meta->mutable_children_columns(1), _segment); + res = ColumnReader::create(meta->mutable_children_columns(1), _segment, nullptr); RETURN_IF_ERROR(res); _sub_readers->emplace_back(std::move(res).value()); // offsets - res = ColumnReader::create(meta->mutable_children_columns(2), _segment); + res = ColumnReader::create(meta->mutable_children_columns(2), _segment, nullptr); RETURN_IF_ERROR(res); _sub_readers->emplace_back(std::move(res).value()); } @@ -287,9 +296,11 @@ Status ColumnReader::_init(ColumnMetaPB* meta) { } else if (_column_type == LogicalType::TYPE_STRUCT) { _sub_readers = std::make_unique(); for (int i = 0; i < meta->children_columns_size(); ++i) { - auto res = ColumnReader::create(meta->mutable_children_columns(i), _segment); + auto sub_column = (column != nullptr) ? column->subcolumn_ptr(i) : nullptr; + auto res = ColumnReader::create(meta->mutable_children_columns(i), _segment, sub_column); RETURN_IF_ERROR(res); _sub_readers->emplace_back(std::move(res).value()); + _update_sub_reader_pos(sub_column, i); } return Status::OK(); } else { @@ -597,7 +608,60 @@ bool ColumnReader::segment_zone_map_filter(const std::vector> ColumnReader::new_iterator(ColumnAccessPath* path) { +void ColumnReader::_update_sub_reader_pos(const TabletColumn* column, int pos) { + if (column == nullptr) { + return; + } + auto name = column->name(); + int id = column->unique_id(); + _sub_reader_pos[{std::string(name), id}] = pos; +} + +StatusOr> ColumnReader::_create_merge_struct_iter(ColumnAccessPath* path, + const TabletColumn* column) { + DCHECK(_column_type == LogicalType::TYPE_STRUCT); + DCHECK(column != nullptr); + auto num_fields = column->subcolumn_count(); + + std::unique_ptr null_iter; + if (is_nullable()) { + ASSIGN_OR_RETURN(null_iter, (*_sub_readers)[_sub_readers->size() - 1]->new_iterator()); + } + + std::vector child_paths(num_fields, nullptr); + if (path != nullptr && !path->children().empty()) { + for (const auto& child : path->children()) { + child_paths[child->index()] = child.get(); + } + } + + std::vector> field_iters; + for (int i = 0; i < num_fields; ++i) { + auto sub_column = column->subcolumn_ptr(i); + auto iter = _sub_reader_pos.find({std::string(sub_column->name()), sub_column->unique_id()}); + if (iter != _sub_reader_pos.end()) { + ASSIGN_OR_RETURN(auto iter, (*_sub_readers)[iter->second]->new_iterator(child_paths[i], sub_column)); + field_iters.emplace_back(std::move(iter)); + } else { + if (!sub_column->has_default_value() && !sub_column->is_nullable()) { + return Status::InternalError( + fmt::format("invalid nonexistent column({}) without default value.", sub_column->name())); + } else { + const TypeInfoPtr& type_info = get_type_info(*sub_column); + auto default_value_iter = std::make_unique( + sub_column->has_default_value(), sub_column->default_value(), sub_column->is_nullable(), + type_info, sub_column->length(), num_rows()); + ColumnIteratorOptions iter_opts; + RETURN_IF_ERROR(default_value_iter->init(iter_opts)); + field_iters.emplace_back(std::move(default_value_iter)); + } + } + } + return create_struct_iter(this, std::move(null_iter), std::move(field_iters), path); +} + +StatusOr> ColumnReader::new_iterator(ColumnAccessPath* path, + const TabletColumn* column) { if (_column_type == LogicalType::TYPE_JSON) { auto json_iter = std::make_unique(this); if (path == nullptr || path->children().empty()) { @@ -663,7 +727,8 @@ StatusOr> ColumnReader::new_iterator(ColumnAcces } } - ASSIGN_OR_RETURN(auto element_iterator, (*_sub_readers)[col++]->new_iterator(value_path)); + auto sub_column = (column != nullptr) ? column->subcolumn_ptr(0) : nullptr; + ASSIGN_OR_RETURN(auto element_iterator, (*_sub_readers)[col++]->new_iterator(value_path, sub_column)); std::unique_ptr null_iterator; if (is_nullable()) { @@ -691,7 +756,7 @@ StatusOr> ColumnReader::new_iterator(ColumnAcces // key must scalar type now ASSIGN_OR_RETURN(auto keys, (*_sub_readers)[col++]->new_iterator()); - ASSIGN_OR_RETURN(auto values, (*_sub_readers)[col++]->new_iterator(value_path)); + ASSIGN_OR_RETURN(auto values, (*_sub_readers)[col++]->new_iterator(value_path, nullptr)); std::unique_ptr nulls; if (is_nullable()) { ASSIGN_OR_RETURN(nulls, (*_sub_readers)[col++]->new_iterator()); @@ -700,6 +765,9 @@ StatusOr> ColumnReader::new_iterator(ColumnAcces return std::make_unique(this, std::move(nulls), std::move(offsets), std::move(keys), std::move(values), path); } else if (_column_type == LogicalType::TYPE_STRUCT) { + if (column != nullptr) { + return _create_merge_struct_iter(path, column); + } auto num_fields = _sub_readers->size(); std::unique_ptr null_iter; diff --git a/be/src/storage/rowset/column_reader.h b/be/src/storage/rowset/column_reader.h index 94cee1ad5df74..e6e74ea82452e 100644 --- a/be/src/storage/rowset/column_reader.h +++ b/be/src/storage/rowset/column_reader.h @@ -87,14 +87,21 @@ struct NgramBloomFilterReaderOptions; // This will cache data shared by all reader class ColumnReader { struct private_type; + struct SubReaderId; public: // Create and initialize a ColumnReader. // This method will not take the ownership of |meta|. // Note that |meta| is mutable, this method may change its internal state. // + // The primary purpose of the |column| currently is to obtain the name and unique ID of the sub_column + // to support the add/drop field functionality of the struct column. + // It is important that the |column| needs to be consistent with the tablet schema corresponding to the segment. + // If you can ensure that this column does not involve a struct column, the |column| can be set to nullptr. + // // To developers: keep this method lightweight, should not incur any I/O. - static StatusOr> create(ColumnMetaPB* meta, Segment* segment); + static StatusOr> create(ColumnMetaPB* meta, Segment* segment, + const TabletColumn* column); ColumnReader(const private_type&, Segment* segment); ~ColumnReader(); @@ -105,7 +112,8 @@ class ColumnReader { void operator=(ColumnReader&&) = delete; // create a new column iterator. - StatusOr> new_iterator(ColumnAccessPath* path = nullptr); + StatusOr> new_iterator(ColumnAccessPath* path = nullptr, + const TabletColumn* column = nullptr); // Caller should free returned iterator after unused. // TODO: StatusOr> new_bitmap_index_iterator() @@ -199,7 +207,7 @@ class ColumnReader { constexpr static uint8_t kHasAllDictEncodedMask = 2; constexpr static uint8_t kAllDictEncodedMask = 4; - Status _init(ColumnMetaPB* meta); + Status _init(ColumnMetaPB* meta, const TabletColumn* column); Status _load_zonemap_index(const IndexReadOptions& opts); Status _load_bitmap_index(const IndexReadOptions& opts); @@ -219,6 +227,11 @@ class ColumnReader { bool _inverted_index_loaded() const { return invoked(_inverted_index_load_once); } + StatusOr> _create_merge_struct_iter(ColumnAccessPath* path, + const TabletColumn* column); + + void _update_sub_reader_pos(const TabletColumn* column, int pos); + // ColumnReader will be resident in memory. When there are many columns in the table, // the meta in ColumnReader takes up a lot of memory, // and now the content that is not needed in Meta is not saved to ColumnReader @@ -247,6 +260,28 @@ class ColumnReader { using SubReaderList = std::vector>; std::unique_ptr _sub_readers; + // Only used for struct column right now + // Use column names and unique IDs to distinguish sub-columns. + // The unnique id is always -1 for historical data, so the column name is needed. + // After support add/drop field for struct column, the following scenarios might occur: + // 1. Drop field v1 + // 2. Add field v1 + // The field `v1` in step 2 is different from the `v1` in step 1 and needs to be distinguished, + // So we also need to unqiue id. + struct SubReaderId { + std::string name; + int32_t id; + + bool operator==(const SubReaderId& other) const { return id == other.id && name == other.name; } + + bool operator<(const SubReaderId& other) const { + if (id != other.id) { + return id < other.id; + } + return name < other.name; + } + }; + std::map _sub_reader_pos; // Pointer to its father segment, as the column reader // is never released before the end of the parent's life cycle, diff --git a/be/src/storage/rowset/segment.cpp b/be/src/storage/rowset/segment.cpp index 4500076feeb32..11f00d6810aaa 100644 --- a/be/src/storage/rowset/segment.cpp +++ b/be/src/storage/rowset/segment.cpp @@ -397,7 +397,7 @@ Status Segment::_create_column_readers(SegmentFooterPB* footer) { continue; } - auto res = ColumnReader::create(footer->mutable_columns(iter->second), this); + auto res = ColumnReader::create(footer->mutable_columns(iter->second), this, &column); if (!res.ok()) { return res.status(); } @@ -410,7 +410,7 @@ StatusOr> Segment::new_column_iterator_or_defaul ColumnAccessPath* path) { auto id = column.unique_id(); if (_column_readers.contains(id)) { - ASSIGN_OR_RETURN(auto source_iter, _column_readers[id]->new_iterator(path)); + ASSIGN_OR_RETURN(auto source_iter, _column_readers[id]->new_iterator(path, &column)); if (_column_readers[id]->column_type() == column.type()) { return source_iter; } else { @@ -439,7 +439,7 @@ StatusOr> Segment::new_column_iterator(const Tab auto id = column.unique_id(); auto iter = _column_readers.find(id); if (iter != _column_readers.end()) { - ASSIGN_OR_RETURN(auto source_iter, iter->second->new_iterator(path)); + ASSIGN_OR_RETURN(auto source_iter, iter->second->new_iterator(path, nullptr)); if (iter->second->column_type() == column.type()) { return source_iter; } else { diff --git a/be/src/storage/tablet_schema.h b/be/src/storage/tablet_schema.h index f0aa0c9c83d10..794d19cd5c344 100644 --- a/be/src/storage/tablet_schema.h +++ b/be/src/storage/tablet_schema.h @@ -162,7 +162,18 @@ class TabletColumn { void add_sub_column(const TabletColumn& sub_column); void add_sub_column(TabletColumn&& sub_column); uint32_t subcolumn_count() const { return _extra_fields ? _extra_fields->sub_columns.size() : 0; } - const TabletColumn& subcolumn(uint32_t i) const { return _extra_fields->sub_columns[i]; } + const TabletColumn& subcolumn(uint32_t i) const { + if (i >= subcolumn_count()) { + throw std::out_of_range("Index i is out of range"); + } + return _extra_fields->sub_columns[i]; + } + const TabletColumn* subcolumn_ptr(uint32_t i) const { + if (i >= subcolumn_count()) { + return nullptr; + } + return &(_extra_fields->sub_columns[i]); + } friend bool operator==(const TabletColumn& a, const TabletColumn& b); friend bool operator!=(const TabletColumn& a, const TabletColumn& b); diff --git a/be/test/storage/rowset/column_reader_writer_test.cpp b/be/test/storage/rowset/column_reader_writer_test.cpp index ce7ebf6979271..dc2112c52f009 100644 --- a/be/test/storage/rowset/column_reader_writer_test.cpp +++ b/be/test/storage/rowset/column_reader_writer_test.cpp @@ -161,7 +161,7 @@ class ColumnReaderWriterTest : public testing::Test { // read and check { // read and check - auto res = ColumnReader::create(&meta, segment.get()); + auto res = ColumnReader::create(&meta, segment.get(), nullptr); ASSERT_TRUE(res.ok()); auto reader = std::move(res).value(); @@ -386,7 +386,7 @@ class ColumnReaderWriterTest : public testing::Test { // read and check { - auto res = ColumnReader::create(&meta, segment.get()); + auto res = ColumnReader::create(&meta, segment.get(), nullptr); ASSERT_TRUE(res.ok()); auto reader = std::move(res).value(); @@ -709,7 +709,7 @@ TEST_F(ColumnReaderWriterTest, test_scalar_column_total_mem_footprint) { // read and check { // read and check - auto res = ColumnReader::create(&meta, segment.get()); + auto res = ColumnReader::create(&meta, segment.get(), nullptr); ASSERT_TRUE(res.ok()); auto reader = std::move(res).value(); ASSERT_EQ(1024, meta.num_rows()); @@ -770,7 +770,7 @@ TEST_F(ColumnReaderWriterTest, test_large_varchar_column_writer) { ASSERT_TRUE(wfile->close().ok()); // read and check result auto segment = create_dummy_segment(fs, fname); - auto res = ColumnReader::create(&meta, segment.get()); + auto res = ColumnReader::create(&meta, segment.get(), nullptr); ASSERT_TRUE(res.ok()); auto reader = std::move(res).value(); ASSIGN_OR_ABORT(auto iter, reader->new_iterator()); diff --git a/be/test/storage/rowset/flat_json_column_rw_test.cpp b/be/test/storage/rowset/flat_json_column_rw_test.cpp index 7093e4f3109cc..c2b0566a99c04 100644 --- a/be/test/storage/rowset/flat_json_column_rw_test.cpp +++ b/be/test/storage/rowset/flat_json_column_rw_test.cpp @@ -96,7 +96,7 @@ class FlatJsonColumnRWTest : public testing::Test { } LOG(INFO) << "Finish writing"; - auto res = ColumnReader::create(&meta, segment.get()); + auto res = ColumnReader::create(&meta, segment.get(), nullptr); ASSERT_TRUE(res.ok()); auto reader = std::move(res).value(); diff --git a/be/test/storage/rowset/map_column_rw_test.cpp b/be/test/storage/rowset/map_column_rw_test.cpp index d566bbea7b9ee..3c6a2a9cf0316 100644 --- a/be/test/storage/rowset/map_column_rw_test.cpp +++ b/be/test/storage/rowset/map_column_rw_test.cpp @@ -145,7 +145,7 @@ class MapColumnRWTest : public testing::Test { } // read and check - auto res = ColumnReader::create(&meta, segment.get()); + auto res = ColumnReader::create(&meta, segment.get(), nullptr); ASSERT_TRUE(res.ok()); auto reader = std::move(res).value(); diff --git a/be/test/storage/rowset/struct_column_rw_test.cpp b/be/test/storage/rowset/struct_column_rw_test.cpp index 53445ad6c0288..1fef3f20d5950 100644 --- a/be/test/storage/rowset/struct_column_rw_test.cpp +++ b/be/test/storage/rowset/struct_column_rw_test.cpp @@ -132,7 +132,9 @@ class StructColumnRWTest : public testing::Test { LOG(INFO) << "Finish writing"; // read and check - auto res = ColumnReader::create(&meta, segment.get()); + ColumnMetaPB meta2 = meta; + ColumnMetaPB meta3 = meta; + auto res = ColumnReader::create(&meta, segment.get(), nullptr); ASSERT_TRUE(res.ok()); auto reader = std::move(res).value(); @@ -165,6 +167,87 @@ class StructColumnRWTest : public testing::Test { ASSERT_EQ("{f1:1,f2:'Column2'}", dst_column->debug_item(0)); } + { + TabletColumn new_struct_column = create_struct(0, true); + std::vector names{"f1", "f3"}; + TabletColumn f1_tablet_column = create_int_value(1, STORAGE_AGGREGATE_NONE, true); + new_struct_column.add_sub_column(f1_tablet_column); + // add new field column f3 + TabletColumn f3_tablet_column = create_int_value(3, STORAGE_AGGREGATE_NONE, true, "2"); + ASSERT_TRUE(f3_tablet_column.has_default_value()); + new_struct_column.add_sub_column(f3_tablet_column); + auto res = ColumnReader::create(&meta2, segment.get(), &struct_column); + ASSERT_TRUE(res.ok()); + auto struct_reader = std::move(res).value(); + ASSIGN_OR_ABORT(auto iter, struct_reader->new_iterator(nullptr, &new_struct_column)); + ASSIGN_OR_ABORT(auto read_file, fs->new_random_access_file(fname)); + + ColumnIteratorOptions iter_opts; + OlapReaderStatistics stats; + iter_opts.stats = &stats; + iter_opts.read_file = read_file.get(); + ASSERT_TRUE(iter->init(iter_opts).ok()); + + // sequence read + auto st = iter->seek_to_first(); + ASSERT_TRUE(st.ok()) << st.to_string(); + + auto dst_f1_column = Int32Column::create(); + auto dst_f3_column = Int32Column::create(); + Columns dst_columns; + dst_columns.emplace_back(std::move(dst_f1_column)); + dst_columns.emplace_back(std::move(dst_f3_column)); + + ColumnPtr dst_column = StructColumn::create(dst_columns, names); + size_t rows_read = src_column->size(); + st = iter->next_batch(&rows_read, dst_column.get()); + ASSERT_TRUE(st.ok()); + ASSERT_EQ(src_column->size(), rows_read); + + ASSERT_EQ("{f1:1,f3:2}", dst_column->debug_item(0)); + } + + { + TabletColumn new_struct_column = create_struct(0, true); + std::vector names{"f1", "f3"}; + TabletColumn f1_tablet_column = create_int_value(1, STORAGE_AGGREGATE_NONE, true, "5"); + f1_tablet_column.set_unique_id(10); + new_struct_column.add_sub_column(f1_tablet_column); + // add new field column f3 + TabletColumn f3_tablet_column = create_int_value(3, STORAGE_AGGREGATE_NONE, true, "2"); + ASSERT_TRUE(f3_tablet_column.has_default_value()); + new_struct_column.add_sub_column(f3_tablet_column); + auto res = ColumnReader::create(&meta3, segment.get(), &struct_column); + ASSERT_TRUE(res.ok()); + auto struct_reader = std::move(res).value(); + ASSIGN_OR_ABORT(auto iter, struct_reader->new_iterator(nullptr, &new_struct_column)); + ASSIGN_OR_ABORT(auto read_file, fs->new_random_access_file(fname)); + + ColumnIteratorOptions iter_opts; + OlapReaderStatistics stats; + iter_opts.stats = &stats; + iter_opts.read_file = read_file.get(); + ASSERT_TRUE(iter->init(iter_opts).ok()); + + // sequence read + auto st = iter->seek_to_first(); + ASSERT_TRUE(st.ok()) << st.to_string(); + + auto dst_f1_column = Int32Column::create(); + auto dst_f3_column = Int32Column::create(); + Columns dst_columns; + dst_columns.emplace_back(std::move(dst_f1_column)); + dst_columns.emplace_back(std::move(dst_f3_column)); + + ColumnPtr dst_column = StructColumn::create(dst_columns, names); + size_t rows_read = src_column->size(); + st = iter->next_batch(&rows_read, dst_column.get()); + ASSERT_TRUE(st.ok()); + ASSERT_EQ(src_column->size(), rows_read); + + ASSERT_EQ("{f1:5,f3:2}", dst_column->debug_item(0)); + } + { ASSIGN_OR_ABORT(auto child_path, ColumnAccessPath::create(TAccessPathType::type::FIELD, "f1", 0)); ASSIGN_OR_ABORT(auto path, ColumnAccessPath::create(TAccessPathType::type::ROOT, "root", 0)); diff --git a/fe/fe-core/src/main/java/com/starrocks/catalog/StructField.java b/fe/fe-core/src/main/java/com/starrocks/catalog/StructField.java index dffbe2b92e97c..1451e3c4155ab 100644 --- a/fe/fe-core/src/main/java/com/starrocks/catalog/StructField.java +++ b/fe/fe-core/src/main/java/com/starrocks/catalog/StructField.java @@ -28,19 +28,29 @@ */ public class StructField { @SerializedName(value = "name") - private final String name; + private String name; @SerializedName(value = "type") - private final Type type; + private Type type; // comment is not used now, it's always null. @SerializedName(value = "comment") - private final String comment; + private String comment; private int position; // in struct - public StructField(String name, Type type, String comment) { + @SerializedName(value = "fieldId") + private int fieldId = -1; + + public StructField() {} + + public StructField(String name, int fieldId, Type type, String comment) { this.name = name; this.type = type; this.comment = comment; + this.fieldId = fieldId; + } + + public StructField(String name, Type type, String comment) { + this(name, -1, type, comment); } public StructField(String name, Type type) { @@ -115,6 +125,7 @@ public void toThrift(TTypeDesc container, TTypeNode node) { TStructField field = new TStructField(); field.setName(name); field.setComment(comment); + field.setId(fieldId); node.struct_fields.add(field); type.toThrift(container); } @@ -136,7 +147,7 @@ public boolean equals(Object other) { @Override public StructField clone() { - return new StructField(name, type.clone(), comment); + return new StructField(name, fieldId, type.clone(), comment); } } diff --git a/gensrc/thrift/Types.thrift b/gensrc/thrift/Types.thrift index 24219bfcfc736..f44b2db338b93 100644 --- a/gensrc/thrift/Types.thrift +++ b/gensrc/thrift/Types.thrift @@ -126,6 +126,7 @@ struct TScalarType { struct TStructField { 1: optional string name 2: optional string comment + 3: optional i32 id } struct TTypeNode {