From af11f451fd8ebe460ae59cd5c2ba745a819dd285 Mon Sep 17 00:00:00 2001 From: Zhichang Yu Date: Fri, 1 Mar 2024 21:06:55 +0800 Subject: [PATCH] Merge IndexDef into IndexBase (#692) ### What problem does this PR solve? Merge IndexDef into IndexBase so that `CREATE INDEX` contains only one index definition. Breaking Changes: - iresearch doesn't support fulltext index on multiple columns Issue link:#551 ### Type of change - [x] Breaking Change (fix or feature that could cause existing functionality not to work as expected) - [x] Refactoring --- python/test/test_index.py | 71 +++--- .../operator/physical_create_index_do.cpp | 1 - .../operator/physical_create_index_do.cppm | 2 +- .../operator/physical_create_index_finish.cpp | 8 +- .../physical_create_index_finish.cppm | 6 +- .../physical_create_index_prepare.cpp | 3 +- .../physical_create_index_prepare.cppm | 6 +- .../operator/physical_create_schema.cppm | 2 +- .../operator/physical_create_table.cppm | 2 +- .../operator/physical_create_view.cppm | 2 +- src/executor/operator/physical_knn_scan.cpp | 1 - src/executor/operator/physical_show.cpp | 1 - src/executor/physical_planner.cpp | 3 +- src/planner/logical_planner.cpp | 90 +++---- src/planner/node/logical_create_index.cpp | 2 +- src/planner/node/logical_create_index.cppm | 10 +- src/storage/definition/index_base.cpp | 24 +- src/storage/definition/index_base.cppm | 7 +- src/storage/definition/index_def.cpp | 125 ---------- src/storage/definition/index_def.cppm | 55 ----- src/storage/definition/index_full_text.cpp | 7 +- src/storage/definition/index_full_text.cppm | 9 +- src/storage/definition/index_hnsw.cpp | 7 +- src/storage/definition/index_hnsw.cppm | 10 +- src/storage/definition/index_ivfflat.cpp | 9 +- src/storage/definition/index_ivfflat.cppm | 10 +- src/storage/definition/index_secondary.cppm | 7 +- src/storage/definition/table_def.cppm | 4 +- .../invertedindex/iresearch/datastore.cpp | 223 +++++++++--------- .../invertedindex/iresearch/datastore.cppm | 4 +- src/storage/meta/catalog.cpp | 22 +- src/storage/meta/catalog.cppm | 4 +- .../meta/entry/column_index_entry.cppm | 1 - .../entry/segment_column_index_entry.cppm | 1 - src/storage/meta/entry/table_entry.cpp | 30 ++- src/storage/meta/entry/table_entry.cppm | 4 +- src/storage/meta/entry/table_index_entry.cpp | 82 +++---- src/storage/meta/entry/table_index_entry.cppm | 15 +- src/storage/meta/table_index_meta.cpp | 18 +- src/storage/meta/table_index_meta.cppm | 6 +- src/storage/txn/txn.cpp | 18 +- src/storage/txn/txn.cppm | 6 +- src/storage/txn/txn_store.cpp | 2 +- src/storage/wal/catalog_delta_entry.cpp | 15 +- src/storage/wal/catalog_delta_entry.cppm | 13 +- src/storage/wal/wal_entry.cpp | 16 +- src/storage/wal/wal_entry.cppm | 8 +- src/storage/wal/wal_manager.cpp | 2 +- .../{index_def.cpp => index_base.cpp} | 84 +------ .../storage/wal/catalog_delta_entry.cpp | 6 +- src/unit_test/storage/wal/wal_entry.cpp | 9 +- src/unit_test/storage/wal/wal_replay.cpp | 20 +- test/sql/dql/fulltext.slt | 12 +- test/sql/dql/fusion.slt | 4 +- 54 files changed, 417 insertions(+), 692 deletions(-) delete mode 100644 src/storage/definition/index_def.cpp delete mode 100644 src/storage/definition/index_def.cppm rename src/unit_test/storage/definition/{index_def.cpp => index_base.cpp} (56%) diff --git a/python/test/test_index.py b/python/test/test_index.py index 74096ce9bb..53578dc918 100644 --- a/python/test/test_index.py +++ b/python/test/test_index.py @@ -59,10 +59,14 @@ def test_create_index_HNSW(self): [index.IndexInfo("c1", index.IndexType.Hnsw, [ - index.InitParameter("M", "16"), - index.InitParameter("ef_construction", "50"), - index.InitParameter("ef", "50"), - index.InitParameter("metric", "l2") + index.InitParameter( + "M", "16"), + index.InitParameter( + "ef_construction", "50"), + index.InitParameter( + "ef", "50"), + index.InitParameter( + "metric", "l2") ])], None) assert res.error_code == ErrorCode.OK @@ -71,7 +75,7 @@ def test_create_index_HNSW(self): assert res.error_code == ErrorCode.OK def test_create_index_fulltext(self): - # CREATE INDEX ft_index ON enwiki(body) USING FULLTEXT WITH(ANALYZER=segmentation) (doctitle, docdate) USING FULLTEXT; + # CREATE INDEX ft_index ON enwiki(body) USING FULLTEXT WITH(ANALYZER=segmentation); infinity_obj = infinity.connect(common_values.TEST_REMOTE_HOST) db_obj = infinity_obj.get_database("default") res = db_obj.drop_table("test_index_fulltext", if_exists=True) @@ -84,12 +88,6 @@ def test_create_index_fulltext(self): [index.IndexInfo("body", index.IndexType.FullText, [index.InitParameter("ANALYZER", "segmentation")]), - index.IndexInfo("doctitle", - index.IndexType.FullText, - []), - index.IndexInfo("docdate", - index.IndexType.FullText, - []), ], None) assert res.error_code == ErrorCode.OK @@ -147,11 +145,14 @@ def test_create_created_index(self): # create / drop index with invalid options @pytest.mark.parametrize("cl_name", [(1, False), (2.2, False), ((1, 2), False), ([1, 2, 3], False), ("c1", True)]) @pytest.mark.parametrize("index_type", [ - (1, False), (2.2, False), ([1, 2], False), ("$#%dfva", False), ((1, 2), False), ({"1": 2}, False), - (index.IndexType.Hnsw, False), (index.IndexType.IVFFlat, True), (index.IndexType.FullText, True) + (1, False), (2.2, False), ([1, 2], False), ("$#%dfva", False), (( + 1, 2), False), ({"1": 2}, False), + (index.IndexType.Hnsw, False), (index.IndexType.IVFFlat, + True), (index.IndexType.FullText, True) ]) @pytest.mark.parametrize("params", [ - (1, False), (2.2, False), ([1, 2], False), ("$#%dfva", False), ((1, 2), False), ({"1": 2}, False), + (1, False), (2.2, False), ([1, 2], False), ("$#%dfva", False), (( + 1, 2), False), ({"1": 2}, False), ([index.InitParameter("centroids_count", "128"), index.InitParameter("metric", "l2")], True) ]) @@ -159,7 +160,8 @@ def test_create_drop_index_invalid_options(self, cl_name, index_type, params): # connect infinity_obj = infinity.connect(common_values.TEST_REMOTE_HOST) db_obj = infinity_obj.get_database("default") - db_obj.drop_table("test_create_drop_index_invalid_options", if_exists=True) + db_obj.drop_table( + "test_create_drop_index_invalid_options", if_exists=True) table_obj = db_obj.create_table("test_create_drop_index_invalid_options", { "c1": "vector,3,float"}, None) @@ -179,18 +181,20 @@ def test_create_index_on_dropped_table(self): # connect infinity_obj = infinity.connect(common_values.TEST_REMOTE_HOST) db_obj = infinity_obj.get_database("default") - db_obj.drop_table("test_create_drop_index_invalid_options", if_exists=True) + db_obj.drop_table( + "test_create_drop_index_invalid_options", if_exists=True) table_obj = db_obj.create_table("test_create_drop_index_invalid_options", { "c1": "vector,3,float"}, None) - db_obj.drop_table("test_create_drop_index_invalid_options", if_exists=True) + db_obj.drop_table( + "test_create_drop_index_invalid_options", if_exists=True) # create created index with pytest.raises(Exception, match="ERROR:3022*"): table_obj.create_index("my_index", - [index.IndexInfo("c1", - index.IndexType.IVFFlat, - [index.InitParameter("centroids_count", "128"), - index.InitParameter("metric", "l2")])], None) + [index.IndexInfo("c1", + index.IndexType.IVFFlat, + [index.InitParameter("centroids_count", "128"), + index.InitParameter("metric", "l2")])], None) # disconnect res = infinity_obj.disconnect() @@ -240,29 +244,31 @@ def test_drop_index_show_index(self): # create index on different type of column and show index @pytest.mark.parametrize("types", ["vector, 3, float"]) @pytest.mark.parametrize("index_type", [ - (index.IndexType.Hnsw, False), (index.IndexType.IVFFlat, True), (index.IndexType.FullText, True) + (index.IndexType.Hnsw, False), (index.IndexType.IVFFlat, + True), (index.IndexType.FullText, True) ]) def test_create_index_on_different_type_of_column(self, types, index_type): # connect infinity_obj = infinity.connect(common_values.TEST_REMOTE_HOST) db_obj = infinity_obj.get_database("default") - db_obj.drop_table("test_create_index_on_different_type_of_column", if_exists=True) + db_obj.drop_table( + "test_create_index_on_different_type_of_column", if_exists=True) table_obj = db_obj.create_table("test_create_index_on_different_type_of_column", { "c1": types}, None) # create created index if not index_type[1]: with pytest.raises(Exception, match="ERROR:3061*"): table_obj.create_index("my_index", - [index.IndexInfo("c1", - index_type[0], - [index.InitParameter("centroids_count", "128"), - index.InitParameter("metric", "l2")])], None) - else: - res = table_obj.create_index("my_index", - [index.IndexInfo("c1", + [index.IndexInfo("c1", index_type[0], [index.InitParameter("centroids_count", "128"), - index.InitParameter("metric", "l2")])], None) + index.InitParameter("metric", "l2")])], None) + else: + res = table_obj.create_index("my_index", + [index.IndexInfo("c1", + index_type[0], + [index.InitParameter("centroids_count", "128"), + index.InitParameter("metric", "l2")])], None) assert res.error_code == ErrorCode.OK # disconnect @@ -307,7 +313,8 @@ def test_import_data_create_index(self, index_type, file_format): "c1": "int", "c2": "vector,3,float"}, None) - table_obj.import_data(os.getcwd() + TEST_DATA_DIR + file_format + "/pysdk_test." + file_format) + table_obj.import_data(os.getcwd() + TEST_DATA_DIR + + file_format + "/pysdk_test." + file_format) res = table_obj.create_index("my_index", [index.IndexInfo("c2", index_type, diff --git a/src/executor/operator/physical_create_index_do.cpp b/src/executor/operator/physical_create_index_do.cpp index f79d93d29a..35a64fc04f 100644 --- a/src/executor/operator/physical_create_index_do.cpp +++ b/src/executor/operator/physical_create_index_do.cpp @@ -23,7 +23,6 @@ import query_context; import operator_state; import load_meta; -import index_def; import create_index_data; import base_table_ref; import status; diff --git a/src/executor/operator/physical_create_index_do.cppm b/src/executor/operator/physical_create_index_do.cppm index 9ea42f979c..9a39a6080c 100644 --- a/src/executor/operator/physical_create_index_do.cppm +++ b/src/executor/operator/physical_create_index_do.cppm @@ -24,7 +24,7 @@ import query_context; import operator_state; import load_meta; -import index_def; +import index_base; import base_table_ref; import internal_types; import data_type; diff --git a/src/executor/operator/physical_create_index_finish.cpp b/src/executor/operator/physical_create_index_finish.cpp index e2149ad8c6..3b7b2db971 100644 --- a/src/executor/operator/physical_create_index_finish.cpp +++ b/src/executor/operator/physical_create_index_finish.cpp @@ -23,7 +23,7 @@ import physical_operator; import query_context; import operator_state; import load_meta; -import index_def; +import index_base; import infinity_exception; module physical_create_index_finish; @@ -33,18 +33,18 @@ PhysicalCreateIndexFinish::PhysicalCreateIndexFinish(u64 id, UniquePtr left, SharedPtr db_name, SharedPtr table_name, - SharedPtr index_def, + SharedPtr index_base, SharedPtr> output_names, SharedPtr>> output_types, SharedPtr> load_metas) : PhysicalOperator(PhysicalOperatorType::kCreateIndexFinish, std::move(left), nullptr, id, load_metas), db_name_(db_name), - table_name_(table_name), index_def_(index_def), output_names_(output_names), output_types_(output_types) {} + table_name_(table_name), index_base_(index_base), output_names_(output_names), output_types_(output_types) {} void PhysicalCreateIndexFinish::Init() {} bool PhysicalCreateIndexFinish::Execute(QueryContext *query_context, OperatorState *operator_state) { auto *txn = query_context->GetTxn(); - auto status = txn->CreateIndexFinish(*db_name_, *table_name_, index_def_); + auto status = txn->CreateIndexFinish(*db_name_, *table_name_, index_base_); if (!status.ok()) { RecoverableError(status); } diff --git a/src/executor/operator/physical_create_index_finish.cppm b/src/executor/operator/physical_create_index_finish.cppm index 3e89f01ad4..854ccc6f11 100644 --- a/src/executor/operator/physical_create_index_finish.cppm +++ b/src/executor/operator/physical_create_index_finish.cppm @@ -23,7 +23,7 @@ import physical_operator; import query_context; import operator_state; import load_meta; -import index_def; +import index_base; import internal_types; import data_type; @@ -34,7 +34,7 @@ public: UniquePtr left, SharedPtr db_name, SharedPtr table_name, - SharedPtr index_def, + SharedPtr index_base, SharedPtr> output_names, SharedPtr>> output_types, SharedPtr> load_metas); @@ -53,7 +53,7 @@ public: public: const SharedPtr db_name_{}; const SharedPtr table_name_{}; - const SharedPtr index_def_{}; + const SharedPtr index_base_{}; const SharedPtr> output_names_{}; const SharedPtr>> output_types_{}; diff --git a/src/executor/operator/physical_create_index_prepare.cpp b/src/executor/operator/physical_create_index_prepare.cpp index 56d1d6f137..4f3dbfe505 100644 --- a/src/executor/operator/physical_create_index_prepare.cpp +++ b/src/executor/operator/physical_create_index_prepare.cpp @@ -26,7 +26,6 @@ import query_context; import operator_state; import load_meta; -import index_def; import status; import infinity_exception; import index_base; @@ -48,7 +47,7 @@ import extra_ddl_info; namespace infinity { PhysicalCreateIndexPrepare::PhysicalCreateIndexPrepare(u64 id, SharedPtr base_table_ref, - SharedPtr index_definition, + SharedPtr index_definition, ConflictType conflict_type, SharedPtr> output_names, SharedPtr>> output_types, diff --git a/src/executor/operator/physical_create_index_prepare.cppm b/src/executor/operator/physical_create_index_prepare.cppm index 9f836d7ea2..d4338bea0d 100644 --- a/src/executor/operator/physical_create_index_prepare.cppm +++ b/src/executor/operator/physical_create_index_prepare.cppm @@ -24,7 +24,7 @@ import query_context; import operator_state; import load_meta; import base_table_ref; -import index_def; +import index_base; import internal_types; import extra_ddl_info; import data_type; @@ -34,7 +34,7 @@ export class PhysicalCreateIndexPrepare : public PhysicalOperator { public: PhysicalCreateIndexPrepare(u64 id, SharedPtr base_table_ref, - SharedPtr index_definition, + SharedPtr index_definition, ConflictType conflict_type, SharedPtr> output_names, SharedPtr>> output_types, @@ -55,7 +55,7 @@ public: public: const SharedPtr base_table_ref_{}; - const SharedPtr index_def_ptr_{}; + const SharedPtr index_def_ptr_{}; const ConflictType conflict_type_{}; const SharedPtr> output_names_{}; diff --git a/src/executor/operator/physical_create_schema.cppm b/src/executor/operator/physical_create_schema.cppm index aa2a310628..ada5cfaf25 100644 --- a/src/executor/operator/physical_create_schema.cppm +++ b/src/executor/operator/physical_create_schema.cppm @@ -22,7 +22,7 @@ import query_context; import operator_state; import physical_operator; import physical_operator_type; -import index_def; +import index_base; import load_meta; import infinity_exception; import internal_types; diff --git a/src/executor/operator/physical_create_table.cppm b/src/executor/operator/physical_create_table.cppm index a720b625e8..6daa4442bf 100644 --- a/src/executor/operator/physical_create_table.cppm +++ b/src/executor/operator/physical_create_table.cppm @@ -22,7 +22,7 @@ import query_context; import operator_state; import physical_operator; import physical_operator_type; -import index_def; +import index_base; import table_def; import load_meta; import infinity_exception; diff --git a/src/executor/operator/physical_create_view.cppm b/src/executor/operator/physical_create_view.cppm index 190d492b2e..c95b7a6240 100644 --- a/src/executor/operator/physical_create_view.cppm +++ b/src/executor/operator/physical_create_view.cppm @@ -22,7 +22,7 @@ import query_context; import operator_state; import physical_operator; import physical_operator_type; -import index_def; +import index_base; import load_meta; import infinity_exception; import internal_types; diff --git a/src/executor/operator/physical_knn_scan.cpp b/src/executor/operator/physical_knn_scan.cpp index c19b40240a..de45d434bf 100644 --- a/src/executor/operator/physical_knn_scan.cpp +++ b/src/executor/operator/physical_knn_scan.cpp @@ -42,7 +42,6 @@ import index_base; import buffer_manager; import merge_knn; import knn_result_handler; -import index_def; import ann_ivf_flat; import annivfflat_index_data; import buffer_handle; diff --git a/src/executor/operator/physical_show.cpp b/src/executor/operator/physical_show.cpp index 051a64e1b3..2b6e695b21 100644 --- a/src/executor/operator/physical_show.cpp +++ b/src/executor/operator/physical_show.cpp @@ -36,7 +36,6 @@ import value; import table_def; import data_table; import third_party; -import index_def; import index_ivfflat; import index_base; import index_hnsw; diff --git a/src/executor/physical_planner.cpp b/src/executor/physical_planner.cpp index 2c72e0eaf1..08e0b62d13 100644 --- a/src/executor/physical_planner.cpp +++ b/src/executor/physical_planner.cpp @@ -325,8 +325,7 @@ UniquePtr PhysicalPlanner::BuildCreateIndex(const SharedPtr schema_name = logical_create_index->base_table_ref()->schema_name(); SharedPtr table_name = logical_create_index->base_table_ref()->table_name(); const auto &index_def_ptr = logical_create_index->index_definition(); - if (false || index_def_ptr->index_array_.size() != 1 || index_def_ptr->index_array_[0]->index_type_ != IndexType::kHnsw) { - // TODO: invalidate multiple index in one statement. + if (index_def_ptr->index_type_ != IndexType::kHnsw) { // TODO: support other index types build in parallel. return MakeUnique(logical_create_index->node_id(), logical_create_index->base_table_ref(), diff --git a/src/planner/logical_planner.cpp b/src/planner/logical_planner.cpp index 9b14f6d88c..9018c84d02 100644 --- a/src/planner/logical_planner.cpp +++ b/src/planner/logical_planner.cpp @@ -61,7 +61,6 @@ import explain_ast; import local_file_system; -import index_def; import status; import default_values; import index_base; @@ -520,15 +519,6 @@ Status LogicalPlanner::BuildCreateView(const CreateStatement *statement, SharedP return Status::OK(); } -// struct IndexInfo { -// ~IndexInfo(); -// IndexType method_type_{}; -// std::string column_name_{}; -// std::vector *index_param_list_{nullptr}; -// -// static std::string IndexTypeToString(IndexType index_type); -// }; - Status LogicalPlanner::BuildCreateIndex(const CreateStatement *statement, SharedPtr &bind_context_ptr) { auto *create_index_info = (CreateIndexInfo *)statement->create_info_.get(); @@ -542,14 +532,6 @@ Status LogicalPlanner::BuildCreateIndex(const CreateStatement *statement, Shared // } SharedPtr index_name = MakeShared(std::move(create_index_info->index_name_)); - SharedPtr index_def_ptr = MakeShared(index_name); - - Vector &index_info_list = *create_index_info->index_info_list_; - index_def_ptr->index_array_.reserve(index_info_list.size()); - if (index_info_list.empty()) { - RecoverableError(Status::SyntaxError("No index info.")); - } - UniquePtr query_binder_ptr = MakeUnique(this->query_context_ptr_, bind_context_ptr); auto base_table_ref = query_binder_ptr->GetTableRef(*schema_name, *table_name); @@ -561,45 +543,51 @@ Status LogicalPlanner::BuildCreateIndex(const CreateStatement *statement, Shared } } - for (IndexInfo *index_info : index_info_list) { - SharedPtr base_index_ptr{nullptr}; - switch (index_info->index_type_) { - case IndexType::kFullText: { - base_index_ptr = IndexFullText::Make(fmt::format("{}_{}", create_index_info->table_name_, *index_name), - {index_info->column_name_}, - *(index_info->index_param_list_)); - break; - } - case IndexType::kHnsw: { - // The following check might affect performance - IndexHnsw::ValidateColumnDataType(base_table_ref, index_info->column_name_); // may throw exception - base_index_ptr = IndexHnsw::Make(fmt::format("{}_{}", create_index_info->table_name_, *index_name), + if (create_index_info->index_info_list_->size() != 1) { + RecoverableError(Status::InvalidIndexDefinition( + fmt::format("Index {} consists of {} IndexInfo however 1 is expected", *index_name, create_index_info->index_info_list_->size()))); + } + IndexInfo *index_info = create_index_info->index_info_list_->at(0); + SharedPtr base_index_ptr{nullptr}; + switch (index_info->index_type_) { + case IndexType::kFullText: { + base_index_ptr = IndexFullText::Make(index_name, + fmt::format("{}_{}", create_index_info->table_name_, *index_name), {index_info->column_name_}, *(index_info->index_param_list_)); - break; - } - case IndexType::kIVFFlat: { - IndexIVFFlat::ValidateColumnDataType(base_table_ref, index_info->column_name_); // may throw exception - base_index_ptr = IndexIVFFlat::Make(fmt::format("{}_{}", create_index_info->table_name_, *index_name), - {index_info->column_name_}, - *(index_info->index_param_list_)); - break; - } - case IndexType::kSecondary: { - IndexSecondary::ValidateColumnDataType(base_table_ref, index_info->column_name_); // may throw exception - base_index_ptr = IndexSecondary::Make(fmt::format("{}_{}", create_index_info->table_name_, *index_name), {index_info->column_name_}); - break; - } - case IndexType::kInvalid: { - UnrecoverableError("Invalid index type."); - break; - } + break; + } + case IndexType::kHnsw: { + // The following check might affect performance + IndexHnsw::ValidateColumnDataType(base_table_ref, index_info->column_name_); // may throw exception + base_index_ptr = IndexHnsw::Make(index_name, + fmt::format("{}_{}", create_index_info->table_name_, *index_name), + {index_info->column_name_}, + *(index_info->index_param_list_)); + break; + } + case IndexType::kIVFFlat: { + IndexIVFFlat::ValidateColumnDataType(base_table_ref, index_info->column_name_); // may throw exception + base_index_ptr = IndexIVFFlat::Make(index_name, + fmt::format("{}_{}", create_index_info->table_name_, *index_name), + {index_info->column_name_}, + *(index_info->index_param_list_)); + break; + } + case IndexType::kSecondary: { + IndexSecondary::ValidateColumnDataType(base_table_ref, index_info->column_name_); // may throw exception + base_index_ptr = + IndexSecondary::Make(index_name, fmt::format("{}_{}", create_index_info->table_name_, *index_name), {index_info->column_name_}); + break; + } + case IndexType::kInvalid: { + UnrecoverableError("Invalid index type."); + break; } - index_def_ptr->index_array_.emplace_back(base_index_ptr); } auto logical_create_index_operator = - MakeShared(bind_context_ptr->GetNewLogicalNodeId(), base_table_ref, index_def_ptr, create_index_info->conflict_type_); + MakeShared(bind_context_ptr->GetNewLogicalNodeId(), base_table_ref, base_index_ptr, create_index_info->conflict_type_); this->logical_plan_ = logical_create_index_operator; this->names_ptr_->emplace_back("OK"); diff --git a/src/planner/node/logical_create_index.cpp b/src/planner/node/logical_create_index.cpp index b85c267e64..af24ddf79a 100644 --- a/src/planner/node/logical_create_index.cpp +++ b/src/planner/node/logical_create_index.cpp @@ -22,7 +22,7 @@ import stl; import column_binding; import base_expression; -import index_def; +import index_base; import logical_type; import internal_types; diff --git a/src/planner/node/logical_create_index.cppm b/src/planner/node/logical_create_index.cppm index af2be43ccf..ef5a23af64 100644 --- a/src/planner/node/logical_create_index.cppm +++ b/src/planner/node/logical_create_index.cppm @@ -22,7 +22,7 @@ import column_binding; import logical_node; import base_table_ref; -import index_def; +import index_base; import internal_types; import extra_ddl_info; import data_type; @@ -42,20 +42,20 @@ public: inline String name() override { return "LogicalCreateIndex"; } public: - inline LogicalCreateIndex(u64 node_id, SharedPtr base_table_ref, SharedPtr index_def, ConflictType conflict_type) - : LogicalNode(node_id, LogicalNodeType::kCreateIndex), base_table_ref_(base_table_ref), index_definition_(index_def), + inline LogicalCreateIndex(u64 node_id, SharedPtr base_table_ref, SharedPtr index_base, ConflictType conflict_type) + : LogicalNode(node_id, LogicalNodeType::kCreateIndex), base_table_ref_(base_table_ref), index_definition_(index_base), conflict_type_(conflict_type) {} public: [[nodiscard]] inline SharedPtr base_table_ref() const { return base_table_ref_; } - [[nodiscard]] inline SharedPtr index_definition() const { return index_definition_; } + [[nodiscard]] inline SharedPtr index_definition() const { return index_definition_; } [[nodiscard]] inline ConflictType conflict_type() const { return conflict_type_; } private: SharedPtr base_table_ref_{}; - SharedPtr index_definition_{}; + SharedPtr index_definition_{}; ConflictType conflict_type_{ConflictType::kInvalid}; }; } // namespace infinity \ No newline at end of file diff --git a/src/storage/definition/index_base.cpp b/src/storage/definition/index_base.cpp index f4ddd41716..72e4e7a5db 100644 --- a/src/storage/definition/index_base.cpp +++ b/src/storage/definition/index_base.cpp @@ -69,6 +69,8 @@ int32_t IndexBase::GetSizeInBytes() const { int32_t size = 0; size += sizeof(index_type_); size += sizeof(int32_t); + size += index_name_->length(); + size += sizeof(int32_t); size += file_name_.length(); size += sizeof(int32_t); for (const String &column_name : column_names_) { @@ -79,6 +81,7 @@ int32_t IndexBase::GetSizeInBytes() const { void IndexBase::WriteAdv(char *&ptr) const { WriteBufAdv(ptr, index_type_); + WriteBufAdv(ptr, *index_name_); WriteBufAdv(ptr, file_name_); WriteBufAdv(ptr, static_cast(column_names_.size())); for (const String &column_name : column_names_) { @@ -93,6 +96,7 @@ SharedPtr IndexBase::ReadAdv(char *&ptr, int32_t maxbytes) { } IndexType index_type = ReadBufAdv(ptr); Vector column_names; + SharedPtr index_name = MakeShared(ReadBufAdv(ptr)); String file_name = ReadBufAdv(ptr); int32_t column_names_size = ReadBufAdv(ptr); for (int32_t i = 0; i < column_names_size; ++i) { @@ -103,7 +107,7 @@ SharedPtr IndexBase::ReadAdv(char *&ptr, int32_t maxbytes) { case IndexType::kIVFFlat: { size_t centroids_count = ReadBufAdv(ptr); MetricType metric_type = ReadBufAdv(ptr); - res = MakeShared(file_name, column_names, centroids_count, metric_type); + res = MakeShared(index_name, file_name, column_names, centroids_count, metric_type); break; } case IndexType::kHnsw: { @@ -112,17 +116,17 @@ SharedPtr IndexBase::ReadAdv(char *&ptr, int32_t maxbytes) { SizeT M = ReadBufAdv(ptr); SizeT ef_construction = ReadBufAdv(ptr); SizeT ef = ReadBufAdv(ptr); - res = MakeShared(file_name, column_names, metric_type, encode_type, M, ef_construction, ef); + res = MakeShared(index_name, file_name, column_names, metric_type, encode_type, M, ef_construction, ef); break; } case IndexType::kFullText: { String analyzer = ReadBufAdv(ptr); u8 is_homebrewed = ReadBufAdv(ptr); - res = MakeShared(file_name, column_names, analyzer, bool(is_homebrewed)); + res = MakeShared(index_name, file_name, column_names, analyzer, bool(is_homebrewed)); break; } case IndexType::kSecondary: { - res = MakeShared(std::move(file_name), std::move(column_names)); + res = MakeShared(index_name, file_name, std::move(column_names)); break; } case IndexType::kInvalid: { @@ -153,8 +157,9 @@ String IndexBase::ToString() const { nlohmann::json IndexBase::Serialize() const { nlohmann::json res; - res["file_name"] = file_name_; res["index_type"] = IndexInfo::IndexTypeToString(index_type_); + res["index_name"] = *index_name_; + res["file_name"] = file_name_; res["column_names"] = column_names_; return res; } @@ -163,13 +168,14 @@ SharedPtr IndexBase::Deserialize(const nlohmann::json &index_def_json SharedPtr res = nullptr; String index_type_name = index_def_json["index_type"]; IndexType index_type = IndexInfo::StringToIndexType(index_type_name); + SharedPtr index_name = MakeShared(index_def_json["index_name"]); String file_name = index_def_json["file_name"]; Vector column_names = index_def_json["column_names"]; switch (index_type) { case IndexType::kIVFFlat: { size_t centroids_count = index_def_json["centroids_count"]; MetricType metric_type = StringToMetricType(index_def_json["metric_type"]); - auto ptr = MakeShared(file_name, std::move(column_names), centroids_count, metric_type); + auto ptr = MakeShared(index_name, file_name, std::move(column_names), centroids_count, metric_type); res = std::static_pointer_cast(ptr); break; } @@ -179,7 +185,7 @@ SharedPtr IndexBase::Deserialize(const nlohmann::json &index_def_json SizeT ef = index_def_json["ef"]; MetricType metric_type = StringToMetricType(index_def_json["metric_type"]); HnswEncodeType encode_type = StringToHnswEncodeType(index_def_json["encode_type"]); - auto ptr = MakeShared(file_name, std::move(column_names), metric_type, encode_type, M, ef_construction, ef); + auto ptr = MakeShared(index_name, file_name, std::move(column_names), metric_type, encode_type, M, ef_construction, ef); res = std::static_pointer_cast(ptr); break; } @@ -191,12 +197,12 @@ SharedPtr IndexBase::Deserialize(const nlohmann::json &index_def_json if (!para_val.empty() && para_val != "false" && para_val != "0") { homebrewed = true; } - auto ptr = MakeShared(file_name, std::move(column_names), analyzer, homebrewed); + auto ptr = MakeShared(index_name, file_name, std::move(column_names), analyzer, homebrewed); res = std::static_pointer_cast(ptr); break; } case IndexType::kSecondary: { - auto ptr = MakeShared(std::move(file_name), std::move(column_names)); + auto ptr = MakeShared(index_name, file_name, std::move(column_names)); res = std::static_pointer_cast(ptr); break; } diff --git a/src/storage/definition/index_base.cppm b/src/storage/definition/index_base.cppm index 0c2e88d732..7a16dde56b 100644 --- a/src/storage/definition/index_base.cppm +++ b/src/storage/definition/index_base.cppm @@ -36,8 +36,8 @@ export MetricType StringToMetricType(const String &str); export class IndexBase { protected: - explicit IndexBase(String file_name, IndexType index_type, Vector column_names) - : index_type_(index_type), file_name_(std::move(file_name)), column_names_(std::move(column_names)){}; + explicit IndexBase(IndexType index_type, SharedPtr index_name, const String &file_name, Vector column_names) + : index_type_(index_type), index_name_(index_name), file_name_(file_name), column_names_(std::move(column_names)){}; public: virtual ~IndexBase() = default; @@ -62,10 +62,11 @@ public: static SharedPtr Deserialize(const nlohmann::json &index_def_json); - inline String column_name() { return column_names_[0]; } + inline String column_name() const { return column_names_[0]; } public: const IndexType index_type_{IndexType::kInvalid}; + SharedPtr index_name_{}; const String file_name_{}; const Vector column_names_{}; }; diff --git a/src/storage/definition/index_def.cpp b/src/storage/definition/index_def.cpp deleted file mode 100644 index 4afc27d0e5..0000000000 --- a/src/storage/definition/index_def.cpp +++ /dev/null @@ -1,125 +0,0 @@ -// Copyright(C) 2023 InfiniFlow, Inc. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -module; - -#include - -module index_def; - -import infinity_exception; - -import stl; -import serialize; -import index_ivfflat; -import index_hnsw; -import index_base; -import third_party; - -//-------------------------------------------------- - -namespace infinity { - -bool IndexDef::operator==(const IndexDef &other) const { - if (*index_name_ != *other.index_name_) { - return false; - } - - if (index_array_.size() != index_array_.size()) { - return false; - } - - SizeT index_count = index_array_.size(); - for (SizeT idx = 0; idx < index_count; ++idx) { - if (*index_array_[idx] != *index_array_[idx]) { - return false; - } - } - - return true; -} - -bool IndexDef::operator!=(const IndexDef &other) const { return !(*this == other); } - -int32_t IndexDef::GetSizeInBytes() const { - int32_t size = 0; - size += sizeof(int32_t) + index_name_->length(); - size += sizeof(int32_t); // for the length of index_array_ - for (const SharedPtr &index_base : index_array_) { - size += sizeof(int32_t); // each base index size - size += index_base->GetSizeInBytes(); - } - return size; -} - -void IndexDef::WriteAdv(char *&ptr) const { - WriteBufAdv(ptr, *index_name_); - WriteBufAdv(ptr, static_cast(index_array_.size())); - for (const SharedPtr &index_base : index_array_) { - WriteBufAdv(ptr, index_base->GetSizeInBytes()); - index_base->WriteAdv(ptr); - } -} - -SharedPtr IndexDef::ReadAdv(char *&ptr, int32_t maxbytes) { -// char *const ptr_end = ptr + maxbytes; - if (maxbytes <= 0) { - UnrecoverableError("maxbytes < 0"); - } - SharedPtr index_name = MakeShared(ReadBufAdv(ptr)); - SharedPtr index_def = MakeShared(index_name); - - int32_t index_array_count = ReadBufAdv(ptr); - index_def->index_array_.reserve(index_array_count); - - for (int32_t i = 0; i < index_array_count; ++i) { - int32_t index_size = ReadBufAdv(ptr); - index_def->index_array_.emplace_back(IndexBase::ReadAdv(ptr, index_size)); - } - return index_def; -} - -String IndexDef::ToString() const { - std::stringstream ss; - ss << "IndexDef(" << *index_name_ << ", ["; - for (size_t i = 0; i < index_array_.size(); ++i) { - ss << index_array_[i]->ToString(); - if (i != index_array_.size() - 1) { - ss << ", "; - } - } - ss << "])"; - return ss.str(); -} - -nlohmann::json IndexDef::Serialize() const { - nlohmann::json res; - res["index_name"] = *index_name_; - for (const auto &index : index_array_) { - res["indexes"].emplace_back(index->Serialize()); - } - return res; -} - -SharedPtr IndexDef::Deserialize(const nlohmann::json &index_def_json) { - SharedPtr index_name = MakeShared(index_def_json["index_name"]); - SharedPtr res = MakeShared(index_name); - for (const auto &index : index_def_json["indexes"]) { - SharedPtr index_base = IndexBase::Deserialize(index); - res->index_array_.emplace_back(std::move(index_base)); - } - return res; -} - -} // namespace infinity \ No newline at end of file diff --git a/src/storage/definition/index_def.cppm b/src/storage/definition/index_def.cppm deleted file mode 100644 index de621013ae..0000000000 --- a/src/storage/definition/index_def.cppm +++ /dev/null @@ -1,55 +0,0 @@ -// Copyright(C) 2023 InfiniFlow, Inc. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -module; - -export module index_def; - -import stl; -import third_party; -import index_base; - -namespace infinity { - -export class IndexDef { -public: - explicit IndexDef(SharedPtr index_name): index_name_(std::move(index_name)) {} - - virtual ~IndexDef() = default; - - bool operator==(const IndexDef &other) const; - - bool operator!=(const IndexDef &other) const; - -public: - // Estimated serialized size in bytes, ensured be no less than Write requires, allowed be larger. - i32 GetSizeInBytes() const; - - // Write to a char buffer - void WriteAdv(char *&ptr) const; - - // Read char from buffer - static SharedPtr ReadAdv(char *&ptr, i32 maxbytes); - - String ToString() const; - - nlohmann::json Serialize() const; - - static SharedPtr Deserialize(const nlohmann::json &index_def_json); - -public: - SharedPtr index_name_{}; - Vector> index_array_{}; -}; -} // namespace infinity \ No newline at end of file diff --git a/src/storage/definition/index_full_text.cpp b/src/storage/definition/index_full_text.cpp index be87c542e5..0813b9f517 100644 --- a/src/storage/definition/index_full_text.cpp +++ b/src/storage/definition/index_full_text.cpp @@ -31,7 +31,10 @@ namespace infinity { void ToLowerString(String &lower) { std::transform(lower.begin(), lower.end(), lower.begin(), ::tolower); } -SharedPtr IndexFullText::Make(String file_name, Vector column_names, const Vector &index_param_list) { +SharedPtr IndexFullText::Make(SharedPtr index_name, + const String &file_name, + Vector column_names, + const Vector &index_param_list) { String analyzer{}; bool homebrewed = false; SizeT param_count = index_param_list.size(); @@ -49,7 +52,7 @@ SharedPtr IndexFullText::Make(String file_name, Vector column } } } - return MakeShared(file_name, std::move(column_names), analyzer, homebrewed); + return MakeShared(index_name, file_name, std::move(column_names), analyzer, homebrewed); } bool IndexFullText::operator==(const IndexFullText &other) const { diff --git a/src/storage/definition/index_full_text.cppm b/src/storage/definition/index_full_text.cppm index fa6a296c93..58f1549e2d 100644 --- a/src/storage/definition/index_full_text.cppm +++ b/src/storage/definition/index_full_text.cppm @@ -17,7 +17,7 @@ module; export module index_full_text; import stl; -import index_def; +import index_base; import third_party; import index_base; @@ -28,10 +28,11 @@ namespace infinity { export class IndexFullText final : public IndexBase { public: - static SharedPtr Make(String file_name, Vector column_names, const Vector &index_param_list); + static SharedPtr + Make(SharedPtr index_name, const String &file_name, Vector column_names, const Vector &index_param_list); - IndexFullText(String file_name, Vector column_names, String analyzer, bool homebrewed) - : IndexBase(file_name, IndexType::kFullText, std::move(column_names)), analyzer_(std::move(analyzer)), homebrewed_(homebrewed) {} + IndexFullText(SharedPtr index_name, const String &file_name, Vector column_names, const String &analyzer, bool homebrewed) + : IndexBase(IndexType::kFullText, index_name, file_name, std::move(column_names)), analyzer_(analyzer), homebrewed_(homebrewed) {} ~IndexFullText() final = default; diff --git a/src/storage/definition/index_hnsw.cpp b/src/storage/definition/index_hnsw.cpp index 62e380fe49..222ae38782 100644 --- a/src/storage/definition/index_hnsw.cpp +++ b/src/storage/definition/index_hnsw.cpp @@ -21,7 +21,7 @@ module index_hnsw; import stl; import status; -import index_def; +import index_base; import third_party; import infinity_exception; import serialize; @@ -53,7 +53,8 @@ String HnswEncodeTypeToString(HnswEncodeType encode_type) { } } -SharedPtr IndexHnsw::Make(String file_name, Vector column_names, const Vector &index_param_list) { +SharedPtr +IndexHnsw::Make(SharedPtr index_name, const String &file_name, Vector column_names, const Vector &index_param_list) { SizeT M = HNSW_M; SizeT ef_construction = HNSW_EF_CONSTRUCTION; SizeT ef = HNSW_EF; @@ -77,7 +78,7 @@ SharedPtr IndexHnsw::Make(String file_name, Vector column_nam if (metric_type == MetricType::kInvalid || encode_type == HnswEncodeType::kInvalid) { RecoverableError(Status::LackIndexParam()); } - return MakeShared(file_name, std::move(column_names), metric_type, encode_type, M, ef_construction, ef); + return MakeShared(index_name, file_name, std::move(column_names), metric_type, encode_type, M, ef_construction, ef); } bool IndexHnsw::operator==(const IndexHnsw &other) const { diff --git a/src/storage/definition/index_hnsw.cppm b/src/storage/definition/index_hnsw.cppm index 7e9527c8b8..75cf75bb69 100644 --- a/src/storage/definition/index_hnsw.cppm +++ b/src/storage/definition/index_hnsw.cppm @@ -17,7 +17,7 @@ module; export module index_hnsw; import stl; -import index_def; +import index_base; import third_party; import index_base; @@ -39,16 +39,18 @@ export HnswEncodeType StringToHnswEncodeType(const String &str); export class IndexHnsw final : public IndexBase { public: - static SharedPtr Make(String file_name, Vector column_names, const Vector &index_param_list); + static SharedPtr + Make(SharedPtr index_name, const String &file_name, Vector column_names, const Vector &index_param_list); - IndexHnsw(String file_name, + IndexHnsw(SharedPtr index_name, + const String &file_name, Vector column_names, MetricType metric_type, HnswEncodeType encode_type, SizeT M, SizeT ef_construction, SizeT ef) - : IndexBase(file_name, IndexType::kHnsw, std::move(column_names)), metric_type_(metric_type), encode_type_(encode_type), M_(M), + : IndexBase(IndexType::kHnsw, index_name, file_name, std::move(column_names)), metric_type_(metric_type), encode_type_(encode_type), M_(M), ef_construction_(ef_construction), ef_(ef) {} ~IndexHnsw() final = default; diff --git a/src/storage/definition/index_ivfflat.cpp b/src/storage/definition/index_ivfflat.cpp index d027158ab2..3ed11805eb 100644 --- a/src/storage/definition/index_ivfflat.cpp +++ b/src/storage/definition/index_ivfflat.cpp @@ -21,7 +21,7 @@ module index_ivfflat; import infinity_exception; import stl; -import index_def; +import index_base; import status; import third_party; import serialize; @@ -31,7 +31,10 @@ import statement_common; namespace infinity { -SharedPtr IndexIVFFlat::Make(String file_name, Vector column_names, const Vector &index_param_list) { +SharedPtr IndexIVFFlat::Make(SharedPtr index_name, + const String &file_name, + Vector column_names, + const Vector &index_param_list) { SizeT centroids_count = 0; MetricType metric_type = MetricType::kInvalid; for (auto para : index_param_list) { @@ -44,7 +47,7 @@ SharedPtr IndexIVFFlat::Make(String file_name, Vector column_ if (metric_type == MetricType::kInvalid) { RecoverableError(Status::LackIndexParam()); } - return MakeShared(std::move(file_name), std::move(column_names), centroids_count, metric_type); + return MakeShared(index_name, file_name, std::move(column_names), centroids_count, metric_type); } bool IndexIVFFlat::operator==(const IndexIVFFlat &other) const { diff --git a/src/storage/definition/index_ivfflat.cppm b/src/storage/definition/index_ivfflat.cppm index a242e9a7c8..03ea876ef8 100644 --- a/src/storage/definition/index_ivfflat.cppm +++ b/src/storage/definition/index_ivfflat.cppm @@ -17,7 +17,7 @@ module; export module index_ivfflat; import stl; -import index_def; +import index_base; import index_base; import third_party; @@ -28,10 +28,12 @@ import statement_common; namespace infinity { export class IndexIVFFlat final : public IndexBase { public: - static SharedPtr Make(String file_name, Vector column_names, const Vector &index_param_list); + static SharedPtr + Make(SharedPtr index_name, const String &file_name, Vector column_names, const Vector &index_param_list); - IndexIVFFlat(String file_name, Vector column_names, SizeT centroids_count, MetricType metric_type) - : IndexBase(file_name, IndexType::kIVFFlat, std::move(column_names)), centroids_count_(centroids_count), metric_type_(metric_type) {} + IndexIVFFlat(SharedPtr index_name, const String &file_name, Vector column_names, SizeT centroids_count, MetricType metric_type) + : IndexBase(IndexType::kIVFFlat, index_name, file_name, std::move(column_names)), centroids_count_(centroids_count), + metric_type_(metric_type) {} ~IndexIVFFlat() final = default; diff --git a/src/storage/definition/index_secondary.cppm b/src/storage/definition/index_secondary.cppm index 644483502a..80678ec5c2 100644 --- a/src/storage/definition/index_secondary.cppm +++ b/src/storage/definition/index_secondary.cppm @@ -27,11 +27,12 @@ namespace infinity { // Does not need any extra member. export class IndexSecondary final : public IndexBase { public: - static SharedPtr Make(String file_name, Vector column_names) { - return MakeShared(std::move(file_name), std::move(column_names)); + static SharedPtr Make(SharedPtr index_name, const String &file_name, Vector column_names) { + return MakeShared(index_name, file_name, std::move(column_names)); } - IndexSecondary(String file_name, Vector column_names) : IndexBase(std::move(file_name), IndexType::kSecondary, std::move(column_names)) {} + IndexSecondary(SharedPtr index_name, const String &file_name, Vector column_names) + : IndexBase(IndexType::kSecondary, index_name, file_name, std::move(column_names)) {} ~IndexSecondary() final = default; diff --git a/src/storage/definition/table_def.cppm b/src/storage/definition/table_def.cppm index 11378685e8..774e152ac9 100644 --- a/src/storage/definition/table_def.cppm +++ b/src/storage/definition/table_def.cppm @@ -17,7 +17,7 @@ export module table_def; import stl; -import index_def; +import index_base; import column_def; @@ -76,7 +76,7 @@ private: SharedPtr table_name_{}; Vector> columns_{}; HashMap column_name2id_{}; - Vector indexes_{}; + Vector indexes_{}; }; } // namespace infinity diff --git a/src/storage/invertedindex/iresearch/datastore.cpp b/src/storage/invertedindex/iresearch/datastore.cpp index e0b8d9de18..8d6b622607 100644 --- a/src/storage/invertedindex/iresearch/datastore.cpp +++ b/src/storage/invertedindex/iresearch/datastore.cpp @@ -54,7 +54,7 @@ import buffer_manager; import default_values; import data_block; import txn; -import index_def; +import index_base; import buffer_handle; import local_file_system; @@ -287,7 +287,7 @@ void IRSDataStore::StopSchedule() { maintenance_state_->cancel_.store(true, std::memory_order::release); } -void IRSDataStore::BatchInsert(TableEntry *table_entry, const IndexDef *index_def, const SegmentEntry *segment_entry, BufferManager *buffer_mgr) { +void IRSDataStore::BatchInsert(TableEntry *table_entry, const IndexBase *index_base, const SegmentEntry *segment_entry, BufferManager *buffer_mgr) { constexpr static Array TEXT_FEATURES{IRSType::id()}; constexpr static Array NUMERIC_FEATURES{IRSType::id()}; @@ -296,25 +296,23 @@ void IRSDataStore::BatchInsert(TableEntry *table_entry, const IndexDef *index_de static Features numeric_features{NUMERIC_FEATURES.data(), NUMERIC_FEATURES.size()}; bool schedule = false; auto segment_id = segment_entry->segment_id(); - Vector> analyzers; - for (const auto &ibase : index_def->index_array_) { - auto index_base = reinterpret_cast(ibase.get()); - if (index_base->analyzer_ == JIEBA) { - UniquePtr stream = AnalyzerPool::instance().Get(JIEBA); - if (!stream.get()) { - UnrecoverableError("Dict path of Jieba analyzer is not valid"); - } - analyzers.push_back(std::move(stream)); - } else if (index_base->analyzer_ == SEGMENT) { - UniquePtr stream = AnalyzerPool::instance().Get(SEGMENT); - analyzers.push_back(std::move(stream)); - } else if (index_base->analyzer_.empty()) { - // TODO use segmentation analyzer if analyzer is not set - UniquePtr stream = AnalyzerPool::instance().Get(SEGMENT); - analyzers.push_back(std::move(stream)); - } else { - UnrecoverableError("Non existing analyzer"); + UniquePtr analyzer; + auto index_fulltext = const_cast(static_cast(index_base)); + if (index_fulltext->analyzer_ == JIEBA) { + UniquePtr stream = AnalyzerPool::instance().Get(JIEBA); + if (!stream.get()) { + UnrecoverableError("Dict path of Jieba analyzer is not valid"); } + analyzer = std::move(stream); + } else if (index_fulltext->analyzer_ == SEGMENT) { + UniquePtr stream = AnalyzerPool::instance().Get(SEGMENT); + analyzer = std::move(stream); + } else if (index_fulltext->analyzer_.empty()) { + // TODO use segmentation analyzer if analyzer is not set + UniquePtr stream = AnalyzerPool::instance().Get(SEGMENT); + analyzer = std::move(stream); + } else { + UnrecoverableError("Non existing analyzer"); } auto block_entry_iter = BlockEntryIter(segment_entry); @@ -323,101 +321,98 @@ void IRSDataStore::BatchInsert(TableEntry *table_entry, const IndexDef *index_de for (SizeT i = 0; i < block_entry->row_count(); ++i) { auto doc = ctx.Insert(RowID2DocID(segment_id, block_entry->block_id(), i)); - for (SizeT col = 0; col < index_def->index_array_.size(); ++col) { - auto index_base = reinterpret_cast(index_def->index_array_[col].get()); - u64 column_id = table_entry->GetColumnIdByName(index_base->column_name()); - auto block_column_entry = block_entry->GetColumnBlockEntry(column_id); - BufferHandle buffer_handle = block_column_entry->buffer()->Load(); - switch (block_column_entry->column_type()->type()) { - case kTinyInt: { - auto block_data_ptr = reinterpret_cast(buffer_handle.GetData()); - auto field = MakeUnique>(index_base->column_name().c_str(), irs::IndexFeatures::NONE, numeric_features); - TinyIntT v = block_data_ptr[i]; - field->value_ = v; - doc.Insert(field.get()); - } break; - case kSmallInt: { - auto block_data_ptr = reinterpret_cast(buffer_handle.GetData()); - auto field = MakeUnique>(index_base->column_name().c_str(), irs::IndexFeatures::NONE, numeric_features); - SmallIntT v = block_data_ptr[i]; - field->value_ = v; - doc.Insert(field.get()); - } break; - case kInteger: { - auto block_data_ptr = reinterpret_cast(buffer_handle.GetData()); - auto field = MakeUnique>(index_base->column_name().c_str(), irs::IndexFeatures::NONE, numeric_features); - IntegerT v = block_data_ptr[i]; - field->value_ = v; - doc.Insert(field.get()); - } break; - case kBigInt: { - auto block_data_ptr = reinterpret_cast(buffer_handle.GetData()); - auto field = MakeUnique>(index_base->column_name().c_str(), irs::IndexFeatures::NONE, numeric_features); - BigIntT v = block_data_ptr[i]; - field->value_ = v; - doc.Insert(field.get()); - } break; - case kHugeInt: { - auto block_data_ptr = reinterpret_cast(buffer_handle.GetData()); - auto field = MakeUnique>(index_base->column_name().c_str(), irs::IndexFeatures::NONE, numeric_features); - HugeIntT v = block_data_ptr[i]; - field->value_ = v.lower; // Lose precision - doc.Insert(field.get()); - } break; - case kFloat: { - auto block_data_ptr = reinterpret_cast(buffer_handle.GetData()); - auto field = MakeUnique>(index_base->column_name().c_str(), irs::IndexFeatures::NONE, numeric_features); - FloatT v = block_data_ptr[i]; - field->value_ = v; - doc.Insert(field.get()); - } break; - case kDouble: { - auto block_data_ptr = reinterpret_cast(buffer_handle.GetData()); - auto field = MakeUnique>(index_base->column_name().c_str(), irs::IndexFeatures::NONE, numeric_features); - DoubleT v = block_data_ptr[i]; - field->value_ = v; - doc.Insert(field.get()); - } - case kDate: { - auto block_data_ptr = reinterpret_cast(buffer_handle.GetData()); - auto field = MakeUnique>(index_base->column_name().c_str(), irs::IndexFeatures::NONE, numeric_features); - DateType v = block_data_ptr[i]; - field->value_ = v.value; - doc.Insert(field.get()); - } break; - case kTime: { - auto block_data_ptr = reinterpret_cast(buffer_handle.GetData()); - auto field = MakeUnique>(index_base->column_name().c_str(), irs::IndexFeatures::NONE, numeric_features); - TimeType v = block_data_ptr[i]; - field->value_ = v.value; - doc.Insert(field.get()); - } break; - case kDateTime: { - auto block_data_ptr = reinterpret_cast(buffer_handle.GetData()); - auto field = MakeUnique>(index_base->column_name().c_str(), irs::IndexFeatures::NONE, numeric_features); - DateTimeType v = block_data_ptr[i]; - field->value_ = ((i64)v.date << 32) + v.time; - doc.Insert(field.get()); - } break; - case kTimestamp: { - auto block_data_ptr = reinterpret_cast(buffer_handle.GetData()); - auto field = MakeUnique>(index_base->column_name().c_str(), irs::IndexFeatures::NONE, numeric_features); - TimestampType v = block_data_ptr[i]; - field->value_ = ((i64)v.date << 32) + v.time; - doc.Insert(field.get()); - } break; - case kVarchar: { - auto field = MakeUnique(index_base->column_name().c_str(), - irs::IndexFeatures::FREQ | irs::IndexFeatures::POS, - text_features, - analyzers[col].get()); - ColumnVector column_vector = block_column_entry->GetColumnVector(buffer_mgr); - field->f_ = column_vector.GetValue(i).GetVarchar(); - doc.Insert(field.get()); - } break; - default: - break; + u64 column_id = table_entry->GetColumnIdByName(index_base->column_name()); + auto block_column_entry = block_entry->GetColumnBlockEntry(column_id); + BufferHandle buffer_handle = block_column_entry->buffer()->Load(); + switch (block_column_entry->column_type()->type()) { + case kTinyInt: { + auto block_data_ptr = reinterpret_cast(buffer_handle.GetData()); + auto field = MakeUnique>(index_base->column_name().c_str(), irs::IndexFeatures::NONE, numeric_features); + TinyIntT v = block_data_ptr[i]; + field->value_ = v; + doc.Insert(field.get()); + } break; + case kSmallInt: { + auto block_data_ptr = reinterpret_cast(buffer_handle.GetData()); + auto field = MakeUnique>(index_base->column_name().c_str(), irs::IndexFeatures::NONE, numeric_features); + SmallIntT v = block_data_ptr[i]; + field->value_ = v; + doc.Insert(field.get()); + } break; + case kInteger: { + auto block_data_ptr = reinterpret_cast(buffer_handle.GetData()); + auto field = MakeUnique>(index_base->column_name().c_str(), irs::IndexFeatures::NONE, numeric_features); + IntegerT v = block_data_ptr[i]; + field->value_ = v; + doc.Insert(field.get()); + } break; + case kBigInt: { + auto block_data_ptr = reinterpret_cast(buffer_handle.GetData()); + auto field = MakeUnique>(index_base->column_name().c_str(), irs::IndexFeatures::NONE, numeric_features); + BigIntT v = block_data_ptr[i]; + field->value_ = v; + doc.Insert(field.get()); + } break; + case kHugeInt: { + auto block_data_ptr = reinterpret_cast(buffer_handle.GetData()); + auto field = MakeUnique>(index_base->column_name().c_str(), irs::IndexFeatures::NONE, numeric_features); + HugeIntT v = block_data_ptr[i]; + field->value_ = v.lower; // Lose precision + doc.Insert(field.get()); + } break; + case kFloat: { + auto block_data_ptr = reinterpret_cast(buffer_handle.GetData()); + auto field = MakeUnique>(index_base->column_name().c_str(), irs::IndexFeatures::NONE, numeric_features); + FloatT v = block_data_ptr[i]; + field->value_ = v; + doc.Insert(field.get()); + } break; + case kDouble: { + auto block_data_ptr = reinterpret_cast(buffer_handle.GetData()); + auto field = MakeUnique>(index_base->column_name().c_str(), irs::IndexFeatures::NONE, numeric_features); + DoubleT v = block_data_ptr[i]; + field->value_ = v; + doc.Insert(field.get()); } + case kDate: { + auto block_data_ptr = reinterpret_cast(buffer_handle.GetData()); + auto field = MakeUnique>(index_base->column_name().c_str(), irs::IndexFeatures::NONE, numeric_features); + DateType v = block_data_ptr[i]; + field->value_ = v.value; + doc.Insert(field.get()); + } break; + case kTime: { + auto block_data_ptr = reinterpret_cast(buffer_handle.GetData()); + auto field = MakeUnique>(index_base->column_name().c_str(), irs::IndexFeatures::NONE, numeric_features); + TimeType v = block_data_ptr[i]; + field->value_ = v.value; + doc.Insert(field.get()); + } break; + case kDateTime: { + auto block_data_ptr = reinterpret_cast(buffer_handle.GetData()); + auto field = MakeUnique>(index_base->column_name().c_str(), irs::IndexFeatures::NONE, numeric_features); + DateTimeType v = block_data_ptr[i]; + field->value_ = ((i64)v.date << 32) + v.time; + doc.Insert(field.get()); + } break; + case kTimestamp: { + auto block_data_ptr = reinterpret_cast(buffer_handle.GetData()); + auto field = MakeUnique>(index_base->column_name().c_str(), irs::IndexFeatures::NONE, numeric_features); + TimestampType v = block_data_ptr[i]; + field->value_ = ((i64)v.date << 32) + v.time; + doc.Insert(field.get()); + } break; + case kVarchar: { + auto field = MakeUnique(index_base->column_name().c_str(), + irs::IndexFeatures::FREQ | irs::IndexFeatures::POS, + text_features, + analyzer.get()); + ColumnVector column_vector = block_column_entry->GetColumnVector(buffer_mgr); + field->f_ = column_vector.GetValue(i).GetVarchar(); + doc.Insert(field.get()); + } break; + default: + break; } } if (!schedule) { diff --git a/src/storage/invertedindex/iresearch/datastore.cppm b/src/storage/invertedindex/iresearch/datastore.cppm index ee0f67dccf..78fd1e5c28 100644 --- a/src/storage/invertedindex/iresearch/datastore.cppm +++ b/src/storage/invertedindex/iresearch/datastore.cppm @@ -23,7 +23,7 @@ import data_block; import operator_state; import column_vector; import query_context; -import index_def; +import index_base; import table_entry; import segment_entry; import buffer_manager; @@ -132,7 +132,7 @@ public: void StopSchedule(); - void BatchInsert(TableEntry *table_entry, const IndexDef *index_def, const SegmentEntry *segment_entry, BufferManager *buffer_mgr); + void BatchInsert(TableEntry *table_entry, const IndexBase *index_base, const SegmentEntry *segment_entry, BufferManager *buffer_mgr); void Reset(); diff --git a/src/storage/meta/catalog.cpp b/src/storage/meta/catalog.cpp index cc79a6f69f..54ea293ae5 100644 --- a/src/storage/meta/catalog.cpp +++ b/src/storage/meta/catalog.cpp @@ -37,7 +37,7 @@ import file_system; import table_def; import table_entry_type; import table_detail; -import index_def; +import index_base; import txn_store; import data_access_state; import catalog_delta_entry; @@ -260,15 +260,15 @@ Status Catalog::RemoveTableEntry(TableEntry *table_entry, TransactionID txn_id, } Tuple Catalog::CreateIndex(TableEntry *table_entry, - const SharedPtr &index_def, - ConflictType conflict_type, - TransactionID txn_id, - TxnTimeStamp begin_ts, - TxnManager *txn_mgr, - bool is_replay, - String replay_table_index_dir) { + const SharedPtr &index_base, + ConflictType conflict_type, + TransactionID txn_id, + TxnTimeStamp begin_ts, + TxnManager *txn_mgr, + bool is_replay, + String replay_table_index_dir) { - return table_entry->CreateIndex(index_def, conflict_type, txn_id, begin_ts, txn_mgr, is_replay, replay_table_index_dir); + return table_entry->CreateIndex(index_base, conflict_type, txn_id, begin_ts, txn_mgr, is_replay, replay_table_index_dir); } Tuple Catalog::DropIndex(const String &db_name, @@ -644,7 +644,7 @@ void Catalog::LoadFromEntry(Catalog *catalog, const String &catalog_path, Buffer auto table_name = add_table_index_entry_op->table_name(); auto index_name = add_table_index_entry_op->index_name(); auto index_dir = add_table_index_entry_op->index_dir(); - auto index_def = add_table_index_entry_op->index_def(); + auto index_base = add_table_index_entry_op->index_base(); auto *db_meta = catalog->db_meta_map().at(db_name).get(); auto [db_entry, db_status] = db_meta->GetEntryReplay(txn_id, begin_ts); @@ -658,7 +658,7 @@ void Catalog::LoadFromEntry(Catalog *catalog, const String &catalog_path, Buffer } auto index_meta = table_entry->index_meta_map().at(index_name).get(); auto table_index_entry = TableIndexEntry::NewReplayTableIndexEntry(index_meta, - index_def, + index_base, MakeUnique(index_dir), txn_id, begin_ts, diff --git a/src/storage/meta/catalog.cppm b/src/storage/meta/catalog.cppm index ddaa460b91..70c92c4267 100644 --- a/src/storage/meta/catalog.cppm +++ b/src/storage/meta/catalog.cppm @@ -43,7 +43,7 @@ import profiler; import status; import default_values; import table_detail; -import index_def; +import index_base; import txn_store; import data_access_state; import extra_ddl_info; @@ -162,7 +162,7 @@ public: // Index Related methods Tuple CreateIndex(TableEntry *table_entry, - const SharedPtr &index_def, + const SharedPtr &index_base, ConflictType conflict_type, TransactionID txn_id, TxnTimeStamp begin_ts, diff --git a/src/storage/meta/entry/column_index_entry.cppm b/src/storage/meta/entry/column_index_entry.cppm index 96a60088f3..7a8add9668 100644 --- a/src/storage/meta/entry/column_index_entry.cppm +++ b/src/storage/meta/entry/column_index_entry.cppm @@ -20,7 +20,6 @@ import stl; import index_base; import third_party; -import index_base; import index_file_worker; import status; import column_def; diff --git a/src/storage/meta/entry/segment_column_index_entry.cppm b/src/storage/meta/entry/segment_column_index_entry.cppm index 9a9c54f112..da8763b1d3 100644 --- a/src/storage/meta/entry/segment_column_index_entry.cppm +++ b/src/storage/meta/entry/segment_column_index_entry.cppm @@ -33,7 +33,6 @@ class Txn; struct TableEntry; class FaissIndexPtr; class BufferManager; -class IndexDef; struct SegmentEntry; export class SegmentColumnIndexEntry : public BaseEntry { diff --git a/src/storage/meta/entry/table_entry.cpp b/src/storage/meta/entry/table_entry.cpp index 1e52dec969..1b64a25198 100644 --- a/src/storage/meta/entry/table_entry.cpp +++ b/src/storage/meta/entry/table_entry.cpp @@ -25,7 +25,6 @@ import table_entry_type; import third_party; import txn; import buffer_manager; -import index_def; import block_index; import data_access_state; import internal_types; @@ -107,14 +106,14 @@ SharedPtr TableEntry::NewReplayTableEntry(TableMeta *table_meta, return table_entry; } -Tuple TableEntry::CreateIndex(const SharedPtr &index_def, +Tuple TableEntry::CreateIndex(const SharedPtr &index_base, ConflictType conflict_type, TransactionID txn_id, TxnTimeStamp begin_ts, TxnManager *txn_mgr, bool is_replay, String replay_table_index_dir) { - if (index_def->index_name_->empty()) { + if (index_base->index_name_->empty()) { // Index name shouldn't be empty UnrecoverableError("Attempt to create no name index."); } @@ -123,14 +122,14 @@ Tuple TableEntry::CreateIndex(const SharedPtrrw_locker().lock_shared(); - if (auto iter = this->index_meta_map().find(*index_def->index_name_); iter != this->index_meta_map().end()) { + if (auto iter = this->index_meta_map().find(*index_base->index_name_); iter != this->index_meta_map().end()) { table_index_meta = iter->second.get(); } this->rw_locker().unlock_shared(); if (table_index_meta == nullptr) { - UniquePtr new_table_index_meta = TableIndexMeta::NewTableIndexMeta(this, index_def->index_name_); + UniquePtr new_table_index_meta = TableIndexMeta::NewTableIndexMeta(this, index_base->index_name_); { if (txn_mgr != nullptr) { @@ -143,17 +142,17 @@ Tuple TableEntry::CreateIndex(const SharedPtrrw_locker().lock(); - if (auto iter = this->index_meta_map().find(*index_def->index_name_); iter != this->index_meta_map().end()) { + if (auto iter = this->index_meta_map().find(*index_base->index_name_); iter != this->index_meta_map().end()) { table_index_meta = iter->second.get(); } else { - this->index_meta_map()[*index_def->index_name_] = std::move(new_table_index_meta); + this->index_meta_map()[*index_base->index_name_] = std::move(new_table_index_meta); } this->rw_locker().unlock(); } - LOG_TRACE(fmt::format("Creating new index: {}", *index_def->index_name_)); - return table_index_meta->CreateTableIndexEntry(index_def, conflict_type, txn_id, begin_ts, txn_mgr, is_replay, replay_table_index_dir); + LOG_TRACE(fmt::format("Creating new index: {}", *index_base->index_name_)); + return table_index_meta->CreateTableIndexEntry(index_base, conflict_type, txn_id, begin_ts, txn_mgr, is_replay, replay_table_index_dir); } Tuple @@ -217,13 +216,12 @@ void TableEntry::GetFullTextAnalyzers(TransactionID txn_id, auto [table_index_entry, status] = table_index_meta->GetEntry(txn_id, begin_ts); if (status.ok()) { fulltext_index_entry = table_index_entry->fulltext_index_entry(); - for (const SharedPtr &index_base : table_index_entry->index_def()->index_array_) { - if (index_base->index_type_ != IndexType::kFullText) - continue; - IndexFullText *index_full_text = static_cast(index_base.get()); - for (auto &column_name : index_full_text->column_names_) { - column2analyzer[column_name] = index_full_text->analyzer_; - } + const IndexBase *index_base = table_index_entry->index_base(); + if (index_base->index_type_ != IndexType::kFullText) + continue; + auto index_full_text = static_cast(index_base); + for (auto &column_name : index_full_text->column_names_) { + column2analyzer[column_name] = index_full_text->analyzer_; } if (!column2analyzer.empty()) { // iresearch requires there is exactly one full index per table. diff --git a/src/storage/meta/entry/table_entry.cppm b/src/storage/meta/entry/table_entry.cppm index d244e44e03..328356928a 100644 --- a/src/storage/meta/entry/table_entry.cppm +++ b/src/storage/meta/entry/table_entry.cppm @@ -41,7 +41,7 @@ import cleanup_scanner; namespace infinity { -class IndexDef; +class IndexBase; struct TableIndexEntry; class FulltextIndexEntry; class TableMeta; @@ -82,7 +82,7 @@ public: SizeT row_count); private: - Tuple CreateIndex(const SharedPtr &index_def, + Tuple CreateIndex(const SharedPtr &index_base, ConflictType conflict_type, TransactionID txn_id, TxnTimeStamp begin_ts, diff --git a/src/storage/meta/entry/table_index_entry.cpp b/src/storage/meta/entry/table_index_entry.cpp index c165775381..398a701a2f 100644 --- a/src/storage/meta/entry/table_index_entry.cpp +++ b/src/storage/meta/entry/table_index_entry.cpp @@ -22,7 +22,6 @@ import third_party; import local_file_system; import default_values; import random; -import index_def; import index_base; import segment_iter; @@ -38,13 +37,13 @@ namespace infinity { TableIndexEntry::TableIndexEntry() : BaseEntry(EntryType::kDummy) {} -TableIndexEntry::TableIndexEntry(const SharedPtr &index_def, +TableIndexEntry::TableIndexEntry(const SharedPtr &index_base, TableIndexMeta *table_index_meta, SharedPtr index_dir, TransactionID txn_id, TxnTimeStamp begin_ts, bool is_replay) - : BaseEntry(EntryType::kTableIndex), table_index_meta_(table_index_meta), index_def_(std::move(index_def)), index_dir_(std::move(index_dir)) { + : BaseEntry(EntryType::kTableIndex), table_index_meta_(table_index_meta), index_base_(index_base), index_dir_(std::move(index_dir)) { begin_ts_ = begin_ts; // TODO:: begin_ts and txn_id should be const and set in BaseEntry txn_id_ = txn_id; } @@ -55,7 +54,7 @@ TableIndexEntry::TableIndexEntry(TableIndexMeta *table_index_meta, TransactionID txn_id_ = txn_id; } -SharedPtr TableIndexEntry::NewTableIndexEntry(const SharedPtr &index_def, +SharedPtr TableIndexEntry::NewTableIndexEntry(const SharedPtr &index_base, TableIndexMeta *table_index_meta, Txn *txn, TransactionID txn_id, @@ -65,14 +64,11 @@ SharedPtr TableIndexEntry::NewTableIndexEntry(const SharedPtr(index_def, table_index_meta, MakeShared(replay_table_index_dir), txn_id, begin_ts); - SizeT index_count = index_def->index_array_.size(); - table_index_entry->column_index_map_.reserve(index_count); - + MakeShared(index_base, table_index_meta, MakeShared(replay_table_index_dir), txn_id, begin_ts); return table_index_entry; } else { - SharedPtr index_dir = DetermineIndexDir(*table_index_meta->GetTableEntry()->TableEntryDir(), *index_def->index_name_); - auto table_index_entry = MakeShared(index_def, table_index_meta, index_dir, txn_id, begin_ts); + SharedPtr index_dir = DetermineIndexDir(*table_index_meta->GetTableEntry()->TableEntryDir(), *index_base->index_name_); + auto table_index_entry = MakeShared(index_base, table_index_meta, index_dir, txn_id, begin_ts); TableIndexEntry *table_index_entry_ptr = table_index_entry.get(); { @@ -80,33 +76,25 @@ SharedPtr TableIndexEntry::NewTableIndexEntry(const SharedPtrAddCatalogDeltaOperation(std::move(operation)); } - SizeT index_count = index_def->index_array_.size(); - table_index_entry->column_index_map_.reserve(index_count); - - HashMap> index_info_map; - for (SizeT idx = 0; idx < index_def->index_array_.size(); ++idx) { - SharedPtr &index_base = index_def->index_array_[idx]; - - // Get column info - if (index_base->column_names_.size() != 1) { - RecoverableError(Status::SyntaxError("Currently, composite index doesn't supported.")); - } - ColumnID column_id = table_index_meta->GetTableEntry()->GetColumnIdByName(index_base->column_names_[0]); - if (index_base->index_type_ == IndexType::kFullText) { - SharedPtr index_fulltext = std::static_pointer_cast(index_base); - index_info_map.emplace(column_id, index_fulltext); + // Get column info + if (index_base->column_names_.size() != 1) { + RecoverableError(Status::SyntaxError("Currently, composite index doesn't supported.")); + } + ColumnID column_id = table_index_meta->GetTableEntry()->GetColumnIdByName(index_base->column_names_[0]); + if (index_base->index_type_ == IndexType::kFullText) { + SharedPtr index_fulltext = std::static_pointer_cast(index_base); + if (index_fulltext->homebrewed_) { + // TODO yzc: remove table_index_entry->fulltext_index_entry_ } else { - SharedPtr column_index_dir = - MakeShared(fmt::format("{}/{}", *table_index_entry->index_dir_, index_base->column_names_[0])); - SharedPtr column_index_entry = - ColumnIndexEntry::NewColumnIndexEntry(index_base, column_id, table_index_entry.get(), txn, txn_id, column_index_dir, begin_ts); - table_index_entry->column_index_map_[column_id] = std::move(column_index_entry); + table_index_entry->fulltext_index_entry_ = + FulltextIndexEntry::NewFulltextIndexEntry(table_index_entry_ptr, txn, txn_id, table_index_entry->index_dir_, begin_ts); } - } - - if (!index_info_map.empty()) { - table_index_entry->fulltext_index_entry_ = - FulltextIndexEntry::NewFulltextIndexEntry(table_index_entry_ptr, txn, txn_id, table_index_entry->index_dir_, begin_ts); + } else { + SharedPtr column_index_dir = + MakeShared(fmt::format("{}/{}", *table_index_entry->index_dir_, index_base->column_names_[0])); + SharedPtr column_index_entry = + ColumnIndexEntry::NewColumnIndexEntry(index_base, column_id, table_index_entry.get(), txn, txn_id, column_index_dir, begin_ts); + table_index_entry->column_index_map_[column_id] = std::move(column_index_entry); } return table_index_entry; @@ -115,27 +103,21 @@ SharedPtr TableIndexEntry::NewTableIndexEntry(const SharedPtrindex_array_.size(); ++idx) { - SharedPtr &index_base = index_def_->index_array_[idx]; - if (index_base->index_type_ == IndexType::kFullText) { - SharedPtr index_fulltext = std::static_pointer_cast(index_base); - homebrewed = index_fulltext->homebrewed_; - break; - } + if (index_base_->index_type_ == IndexType::kFullText) { + SharedPtr index_fulltext = std::static_pointer_cast(index_base_); + homebrewed = index_fulltext->homebrewed_; } return homebrewed; } SharedPtr TableIndexEntry::NewReplayTableIndexEntry(TableIndexMeta *table_index_meta, - const SharedPtr &index_def, + const SharedPtr &index_base, const SharedPtr &index_dir, TransactionID txn_id, TxnTimeStamp begin_ts, TxnTimeStamp commit_ts, bool is_delete) { - auto table_index_entry = MakeShared(index_def, table_index_meta, index_dir, txn_id, begin_ts); - SizeT index_count = index_def->index_array_.size(); - table_index_entry->column_index_map_.reserve(index_count); + auto table_index_entry = MakeShared(index_base, table_index_meta, index_dir, txn_id, begin_ts); table_index_entry->commit_ts_.store(commit_ts); table_index_entry->deleted_ = is_delete; return table_index_entry; @@ -182,7 +164,7 @@ nlohmann::json TableIndexEntry::Serialize(TxnTimeStamp max_commit_ts) { } json["index_dir"] = *this->index_dir_; - json["index_def"] = this->index_def_->Serialize(); + json["index_base"] = this->index_base_->Serialize(); for (const auto &[index_name, column_index_entry] : this->column_index_map_) { column_index_entry_candidates.emplace_back((ColumnIndexEntry *)column_index_entry.get()); @@ -220,9 +202,9 @@ SharedPtr TableIndexEntry::Deserialize(const nlohmann::json &in } auto index_dir = MakeShared(index_def_entry_json["index_dir"]); - auto index_def = IndexDef::Deserialize(index_def_entry_json["index_def"]); + auto index_base = IndexBase::Deserialize(index_def_entry_json["index_base"]); - SharedPtr table_index_entry = MakeShared(index_def, table_index_meta, index_dir, txn_id, begin_ts, true); + SharedPtr table_index_entry = MakeShared(index_base, table_index_meta, index_dir, txn_id, begin_ts, true); table_index_entry->commit_ts_.store(commit_ts); table_index_entry->begin_ts_ = begin_ts; @@ -258,7 +240,7 @@ Status TableIndexEntry::CreateIndexPrepare(TableEntry *table_entry, BlockIndex * if (fulltext_index_entry->irs_index_.get() != nullptr) { auto *buffer_mgr = txn->GetBufferMgr(); for (const auto *segment_entry : block_index->segments_) { - fulltext_index_entry->irs_index_->BatchInsert(table_entry, index_def_.get(), segment_entry, buffer_mgr); + fulltext_index_entry->irs_index_->BatchInsert(table_entry, index_base_.get(), segment_entry, buffer_mgr); } fulltext_index_entry->irs_index_->Commit(); fulltext_index_entry->irs_index_->StopSchedule(); diff --git a/src/storage/meta/entry/table_index_entry.cppm b/src/storage/meta/entry/table_index_entry.cppm index 0f2e09a669..046808fc7d 100644 --- a/src/storage/meta/entry/table_index_entry.cppm +++ b/src/storage/meta/entry/table_index_entry.cppm @@ -23,7 +23,7 @@ import fulltext_index_entry; import column_index_entry; import segment_column_index_entry; import base_entry; -import index_def; +import index_base; import block_index; import third_party; import status; @@ -46,7 +46,7 @@ export struct TableIndexEntry : public BaseEntry, public EntryInterface { public: TableIndexEntry(); - TableIndexEntry(const SharedPtr &index_def, + TableIndexEntry(const SharedPtr &index_base, TableIndexMeta *table_index_meta, SharedPtr index_dir, TransactionID txn_id, @@ -55,7 +55,7 @@ public: TableIndexEntry(TableIndexMeta *table_index_meta, TransactionID txn_id, TxnTimeStamp begin_ts); - static SharedPtr NewTableIndexEntry(const SharedPtr &index_def, + static SharedPtr NewTableIndexEntry(const SharedPtr &index_base, TableIndexMeta *table_index_meta, Txn *txn, TransactionID txn_id, @@ -66,7 +66,7 @@ public: static SharedPtr NewDropTableIndexEntry(TableIndexMeta *table_index_meta, TransactionID txn_id, TxnTimeStamp begin_ts); static SharedPtr NewReplayTableIndexEntry(TableIndexMeta *table_index_meta, - const SharedPtr &index_def, + const SharedPtr &index_base, const SharedPtr &index_dir, TransactionID txn_id, TxnTimeStamp begin_ts, @@ -81,8 +81,8 @@ public: public: // Getter inline const TableIndexMeta *table_index_meta() const { return table_index_meta_; } - inline const IndexDef *index_def() const { return index_def_.get(); } - const SharedPtr &table_index_def() { return index_def_; } + inline const IndexBase *index_base() const { return index_base_.get(); } + const SharedPtr &table_index_def() { return index_base_; } SharedPtr &fulltext_index_entry() { return fulltext_index_entry_; } HashMap> &column_index_map() { return column_index_map_; } SharedPtr index_dir() { return index_dir_; } @@ -103,9 +103,10 @@ private: private: std::shared_mutex rw_locker_{}; TableIndexMeta *table_index_meta_{}; - const SharedPtr index_def_{}; + const SharedPtr index_base_{}; SharedPtr index_dir_{}; + // TODO yzc: replace column_index_map_ and fulltext_index_entry_ with segment_index_map_ HashMap> column_index_map_{}; SharedPtr fulltext_index_entry_{}; diff --git a/src/storage/meta/table_index_meta.cpp b/src/storage/meta/table_index_meta.cpp index 4291db9e8c..3dc3635129 100644 --- a/src/storage/meta/table_index_meta.cpp +++ b/src/storage/meta/table_index_meta.cpp @@ -20,7 +20,7 @@ module table_index_meta; import stl; -import index_def; +import index_base; import txn_manager; import default_values; import txn_state; @@ -46,14 +46,14 @@ UniquePtr TableIndexMeta::NewTableIndexMeta(TableEntry *table_en return table_index_meta; } -Tuple TableIndexMeta::CreateTableIndexEntry(const SharedPtr &index_def, +Tuple TableIndexMeta::CreateTableIndexEntry(const SharedPtr &index_base, ConflictType conflict_type, TransactionID txn_id, TxnTimeStamp begin_ts, TxnManager *txn_mgr, bool is_replay, String replay_table_index_dir) { - auto [table_index_entry, status] = CreateTableIndexEntryInternal(index_def, txn_id, begin_ts, txn_mgr, is_replay, replay_table_index_dir); + auto [table_index_entry, status] = CreateTableIndexEntryInternal(index_base, txn_id, begin_ts, txn_mgr, is_replay, replay_table_index_dir); switch (conflict_type) { case ConflictType::kError: { return {table_index_entry, status}; @@ -72,7 +72,7 @@ Tuple TableIndexMeta::CreateTableIndexEntry(const Sha } } -Tuple TableIndexMeta::CreateTableIndexEntryInternal(const SharedPtr &index_def, +Tuple TableIndexMeta::CreateTableIndexEntryInternal(const SharedPtr &index_base, TransactionID txn_id, TxnTimeStamp begin_ts, TxnManager *txn_mgr, @@ -95,7 +95,7 @@ Tuple TableIndexMeta::CreateTableIndexEntryInternal(c this->index_entry_list().emplace_back(std::move(dummy_entry)); // Create a new table index entry - auto table_index_entry = TableIndexEntry::NewTableIndexEntry(index_def, this, txn, txn_id, begin_ts, is_replay, replay_table_index_dir); + auto table_index_entry = TableIndexEntry::NewTableIndexEntry(index_base, this, txn, txn_id, begin_ts, is_replay, replay_table_index_dir); table_index_entry_ptr = table_index_entry.get(); this->index_entry_list().emplace_front(std::move(table_index_entry)); LOG_TRACE("New table index entry is added."); @@ -104,7 +104,7 @@ Tuple TableIndexMeta::CreateTableIndexEntryInternal(c // Already have a db_entry, check if the db_entry is valid here. BaseEntry *header_base_entry = this->index_entry_list().front().get(); if (header_base_entry->entry_type_ == EntryType::kDummy) { - auto table_index_entry = TableIndexEntry::NewTableIndexEntry(index_def, this, txn, txn_id, begin_ts, is_replay, replay_table_index_dir); + auto table_index_entry = TableIndexEntry::NewTableIndexEntry(index_base, this, txn, txn_id, begin_ts, is_replay, replay_table_index_dir); table_index_entry_ptr = table_index_entry.get(); this->index_entry_list().emplace_front(std::move(table_index_entry)); LOG_TRACE("New table index entry is added."); @@ -117,7 +117,7 @@ Tuple TableIndexMeta::CreateTableIndexEntryInternal(c if (header_entry->deleted_) { // No conflict auto table_index_entry = - TableIndexEntry::NewTableIndexEntry(index_def, this, txn, txn_id, begin_ts, is_replay, replay_table_index_dir); + TableIndexEntry::NewTableIndexEntry(index_base, this, txn, txn_id, begin_ts, is_replay, replay_table_index_dir); table_index_entry_ptr = table_index_entry.get(); this->index_entry_list().emplace_front(std::move(table_index_entry)); LOG_TRACE("New table index entry is added."); @@ -147,7 +147,7 @@ Tuple TableIndexMeta::CreateTableIndexEntryInternal(c if (header_entry->deleted_) { // No conflict auto table_index_entry = - TableIndexEntry::NewTableIndexEntry(index_def, this, txn, txn_id, begin_ts, is_replay, replay_table_index_dir); + TableIndexEntry::NewTableIndexEntry(index_base, this, txn, txn_id, begin_ts, is_replay, replay_table_index_dir); table_index_entry_ptr = table_index_entry.get(); this->index_entry_list().emplace_front(std::move(table_index_entry)); LOG_TRACE("New table index entry is added."); @@ -178,7 +178,7 @@ Tuple TableIndexMeta::CreateTableIndexEntryInternal(c // Append new one auto table_index_entry = - TableIndexEntry::NewTableIndexEntry(index_def, this, txn, txn_id, begin_ts, is_replay, replay_table_index_dir); + TableIndexEntry::NewTableIndexEntry(index_base, this, txn, txn_id, begin_ts, is_replay, replay_table_index_dir); table_index_entry_ptr = table_index_entry.get(); this->index_entry_list().emplace_front(std::move(table_index_entry)); LOG_TRACE("New table index entry is added."); diff --git a/src/storage/meta/table_index_meta.cppm b/src/storage/meta/table_index_meta.cppm index 4af12d98da..1ac9198326 100644 --- a/src/storage/meta/table_index_meta.cppm +++ b/src/storage/meta/table_index_meta.cppm @@ -20,7 +20,7 @@ import table_index_entry; import base_entry; import stl; import third_party; -import index_def; +import index_base; import status; import extra_ddl_info; import entry_list; @@ -53,7 +53,7 @@ public: Tuple GetEntryReplay(TransactionID txn_id, TxnTimeStamp begin_ts); private: - Tuple CreateTableIndexEntry(const SharedPtr &index_def, + Tuple CreateTableIndexEntry(const SharedPtr &index_base, ConflictType conflict_type, TransactionID txn_id, TxnTimeStamp begin_ts, @@ -74,7 +74,7 @@ private: void MergeFrom(TableIndexMeta &other); - Tuple CreateTableIndexEntryInternal(const SharedPtr &index_def, + Tuple CreateTableIndexEntryInternal(const SharedPtr &index_base, TransactionID txn_id, TxnTimeStamp begin_ts, TxnManager *txn_mgr, diff --git a/src/storage/txn/txn.cpp b/src/storage/txn/txn.cpp index 538719970c..1d8bf96ddd 100644 --- a/src/storage/txn/txn.cpp +++ b/src/storage/txn/txn.cpp @@ -41,7 +41,7 @@ import table_entry_type; import database_detail; import status; import table_def; -import index_def; +import index_base; import catalog_delta_entry; import bg_task; import background_process; @@ -300,7 +300,7 @@ Status Txn::DropTableCollectionByName(const String &db_name, const String &table return Status::OK(); } -Tuple Txn::CreateIndexDef(TableEntry *table_entry, const SharedPtr &index_def, ConflictType conflict_type) { +Tuple Txn::CreateIndexDef(TableEntry *table_entry, const SharedPtr &index_base, ConflictType conflict_type) { TxnState txn_state = txn_context_.GetTxnState(); if (txn_state != TxnState::kStarted) { @@ -308,11 +308,11 @@ Tuple Txn::CreateIndexDef(TableEntry *table_entry, co } TxnTimeStamp begin_ts = txn_context_.GetBeginTS(); - auto [table_index_entry, index_status] = catalog_->CreateIndex(table_entry, index_def, conflict_type, txn_id_, begin_ts, txn_mgr_); + auto [table_index_entry, index_status] = catalog_->CreateIndex(table_entry, index_base, conflict_type, txn_id_, begin_ts, txn_mgr_); if (!index_status.ok() || (index_status.ok() && table_index_entry == nullptr && conflict_type == ConflictType::kIgnore)) { return {nullptr, index_status}; } - txn_indexes_.emplace(*index_def->index_name_, table_index_entry); + txn_indexes_.emplace(*index_base->index_name_, table_index_entry); return {table_index_entry, index_status}; } @@ -322,8 +322,8 @@ Status Txn::CreateIndexPrepare(TableIndexEntry *table_index_entry, BaseTableRef if (!prepare) { String index_dir = *table_index_entry->index_dir(); - auto index_def = table_index_entry->table_index_def(); - wal_entry_->cmds_.push_back(MakeShared(*table_entry->GetDBName(), *table_entry->GetTableName(), index_dir, index_def)); + auto index_base = table_index_entry->table_index_def(); + wal_entry_->cmds_.push_back(MakeShared(*table_entry->GetDBName(), *table_entry->GetTableName(), index_dir, index_base)); } return Status::OK(); } @@ -344,13 +344,13 @@ Status Txn::CreateIndexDo(BaseTableRef *table_ref, const String &index_name, Has return table_index_entry->CreateIndexDo(table_entry, create_index_idxes); } -Status Txn::CreateIndexFinish(const String &db_name, const String &table_name, const SharedPtr &index_def) { - String key = *index_def->index_name_; +Status Txn::CreateIndexFinish(const String &db_name, const String &table_name, const SharedPtr &index_base) { + String key = *index_base->index_name_; auto it = txn_indexes_.find(key); if (it != txn_indexes_.end()) { // Key found, it -> second is the value TableIndexEntry *found_entry = it->second; - this->AddWalCmd(MakeShared(db_name, table_name, *found_entry->index_dir(), index_def)); + this->AddWalCmd(MakeShared(db_name, table_name, *found_entry->index_dir(), index_base)); LOG_TRACE(fmt::format("The key {} exists in the map", key)); } else { // Key not found diff --git a/src/storage/txn/txn.cppm b/src/storage/txn/txn.cppm index 894abe80b4..aac15174c8 100644 --- a/src/storage/txn/txn.cppm +++ b/src/storage/txn/txn.cppm @@ -19,7 +19,7 @@ import stl; import table_detail; import table_def; -import index_def; +import index_base; import data_block; import meta_state; import data_access_state; @@ -112,14 +112,14 @@ public: // If `prepare` is false, the index will be created in single thread. (called by `FsPhysicalCreateIndex`) // Else, only data is stored in index (Called by `PhysicalCreateIndexPrepare`). And the index will be created by multiple threads in next // operator. (called by `PhysicalCreateIndexDo`) - Tuple CreateIndexDef(TableEntry *table_entry, const SharedPtr &index_def, ConflictType conflict_type); + Tuple CreateIndexDef(TableEntry *table_entry, const SharedPtr &index_base, ConflictType conflict_type); Status CreateIndexPrepare(TableIndexEntry *table_index_entry, BaseTableRef *table_ref, bool prepare, bool check_ts = true); Status CreateIndexDo(BaseTableRef *table_ref, const String &index_name, HashMap &create_index_idxes); // write wal - Status CreateIndexFinish(const String &db_name, const String &table_name, const SharedPtr &indef); + Status CreateIndexFinish(const String &db_name, const String &table_name, const SharedPtr &indef); Status DropIndexByName(const String &db_name, const String &table_name, const String &index_name, ConflictType conflict_type); diff --git a/src/storage/txn/txn_store.cpp b/src/storage/txn/txn_store.cpp index 3174f02682..b90a576e9f 100644 --- a/src/storage/txn/txn_store.cpp +++ b/src/storage/txn/txn_store.cpp @@ -93,7 +93,7 @@ Tuple, Status> TxnTableStore::Import(const SharedPtr, Status> TxnTableStore::CreateIndexFile(TableIndexEntry *table_index_entry, u64 column_id, u32 segment_id, SharedPtr index) { - const String &index_name = *table_index_entry->index_def()->index_name_; + const String &index_name = *table_index_entry->index_base()->index_name_; if (auto column_index_iter = txn_indexes_store_.find(index_name); column_index_iter != txn_indexes_store_.end()) { column_index_iter->second.index_entry_map_[column_id][segment_id] = index; } else { diff --git a/src/storage/wal/catalog_delta_entry.cpp b/src/storage/wal/catalog_delta_entry.cpp index bd4f48d2a3..aafaf96c1b 100644 --- a/src/storage/wal/catalog_delta_entry.cpp +++ b/src/storage/wal/catalog_delta_entry.cpp @@ -22,7 +22,7 @@ import crc; import serialize; import data_block; import table_def; -import index_def; +import index_base; import infinity_exception; import internal_types; import stl; @@ -166,10 +166,9 @@ UniquePtr CatalogDeltaOperation::ReadAdv(char *&ptr, i32 String table_name = ReadBufAdv(ptr); String index_name = ReadBufAdv(ptr); String index_dir = ReadBufAdv(ptr); - SharedPtr index_def = IndexDef::ReadAdv(ptr, ptr_end - ptr); - // TODO: index_def + SharedPtr index_base = IndexBase::ReadAdv(ptr, ptr_end - ptr); operation = - MakeUnique(begin_ts, is_delete, txn_id, commit_ts, db_name, table_name, index_name, index_dir, index_def); + MakeUnique(begin_ts, is_delete, txn_id, commit_ts, db_name, table_name, index_name, index_dir, index_base); break; } case CatalogDeltaOpType::ADD_FULLTEXT_INDEX_ENTRY: { @@ -320,7 +319,7 @@ void AddTableIndexEntryOp::WriteAdv(char *&buf) const { WriteBufAdv(buf, this->table_name_); WriteBufAdv(buf, this->index_name_); WriteBufAdv(buf, this->index_dir_); - index_def_->WriteAdv(buf); + index_base_->WriteAdv(buf); } void AddFulltextIndexEntryOp::WriteAdv(char *&buf) const { @@ -444,7 +443,7 @@ void AddTableIndexEntryOp::SaveSate() { this->table_name_ = *this->table_index_entry_->table_index_meta()->GetTableEntry()->GetTableName(); this->index_name_ = this->table_index_entry_->table_index_meta()->index_name(); this->index_dir_ = *this->table_index_entry_->index_dir(); - this->index_def_ = this->table_index_entry_->table_index_def(); + this->index_base_ = this->table_index_entry_->table_index_def(); is_saved_sate_ = true; } @@ -555,12 +554,12 @@ const String AddIndexMetaOp::ToString() const { } const String AddTableIndexEntryOp::ToString() const { - return fmt::format("AddTableIndexEntryOp db_name: {} table_name: {} index_name: {} index_dir: {} index_def: {}", + return fmt::format("AddTableIndexEntryOp db_name: {} table_name: {} index_name: {} index_dir: {} index_base: {}", db_name_, table_name_, index_name_, index_dir_, - index_def_->ToString()); + index_base_->ToString()); } const String AddFulltextIndexEntryOp::ToString() const { diff --git a/src/storage/wal/catalog_delta_entry.cppm b/src/storage/wal/catalog_delta_entry.cppm index e42a819a1b..c3ce1bbab0 100644 --- a/src/storage/wal/catalog_delta_entry.cppm +++ b/src/storage/wal/catalog_delta_entry.cppm @@ -17,7 +17,6 @@ module; export module catalog_delta_entry; import table_def; -import index_def; import data_block; import stl; @@ -587,10 +586,10 @@ public: String table_name, String index_name, String index_dir, - SharedPtr index_def) + SharedPtr index_base) : CatalogDeltaOperation(CatalogDeltaOpType::ADD_TABLE_INDEX_ENTRY, begin_ts, is_delete, txn_id, commit_ts), db_name_(std::move(db_name)), - table_name_(std::move(table_name)), index_name_(std::move(index_name)), index_dir_(std::move(index_dir)), index_def_(std::move(index_def)) { - } + table_name_(std::move(table_name)), index_name_(std::move(index_name)), index_dir_(std::move(index_dir)), + index_base_(std::move(index_base)) {} explicit AddTableIndexEntryOp(SharedPtr table_index_entry) : CatalogDeltaOperation(CatalogDeltaOpType::ADD_TABLE_INDEX_ENTRY), table_index_entry_(table_index_entry) {} CatalogDeltaOpType GetType() const final { return CatalogDeltaOpType::ADD_TABLE_INDEX_ENTRY; } @@ -601,7 +600,7 @@ public: total_size += sizeof(i32) + this->table_name_.size(); total_size += sizeof(i32) + this->index_name_.size(); total_size += sizeof(i32) + this->index_dir_.size(); - total_size += this->index_def_->GetSizeInBytes(); + total_size += this->index_base_->GetSizeInBytes(); return total_size; } void WriteAdv(char *&buf) const final; @@ -619,14 +618,14 @@ public: const String &table_name() const { return table_name_; } const String &index_name() const { return index_name_; } const String &index_dir() const { return index_dir_; } - SharedPtr index_def() const { return index_def_; } + SharedPtr index_base() const { return index_base_; } private: String db_name_{}; String table_name_{}; String index_name_{}; String index_dir_{}; - SharedPtr index_def_{}; + SharedPtr index_base_{}; }; /// class AddFulltextIndexEntryOp diff --git a/src/storage/wal/wal_entry.cpp b/src/storage/wal/wal_entry.cpp index 9a452084f8..82fbbb2675 100644 --- a/src/storage/wal/wal_entry.cpp +++ b/src/storage/wal/wal_entry.cpp @@ -22,7 +22,7 @@ import crc; import serialize; import data_block; import table_def; -import index_def; +import index_base; import infinity_exception; import stl; @@ -120,8 +120,8 @@ SharedPtr WalCmd::ReadAdv(char *&ptr, i32 max_bytes) { String db_name = ReadBufAdv(ptr); String table_name = ReadBufAdv(ptr); String table_index_dir = ReadBufAdv(ptr); - SharedPtr index_def = IndexDef::ReadAdv(ptr, ptr_end - ptr); - cmd = MakeShared(db_name, table_name, table_index_dir, index_def); + SharedPtr index_base = IndexBase::ReadAdv(ptr, ptr_end - ptr); + cmd = MakeShared(db_name, table_name, table_index_dir, index_base); break; } case WalCommandType::DROP_INDEX: { @@ -167,8 +167,8 @@ bool WalCmdCreateTable::operator==(const WalCmd &other) const { bool WalCmdCreateIndex::operator==(const WalCmd &other) const { auto other_cmd = dynamic_cast(&other); - return other_cmd != nullptr && db_name_ == other_cmd->db_name_ && table_name_ == other_cmd->table_name_ && index_def_.get() != nullptr && - other_cmd->index_def_.get() != nullptr && *index_def_ == *other_cmd->index_def_ && table_index_dir_ == other_cmd->table_index_dir_; + return other_cmd != nullptr && db_name_ == other_cmd->db_name_ && table_name_ == other_cmd->table_name_ && index_base_.get() != nullptr && + other_cmd->index_base_.get() != nullptr && *index_base_ == *other_cmd->index_base_ && table_index_dir_ == other_cmd->table_index_dir_; } bool WalCmdDropIndex::operator==(const WalCmd &other) const { @@ -235,7 +235,7 @@ i32 WalCmdCreateTable::GetSizeInBytes() const { i32 WalCmdCreateIndex::GetSizeInBytes() const { return sizeof(WalCommandType) + sizeof(i32) + this->db_name_.size() + sizeof(i32) + this->table_name_.size() + sizeof(i32) + - this->table_index_dir_.size() + this->index_def_->GetSizeInBytes(); + this->table_index_dir_.size() + this->index_base_->GetSizeInBytes(); } i32 WalCmdDropTable::GetSizeInBytes() const { @@ -291,7 +291,7 @@ void WalCmdCreateIndex::WriteAdv(char *&buf) const { WriteBufAdv(buf, this->db_name_); WriteBufAdv(buf, this->table_name_); WriteBufAdv(buf, this->table_index_dir_); - index_def_->WriteAdv(buf); + index_base_->WriteAdv(buf); } void WalCmdDropTable::WriteAdv(char *&buf) const { @@ -525,7 +525,7 @@ String WalEntry::ToString() const { ss << "db name: " << create_index_cmd->db_name_ << std::endl; ss << "table name: " << create_index_cmd->table_name_ << std::endl; ss << "table index dir: " << create_index_cmd->table_index_dir_ << std::endl; - ss << "index def: " << create_index_cmd->index_def_->ToString() << std::endl; + ss << "index def: " << create_index_cmd->index_base_->ToString() << std::endl; } } ss << "========================" << std::endl; diff --git a/src/storage/wal/wal_entry.cppm b/src/storage/wal/wal_entry.cppm index dfa8cdfded..197feab453 100644 --- a/src/storage/wal/wal_entry.cppm +++ b/src/storage/wal/wal_entry.cppm @@ -19,7 +19,7 @@ module; export module wal_entry; import table_def; -import index_def; +import index_base; import data_block; import stl; @@ -129,9 +129,9 @@ export struct WalCmdCreateTable : public WalCmd { }; export struct WalCmdCreateIndex : public WalCmd { - WalCmdCreateIndex(String db_name, String table_name, String table_index_dir, SharedPtr index_def) + WalCmdCreateIndex(String db_name, String table_name, String table_index_dir, SharedPtr index_base) : db_name_(std::move(db_name)), table_name_(std::move(table_name)), table_index_dir_(std::move(table_index_dir)), - index_def_(std::move(index_def)) {} + index_base_(std::move(index_base)) {} WalCommandType GetType() override { return WalCommandType::CREATE_INDEX; } @@ -144,7 +144,7 @@ export struct WalCmdCreateIndex : public WalCmd { String db_name_{}; String table_name_{}; String table_index_dir_{}; - SharedPtr index_def_{}; + SharedPtr index_base_{}; }; export struct WalCmdDropTable : public WalCmd { diff --git a/src/storage/wal/wal_manager.cpp b/src/storage/wal/wal_manager.cpp index 285c93d7bb..874b38151d 100644 --- a/src/storage/wal/wal_manager.cpp +++ b/src/storage/wal/wal_manager.cpp @@ -643,7 +643,7 @@ void WalManager::WalCmdCreateIndexReplay(const WalCmdCreateIndex &cmd, Transacti } auto [table_index_entry, index_def_entry_status] = storage_->catalog()->CreateIndex(table_entry, - cmd.index_def_, + cmd.index_base_, ConflictType::kError, txn_id, commit_ts, diff --git a/src/unit_test/storage/definition/index_def.cpp b/src/unit_test/storage/definition/index_base.cpp similarity index 56% rename from src/unit_test/storage/definition/index_def.cpp rename to src/unit_test/storage/definition/index_base.cpp index a50c971034..fc685f0cdf 100644 --- a/src/unit_test/storage/definition/index_def.cpp +++ b/src/unit_test/storage/definition/index_base.cpp @@ -19,15 +19,13 @@ import stl; import index_base; import index_ivfflat; import index_hnsw; -import index_base; import index_full_text; -import index_def; import statement_common; -class IndexDefTest : public BaseTest {}; +class IndexBaseTest : public BaseTest {}; -TEST_F(IndexDefTest, ivfflat_readwrite) { +TEST_F(IndexBaseTest, ivfflat_readwrite) { using namespace infinity; Vector columns{"col1", "col2"}; @@ -35,7 +33,7 @@ TEST_F(IndexDefTest, ivfflat_readwrite) { parameters.emplace_back(new InitParameter("centroids_count", "100")); parameters.emplace_back(new InitParameter("metric", "l2")); - auto index_base = IndexIVFFlat::Make("idx1", columns, parameters); + auto index_base = IndexIVFFlat::Make(MakeShared("idx1"), "tbl1_idx1", columns, parameters); std::cout << "index_base: " << index_base->ToString() << std::endl; for (auto parameter : parameters) { delete parameter; @@ -57,7 +55,7 @@ TEST_F(IndexDefTest, ivfflat_readwrite) { EXPECT_EQ(*index_base, *index_base1); } -TEST_F(IndexDefTest, hnsw_readwrite) { +TEST_F(IndexBaseTest, hnsw_readwrite) { using namespace infinity; Vector columns{"col1", "col2"}; @@ -68,7 +66,7 @@ TEST_F(IndexDefTest, hnsw_readwrite) { parameters.emplace_back(new InitParameter("ef", "200")); parameters.emplace_back(new InitParameter("encode", "plain")); - auto index_base = IndexHnsw::Make("idx1", columns, parameters); + auto index_base = IndexHnsw::Make(MakeShared("idx1"), "tbl1_idx1", columns, parameters); std::cout << "index_base: " << index_base->ToString() << std::endl; for (auto parameter : parameters) { @@ -91,14 +89,14 @@ TEST_F(IndexDefTest, hnsw_readwrite) { EXPECT_EQ(*index_base, *index_base1); } -TEST_F(IndexDefTest, full_text_readwrite) { +TEST_F(IndexBaseTest, full_text_readwrite) { using namespace infinity; Vector columns{"col1", "col2"}; Vector parameters; parameters.emplace_back(new InitParameter("analyzer", "jieba")); - auto index_base = IndexFullText::Make("idx1", columns, parameters); + auto index_base = IndexFullText::Make(MakeShared("idx1"), "tbl1_idx1", columns, parameters); std::cout << "index_base: " << index_base->ToString() << std::endl; for (auto parameter : parameters) { @@ -120,71 +118,3 @@ TEST_F(IndexDefTest, full_text_readwrite) { EXPECT_NE(index_base.get(), nullptr); EXPECT_EQ(*index_base, *index_base1); } - -TEST_F(IndexDefTest, index_def) { - using namespace infinity; - - Vector columns1{"col1", "col2"}; - Vector parameters1; - parameters1.emplace_back(new InitParameter("centroids_count", "100")); - parameters1.emplace_back(new InitParameter("metric", "l2")); - - auto index_base_ivf = IndexIVFFlat::Make("name1", columns1, parameters1); - for(auto* init_parameter: parameters1) { - delete init_parameter; - } - - Vector columns2{"col3"}; - Vector parameters2; - parameters2.emplace_back(new InitParameter("metric", "l2")); - parameters2.emplace_back(new InitParameter("encode", "plain")); - parameters2.emplace_back(new InitParameter("M", "16")); - parameters2.emplace_back(new InitParameter("ef_construction", "200")); - parameters2.emplace_back(new InitParameter("ef", "200")); - - auto index_base_hnsw = IndexHnsw::Make("name2", columns2, parameters2); - for(auto* init_parameter: parameters2) { - delete init_parameter; - } - - Vector columns3{"col4", "col5"}; - Vector parameters3; - parameters3.emplace_back(new InitParameter("analyzer", "jieba")); - - auto index_base_ft = IndexFullText::Make("name3", columns3, parameters3); - for(auto* init_parameter: parameters3) { - delete init_parameter; - } - - Vector columns4{"col6", "col7"}; - Vector parameters4; - - auto index_base_ft1 = IndexFullText::Make("name4", columns4, parameters4); - for (auto parameter : parameters4) { - delete parameter; - } - - auto index_def = MakeShared(MakeShared("index1")); - index_def->index_array_.emplace_back(index_base_ivf); - index_def->index_array_.emplace_back(index_base_hnsw); - index_def->index_array_.emplace_back(index_base_ft); - index_def->index_array_.emplace_back(index_base_ft1); - - EXPECT_EQ(*index_def->index_name_, "index1"); - std::cout << "index_def: " << index_def->ToString() << std::endl; - - int32_t exp_size = index_def->GetSizeInBytes(); - Vector buf(exp_size, char(0)); - char *buf_beg = buf.data(); - char *ptr = buf_beg; - index_def->WriteAdv(ptr); - EXPECT_EQ(ptr - buf_beg, exp_size); - - ptr = buf_beg; - int32_t maxbytes = exp_size; - SharedPtr index_def1 = IndexDef::ReadAdv(ptr, maxbytes); - std::cout << "index_def1: " << index_def1->ToString() << std::endl; - EXPECT_EQ(ptr - buf_beg, exp_size); - EXPECT_NE(index_def1.get(), nullptr); - EXPECT_EQ(*index_def, *index_def1); -} diff --git a/src/unit_test/storage/wal/catalog_delta_entry.cpp b/src/unit_test/storage/wal/catalog_delta_entry.cpp index db2e86408d..a6015dec37 100644 --- a/src/unit_test/storage/wal/catalog_delta_entry.cpp +++ b/src/unit_test/storage/wal/catalog_delta_entry.cpp @@ -17,7 +17,6 @@ import infinity_exception; import index_base; -import index_def; import logger; import third_party; import stl; @@ -47,7 +46,6 @@ TEST_F(CatalogDeltaEntryTest, MergeEntries) { String index_dir{"data/db_test/table_test/0/0/index_test"}; String col_index_dir{"data/db_test/table_test/0/0/index_test"}; Vector> column_defs{}; - SharedPtr index_def{nullptr}; SharedPtr index_base{nullptr}; // db meta @@ -103,8 +101,8 @@ TEST_F(CatalogDeltaEntryTest, MergeEntries) { local_catalog_delta_entry->operations().push_back(std::move(op10_same_name)); // table index entry - auto op11 = MakeUnique(11, false, 0, 0, db_name, table_name, index_name, index_dir, index_def); - auto op11_same_name = MakeUnique(11, false, 0, 0, db_name, table_name, index_name, index_dir, index_def); + auto op11 = MakeUnique(11, false, 0, 0, db_name, table_name, index_name, index_dir, index_base); + auto op11_same_name = MakeUnique(11, false, 0, 0, db_name, table_name, index_name, index_dir, index_base); local_catalog_delta_entry->operations().push_back(std::move(op11)); local_catalog_delta_entry->operations().push_back(std::move(op11_same_name)); diff --git a/src/unit_test/storage/wal/wal_entry.cpp b/src/unit_test/storage/wal/wal_entry.cpp index c687e172c0..0c55d0ff18 100644 --- a/src/unit_test/storage/wal/wal_entry.cpp +++ b/src/unit_test/storage/wal/wal_entry.cpp @@ -30,7 +30,7 @@ import value; import data_block; import default_values; import index_ivfflat; -import index_def; +import index_base; import logical_type; import internal_types; import column_def; @@ -192,14 +192,13 @@ TEST_F(WalEntryTest, ReadWrite) { Vector parameters = {new InitParameter("centroids_count", "100"), new InitParameter("metric", "l2")}; - SharedPtr index_def = MakeShared(MakeShared("idx1")); - - auto index_base = IndexIVFFlat::Make("idx1", Vector{"col1", "col2"}, parameters); + SharedPtr index_name = MakeShared("idx1"); + auto index_base = IndexIVFFlat::Make(index_name, "idx1_tbl1", Vector{"col1", "col2"}, parameters); for (auto parameter : parameters) { delete parameter; } - entry->cmds_.push_back(MakeShared("db1", "tbl1", "", index_def)); + entry->cmds_.push_back(MakeShared("db1", "tbl1", "", index_base)); entry->cmds_.push_back(MakeShared("db1", "tbl1", "idx1")); diff --git a/src/unit_test/storage/wal/wal_replay.cpp b/src/unit_test/storage/wal/wal_replay.cpp index e092367dcf..6eab60862e 100644 --- a/src/unit_test/storage/wal/wal_replay.cpp +++ b/src/unit_test/storage/wal/wal_replay.cpp @@ -38,9 +38,7 @@ import catalog; import index_base; import index_ivfflat; import index_hnsw; -import index_base; import index_full_text; -import index_def; import bg_task; import background_process; import compact_segments_task; @@ -824,22 +822,21 @@ TEST_F(WalReplayTest, WalReplayCreateIndexIvfFlat) { parameters1.emplace_back(new InitParameter("centroids_count", "100")); parameters1.emplace_back(new InitParameter("metric", "l2")); - auto index_base_ivf = IndexIVFFlat::Make("name1", columns1, parameters1); + SharedPtr index_name = MakeShared("idx1"); + auto index_base_ivf = IndexIVFFlat::Make(index_name, "idx1_tbl1", columns1, parameters1); for (auto *init_parameter : parameters1) { delete init_parameter; } const String &db_name = "default"; const String &table_name = "test_annivfflat"; - const SharedPtr index_def = MakeShared(MakeShared("idx1")); - index_def->index_array_.emplace_back(index_base_ivf); ConflictType conflict_type = ConflictType::kError; bool prepare = false; auto [table_entry, table_status] = txn->GetTableByName(db_name, table_name); EXPECT_EQ(table_status.ok(), true); { auto table_ref = BaseTableRef::FakeTableRef(table_entry, txn->BeginTS()); - auto result = txn->CreateIndexDef(table_entry, index_def, conflict_type); + auto result = txn->CreateIndexDef(table_entry, index_base_ivf, conflict_type); auto *table_index_entry = std::get<0>(result); auto status = std::get<1>(result); EXPECT_EQ(status.ok(), true); @@ -876,7 +873,7 @@ TEST_F(WalReplayTest, WalReplayCreateIndexIvfFlat) { EXPECT_EQ(table_index_meta->index_name(), "idx1"); EXPECT_EQ(table_index_meta->index_entry_list().size(), 2); auto table_index_entry_front = static_cast(table_index_meta->index_entry_list().front().get()); - EXPECT_EQ(*table_index_entry_front->index_def()->index_name_, "idx1"); + EXPECT_EQ(*table_index_entry_front->index_base()->index_name_, "idx1"); auto entry_back = table_index_meta->index_entry_list().back().get(); EXPECT_EQ(entry_back->entry_type_, EntryType::kDummy); txn_mgr->CommitTxn(txn); @@ -931,22 +928,21 @@ TEST_F(WalReplayTest, WalReplayCreateIndexHnsw) { parameters1.emplace_back(new InitParameter("ef_construction", "200")); parameters1.emplace_back(new InitParameter("ef", "200")); - auto index_base_hnsw = IndexHnsw::Make("hnsw_index", columns1, parameters1); + SharedPtr index_name = MakeShared("hnsw_index"); + auto index_base_hnsw = IndexHnsw::Make(index_name, "hnsw_index_test_hnsw", columns1, parameters1); for (auto *init_parameter : parameters1) { delete init_parameter; } const String &db_name = "default"; const String &table_name = "test_hnsw"; - const SharedPtr index_def = MakeShared(MakeShared("hnsw_index")); - index_def->index_array_.emplace_back(index_base_hnsw); ConflictType conflict_type = ConflictType::kError; bool prepare = false; auto [table_entry, table_status] = txn->GetTableByName(db_name, table_name); EXPECT_EQ(table_status.ok(), true); { auto table_ref = BaseTableRef::FakeTableRef(table_entry, txn->BeginTS()); - auto result = txn->CreateIndexDef(table_entry, index_def, conflict_type); + auto result = txn->CreateIndexDef(table_entry, index_base_hnsw, conflict_type); auto *table_index_entry = std::get<0>(result); auto status = std::get<1>(result); EXPECT_EQ(status.ok(), true); @@ -983,7 +979,7 @@ TEST_F(WalReplayTest, WalReplayCreateIndexHnsw) { EXPECT_EQ(table_index_meta->index_name(), "hnsw_index"); EXPECT_EQ(table_index_meta->index_entry_list().size(), 2); auto table_index_entry_front = static_cast(table_index_meta->index_entry_list().front().get()); - EXPECT_EQ(*table_index_entry_front->index_def()->index_name_, "hnsw_index"); + EXPECT_EQ(*table_index_entry_front->index_base()->index_name_, "hnsw_index"); auto entry_back = table_index_meta->index_entry_list().back().get(); EXPECT_EQ(entry_back->entry_type_, EntryType::kDummy); txn_mgr->CommitTxn(txn); diff --git a/test/sql/dql/fulltext.slt b/test/sql/dql/fulltext.slt index eb5d2410cc..a0eac23e32 100644 --- a/test/sql/dql/fulltext.slt +++ b/test/sql/dql/fulltext.slt @@ -14,31 +14,31 @@ COPY enwiki FROM '/tmp/infinity/test_data/enwiki_9999.csv' WITH ( DELIMITER '\t' ---- statement ok -CREATE INDEX ft_index ON enwiki(body) USING FULLTEXT WITH(ANALYZER=segmentation) (doctitle, docdate) USING FULLTEXT; +CREATE INDEX ft_index ON enwiki(body) USING FULLTEXT WITH(ANALYZER=segmentation); query TTT -SELECT doctitle, docdate, body FROM enwiki SEARCH MATCH('doctitle^2,body^5', 'harmful chemical', 'topn=3'); +SELECT doctitle, docdate, body FROM enwiki SEARCH MATCH('body^5', 'harmful chemical', 'topn=3'); ---- Avicenna 29-APR-2012 07:47:29.000 [[saliva]]tion, [[Somnolence|sleepiness]] | [[insomnia]], [[wakefulness]] |- | Physical signs | high [[pulse]] rate, lassitude | [[flaccid]] joints | [[diarrhea]], [[eye puffiness|swollen eyelids]], rough skin, acquired [[habit (psychology)|habit]] | rough skin, acquired habit |- | Foods & medicines | [http://www.thefreedictionary.com/calefacient calefacients] harmful, [http://www.thefreedictionary.com/Infrigidate infrigidants] beneficial | [http://www.thefreedictionary.com/Infrigidate infrigidants] harmful, [http://www.thefreedictionary.com/calefacient calefacients] beneficial | [[moisture|moist]] articles harmful | [[wikt:dry|dry]] regimen harmful, [[humectant]]s beneficial |- | Relation to weather | worse in summer | worse in winter | | bad in autumn |} ===Physical Exercise: the Key to Health=== {{Refimprove section|please provide publication information|date=June 2010}} '''The Canon of Medicine: Volume 1 of 5; Part 4 of 5: The Preservation of Health ''' Of Ibn Sina's Canon of Medicine which is written Alkali metal 30-APR-2012 05:35:44.000 royale des Sciences| title = Sur la Base de Sel Marine| last = du Monceau|first = H. L. D.| pages = 65–68| language = French}} The exact chemical composition of potassium and sodium compounds, and the status as chemical element of potassium and sodium, was not known then, and thus [[Antoine Lavoisier]] did include the alkali in his list of chemical elements in 1789.{{cite journal|doi = 10.1021/ed009p1035|title = The discovery of the elements. IX. Three alkali metals: Potassium, sodium, and lithium|year = 1932|last1 = Weeks|first1 = Mary Elvira|journal = Journal of Chemical Education|volume = 9|issue = 6|pages = 1035|bibcode = 1932JChEd...9.1035W}}{{cite journal|jstor = 228541|pages = 247–258|last1 = Siegfried|first1 = R.|title = The Discovery of Potassium and Sodium, and the Problem of the Chemical Elements|volume = 54|issue = 2|journal = Isis|year = 1963|doi = 10.1086/349704}} Pure potassium was first isolated in 1807 in England by Sir [[Humphry Atom 20-APR-2012 03:53:14.000 beam of light energy in a narrow frequency band. ===Valence and bonding behavior=== {{Main|Valence (chemistry)|Chemical bond}} The outermost electron shell of an atom in its uncombined state is known as the valence shell, and the electrons in that shell are called [[valence electron]]s. The number of valence electrons determines the [[chemical bond|bonding]] behavior with other atoms. Atoms tend to [[Chemical reaction|chemically react]] with each other in a manner that fills (or empties) their outer valence shells. For example, a transfer of a single electron between atoms is a useful approximation for bonds that form between atoms with one-electron more than a filled shell, and others that are one-electron short of a full shell, such as occurs in the compound [[sodium chloride]] and other chemical ionic salts. However, many elements display multiple valences, or tendencies to share differing numbers of electrons in different compounds. Thus, [[chemical query TTT -SELECT doctitle, docdate, ROW_ID(), SCORE() FROM enwiki SEARCH MATCH('doctitle^2,body^5', 'harmful chemical', 'topn=3'); +SELECT doctitle, docdate, ROW_ID(), SCORE() FROM enwiki SEARCH MATCH('body^5', 'harmful chemical', 'topn=3'); ---- Avicenna 29-APR-2012 07:47:29.000 9893 6.723495 Alkali metal 30-APR-2012 05:35:44.000 2681 4.617455 Atom 20-APR-2012 03:53:14.000 7207 4.617455 query TTT -SELECT doctitle, docdate, body FROM enwiki SEARCH MATCH('doctitle^2,body^5', 'harmful chemical', 'topn=3'), FUSION('rrf'); +SELECT doctitle, docdate, body FROM enwiki SEARCH MATCH('body^5', 'harmful chemical', 'topn=3'), FUSION('rrf'); ---- Avicenna 29-APR-2012 07:47:29.000 [[saliva]]tion, [[Somnolence|sleepiness]] | [[insomnia]], [[wakefulness]] |- | Physical signs | high [[pulse]] rate, lassitude | [[flaccid]] joints | [[diarrhea]], [[eye puffiness|swollen eyelids]], rough skin, acquired [[habit (psychology)|habit]] | rough skin, acquired habit |- | Foods & medicines | [http://www.thefreedictionary.com/calefacient calefacients] harmful, [http://www.thefreedictionary.com/Infrigidate infrigidants] beneficial | [http://www.thefreedictionary.com/Infrigidate infrigidants] harmful, [http://www.thefreedictionary.com/calefacient calefacients] beneficial | [[moisture|moist]] articles harmful | [[wikt:dry|dry]] regimen harmful, [[humectant]]s beneficial |- | Relation to weather | worse in summer | worse in winter | | bad in autumn |} ===Physical Exercise: the Key to Health=== {{Refimprove section|please provide publication information|date=June 2010}} '''The Canon of Medicine: Volume 1 of 5; Part 4 of 5: The Preservation of Health ''' Of Ibn Sina's Canon of Medicine which is written Alkali metal 30-APR-2012 05:35:44.000 royale des Sciences| title = Sur la Base de Sel Marine| last = du Monceau|first = H. L. D.| pages = 65–68| language = French}} The exact chemical composition of potassium and sodium compounds, and the status as chemical element of potassium and sodium, was not known then, and thus [[Antoine Lavoisier]] did include the alkali in his list of chemical elements in 1789.{{cite journal|doi = 10.1021/ed009p1035|title = The discovery of the elements. IX. Three alkali metals: Potassium, sodium, and lithium|year = 1932|last1 = Weeks|first1 = Mary Elvira|journal = Journal of Chemical Education|volume = 9|issue = 6|pages = 1035|bibcode = 1932JChEd...9.1035W}}{{cite journal|jstor = 228541|pages = 247–258|last1 = Siegfried|first1 = R.|title = The Discovery of Potassium and Sodium, and the Problem of the Chemical Elements|volume = 54|issue = 2|journal = Isis|year = 1963|doi = 10.1086/349704}} Pure potassium was first isolated in 1807 in England by Sir [[Humphry Atom 20-APR-2012 03:53:14.000 beam of light energy in a narrow frequency band. ===Valence and bonding behavior=== {{Main|Valence (chemistry)|Chemical bond}} The outermost electron shell of an atom in its uncombined state is known as the valence shell, and the electrons in that shell are called [[valence electron]]s. The number of valence electrons determines the [[chemical bond|bonding]] behavior with other atoms. Atoms tend to [[Chemical reaction|chemically react]] with each other in a manner that fills (or empties) their outer valence shells. For example, a transfer of a single electron between atoms is a useful approximation for bonds that form between atoms with one-electron more than a filled shell, and others that are one-electron short of a full shell, such as occurs in the compound [[sodium chloride]] and other chemical ionic salts. However, many elements display multiple valences, or tendencies to share differing numbers of electrons in different compounds. Thus, [[chemical query TTT -SELECT doctitle, docdate, body FROM enwiki SEARCH MATCH('doctitle^2,body^5', 'harmful chemical', 'topn=3'), MATCH('doctitle^2,body^5', 'society', 'topn=3'), FUSION('rrf'); +SELECT doctitle, docdate, body FROM enwiki SEARCH MATCH('body^5', 'harmful chemical', 'topn=3'), MATCH('doctitle^2,body^5', 'society', 'topn=3'), FUSION('rrf'); ---- Avicenna 29-APR-2012 07:47:29.000 [[saliva]]tion, [[Somnolence|sleepiness]] | [[insomnia]], [[wakefulness]] |- | Physical signs | high [[pulse]] rate, lassitude | [[flaccid]] joints | [[diarrhea]], [[eye puffiness|swollen eyelids]], rough skin, acquired [[habit (psychology)|habit]] | rough skin, acquired habit |- | Foods & medicines | [http://www.thefreedictionary.com/calefacient calefacients] harmful, [http://www.thefreedictionary.com/Infrigidate infrigidants] beneficial | [http://www.thefreedictionary.com/Infrigidate infrigidants] harmful, [http://www.thefreedictionary.com/calefacient calefacients] beneficial | [[moisture|moist]] articles harmful | [[wikt:dry|dry]] regimen harmful, [[humectant]]s beneficial |- | Relation to weather | worse in summer | worse in winter | | bad in autumn |} ===Physical Exercise: the Key to Health=== {{Refimprove section|please provide publication information|date=June 2010}} '''The Canon of Medicine: Volume 1 of 5; Part 4 of 5: The Preservation of Health ''' Of Ibn Sina's Canon of Medicine which is written Anarcho-capitalism 24-APR-2012 15:46:17.000 [http://www.mises.org/story/1787 ''Authentic German Liberalism of the 19th Century''] Ecole Polytechnique, [http://www.crea.polytechnique.fr/index.htm Centre de Recherce en Epistemologie Appliquee], Unité associée au CNRS Unlike the liberalism of Locke, which saw the state as evolving from society, the anti-state liberals saw a fundamental conflict between the voluntary interactions of people – society – and the institutions of force – the State. This ''society versus state'' idea was expressed in various ways: natural society vs. artificial society, liberty vs. authority, society of contract vs. society of authority, and industrial society vs. militant society, just to name a few. The anti-state liberal tradition in Europe and the United States continued after Molinari in the early writings of [[Herbert Spencer]], as well as in thinkers such as [[Paul Émile de Puydt]] and [[Auberon Herbert]]. Ulrike Heider, in discussing the "anarcho-capitalists @@ -48,7 +48,7 @@ Atom 20-APR-2012 03:53:14.000 beam of light energy in a narrow frequency band.< Astronomer 17-APR-2012 19:09:32.000 York]]|ref=harv|postscript=|isbn=0300159110 }} * {{Cite journal|last=Kennedy|first=E.S.|title=A Survey of Islamic Astronomical Tables; Transactions of the American Philosophical Society| year=1956|location=[[Philadelphia]]| publisher=[[American Philosophical Society]]|volume=46|issue=2|ref=harv|postscript=}} * {{Cite encyclopedia | last = Toomer | first = Gerald | title = Al-Khwārizmī, Abu Jaʿfar Muḥammad ibn Mūsā | encyclopedia = [[Dictionary of Scientific Biography]] | volume = 7 | editor = Gillispie, Charles Coulston | publisher = Charles Scribner's Sons | location = New York | year = 1990 | isbn = 0-684-16962-2 | ref=harv }} ==External links== * [http://www.aas.org American Astronomical Society] * [http://www.iau.org International Astronomical Union] * [http://www.astrosociety.org Astronomical Society of the Pacific] [[Category:Astronomy| ]] [[Category:Astronomers| ]] [[Category:Science occupations]] [[als:Astronom]] [[ar:عالم فلك]] query TTT -SELECT doctitle, docdate, ROW_ID(), SCORE() FROM enwiki SEARCH MATCH('doctitle^2,body^5', 'harmful chemical', 'topn=3'), MATCH('doctitle^2,body^5', 'society', 'topn=3'), FUSION('rrf'); +SELECT doctitle, docdate, ROW_ID(), SCORE() FROM enwiki SEARCH MATCH('body^5', 'harmful chemical', 'topn=3'), MATCH('body^5', 'society', 'topn=3'), FUSION('rrf'); ---- Avicenna 29-APR-2012 07:47:29.000 9893 0.016393 Anarcho-capitalism 24-APR-2012 15:46:17.000 8781 0.016393 diff --git a/test/sql/dql/fusion.slt b/test/sql/dql/fusion.slt index d9daab956e..0f01c45603 100644 --- a/test/sql/dql/fusion.slt +++ b/test/sql/dql/fusion.slt @@ -15,7 +15,7 @@ COPY enwiki_embedding FROM '/tmp/infinity/test_data/enwiki_embedding_9999.csv' W ---- statement ok -CREATE INDEX ft_index ON enwiki_embedding(body) USING FULLTEXT WITH(ANALYZER=segmentation) (doctitle, docdate) USING FULLTEXT; +CREATE INDEX ft_index ON enwiki_embedding(body) USING FULLTEXT WITH(ANALYZER=segmentation); query TTT SELECT doctitle, docdate, body, num, vec FROM enwiki_embedding SEARCH MATCH('doctitle^2,body^5', 'harmful chemical', 'topn=3'), KNN(vec, [0.0, 0.0, 0.0, 0.0], 'float', 'l2', 3), FUSION('rrf'); @@ -29,7 +29,7 @@ Anarchism 30-APR-2012 03:25:17.000 [[hierarchical organization]] in the conduct query I -SELECT num FROM enwiki_embedding SEARCH MATCH('doctitle^2,body^5', 'harmful chemical', 'topn=3'), KNN(vec, [0.0, 0.0, 0.0, 0.0], 'float', 'l2', 3), FUSION('rrf'); +SELECT num FROM enwiki_embedding SEARCH MATCH('body^5', 'harmful chemical', 'topn=3'), KNN(vec, [0.0, 0.0, 0.0, 0.0], 'float', 'l2', 3), FUSION('rrf'); ---- 9893 0