Merge IndexDef into IndexBase (#692)

### What problem does this PR solve? Merge IndexDef into IndexBase so that `CREATE INDEX` contains only one index definition. Breaking Changes: - iresearch doesn't support fulltext index on multiple columns Issue link:#551 ### Type of change - [x] Breaking Change (fix or feature that could cause existing functionality not to work as expected) - [x] Refactoring
infiniflow · Mar 1, 2024 · af11f45 · af11f45
1 parent a217862
commit af11f45
Show file tree

Hide file tree

Showing 54 changed files with 417 additions and 692 deletions.
diff --git a/python/test/test_index.py b/python/test/test_index.py
@@ -59,10 +59,14 @@ def test_create_index_HNSW(self):
                                      [index.IndexInfo("c1",
                                                       index.IndexType.Hnsw,
                                                       [
-                                                          index.InitParameter("M", "16"),
-                                                          index.InitParameter("ef_construction", "50"),
-                                                          index.InitParameter("ef", "50"),
-                                                          index.InitParameter("metric", "l2")
+                                                          index.InitParameter(
+                                                              "M", "16"),
+                                                          index.InitParameter(
+                                                              "ef_construction", "50"),
+                                                          index.InitParameter(
+                                                              "ef", "50"),
+                                                          index.InitParameter(
+                                                              "metric", "l2")
                                                       ])], None)
 
         assert res.error_code == ErrorCode.OK
@@ -71,7 +75,7 @@ def test_create_index_HNSW(self):
         assert res.error_code == ErrorCode.OK
 
     def test_create_index_fulltext(self):
-        # CREATE INDEX ft_index ON enwiki(body) USING FULLTEXT WITH(ANALYZER=segmentation) (doctitle, docdate) USING FULLTEXT;
+        # CREATE INDEX ft_index ON enwiki(body) USING FULLTEXT WITH(ANALYZER=segmentation);
         infinity_obj = infinity.connect(common_values.TEST_REMOTE_HOST)
         db_obj = infinity_obj.get_database("default")
         res = db_obj.drop_table("test_index_fulltext", if_exists=True)
@@ -84,12 +88,6 @@ def test_create_index_fulltext(self):
                                      [index.IndexInfo("body",
                                                       index.IndexType.FullText,
                                                       [index.InitParameter("ANALYZER", "segmentation")]),
-                                      index.IndexInfo("doctitle",
-                                                      index.IndexType.FullText,
-                                                      []),
-                                      index.IndexInfo("docdate",
-                                                      index.IndexType.FullText,
-                                                      []),
                                       ], None)
 
         assert res.error_code == ErrorCode.OK
@@ -147,19 +145,23 @@ def test_create_created_index(self):
     # create / drop index with invalid options
     @pytest.mark.parametrize("cl_name", [(1, False), (2.2, False), ((1, 2), False), ([1, 2, 3], False), ("c1", True)])
     @pytest.mark.parametrize("index_type", [
-        (1, False), (2.2, False), ([1, 2], False), ("$#%dfva", False), ((1, 2), False), ({"1": 2}, False),
-        (index.IndexType.Hnsw, False), (index.IndexType.IVFFlat, True), (index.IndexType.FullText, True)
+        (1, False), (2.2, False), ([1, 2], False), ("$#%dfva", False), ((
+            1, 2), False), ({"1": 2}, False),
+        (index.IndexType.Hnsw, False), (index.IndexType.IVFFlat,
+                                        True), (index.IndexType.FullText, True)
     ])
     @pytest.mark.parametrize("params", [
-        (1, False), (2.2, False), ([1, 2], False), ("$#%dfva", False), ((1, 2), False), ({"1": 2}, False),
+        (1, False), (2.2, False), ([1, 2], False), ("$#%dfva", False), ((
+            1, 2), False), ({"1": 2}, False),
         ([index.InitParameter("centroids_count", "128"),
          index.InitParameter("metric", "l2")], True)
     ])
     def test_create_drop_index_invalid_options(self, cl_name, index_type, params):
         # connect
         infinity_obj = infinity.connect(common_values.TEST_REMOTE_HOST)
         db_obj = infinity_obj.get_database("default")
-        db_obj.drop_table("test_create_drop_index_invalid_options", if_exists=True)
+        db_obj.drop_table(
+            "test_create_drop_index_invalid_options", if_exists=True)
         table_obj = db_obj.create_table("test_create_drop_index_invalid_options", {
             "c1": "vector,3,float"}, None)
 
@@ -179,18 +181,20 @@ def test_create_index_on_dropped_table(self):
         # connect
         infinity_obj = infinity.connect(common_values.TEST_REMOTE_HOST)
         db_obj = infinity_obj.get_database("default")
-        db_obj.drop_table("test_create_drop_index_invalid_options", if_exists=True)
+        db_obj.drop_table(
+            "test_create_drop_index_invalid_options", if_exists=True)
         table_obj = db_obj.create_table("test_create_drop_index_invalid_options", {
             "c1": "vector,3,float"}, None)
-        db_obj.drop_table("test_create_drop_index_invalid_options", if_exists=True)
+        db_obj.drop_table(
+            "test_create_drop_index_invalid_options", if_exists=True)
 
         # create created index
         with pytest.raises(Exception, match="ERROR:3022*"):
             table_obj.create_index("my_index",
-                                     [index.IndexInfo("c1",
-                                                      index.IndexType.IVFFlat,
-                                                      [index.InitParameter("centroids_count", "128"),
-                                                       index.InitParameter("metric", "l2")])], None)
+                                   [index.IndexInfo("c1",
+                                                    index.IndexType.IVFFlat,
+                                                    [index.InitParameter("centroids_count", "128"),
+                                                     index.InitParameter("metric", "l2")])], None)
 
         # disconnect
         res = infinity_obj.disconnect()
@@ -240,29 +244,31 @@ def test_drop_index_show_index(self):
     # create index on different type of column and show index
     @pytest.mark.parametrize("types", ["vector, 3, float"])
     @pytest.mark.parametrize("index_type", [
-        (index.IndexType.Hnsw, False), (index.IndexType.IVFFlat, True), (index.IndexType.FullText, True)
+        (index.IndexType.Hnsw, False), (index.IndexType.IVFFlat,
+                                        True), (index.IndexType.FullText, True)
     ])
     def test_create_index_on_different_type_of_column(self, types, index_type):
         # connect
         infinity_obj = infinity.connect(common_values.TEST_REMOTE_HOST)
         db_obj = infinity_obj.get_database("default")
-        db_obj.drop_table("test_create_index_on_different_type_of_column", if_exists=True)
+        db_obj.drop_table(
+            "test_create_index_on_different_type_of_column", if_exists=True)
         table_obj = db_obj.create_table("test_create_index_on_different_type_of_column", {
             "c1": types}, None)
         # create created index
         if not index_type[1]:
             with pytest.raises(Exception, match="ERROR:3061*"):
                 table_obj.create_index("my_index",
-                                      [index.IndexInfo("c1",
-                                                       index_type[0],
-                                                       [index.InitParameter("centroids_count", "128"),
-                                                        index.InitParameter("metric", "l2")])], None)
-        else:
-            res = table_obj.create_index("my_index",
-                                        [index.IndexInfo("c1",
+                                       [index.IndexInfo("c1",
                                                         index_type[0],
                                                         [index.InitParameter("centroids_count", "128"),
-                                                        index.InitParameter("metric", "l2")])], None)
+                                                         index.InitParameter("metric", "l2")])], None)
+        else:
+            res = table_obj.create_index("my_index",
+                                         [index.IndexInfo("c1",
+                                                          index_type[0],
+                                                          [index.InitParameter("centroids_count", "128"),
+                                                           index.InitParameter("metric", "l2")])], None)
             assert res.error_code == ErrorCode.OK
 
         # disconnect
@@ -307,7 +313,8 @@ def test_import_data_create_index(self, index_type, file_format):
             "c1": "int",
             "c2": "vector,3,float"}, None)
 
-        table_obj.import_data(os.getcwd() + TEST_DATA_DIR + file_format + "/pysdk_test." + file_format)
+        table_obj.import_data(os.getcwd() + TEST_DATA_DIR +
+                              file_format + "/pysdk_test." + file_format)
         res = table_obj.create_index("my_index",
                                      [index.IndexInfo("c2",
                                                       index_type,

diff --git a/src/executor/operator/physical_create_index_do.cpp b/src/executor/operator/physical_create_index_do.cpp
@@ -23,7 +23,6 @@ import query_context;
 import operator_state;
 import load_meta;
 
-import index_def;
 import create_index_data;
 import base_table_ref;
 import status;

diff --git a/src/executor/operator/physical_create_index_do.cppm b/src/executor/operator/physical_create_index_do.cppm
@@ -24,7 +24,7 @@ import query_context;
 import operator_state;
 import load_meta;
 
-import index_def;
+import index_base;
 import base_table_ref;
 import internal_types;
 import data_type;

diff --git a/src/executor/operator/physical_create_index_finish.cpp b/src/executor/operator/physical_create_index_finish.cpp
@@ -23,7 +23,7 @@ import physical_operator;
 import query_context;
 import operator_state;
 import load_meta;
-import index_def;
+import index_base;
 import infinity_exception;
 
 module physical_create_index_finish;
@@ -33,18 +33,18 @@ PhysicalCreateIndexFinish::PhysicalCreateIndexFinish(u64 id,
                                                      UniquePtr<PhysicalOperator> left,
                                                      SharedPtr<String> db_name,
                                                      SharedPtr<String> table_name,
-                                                     SharedPtr<IndexDef> index_def,
+                                                     SharedPtr<IndexBase> index_base,
                                                      SharedPtr<Vector<String>> output_names,
                                                      SharedPtr<Vector<SharedPtr<DataType>>> output_types,
                                                      SharedPtr<Vector<LoadMeta>> load_metas)
     : PhysicalOperator(PhysicalOperatorType::kCreateIndexFinish, std::move(left), nullptr, id, load_metas), db_name_(db_name),
-      table_name_(table_name), index_def_(index_def), output_names_(output_names), output_types_(output_types) {}
+      table_name_(table_name), index_base_(index_base), output_names_(output_names), output_types_(output_types) {}
 
 void PhysicalCreateIndexFinish::Init() {}
 
 bool PhysicalCreateIndexFinish::Execute(QueryContext *query_context, OperatorState *operator_state) {
     auto *txn = query_context->GetTxn();
-    auto status = txn->CreateIndexFinish(*db_name_, *table_name_, index_def_);
+    auto status = txn->CreateIndexFinish(*db_name_, *table_name_, index_base_);
     if (!status.ok()) {
         RecoverableError(status);
     }

diff --git a/src/executor/operator/physical_create_index_finish.cppm b/src/executor/operator/physical_create_index_finish.cppm
@@ -23,7 +23,7 @@ import physical_operator;
 import query_context;
 import operator_state;
 import load_meta;
-import index_def;
+import index_base;
 import internal_types;
 import data_type;
 
@@ -34,7 +34,7 @@ public:
                               UniquePtr<PhysicalOperator> left,
                               SharedPtr<String> db_name,
                               SharedPtr<String> table_name,
-                              SharedPtr<IndexDef> index_def,
+                              SharedPtr<IndexBase> index_base,
                               SharedPtr<Vector<String>> output_names,
                               SharedPtr<Vector<SharedPtr<DataType>>> output_types,
                               SharedPtr<Vector<LoadMeta>> load_metas);
@@ -53,7 +53,7 @@ public:
 public:
     const SharedPtr<String> db_name_{};
     const SharedPtr<String> table_name_{};
-    const SharedPtr<IndexDef> index_def_{};
+    const SharedPtr<IndexBase> index_base_{};
 
     const SharedPtr<Vector<String>> output_names_{};
     const SharedPtr<Vector<SharedPtr<DataType>>> output_types_{};

diff --git a/src/executor/operator/physical_create_index_prepare.cpp b/src/executor/operator/physical_create_index_prepare.cpp
@@ -26,7 +26,6 @@ import query_context;
 import operator_state;
 import load_meta;
 
-import index_def;
 import status;
 import infinity_exception;
 import index_base;
@@ -48,7 +47,7 @@ import extra_ddl_info;
 namespace infinity {
 PhysicalCreateIndexPrepare::PhysicalCreateIndexPrepare(u64 id,
                                                        SharedPtr<BaseTableRef> base_table_ref,
-                                                       SharedPtr<IndexDef> index_definition,
+                                                       SharedPtr<IndexBase> index_definition,
                                                        ConflictType conflict_type,
                                                        SharedPtr<Vector<String>> output_names,
                                                        SharedPtr<Vector<SharedPtr<DataType>>> output_types,

diff --git a/src/executor/operator/physical_create_index_prepare.cppm b/src/executor/operator/physical_create_index_prepare.cppm
@@ -24,7 +24,7 @@ import query_context;
 import operator_state;
 import load_meta;
 import base_table_ref;
-import index_def;
+import index_base;
 import internal_types;
 import extra_ddl_info;
 import data_type;
@@ -34,7 +34,7 @@ export class PhysicalCreateIndexPrepare : public PhysicalOperator {
 public:
     PhysicalCreateIndexPrepare(u64 id,
                                SharedPtr<BaseTableRef> base_table_ref,
-                               SharedPtr<IndexDef> index_definition,
+                               SharedPtr<IndexBase> index_definition,
                                ConflictType conflict_type,
                                SharedPtr<Vector<String>> output_names,
                                SharedPtr<Vector<SharedPtr<DataType>>> output_types,
@@ -55,7 +55,7 @@ public:
 public:
     const SharedPtr<BaseTableRef> base_table_ref_{};
 
-    const SharedPtr<IndexDef> index_def_ptr_{};
+    const SharedPtr<IndexBase> index_def_ptr_{};
     const ConflictType conflict_type_{};
 
     const SharedPtr<Vector<String>> output_names_{};

diff --git a/src/executor/operator/physical_create_schema.cppm b/src/executor/operator/physical_create_schema.cppm
@@ -22,7 +22,7 @@ import query_context;
 import operator_state;
 import physical_operator;
 import physical_operator_type;
-import index_def;
+import index_base;
 import load_meta;
 import infinity_exception;
 import internal_types;

diff --git a/src/executor/operator/physical_create_table.cppm b/src/executor/operator/physical_create_table.cppm
@@ -22,7 +22,7 @@ import query_context;
 import operator_state;
 import physical_operator;
 import physical_operator_type;
-import index_def;
+import index_base;
 import table_def;
 import load_meta;
 import infinity_exception;

diff --git a/src/executor/operator/physical_create_view.cppm b/src/executor/operator/physical_create_view.cppm
@@ -22,7 +22,7 @@ import query_context;
 import operator_state;
 import physical_operator;
 import physical_operator_type;
-import index_def;
+import index_base;
 import load_meta;
 import infinity_exception;
 import internal_types;

diff --git a/src/executor/operator/physical_knn_scan.cpp b/src/executor/operator/physical_knn_scan.cpp
@@ -42,7 +42,6 @@ import index_base;
 import buffer_manager;
 import merge_knn;
 import knn_result_handler;
-import index_def;
 import ann_ivf_flat;
 import annivfflat_index_data;
 import buffer_handle;

diff --git a/src/executor/operator/physical_show.cpp b/src/executor/operator/physical_show.cpp
@@ -36,7 +36,6 @@ import value;
 import table_def;
 import data_table;
 import third_party;
-import index_def;
 import index_ivfflat;
 import index_base;
 import index_hnsw;

diff --git a/src/executor/physical_planner.cpp b/src/executor/physical_planner.cpp
@@ -325,8 +325,7 @@ UniquePtr<PhysicalOperator> PhysicalPlanner::BuildCreateIndex(const SharedPtr<Lo
     SharedPtr<String> schema_name = logical_create_index->base_table_ref()->schema_name();
     SharedPtr<String> table_name = logical_create_index->base_table_ref()->table_name();
     const auto &index_def_ptr = logical_create_index->index_definition();
-    if (false || index_def_ptr->index_array_.size() != 1 || index_def_ptr->index_array_[0]->index_type_ != IndexType::kHnsw) {
-        // TODO: invalidate multiple index in one statement.
+    if (index_def_ptr->index_type_ != IndexType::kHnsw) {
         // TODO: support other index types build in parallel.
         return MakeUnique<PhysicalCreateIndexPrepare>(logical_create_index->node_id(),
                                                       logical_create_index->base_table_ref(),