Create an specific embedded Infinity python module (#1792)

### What problem does this PR solve?

- Create an specific embedded Infinity python module 
- delete embedded infinity in old infinity_sdk

Issue link:#1786

### Type of change

- [x] New Feature (non-breaking change which adds functionality)
- [x] Python SDK impacted, Need to update PyPI

---------

Co-authored-by: Zhichang Yu <[email protected]>

pyproject.toml

@@ -36,7 +36,7 @@ cmake.build-type = "Release"
 build-dir = "build/{build_type}"
 
 wheel.py-api = "cp310"
-wheel.packages = ["python/infinity"]
+wheel.packages = ["python/infinity","python/infinity_embedded"]
 
 [tool.pytest.ini_options]
 addopts = "--strict-markers"

python/infinity/__init__.py

@@ -23,13 +23,10 @@
 from infinity.common import URI, NetworkAddress, LOCAL_HOST, LOCAL_INFINITY_PATH, InfinityException
 from infinity.infinity import InfinityConnection
 from infinity.remote_thrift.infinity import RemoteThriftInfinityConnection
-from infinity.local_infinity.infinity import LocalInfinityConnection
 from infinity.errors import ErrorCode
 
 def connect(uri) -> InfinityConnection:
     if isinstance(uri, NetworkAddress):
         return RemoteThriftInfinityConnection(uri)
-    elif isinstance(uri, str) and len(uri) != 0:
-        return LocalInfinityConnection(uri)
     else:
         raise InfinityException(ErrorCode.INVALID_SERVER_ADDRESS, f"Unknown uri: {uri}")

python/infinity/index.py

@@ -16,10 +16,6 @@
 
 import infinity.remote_thrift.infinity_thrift_rpc.ttypes as ttypes
 from infinity.common import InfinityException
-
-from infinity.embedded_infinity_ext import IndexType as LocalIndexType, WrapIndexInfo
-from infinity.embedded_infinity_ext import InitParameter as LocalInitParameter
-from infinity.embedded_infinity_ext import WrapIndexInfo as LocalIndexInfo
 from infinity.errors import ErrorCode
 
 
@@ -51,25 +47,6 @@ def to_ttype(self):
             case _:
                 raise InfinityException(ErrorCode.INVALID_INDEX_TYPE, "Unknown index type")
 
-    def to_local_type(self):
-        match self:
-            case IndexType.IVFFlat:
-                return LocalIndexType.kIVFFlat
-            case IndexType.Hnsw:
-                return LocalIndexType.kHnsw
-            case IndexType.FullText:
-                return LocalIndexType.kFullText
-            case IndexType.Secondary:
-                return LocalIndexType.kSecondary
-            case IndexType.EMVB:
-                return LocalIndexType.kEMVB
-            case IndexType.BMP:
-                return LocalIndexType.kBMP
-            case IndexType.DiskAnn:
-                return LocalIndexType.kDiskAnn
-            case _:
-                raise InfinityException(ErrorCode.INVALID_INDEX_TYPE, "Unknown index type")
-
 
 class InitParameter:
     def __init__(self, param_name: str, param_value: str):
@@ -85,12 +62,6 @@ def __repr__(self):
     def to_ttype(self):
         return ttypes.InitParameter(self.param_name, self.param_value)
 
-    def to_local_type(self):
-        local_init_parameter = LocalInitParameter()
-        local_init_parameter.param_name = self.param_name
-        local_init_parameter.param_value = self.param_value
-        return local_init_parameter
-
 
 class IndexInfo:
     def __init__(self, column_name: str, index_type: IndexType, params: dict = None):
@@ -130,22 +101,3 @@ def to_ttype(self):
             self.index_type.to_ttype(),
             init_params_list
         )
-
-    def to_local_type(self):
-        index_info_to_use = WrapIndexInfo()
-        index_info_to_use.index_type = self.index_type.to_local_type()
-        index_info_to_use.column_name = self.column_name.strip()
-
-        index_param_list = []
-        if self.params is not None:
-            for key, value in self.params.items():
-                if isinstance(value, str):
-                    local_init_parameter = LocalInitParameter()
-                    local_init_parameter.param_name = key
-                    local_init_parameter.param_value = value
-                    index_param_list.append(local_init_parameter)
-                else:
-                    raise InfinityException(ErrorCode.INVALID_INDEX_PARAM, f"{value} should be string type")
-
-        index_info_to_use.index_param_list = index_param_list
-        return index_info_to_use

python/infinity/remote_thrift/table.py

@@ -28,12 +28,12 @@
 from infinity.remote_thrift.types import build_result
 from infinity.remote_thrift.utils import traverse_conditions, name_validity_check, select_res_to_polars
 from infinity.remote_thrift.utils import get_remote_constant_expr_from_python_value
-from infinity.table import Table, ExplainType
+from infinity.table import ExplainType
 from infinity.common import ConflictType, DEFAULT_MATCH_VECTOR_TOPN
 from infinity.utils import deprecated_api
 
 
-class RemoteTable(Table, ABC):
+class RemoteTable():
 
     def __init__(self, conn, db_name, table_name):
         self._conn = conn

python/infinity/table.py

@@ -19,7 +19,6 @@
 import infinity.remote_thrift.infinity_thrift_rpc.ttypes as ttypes
 from infinity.index import IndexInfo
 from infinity.common import InfinityException, INSERT_DATA
-from infinity.embedded_infinity_ext import ExplainType as LocalExplainType
 from infinity.errors import ErrorCode
 
 class ExplainType(Enum):
@@ -47,58 +46,4 @@ def to_ttype(self):
         elif self is ExplainType.Fragment:
             return ttypes.ExplainType.Fragment
         else:
-            raise InfinityException(ErrorCode.INVALID_EXPLAIN_TYPE, "Unknown explain type")
-    def to_local_ttype(self):
-        if self is ExplainType.Ast:
-            return LocalExplainType.kAst
-        elif self is ExplainType.Analyze:
-            return LocalExplainType.kAnalyze
-        elif self is ExplainType.UnOpt:
-            return LocalExplainType.kUnOpt
-        elif self is ExplainType.Opt:
-            return LocalExplainType.kOpt
-        elif self is ExplainType.Physical:
-            return LocalExplainType.kPhysical
-        elif self is ExplainType.Pipeline:
-            return LocalExplainType.kPipeline
-        elif self is ExplainType.Fragment:
-            return LocalExplainType.kFragment
-        else:
-            raise InfinityException(ErrorCode.INVALID_EXPLAIN_TYPE, "Unknown explain type")
-class Table(ABC):
-
-    @abstractmethod
-    def create_index(self, index_name: str, index_info: IndexInfo, options=None):
-        pass
-
-    @abstractmethod
-    def drop_index(self, index_name: str):
-        pass
-
-    @abstractmethod
-    def insert(self, data: Union[INSERT_DATA, list[INSERT_DATA]]):
-        pass
-
-    @abstractmethod
-    def import_data(self, file_path: str, import_options: {} = None):
-        pass
-
-    @abstractmethod
-    def export_data(self, file_path: str, export_options: {} = None, columns: [str] = None):
-        pass
-
-    @abstractmethod
-    def delete(self, cond: Optional[str] = None):
-        pass
-
-    @abstractmethod
-    def update(self, cond: str, data: dict[str, Any]):
-        pass
-
-    @abstractmethod
-    def _execute_query(self, query):
-        pass
-
-    @abstractmethod
-    def _explain_query(self, query):
-        pass
+            raise InfinityException(ErrorCode.INVALID_EXPLAIN_TYPE, "Unknown explain type")

python/infinity_embedded/__init__.py

@@ -0,0 +1,32 @@
+# Copyright(C) 2023 InfiniFlow, Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# import importlib.metadata
+#
+# __version__ = importlib.metadata.version("infinity_sdk")
+
+import os
+# import pkg_resources
+# __version__ = pkg_resources.get_distribution("infinity_sdk").version
+
+from infinity_embedded.common import URI, NetworkAddress, LOCAL_HOST, LOCAL_INFINITY_PATH, InfinityException
+from infinity_embedded.infinity import InfinityConnection
+from infinity_embedded.local_infinity.infinity import LocalInfinityConnection
+from infinity_embedded.errors import ErrorCode
+
+def connect(uri) -> InfinityConnection:
+    if isinstance(uri, str) and len(uri) != 0:
+        return LocalInfinityConnection(uri)
+    else:
+        raise InfinityException(ErrorCode.INVALID_SERVER_ADDRESS, f"Unknown uri: {uri}")

python/infinity_embedded/common.py

@@ -0,0 +1,85 @@
+# Copyright(C) 2023 InfiniFlow, Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from pathlib import Path
+from typing import Union
+from dataclasses import dataclass
+import numpy as np
+
+
+class NetworkAddress:
+    def __init__(self, ip, port):
+        self.ip = ip
+        self.port = port
+
+    def __str__(self):
+        return f'IP: {self.ip}, Port: {self.port}'
+
+
+@dataclass
+class SparseVector:
+    indices: list[int]
+    values: Union[list[float], list[int], None] = None
+
+    def __post_init__(self):
+        assert (self.values is None) or (len(self.indices) == len(self.values))
+
+    def to_dict_old(self):
+        d = {"indices": self.indices}
+        if self.values is not None:
+            d["values"] = self.values
+        return d
+
+    def to_dict(self):
+        if self.values is None:
+            raise ValueError("SparseVector.values is None")
+        result = {}
+        for i, v in zip(self.indices, self.values):
+            result[str(i)] = v
+        return result
+
+    @staticmethod
+    def from_dict(d):
+        return SparseVector(d["indices"], d.get("values"))
+
+    def __str__(self):
+        return f"SparseVector(indices={self.indices}{'' if self.values is None else f', values={self.values}'})"
+
+    def __repr__(self):
+        return str(self)
+
+
+URI = Union[NetworkAddress, Path]
+VEC = Union[list, np.ndarray]
+INSERT_DATA = dict[str, Union[str, int, float, list[Union[int, float]]], SparseVector, dict]
+
+LOCAL_HOST = NetworkAddress("127.0.0.1", 23817)
+
+# test embedded_infinity
+LOCAL_INFINITY_PATH = "/var/infinity"
+
+
+class ConflictType(object):
+    Ignore = 0
+    Error = 1
+    Replace = 2
+
+
+class InfinityException(Exception):
+    def __init__(self, error_code=0, error_message=None):
+        self.error_code = error_code
+        self.error_message = error_message
+
+
+DEFAULT_MATCH_VECTOR_TOPN = 10
+DEFAULT_MATCH_SPARSE_TOPN = 10

python/infinity_embedded/db.py

@@ -0,0 +1,45 @@
+# Copyright(C) 2023 InfiniFlow, Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from abc import ABC, abstractmethod
+
+class Database(ABC):
+
+    @abstractmethod
+    def create_table(self, table_name, schema, options):
+        pass  # implement create table logic here
+
+    @abstractmethod
+    def drop_table(self, table_name):
+        pass  # implement drop table logic here
+
+    @abstractmethod
+    def list_tables(self):
+        pass  # implement list tables logic here
+
+    @abstractmethod
+    def show_table(self, table_name):
+        pass  # implement describe table logic here
+
+    @abstractmethod
+    def show_columns(self, table_name):
+        pass  # implement describe table logic here
+
+    @abstractmethod
+    def get_table(self, table_name):
+        pass  # implement get table logic here
+
+    @abstractmethod
+    def show_tables(self):
+        pass