Add pytest mark for different function tests. (#694)

### What problem does this PR solve?
Add pytest mark for different function tests, update the sqllogictest
file, and add python_sdk_test parameters.

### Type of change

- [x] New Feature (non-breaking change which adds functionality)
- [x] Test cases

---------

Co-authored-by: chrysanthemum-boy <[email protected]>

python/test/config/conftest.py

@@ -0,0 +1,31 @@
+# Copyright(C) 2023 InfiniFlow, Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import pytest
+
+
+def disable_items_with_mark(items, mark, reason):
+    skipper = pytest.mark.skip(reason=reason)
+    for item in items:
+        if mark in item.keywords:
+            item.add_marker(skipper)
+
+
+def pytest_addoption(parser):
+    parser.addoption(
+        "--run-integration",
+        action="store_true",
+        default=False,
+        help="Run integration tests (requires S3 buckets to be setup with access)",
+    )

python/test/pytest.ini

@@ -0,0 +1,6 @@
+[pytest]
+addopts = --strict-markers
+markers =
+    complex: marks tests as complex, with start or shutdown service
+    slow: marks tests as slow
+    nightly: test in night

python/test/test_basic.py

@@ -16,7 +16,7 @@
 import pandas as pd
 from numpy import dtype
 
-from python.test.common import common_values
+from common import common_values
 import infinity
 import infinity.index as index
 

python/test/test_connect.py

@@ -13,7 +13,7 @@
 # limitations under the License.
 
 import pytest
-from python.test.common import common_values
+from common import common_values
 import infinity
 from infinity.common import NetworkAddress
 

python/test/test_convert.py

@@ -2,8 +2,10 @@
 from infinity.errors import ErrorCode
 
 from common import common_values
-from python import infinity
-from python.infinity.remote_thrift.query_builder import InfinityThriftQueryBuilder
+
+import infinity
+from infinity.remote_thrift.query_builder import InfinityThriftQueryBuilder
+
 
 
 class TestConvert:

python/test/test_database.py

@@ -14,7 +14,7 @@
 
 import threading
 import infinity
-from python.test.common import common_values
+from common import common_values
 import pytest
 from infinity.errors import ErrorCode
 from utils import trace_expected_exceptions
@@ -166,6 +166,7 @@ def test_create_drop_show_1K_databases(self):
 
         assert res.error_code == ErrorCode.OK
 
+    @pytest.mark.slow
     @pytest.mark.skip(reason="Cost too much times")
     def test_create_drop_show_1M_databases(self):
 

python/test/test_delete.py

@@ -16,7 +16,7 @@
 import pandas as pd
 import pytest
 from numpy import dtype
-from python.test.common import common_values
+from common import common_values
 import infinity
 from infinity.errors import ErrorCode
 from utils import trace_expected_exceptions
@@ -289,6 +289,7 @@ def test_delete_insert_data(self):
         assert res.error_code == ErrorCode.OK
 
     # delete inserted long before and select to check
+    @pytest.mark.slow
     @pytest.mark.skip(reason="Cost too much time.")
     def test_delete_inserted_long_before_data(self):
         # connect

python/test/test_describe.py

@@ -13,7 +13,7 @@
 # limitations under the License.
 import polars as pl
 
-from python.test.common import common_values
+from common import common_values
 import infinity
 
 

python/test/test_explain.py

@@ -15,7 +15,7 @@
 import polars as pl
 import pytest
 
-from python.test.common import common_values
+from common import common_values
 import infinity
 from infinity.table import ExplainType
 

python/test/test_import.py

@@ -14,7 +14,7 @@
 
 import os
 import pytest
-from python.test.common import common_values
+from common import common_values
 import infinity
 from infinity.errors import ErrorCode
 

python/test/test_index.py

@@ -15,7 +15,7 @@
 
 import pytest
 
-from python.test.common import common_values
+from common import common_values
 import infinity
 import infinity.index as index
 from infinity.errors import ErrorCode

python/test/test_infinity.py

@@ -14,8 +14,8 @@
 import infinity
 from infinity.errors import ErrorCode
 
-from python.infinity.infinity import ShowVariable
-from python.test.common import common_values
+from infinity.infinity import ShowVariable
+from common import common_values
 
 
 class TestInfinity:

python/test/test_insert.py

@@ -13,6 +13,8 @@
 # limitations under the License.
 import os
 import signal
+import time
+
 
 import pandas as pd
 import pytest
@@ -537,6 +539,8 @@ def test_various_insert_types(self, column_types, column_types_example):
         res = infinity_obj.disconnect()
         assert res.error_code == ErrorCode.OK
 
+    @pytest.mark.complex
+    # @pytest.mark.skip(reason="complex")
     # @pytest.mark.tag(ct.CaseLabel.L0)
     @pytest.mark.skip(reason="TODO")
     def test_insert_and_shutdown_output(self):
@@ -564,6 +568,8 @@ def test_insert_and_shutdown_output(self):
         # shutdown service
         os.kill(infinity_service_1.pid, signal.SIGINT)
 
+        time.sleep(1)
+
         # restart
         infinity_service_2 = start_infinity_service_in_subporcess()
 

python/test/test_knn.py

@@ -14,7 +14,7 @@
 
 import pytest
 
-from python.test.common import common_values
+from common import common_values
 import infinity
 
 

python/test/test_query.py

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from python.test.common import common_values
+from common import common_values
 import infinity
 import infinity.index as index
 from infinity.errors import ErrorCode

python/test/test_select.py

@@ -14,7 +14,7 @@
 import os
 
 import pandas as pd
-from python.test.common import common_values
+from common import common_values
 import infinity
 from numpy import dtype
 from infinity.errors import ErrorCode

python/test/test_table.py

@@ -15,7 +15,7 @@
 import pytest
 import polars as pl
 
-from python.test.common import common_values
+from common import common_values
 import infinity
 from infinity.errors import ErrorCode
 from utils import trace_expected_exceptions
@@ -432,6 +432,7 @@ def test_table_with_different_invalid_options(self):
             print(e)
 
     # create/drop/describe/get 1000 tables with 10000 columns with various column types.
+    @pytest.mark.slow
     @pytest.mark.skip(reason="Cost too much times,and may cause the serve to terminate")
     def test_various_tables_with_various_columns(self):
         # connect
@@ -624,6 +625,7 @@ def test_create_1K_table(self):
         assert res.error_code == ErrorCode.OK
 
     # create/drop/list/get 1M table to reach the limit
+    @pytest.mark.slow
     @pytest.mark.skip(reason="Cost too much times")
     def test_create_1M_table(self):
         """

python/test/test_update.py

@@ -17,7 +17,7 @@
 import pytest
 from numpy import dtype
 
-from python.test.common import common_values
+from common import common_values
 import infinity
 from infinity.errors import ErrorCode
 from utils import trace_expected_exceptions

python/test/utils.py

@@ -1,6 +1,7 @@
 import functools
 import os
 import subprocess
+import time
 import traceback
 
 import numpy as np
@@ -29,8 +30,8 @@ def read_fvecs_file(filename):
 
 
 def start_infinity_service_in_subporcess():
-    # os.system("rm -fr /tmp/infinity")
     shell = "/home/fann/infinity/build/src/infinity"
     with open("./tmp.txt", "w") as f:
         infinity = subprocess.Popen(shell, stdout=f)
+    time.sleep(1)
     return infinity

tools/sqllogictest.py

@@ -6,6 +6,7 @@
 import time
 from shutil import copyfile
 import subprocess
+import signal
 
 from generate_big import generate as generate1
 from generate_fvecs import generate as generate2
@@ -35,7 +36,7 @@ def stop_spinner(self):
         self.stop = True
 
 
-def python_sdk_test(python_test_dir: str):
+def python_sdk_test(python_test_dir: str, pytest_mark: str):
     print("python test path is {}".format(python_test_dir))
     # os.system(f"cd {python_test_dir}/test")
     # check if infinity_sdk is installed
@@ -46,7 +47,7 @@ def python_sdk_test(python_test_dir: str):
     # run test
     print("start pysdk test...")
     process = subprocess.Popen(
-        ["python", "-m", "pytest", f"{python_test_dir}/test"],
+        ["python", "-m", "pytest", '-m', pytest_mark, f'{python_test_dir}/test'],
         stdout=subprocess.PIPE,
         stderr=subprocess.PIPE,
         universal_newlines=True,
@@ -150,6 +151,13 @@ def copy_all(data_dir, copy_dir):
         default=data_dir,
         dest="data",
     )
+    parser.add_argument(
+        "-m",
+        "--pytest_mark",
+        type=str,
+        default='not complex and not slow',
+        dest="pytest_mark",
+    )
     parser.add_argument(
         "-c",
         "--copy",
@@ -189,7 +197,7 @@ def copy_all(data_dir, copy_dir):
         print("Start testing...")
         start = time.time()
         try:
-            python_sdk_test(python_test_dir)
+            python_sdk_test(python_test_dir, args.pytest_mark)
             test_process(args.path, args.test, args.data, args.copy)
         except Exception as e:
             print(e)