From 15b125091447b0fa86deac503b00ae19c48741fe Mon Sep 17 00:00:00 2001 From: Simone Lazzaris Date: Fri, 3 Nov 2023 08:52:14 +0100 Subject: [PATCH] fix: make pytest works again (#71) * fix: removed UnknownFields call which is not implemented * fix: updated varchar description post 1.9 * fix: more fixes, bumped immudb test versions * deprecate old query column mode, fix other tests * fix: autopep8 --- .github/workflows/ci.yml | 8 +++---- immudb/client.py | 17 +++++++++++++-- immudb/handler/sqlquery.py | 32 +++++++++++++++++++--------- immudb/handler/verifiedSet.py | 2 -- tests/immu/test_export_tx.py | 9 ++++++-- tests/immu/test_massive_operation.py | 9 +++++++- tests/immu/test_session.py | 6 +++++- tests/immu/test_sql.py | 8 +++++-- tests/immuTestClient.py | 10 +++++---- 9 files changed, 73 insertions(+), 28 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ef5b20c..c43c5f4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,10 +21,10 @@ jobs: make dev - name: Start immudb container run: | - docker run -d --health-cmd "immuadmin status" --health-interval 10s --health-timeout 5s --health-retries 5 -v ${{ github.workspace }}/tests/certs/my.key.pem:/key.pem -p 3322:3322 codenotary/immudb:1.4.0 --signingKey=/key.pem - docker run -d --health-cmd "immuadmin status" --health-interval 10s --health-timeout 5s --health-retries 5 -v ${{ github.workspace }}/tests/certs/my.key.pem:/key.pem -p 3333:3322 codenotary/immudb:1.3.2 --signingKey=/key.pem - docker run -d --health-cmd "immuadmin status" --health-interval 10s --health-timeout 5s --health-retries 5 -v ${{ github.workspace }}/tests/certs/my.key.pem:/key.pem -p 3344:3322 codenotary/immudb:1.2.4 --signingKey=/key.pem - docker run -d --health-cmd "immuadmin status" --health-interval 10s --health-timeout 5s --health-retries 5 -v ${{ github.workspace }}/tests/certs/my.key.pem:/key.pem -p 3355:3322 codenotary/immudb:1.1.0 --signingKey=/key.pem + docker run -d --health-cmd "immuadmin status" --health-interval 10s --health-timeout 5s --health-retries 5 -v ${{ github.workspace }}/tests/certs/my.key.pem:/key.pem -p 3322:3322 codenotary/immudb:1.9DOM --signingKey=/key.pem + docker run -d --health-cmd "immuadmin status" --health-interval 10s --health-timeout 5s --health-retries 5 -v ${{ github.workspace }}/tests/certs/my.key.pem:/key.pem -p 3333:3322 codenotary/immudb:1.5.0 --signingKey=/key.pem + docker run -d --health-cmd "immuadmin status" --health-interval 10s --health-timeout 5s --health-retries 5 -v ${{ github.workspace }}/tests/certs/my.key.pem:/key.pem -p 3344:3322 codenotary/immudb:1.4.1 --signingKey=/key.pem + docker run -d --health-cmd "immuadmin status" --health-interval 10s --health-timeout 5s --health-retries 5 -v ${{ github.workspace }}/tests/certs/my.key.pem:/key.pem -p 3355:3322 codenotary/immudb:1.4.0 --signingKey=/key.pem - name: Run tests run: | make test diff --git a/immudb/client.py b/immudb/client.py index dfb1a55..f12eb8c 100644 --- a/immudb/client.py +++ b/immudb/client.py @@ -79,6 +79,7 @@ def __init__(self, immudUrl=None, rs: RootService = None, publicKeyFile: str = N self._rs = rs self._url = immudUrl self._vk = None + self._currentdb = None if publicKeyFile: self.loadKey(publicKeyFile) @@ -216,6 +217,7 @@ def login(self, username, password, database=b"defaultdb"): self._stub = self._set_token_header_interceptor(resp) self._rs.init("{}/{}".format(self._url, database), self._stub) + self._currentdb = convertedDatabase return login_response def logout(self): @@ -223,6 +225,7 @@ def logout(self): """ self._stub.Logout(google_dot_protobuf_dot_empty__pb2.Empty()) self._resetStub() + self._currentdb = None def _resetStub(self): self.headersInterceptors = [] @@ -511,6 +514,7 @@ def useDatabase(self, dbName: bytes): # modifies header token accordingly self._stub = self._set_token_header_interceptor(resp) self._rs.init(dbName, self._stub) + self._currentdb = dbName return resp def getDatabaseSettingsV2(self) -> datatypesv2.DatabaseSettingsResponseV2: @@ -1620,7 +1624,17 @@ def sqlQuery(self, query, params={}, columnNameMode=constants.COLUMN_NAME_MODE_N ['table1', 'table2'] """ - return sqlquery.call(self._stub, self._rs, query, params, columnNameMode) + ret = sqlquery.call(self._stub, self._rs, query, + params, columnNameMode) + if columnNameMode in [constants.COLUMN_NAME_MODE_DATABASE, constants.COLUMN_NAME_MODE_FULL]: + # newer DB version don't insert database name anymore, we need to + # process it manually + for i, t in enumerate(ret): + newkeys = [ + x.replace("[@DB]", self._currentdb.decode("utf-8")) for x in t.keys()] + k = dict(zip(newkeys, list(t.values()))) + ret[i] = k + return ret def listTables(self): """List all tables in the current database @@ -1696,7 +1710,6 @@ def verifiableSQLGet(self, table: str, primaryKeys: List[datatypesv2.PrimaryKey] # immudb-py only - def getAllValues(self, keys: list): # immudb-py only resp = batchGet.call(self._stub, self._rs, keys) return resp diff --git a/immudb/handler/sqlquery.py b/immudb/handler/sqlquery.py index d660aec..097318a 100644 --- a/immudb/handler/sqlquery.py +++ b/immudb/handler/sqlquery.py @@ -35,7 +35,6 @@ def _call_with_executor(query, params, columnNameMode, executor): resp = executor(request) result = [] - columnNames = getColumnNames(resp, columnNameMode) for row in resp.rows: @@ -51,14 +50,27 @@ def getColumnNames(resp, columnNameMode): columnNames = [] if columnNameMode: for column in resp.columns: + # note that depending on the version parts can be + # '(dbname.tablename.fieldname)' *or* + # '(tablename.fieldname)' without dbnname. + # In that case we mimic the old behavior by using [@DB] as placeholder + # that will be replaced at higher level. + parts = column.name.strip("()").split(".") if columnNameMode == constants.COLUMN_NAME_MODE_FIELD: - columnNames.append(column.name.strip("()").split(".")[2]) - elif columnNameMode == constants.COLUMN_NAME_MODE_TABLE: - columnNames.append(column.name.strip("()").split(".", 1)[1]) - elif columnNameMode == constants.COLUMN_NAME_MODE_DATABASE: - columnNames.append(column.name.strip("()")) - elif columnNameMode == constants.COLUMN_NAME_MODE_FULL: - columnNames.append(column.name) - else: - raise ErrPySDKInvalidColumnMode + columnNames.append(parts[-1]) + continue + if columnNameMode == constants.COLUMN_NAME_MODE_TABLE: + columnNames.append(".".join(parts[-2:])) + continue + print( + "Use of COLUMN_NAME_MODE_DATABASE and COLUMN_NAME_MODE_FULL is deprecated") + if len(parts) == 2: + parts.insert(0, "[@DB]") + if columnNameMode == constants.COLUMN_NAME_MODE_DATABASE: + columnNames.append(".".join(parts)) + continue + if columnNameMode == constants.COLUMN_NAME_MODE_FULL: + columnNames.append("("+".".join(parts)+")") + continue + raise ErrPySDKInvalidColumnMode return columnNames diff --git a/immudb/handler/verifiedSet.py b/immudb/handler/verifiedSet.py index 7ac7503..e3fbc61 100644 --- a/immudb/handler/verifiedSet.py +++ b/immudb/handler/verifiedSet.py @@ -21,8 +21,6 @@ import immudb.schema as schema from immudb.typeconv import MetadataToProto -#import base64 - def call(service: schema_pb2_grpc.ImmuServiceStub, rs: RootService, key: bytes, value: bytes, verifying_key=None, metadata=None): schemaMetadata = MetadataToProto(metadata) diff --git a/tests/immu/test_export_tx.py b/tests/immu/test_export_tx.py index 696eba0..aedc51a 100644 --- a/tests/immu/test_export_tx.py +++ b/tests/immu/test_export_tx.py @@ -9,6 +9,11 @@ def test_export_tx_replicate_tx(wrappedClient: ImmudbClient): client = wrappedClient.client if(not wrappedClient.serverHigherOrEqualsToVersion("1.2.0")): pytest.skip("Immudb version too low") + if wrappedClient.serverHigherOrEqualsToVersion("1.5.0"): + offset = 0 + else: + offset = 1 + newuuid1 = str(uuid.uuid4()).replace("-", "") client.createDatabaseV2(newuuid1, settings = datatypesv2.DatabaseSettingsV2( ), ifNotExists=False) @@ -32,7 +37,7 @@ def test_export_tx_replicate_tx(wrappedClient: ImmudbClient): assert txHeader.id > 0 assert txHeader.nentries == 1 if(index > 1): # First transaction is not db set - assert client.get(b'kisz123123kaaaa').value.decode("utf-8") == str(index - 1) + assert client.get(b'kisz123123kaaaa').value.decode("utf-8") == str(index - offset) client.useDatabase(newuuid1) client.useDatabase(newuuid2) - assert client.get(b'kisz123123kaaaa').value == b'5' \ No newline at end of file + assert client.get(b'kisz123123kaaaa').value == b'5' diff --git a/tests/immu/test_massive_operation.py b/tests/immu/test_massive_operation.py index 437ce4d..9aeb5fa 100644 --- a/tests/immu/test_massive_operation.py +++ b/tests/immu/test_massive_operation.py @@ -15,6 +15,7 @@ import time from tests.immuTestClient import ImmuTestClient import pytest +import grpc def get_random_string(length): @@ -46,6 +47,12 @@ def test_get_set_massive(self, client): def test_compact(self, wrappedClient: ImmuTestClient): # Snapshot compaction fixed from 1.3.1 if(wrappedClient.serverHigherOrEqualsToVersion("1.3.0")): - wrappedClient.client.compactIndex() + try: + wrappedClient.client.compactIndex() + except grpc._channel._InactiveRpcError as e: + if e.details() == "tbtree: compaction threshold not yet reached": + print(e) + else: + raise e else: pytest.skip("Server version too low") diff --git a/tests/immu/test_session.py b/tests/immu/test_session.py index 76bbe77..63c7323 100644 --- a/tests/immu/test_session.py +++ b/tests/immu/test_session.py @@ -32,6 +32,8 @@ def test_simple_unmanaged_session(self, wrappedClient: ImmuTestClient): newTx = txInterface.newTx() newTx.sqlExec(f"INSERT INTO {table} (tester) VALUES(@testParam)", params = {"testParam": "123"}) what = newTx.sqlQuery(f"SELECT * FROM {table}", dict(), columnNameMode=constants.COLUMN_NAME_MODE_FIELD) + if wrappedClient.serverVersionEqual("1.9DOM.0") and what==[]: + pytest.xfail("Known bug #1854") assert what == [{"id": 1, "tester": '123'}] commit = newTx.commit() assert commit.header.id != None @@ -66,6 +68,8 @@ def test_simple_managed_session(self, wrappedClient: ImmuTestClient): newTx = session.newTx() newTx.sqlExec(f"INSERT INTO {table} (tester) VALUES(@testParam)", params = {"testParam": "123"}) what = newTx.sqlQuery(f"SELECT * FROM {table}", dict(), columnNameMode=constants.COLUMN_NAME_MODE_FIELD) + if wrappedClient.serverVersionEqual("1.9DOM.0") and what==[]: + pytest.xfail("Known bug #1854") assert what == [{"id": 1, "tester": '123'}] commit = newTx.commit() assert commit.header.id != None @@ -203,4 +207,4 @@ def test_managed_session(self, wrappedClient: ImmuTestClient): assert concatenated == ["3", "4", "5", "6", "7", "8"] what = wrappedClient.commit() - \ No newline at end of file + diff --git a/tests/immu/test_sql.py b/tests/immu/test_sql.py index e6228d8..1ee19e5 100644 --- a/tests/immu/test_sql.py +++ b/tests/immu/test_sql.py @@ -41,8 +41,12 @@ def test_exec_query(self, wrappedClient: ImmuTestClient): assert(result == [(1, "Joe")]) def test_describe(self, wrappedClient: ImmuTestClient): + if wrappedClient.serverHigherOrEqualsToVersion("1.9.0"): + varchar100 = "VARCHAR(100)" + else: + varchar100 = "VARCHAR[100]" tbname = wrappedClient.createTestTable("id INTEGER", "name VARCHAR[100]", "PRIMARY KEY id") - + response = wrappedClient.client.describeTable(tbname) assert response == [ColumnDescription(name='id', type='INTEGER', nullable=True, index='PRIMARY KEY', autoincrement=False, unique=True), ColumnDescription( - name='name', type='VARCHAR[100]', nullable=True, index='NO', autoincrement=False, unique=False)] + name='name', type=varchar100, nullable=True, index='NO', autoincrement=False, unique=False)] diff --git a/tests/immuTestClient.py b/tests/immuTestClient.py index 70dba70..68c7ce6 100644 --- a/tests/immuTestClient.py +++ b/tests/immuTestClient.py @@ -46,14 +46,17 @@ def serverHigherOrEqualsToVersion(self, version: str): health = self.client.health() return self.compare_version(health.version, version) > -1 + def serverVersionEqual(self, version: str): + health = self.client.health() + return self.compare_version(health.version, version) == 0 + def executeWithTransaction(self, concatenatedParams: dict, queries: List[str], separator="\n"): toExecute = [self.transactionStart] toExecute.extend(queries) toExecute.append(self.transactionEnd) multiLineQuery = separator.join(toExecute) resp = self.client.sqlExec(multiLineQuery, concatenatedParams) - assert((len(resp.txs) > 0 and not resp.ongoingTx and not resp.UnknownFields()) - or len(resp.UnknownFields()) > 0) + assert(len(resp.txs) > 0 and not resp.ongoingTx) return resp def _generateTableName(self): @@ -104,8 +107,7 @@ def insertToTable(self, table: str, fields: List[str], values: List, params: dic return self.currentTx.sqlExec(preparedQuery, params) resp = self.client.sqlExec(preparedQuery, params) - assert((len(resp.txs) > 0 and not resp.ongoingTx and not resp.UnknownFields()) - or len(resp.UnknownFields()) > 0) + assert(len(resp.txs) > 0 and not resp.ongoingTx) return resp def simpleSelect(self, fromWhat: str, whatToSelect: List[str], params: dict, *conditions: List[str], columnNameMode = constants.COLUMN_NAME_MODE_NONE):