Merge remote-tracking branch 'origin/master' into test/no_dataclasses

.github/workflows/ci.yml

@@ -21,10 +21,10 @@ jobs:
           make dev
       - name: Start immudb container
         run: |
-          docker run -d --health-cmd "immuadmin status" --health-interval 10s --health-timeout 5s --health-retries 5 -v ${{ github.workspace }}/tests/certs/my.key.pem:/key.pem -p 3322:3322 codenotary/immudb:1.4.0 --signingKey=/key.pem
-          docker run -d --health-cmd "immuadmin status" --health-interval 10s --health-timeout 5s --health-retries 5 -v ${{ github.workspace }}/tests/certs/my.key.pem:/key.pem -p 3333:3322 codenotary/immudb:1.3.2 --signingKey=/key.pem
-          docker run -d --health-cmd "immuadmin status" --health-interval 10s --health-timeout 5s --health-retries 5 -v ${{ github.workspace }}/tests/certs/my.key.pem:/key.pem -p 3344:3322 codenotary/immudb:1.2.4 --signingKey=/key.pem
-          docker run -d --health-cmd "immuadmin status" --health-interval 10s --health-timeout 5s --health-retries 5 -v ${{ github.workspace }}/tests/certs/my.key.pem:/key.pem -p 3355:3322 codenotary/immudb:1.1.0 --signingKey=/key.pem
+          docker run -d --health-cmd "immuadmin status" --health-interval 10s --health-timeout 5s --health-retries 5 -v ${{ github.workspace }}/tests/certs/my.key.pem:/key.pem -p 3322:3322 codenotary/immudb:1.9DOM --signingKey=/key.pem
+          docker run -d --health-cmd "immuadmin status" --health-interval 10s --health-timeout 5s --health-retries 5 -v ${{ github.workspace }}/tests/certs/my.key.pem:/key.pem -p 3333:3322 codenotary/immudb:1.5.0 --signingKey=/key.pem
+          docker run -d --health-cmd "immuadmin status" --health-interval 10s --health-timeout 5s --health-retries 5 -v ${{ github.workspace }}/tests/certs/my.key.pem:/key.pem -p 3344:3322 codenotary/immudb:1.4.1 --signingKey=/key.pem
+          docker run -d --health-cmd "immuadmin status" --health-interval 10s --health-timeout 5s --health-retries 5 -v ${{ github.workspace }}/tests/certs/my.key.pem:/key.pem -p 3355:3322 codenotary/immudb:1.4.0 --signingKey=/key.pem
       - name: Run tests
         run: |
           make test

immudb/client.py

@@ -79,6 +79,7 @@ def __init__(self, immudUrl=None, rs: RootService = None, publicKeyFile: str = N
             self._rs = rs
         self._url = immudUrl
         self._vk = None
+        self._currentdb = None
         if publicKeyFile:
             self.loadKey(publicKeyFile)
 
@@ -216,13 +217,15 @@ def login(self, username, password, database=b"defaultdb"):
         self._stub = self._set_token_header_interceptor(resp)
 
         self._rs.init("{}/{}".format(self._url, database), self._stub)
+        self._currentdb = convertedDatabase
         return login_response
 
     def logout(self):
         """Logouts all sessions
         """
         self._stub.Logout(google_dot_protobuf_dot_empty__pb2.Empty())
         self._resetStub()
+        self._currentdb = None
 
     def _resetStub(self):
         self.headersInterceptors = []
@@ -511,6 +514,7 @@ def useDatabase(self, dbName: bytes):
         # modifies header token accordingly
         self._stub = self._set_token_header_interceptor(resp)
         self._rs.init(dbName, self._stub)
+        self._currentdb = dbName
         return resp
 
     def getDatabaseSettingsV2(self) -> datatypesv2.DatabaseSettingsResponseV2:
@@ -1620,7 +1624,17 @@ def sqlQuery(self, query, params={}, columnNameMode=constants.COLUMN_NAME_MODE_N
 
             ['table1', 'table2']
         """
-        return sqlquery.call(self._stub, self._rs, query, params, columnNameMode)
+        ret = sqlquery.call(self._stub, self._rs, query,
+                            params, columnNameMode)
+        if columnNameMode in [constants.COLUMN_NAME_MODE_DATABASE, constants.COLUMN_NAME_MODE_FULL]:
+            # newer DB version don't insert database name anymore, we need to
+            # process it manually
+            for i, t in enumerate(ret):
+                newkeys = [
+                    x.replace("[@DB]", self._currentdb.decode("utf-8")) for x in t.keys()]
+                k = dict(zip(newkeys, list(t.values())))
+                ret[i] = k
+        return ret
 
     def listTables(self):
         """List all tables in the current database
@@ -1696,7 +1710,6 @@ def verifiableSQLGet(self, table: str, primaryKeys: List[datatypesv2.PrimaryKey]
 
 # immudb-py only
 
-
     def getAllValues(self, keys: list):  # immudb-py only
         resp = batchGet.call(self._stub, self._rs, keys)
         return resp

immudb/handler/sqlquery.py

@@ -35,7 +35,6 @@ def _call_with_executor(query, params, columnNameMode, executor):
 
     resp = executor(request)
     result = []
-
     columnNames = getColumnNames(resp, columnNameMode)
 
     for row in resp.rows:
@@ -51,14 +50,27 @@ def getColumnNames(resp, columnNameMode):
     columnNames = []
     if columnNameMode:
         for column in resp.columns:
+            # note that depending on the version parts can be
+            # '(dbname.tablename.fieldname)' *or*
+            # '(tablename.fieldname)' without dbnname.
+            # In that case we mimic the old behavior by using [@DB] as placeholder
+            # that will be replaced at higher level.
+            parts = column.name.strip("()").split(".")
             if columnNameMode == constants.COLUMN_NAME_MODE_FIELD:
-                columnNames.append(column.name.strip("()").split(".")[2])
-            elif columnNameMode == constants.COLUMN_NAME_MODE_TABLE:
-                columnNames.append(column.name.strip("()").split(".", 1)[1])
-            elif columnNameMode == constants.COLUMN_NAME_MODE_DATABASE:
-                columnNames.append(column.name.strip("()"))
-            elif columnNameMode == constants.COLUMN_NAME_MODE_FULL:
-                columnNames.append(column.name)
-            else:
-                raise ErrPySDKInvalidColumnMode
+                columnNames.append(parts[-1])
+                continue
+            if columnNameMode == constants.COLUMN_NAME_MODE_TABLE:
+                columnNames.append(".".join(parts[-2:]))
+                continue
+            print(
+                "Use of COLUMN_NAME_MODE_DATABASE and COLUMN_NAME_MODE_FULL is deprecated")
+            if len(parts) == 2:
+                parts.insert(0, "[@DB]")
+            if columnNameMode == constants.COLUMN_NAME_MODE_DATABASE:
+                columnNames.append(".".join(parts))
+                continue
+            if columnNameMode == constants.COLUMN_NAME_MODE_FULL:
+                columnNames.append("("+".".join(parts)+")")
+                continue
+            raise ErrPySDKInvalidColumnMode
     return columnNames

immudb/handler/verifiedSet.py

@@ -21,8 +21,6 @@
 import immudb.schema as schema
 from immudb.typeconv import MetadataToProto
 
-#import base64
-
 
 def call(service: schema_pb2_grpc.ImmuServiceStub, rs: RootService, key: bytes, value: bytes, verifying_key=None, metadata=None):
     schemaMetadata = MetadataToProto(metadata)

tests/immu/test_export_tx.py

@@ -9,6 +9,11 @@ def test_export_tx_replicate_tx(wrappedClient: ImmudbClient):
     client = wrappedClient.client
     if(not wrappedClient.serverHigherOrEqualsToVersion("1.2.0")):
         pytest.skip("Immudb version too low")
+    if wrappedClient.serverHigherOrEqualsToVersion("1.5.0"):
+        offset = 0
+    else:
+        offset = 1
+
     newuuid1 = str(uuid.uuid4()).replace("-", "")
     client.createDatabaseV2(newuuid1, settings = datatypesv2.DatabaseSettingsV2(
     ), ifNotExists=False)
@@ -32,7 +37,7 @@ def test_export_tx_replicate_tx(wrappedClient: ImmudbClient):
         assert txHeader.id > 0
         assert txHeader.nentries == 1
         if(index > 1): # First transaction is not db set
-            assert client.get(b'kisz123123kaaaa').value.decode("utf-8") == str(index - 1) 
+            assert client.get(b'kisz123123kaaaa').value.decode("utf-8") == str(index - offset)
         client.useDatabase(newuuid1)
     client.useDatabase(newuuid2)
-    assert client.get(b'kisz123123kaaaa').value == b'5'
+    assert client.get(b'kisz123123kaaaa').value == b'5'

tests/immu/test_massive_operation.py

@@ -15,6 +15,7 @@
 import time
 from tests.immuTestClient import ImmuTestClient
 import pytest
+import grpc
 
 
 def get_random_string(length):
@@ -46,6 +47,12 @@ def test_get_set_massive(self, client):
     def test_compact(self, wrappedClient: ImmuTestClient):
         # Snapshot compaction fixed from 1.3.1
         if(wrappedClient.serverHigherOrEqualsToVersion("1.3.0")):
-            wrappedClient.client.compactIndex()
+            try:
+                wrappedClient.client.compactIndex()
+            except grpc._channel._InactiveRpcError as e:
+                if e.details() == "tbtree: compaction threshold not yet reached":
+                    print(e)
+                else:
+                    raise e
         else:
             pytest.skip("Server version too low")

tests/immu/test_session.py

@@ -32,6 +32,8 @@ def test_simple_unmanaged_session(self, wrappedClient: ImmuTestClient):
             newTx = txInterface.newTx()
             newTx.sqlExec(f"INSERT INTO {table} (tester) VALUES(@testParam)", params = {"testParam": "123"})
             what = newTx.sqlQuery(f"SELECT * FROM {table}", dict(), columnNameMode=constants.COLUMN_NAME_MODE_FIELD)
+            if wrappedClient.serverVersionEqual("1.9DOM.0") and what==[]:
+                pytest.xfail("Known bug #1854")
             assert what == [{"id": 1, "tester": '123'}]
             commit = newTx.commit()
             assert commit.header.id != None
@@ -66,6 +68,8 @@ def test_simple_managed_session(self, wrappedClient: ImmuTestClient):
             newTx = session.newTx()
             newTx.sqlExec(f"INSERT INTO {table} (tester) VALUES(@testParam)", params = {"testParam": "123"})
             what = newTx.sqlQuery(f"SELECT * FROM {table}", dict(), columnNameMode=constants.COLUMN_NAME_MODE_FIELD)
+            if wrappedClient.serverVersionEqual("1.9DOM.0") and what==[]:
+                pytest.xfail("Known bug #1854")
             assert what == [{"id": 1, "tester": '123'}]
             commit = newTx.commit()
             assert commit.header.id != None
@@ -203,4 +207,4 @@ def test_managed_session(self, wrappedClient: ImmuTestClient):
             assert concatenated == ["3", "4", "5", "6", "7", "8"]
             what = wrappedClient.commit()
 
-
+

tests/immu/test_sql.py

@@ -41,8 +41,12 @@ def test_exec_query(self, wrappedClient: ImmuTestClient):
         assert(result == [(1, "Joe")])
 
     def test_describe(self, wrappedClient: ImmuTestClient):
+        if wrappedClient.serverHigherOrEqualsToVersion("1.9.0"):
+            varchar100 = "VARCHAR(100)"
+        else:
+            varchar100 = "VARCHAR[100]"
         tbname = wrappedClient.createTestTable("id INTEGER", "name VARCHAR[100]", "PRIMARY KEY id")
-        
+
         response = wrappedClient.client.describeTable(tbname)
         assert response == [ColumnDescription(name='id', type='INTEGER', nullable=True, index='PRIMARY KEY', autoincrement=False, unique=True), ColumnDescription(
-            name='name', type='VARCHAR[100]', nullable=True, index='NO', autoincrement=False, unique=False)]
+            name='name', type=varchar100, nullable=True, index='NO', autoincrement=False, unique=False)]

tests/immuTestClient.py

@@ -46,14 +46,17 @@ def serverHigherOrEqualsToVersion(self, version: str):
         health = self.client.health()
         return self.compare_version(health.version, version) > -1
 
+    def serverVersionEqual(self, version: str):
+        health = self.client.health()
+        return self.compare_version(health.version, version) == 0
+
     def executeWithTransaction(self, concatenatedParams: dict, queries: List[str], separator="\n"):
         toExecute = [self.transactionStart]
         toExecute.extend(queries)
         toExecute.append(self.transactionEnd)
         multiLineQuery = separator.join(toExecute)
         resp = self.client.sqlExec(multiLineQuery, concatenatedParams)
-        assert((len(resp.txs) > 0 and not resp.ongoingTx and not resp.UnknownFields())
-               or len(resp.UnknownFields()) > 0)
+        assert(len(resp.txs) > 0 and not resp.ongoingTx)
         return resp
 
     def _generateTableName(self):
@@ -104,8 +107,7 @@ def insertToTable(self, table: str, fields: List[str], values: List, params: dic
             return self.currentTx.sqlExec(preparedQuery, params)
 
         resp = self.client.sqlExec(preparedQuery, params)
-        assert((len(resp.txs) > 0 and not resp.ongoingTx and not resp.UnknownFields())
-               or len(resp.UnknownFields()) > 0)
+        assert(len(resp.txs) > 0 and not resp.ongoingTx)
         return resp
 
     def simpleSelect(self, fromWhat: str, whatToSelect: List[str], params: dict, *conditions: List[str], columnNameMode = constants.COLUMN_NAME_MODE_NONE):