From 0212487efd09af9b433404f777545f7b61507f90 Mon Sep 17 00:00:00 2001 From: Aymaru Date: Mon, 22 Feb 2021 09:57:50 -0600 Subject: [PATCH 01/25] return a list instead of a set on get_pk_constraint This is to fix the apache superset bug, that doesn't show the cratedb table metadata in the sqleditor --- src/crate/client/sqlalchemy/dialect.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/crate/client/sqlalchemy/dialect.py b/src/crate/client/sqlalchemy/dialect.py index 8c3bc91a..d680e4b0 100644 --- a/src/crate/client/sqlalchemy/dialect.py +++ b/src/crate/client/sqlalchemy/dialect.py @@ -43,6 +43,7 @@ "smallint": sqltypes.SmallInteger, "timestamp": sqltypes.TIMESTAMP, "timestamp with time zone": sqltypes.TIMESTAMP, + "timestamp without time zone": sqltypes.TIMESTAMP, "object": Object, "integer": sqltypes.Integer, "long": sqltypes.NUMERIC, @@ -64,6 +65,7 @@ TYPES_MAP["smallint_array"] = ARRAY(sqltypes.SmallInteger) TYPES_MAP["timestamp_array"] = ARRAY(sqltypes.TIMESTAMP) TYPES_MAP["timestamp with time zone_array"] = ARRAY(sqltypes.TIMESTAMP) + TYPES_MAP["timestamp without time zone_array"] = ARRAY(sqltypes.TIMESTAMP) TYPES_MAP["long_array"] = ARRAY(sqltypes.NUMERIC) TYPES_MAP["bigint_array"] = ARRAY(sqltypes.NUMERIC) TYPES_MAP["double_array"] = ARRAY(sqltypes.DECIMAL) @@ -75,7 +77,6 @@ except Exception: pass - log = logging.getLogger(__name__) @@ -261,7 +262,7 @@ def get_pk_constraint(self, engine, table_name, schema=None, **kw): def result_fun(result): rows = result.fetchall() - return set(map(lambda el: el[0], rows)) + return list(set(map(lambda el: el[0], rows))) else: query = """SELECT constraint_name FROM information_schema.table_constraints From f9bd08c8f1fbb5e738324346ad8303887da8ce1e Mon Sep 17 00:00:00 2001 From: Aymaru Date: Mon, 22 Feb 2021 09:59:15 -0600 Subject: [PATCH 02/25] Changed the sql_path This returns the data type of the columns in a query, to allow us to transform the dates from cratedb (long int) to python datetime --- src/crate/client/http.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/crate/client/http.py b/src/crate/client/http.py index 07e31e7b..29927762 100644 --- a/src/crate/client/http.py +++ b/src/crate/client/http.py @@ -315,7 +315,7 @@ class Client(object): Crate connection client using crate's HTTP API. """ - SQL_PATH = '/_sql' + SQL_PATH = '/_sql?types' """Crate URI path for issuing SQL statements.""" retry_interval = 30 From 66e7b512d3c5cd44f04324a51ba410a1fea0b767 Mon Sep 17 00:00:00 2001 From: Aymaru Date: Mon, 22 Feb 2021 10:00:48 -0600 Subject: [PATCH 03/25] Transform dates from crate to python datetime In execute(), transform the columns with type timestamp and timestamp without time zone to python datetime, this will correctly display dates in apache superset --- src/crate/client/cursor.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/src/crate/client/cursor.py b/src/crate/client/cursor.py index 329cb9ff..f63043f8 100644 --- a/src/crate/client/cursor.py +++ b/src/crate/client/cursor.py @@ -22,6 +22,7 @@ from .exceptions import ProgrammingError from distutils.version import StrictVersion import warnings +from datetime import datetime BULK_INSERT_MIN_VERSION = StrictVersion("0.42.0") @@ -53,8 +54,35 @@ def execute(self, sql, parameters=None, bulk_parameters=None): self._result = self.connection.client.sql(sql, parameters, bulk_parameters) if "rows" in self._result: + if "col_types" in self._result: + col_types = self._result["col_types"] + tmp_data = self._result["rows"] + + rows_to_convert = self._get_rows_to_convert_to_date(col_types) + tmp_data = self._convert_dates_to_datetime(tmp_data, rows_to_convert) + + self._result["rows"] = tmp_data + self.rows = iter(self._result["rows"]) + @staticmethod + def _get_rows_to_convert_to_date(col_types): + return [True if col_type == 11 or col_type == 15 else False for col_type in col_types] + + @staticmethod + def _date_to_datetime(row, rows_to_convert): + return list( + map(lambda x, y: + datetime.fromtimestamp(float(str(x)[0:10])) if y else x, + row, + rows_to_convert)) + + def _convert_dates_to_datetime(self, rows, rows_to_convert): + return list( + map(lambda x: + self._date_to_datetime(x, rows_to_convert), + rows)) + def executemany(self, sql, seq_of_parameters): """ Prepare a database operation (query or command) and then execute it From 3dc5cb7cedb698a8002baf233934eb2c92d579e1 Mon Sep 17 00:00:00 2001 From: Aymaru Date: Tue, 2 Mar 2021 15:53:05 -0600 Subject: [PATCH 04/25] Datetime conversion implemented using map and generator --- src/crate/client/cursor.py | 38 ++++++++++++++++++++------------------ 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/src/crate/client/cursor.py b/src/crate/client/cursor.py index f63043f8..70d9c644 100644 --- a/src/crate/client/cursor.py +++ b/src/crate/client/cursor.py @@ -55,33 +55,35 @@ def execute(self, sql, parameters=None, bulk_parameters=None): bulk_parameters) if "rows" in self._result: if "col_types" in self._result: - col_types = self._result["col_types"] - tmp_data = self._result["rows"] - - rows_to_convert = self._get_rows_to_convert_to_date(col_types) - tmp_data = self._convert_dates_to_datetime(tmp_data, rows_to_convert) - - self._result["rows"] = tmp_data - - self.rows = iter(self._result["rows"]) + rows_to_convert = self._get_rows_to_convert_to_date(self._result["col_types"]) + self.rows = self._convert_dates_to_datetime(self._result["rows"], rows_to_convert) + else: + self.rows = iter(self._result["rows"]) @staticmethod def _get_rows_to_convert_to_date(col_types): + """ + Generates a list of boolean. True if the column is type timestamp (11 - 15) + """ return [True if col_type == 11 or col_type == 15 else False for col_type in col_types] @staticmethod def _date_to_datetime(row, rows_to_convert): - return list( - map(lambda x, y: - datetime.fromtimestamp(float(str(x)[0:10])) if y else x, - row, - rows_to_convert)) + """ + Converts all values epoch to a datetime object in a given row + """ + return map(lambda value, flag: + datetime.fromtimestamp(float(str(value)[0:10])) if (flag and value is not None) else value, + row, + rows_to_convert) def _convert_dates_to_datetime(self, rows, rows_to_convert): - return list( - map(lambda x: - self._date_to_datetime(x, rows_to_convert), - rows)) + """ + Takes a list of rows and map each row to convert date columns to timestamp + """ + return map(lambda x: + self._date_to_datetime(x, (flag for flag in rows_to_convert)), + rows) def executemany(self, sql, seq_of_parameters): """ From 8dfcf80737ae0b7ce0aa81d762de778f9167f3ef Mon Sep 17 00:00:00 2001 From: Aymaru Date: Tue, 2 Mar 2021 16:30:32 -0600 Subject: [PATCH 05/25] Updated tests --- src/crate/client/sqlalchemy/tests/dialect_test.py | 2 +- src/crate/client/test_http.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/crate/client/sqlalchemy/tests/dialect_test.py b/src/crate/client/sqlalchemy/tests/dialect_test.py index 8a6f0006..754eade0 100644 --- a/src/crate/client/sqlalchemy/tests/dialect_test.py +++ b/src/crate/client/sqlalchemy/tests/dialect_test.py @@ -76,7 +76,7 @@ def test_pks_are_retrieved_depending_on_version_set(self): self.engine.dialect.server_version_info = (2, 3, 0) fake_cursor.rowcount = 3 fake_cursor.fetchall = MagicMock(return_value=[["id"], ["id2"], ["id3"]]) - eq_(insp.get_pk_constraint("characters")['constrained_columns'], {"id", "id2", "id3"}) + eq_(insp.get_pk_constraint("characters")['constrained_columns'], ["id", "id2", "id3"]) fake_cursor.fetchall.assert_called_once_with() in_("information_schema.key_column_usage", self.executed_statement) diff --git a/src/crate/client/test_http.py b/src/crate/client/test_http.py index aa115714..5cde667e 100644 --- a/src/crate/client/test_http.py +++ b/src/crate/client/test_http.py @@ -433,7 +433,7 @@ def test_params(self): def test_no_params(self): client = Client() - self.assertEqual(client.path, "/_sql") + self.assertEqual(client.path, "/_sql?types") client.close() From b08cb166263f2511858ce9109d488f6d3ae9c4c4 Mon Sep 17 00:00:00 2001 From: Aymaru Date: Wed, 3 Mar 2021 14:46:20 -0600 Subject: [PATCH 06/25] Using generators to work with large datasets --- src/crate/client/cursor.py | 37 ++++++++++++++++++------------------- 1 file changed, 18 insertions(+), 19 deletions(-) diff --git a/src/crate/client/cursor.py b/src/crate/client/cursor.py index 70d9c644..b8098116 100644 --- a/src/crate/client/cursor.py +++ b/src/crate/client/cursor.py @@ -56,34 +56,33 @@ def execute(self, sql, parameters=None, bulk_parameters=None): if "rows" in self._result: if "col_types" in self._result: rows_to_convert = self._get_rows_to_convert_to_date(self._result["col_types"]) - self.rows = self._convert_dates_to_datetime(self._result["rows"], rows_to_convert) - else: - self.rows = iter(self._result["rows"]) + for flag in rows_to_convert: + if flag: + t_rows = (row for row in self._result["rows"]) + t_values = (self._transform_date_columns(row, rows_to_convert) for row in t_rows) + self._result["rows"] = [value for value in t_values] + break + self.rows = iter(self._result["rows"]) @staticmethod - def _get_rows_to_convert_to_date(col_types): + def _transform_date_columns(row, flags): """ Generates a list of boolean. True if the column is type timestamp (11 - 15) """ - return [True if col_type == 11 or col_type == 15 else False for col_type in col_types] + gen_flags = (flag for flag in flags) + for value in row: + flag = next(gen_flags) + if not flag or value is None: + yield value + else: + yield datetime.fromtimestamp(float(str(value)[0:10])) @staticmethod - def _date_to_datetime(row, rows_to_convert): - """ - Converts all values epoch to a datetime object in a given row - """ - return map(lambda value, flag: - datetime.fromtimestamp(float(str(value)[0:10])) if (flag and value is not None) else value, - row, - rows_to_convert) - - def _convert_dates_to_datetime(self, rows, rows_to_convert): + def _get_rows_to_convert_to_date(col_types): """ - Takes a list of rows and map each row to convert date columns to timestamp + Generates a list of boolean. True if the column is type timestamp (11 - 15) """ - return map(lambda x: - self._date_to_datetime(x, (flag for flag in rows_to_convert)), - rows) + return [True if col_type == 11 or col_type == 15 else False for col_type in col_types] def executemany(self, sql, seq_of_parameters): """ From efe8bfbe30f18d8e9a801f140ba96edefc53f0ef Mon Sep 17 00:00:00 2001 From: Aymaru Date: Wed, 3 Mar 2021 14:51:24 -0600 Subject: [PATCH 07/25] changed tests --- src/crate/client/sqlalchemy/tests/dialect_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/crate/client/sqlalchemy/tests/dialect_test.py b/src/crate/client/sqlalchemy/tests/dialect_test.py index 754eade0..105a297e 100644 --- a/src/crate/client/sqlalchemy/tests/dialect_test.py +++ b/src/crate/client/sqlalchemy/tests/dialect_test.py @@ -67,7 +67,7 @@ def test_pks_are_retrieved_depending_on_version_set(self): self.engine.dialect.server_version_info = (0, 54, 0) fake_cursor.rowcount = 1 fake_cursor.fetchone = MagicMock(return_value=[["id", "id2", "id3"]]) - eq_(insp.get_pk_constraint("characters")['constrained_columns'], {"id", "id2", "id3"}) + eq_(insp.get_pk_constraint("characters")['constrained_columns'], ["id", "id2", "id3"]) fake_cursor.fetchone.assert_called_once_with() in_("information_schema.table_constraints", self.executed_statement) From 2d1f41f427f4aa136a2c63bb1118a09c55da1c19 Mon Sep 17 00:00:00 2001 From: Aymaru Date: Wed, 3 Mar 2021 15:01:02 -0600 Subject: [PATCH 08/25] fix --- src/crate/client/cursor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/crate/client/cursor.py b/src/crate/client/cursor.py index b8098116..82827907 100644 --- a/src/crate/client/cursor.py +++ b/src/crate/client/cursor.py @@ -59,7 +59,7 @@ def execute(self, sql, parameters=None, bulk_parameters=None): for flag in rows_to_convert: if flag: t_rows = (row for row in self._result["rows"]) - t_values = (self._transform_date_columns(row, rows_to_convert) for row in t_rows) + t_values = [self._transform_date_columns(row, rows_to_convert) for row in t_rows] self._result["rows"] = [value for value in t_values] break self.rows = iter(self._result["rows"]) From 246d917d66726ce9a08ce07e113bf3fcc3762815 Mon Sep 17 00:00:00 2001 From: Aymaru Date: Wed, 3 Mar 2021 15:15:25 -0600 Subject: [PATCH 09/25] test --- src/crate/client/doctests/client.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/crate/client/doctests/client.txt b/src/crate/client/doctests/client.txt index b564c04d..3b1f59d3 100644 --- a/src/crate/client/doctests/client.txt +++ b/src/crate/client/doctests/client.txt @@ -179,7 +179,7 @@ supported, all other fields are 'None':: >>> result = cursor.fetchone() >>> pprint(result) ['Aldebaran', - 1373932800000, + datetime.datetime(2013, 7, 15, 18, 0), None, None, None, From 94b3d1eeb762032cd8385054a40bed932d205341 Mon Sep 17 00:00:00 2001 From: Aymaru Date: Thu, 4 Mar 2021 09:42:30 -0600 Subject: [PATCH 10/25] updated datetime transformation using generators and updated test cases --- src/crate/client/cursor.py | 11 ++++++++--- src/crate/client/doctests/http.txt | 3 ++- src/crate/client/test_http.py | 7 ++++++- 3 files changed, 16 insertions(+), 5 deletions(-) diff --git a/src/crate/client/cursor.py b/src/crate/client/cursor.py index 82827907..4fac96a8 100644 --- a/src/crate/client/cursor.py +++ b/src/crate/client/cursor.py @@ -54,15 +54,19 @@ def execute(self, sql, parameters=None, bulk_parameters=None): self._result = self.connection.client.sql(sql, parameters, bulk_parameters) if "rows" in self._result: + print(self._result["rows"]) if "col_types" in self._result: + print('testing') rows_to_convert = self._get_rows_to_convert_to_date(self._result["col_types"]) for flag in rows_to_convert: if flag: t_rows = (row for row in self._result["rows"]) - t_values = [self._transform_date_columns(row, rows_to_convert) for row in t_rows] - self._result["rows"] = [value for value in t_values] + t_values = (self._transform_date_columns(row, rows_to_convert) for row in t_rows) + self._result["rows"] = [[value for value in row] for row in t_values] break self.rows = iter(self._result["rows"]) + print(self._result["rows"]) + print(self.rows) @staticmethod def _transform_date_columns(row, flags): @@ -75,7 +79,8 @@ def _transform_date_columns(row, flags): if not flag or value is None: yield value else: - yield datetime.fromtimestamp(float(str(value)[0:10])) + value = datetime.fromtimestamp(float(str(value)[0:10])) + yield value @staticmethod def _get_rows_to_convert_to_date(col_types): diff --git a/src/crate/client/doctests/http.txt b/src/crate/client/doctests/http.txt index 5382b5b6..1991d4ce 100644 --- a/src/crate/client/doctests/http.txt +++ b/src/crate/client/doctests/http.txt @@ -69,7 +69,8 @@ Issue a select statement against our with test data pre-filled crate instance:: >>> http_client = HttpClient(crate_host) >>> result = http_client.sql('select name from locations order by name') >>> pprint(result) - {'cols': ['name'], + {'col_types': [4], + 'cols': ['name'], 'duration': ..., 'rowcount': 13, 'rows': [['Aldebaran'], diff --git a/src/crate/client/test_http.py b/src/crate/client/test_http.py index 5cde667e..7b02771f 100644 --- a/src/crate/client/test_http.py +++ b/src/crate/client/test_http.py @@ -428,7 +428,8 @@ def test_params(self): client = Client(['127.0.0.1:4200'], error_trace=True) parsed = urlparse(client.path) params = parse_qs(parsed.query) - self.assertEqual(params["error_trace"], ["true"]) + print(params) + self.assertEqual(params["types?error_trace"], ["true"]) client.close() def test_no_params(self): @@ -623,3 +624,7 @@ def test_username(self): self.assertEqual(TestingHTTPServer.SHARED['usernameFromXUser'], 'testDBUser') self.assertEqual(TestingHTTPServer.SHARED['username'], 'testDBUser') self.assertEqual(TestingHTTPServer.SHARED['password'], 'test:password') + +pt = ParamsTest() +pt.test_params() +pt.test_no_params() \ No newline at end of file From 85b7c7fbaefa6a383a2f42a6b9b4c05c2ff718a9 Mon Sep 17 00:00:00 2001 From: Aymaru Date: Thu, 4 Mar 2021 09:44:02 -0600 Subject: [PATCH 11/25] cleaning debug prints --- src/crate/client/cursor.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/crate/client/cursor.py b/src/crate/client/cursor.py index 4fac96a8..296c8292 100644 --- a/src/crate/client/cursor.py +++ b/src/crate/client/cursor.py @@ -54,9 +54,7 @@ def execute(self, sql, parameters=None, bulk_parameters=None): self._result = self.connection.client.sql(sql, parameters, bulk_parameters) if "rows" in self._result: - print(self._result["rows"]) if "col_types" in self._result: - print('testing') rows_to_convert = self._get_rows_to_convert_to_date(self._result["col_types"]) for flag in rows_to_convert: if flag: @@ -65,8 +63,6 @@ def execute(self, sql, parameters=None, bulk_parameters=None): self._result["rows"] = [[value for value in row] for row in t_values] break self.rows = iter(self._result["rows"]) - print(self._result["rows"]) - print(self.rows) @staticmethod def _transform_date_columns(row, flags): From 2cf32e279a1ccaef52e0afc71729caaa0d518e01 Mon Sep 17 00:00:00 2001 From: Aymaru Date: Thu, 4 Mar 2021 10:26:51 -0600 Subject: [PATCH 12/25] Passing a generator of the flags instead of passing the list of values --- src/crate/client/cursor.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/crate/client/cursor.py b/src/crate/client/cursor.py index 296c8292..aae621fe 100644 --- a/src/crate/client/cursor.py +++ b/src/crate/client/cursor.py @@ -59,17 +59,17 @@ def execute(self, sql, parameters=None, bulk_parameters=None): for flag in rows_to_convert: if flag: t_rows = (row for row in self._result["rows"]) - t_values = (self._transform_date_columns(row, rows_to_convert) for row in t_rows) + t_values = (self._transform_date_columns(row, (flag for flag in rows_to_convert)) + for row in t_rows) self._result["rows"] = [[value for value in row] for row in t_values] break self.rows = iter(self._result["rows"]) @staticmethod - def _transform_date_columns(row, flags): + def _transform_date_columns(row, gen_flags): """ Generates a list of boolean. True if the column is type timestamp (11 - 15) """ - gen_flags = (flag for flag in flags) for value in row: flag = next(gen_flags) if not flag or value is None: From fbef4487a51c5141b315d2368486115fa0df3997 Mon Sep 17 00:00:00 2001 From: Aymaru Date: Thu, 4 Mar 2021 10:36:06 -0600 Subject: [PATCH 13/25] Removed tests --- src/crate/client/test_http.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/crate/client/test_http.py b/src/crate/client/test_http.py index 7b02771f..bf8e7134 100644 --- a/src/crate/client/test_http.py +++ b/src/crate/client/test_http.py @@ -624,7 +624,3 @@ def test_username(self): self.assertEqual(TestingHTTPServer.SHARED['usernameFromXUser'], 'testDBUser') self.assertEqual(TestingHTTPServer.SHARED['username'], 'testDBUser') self.assertEqual(TestingHTTPServer.SHARED['password'], 'test:password') - -pt = ParamsTest() -pt.test_params() -pt.test_no_params() \ No newline at end of file From 57ec0908ea19f7601382996a2958e70fe8be33fc Mon Sep 17 00:00:00 2001 From: Aymaru Date: Tue, 30 Mar 2021 12:34:34 -0600 Subject: [PATCH 14/25] updated conversion of timestamps --- src/crate/client/cursor.py | 41 +++++++++++++++++++------------------- 1 file changed, 21 insertions(+), 20 deletions(-) diff --git a/src/crate/client/cursor.py b/src/crate/client/cursor.py index aae621fe..49bdef76 100644 --- a/src/crate/client/cursor.py +++ b/src/crate/client/cursor.py @@ -22,7 +22,7 @@ from .exceptions import ProgrammingError from distutils.version import StrictVersion import warnings -from datetime import datetime +import pandas as pd BULK_INSERT_MIN_VERSION = StrictVersion("0.42.0") @@ -55,35 +55,36 @@ def execute(self, sql, parameters=None, bulk_parameters=None): bulk_parameters) if "rows" in self._result: if "col_types" in self._result: - rows_to_convert = self._get_rows_to_convert_to_date(self._result["col_types"]) - for flag in rows_to_convert: - if flag: - t_rows = (row for row in self._result["rows"]) - t_values = (self._transform_date_columns(row, (flag for flag in rows_to_convert)) - for row in t_rows) - self._result["rows"] = [[value for value in row] for row in t_values] - break - self.rows = iter(self._result["rows"]) + self.rows = self.result_set_transformed() + if self.rows is None: + self.rows = iter(self._result["rows"]) + + def result_set_transformed(self): + """ + Generator that iterates over each row from the result set + """ + rows_to_convert = [True if col_type == 11 or col_type == 15 else False for col_type in + self._result["col_types"]] + for row in self._result["rows"]: + gen_flags = (flag for flag in rows_to_convert) + yield self._transform_date_columns(row, gen_flags) @staticmethod def _transform_date_columns(row, gen_flags): """ - Generates a list of boolean. True if the column is type timestamp (11 - 15) + Generates iterates over each value from a row and converts timestamps to pandas TIMESTAMP """ for value in row: flag = next(gen_flags) + if not flag or value is None: yield value else: - value = datetime.fromtimestamp(float(str(value)[0:10])) - yield value - - @staticmethod - def _get_rows_to_convert_to_date(col_types): - """ - Generates a list of boolean. True if the column is type timestamp (11 - 15) - """ - return [True if col_type == 11 or col_type == 15 else False for col_type in col_types] + if value < 0: + yield None + else: + value = pd.Timestamp(float(str(value)[0:13]), unit='ms') + yield value def executemany(self, sql, seq_of_parameters): """ From 9b82ac243b8fd1b1c01e9f271d55fc8e32c38bfc Mon Sep 17 00:00:00 2001 From: Aymaru Date: Tue, 30 Mar 2021 12:53:54 -0600 Subject: [PATCH 15/25] Added pandas dependency --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index cdd445ef..b9548291 100644 --- a/setup.py +++ b/setup.py @@ -26,6 +26,7 @@ requirements = [ 'urllib3>=1.9', + "pandas>=1.2.2, <1.3", ] From 5b4d5ec5a741c25fdd5d5ebd78e972d898b4ca66 Mon Sep 17 00:00:00 2001 From: Aymaru Date: Tue, 30 Mar 2021 16:00:29 -0600 Subject: [PATCH 16/25] Changed pandas timestamp to python datetime && deleted pandas dependecy --- setup.py | 1 - src/crate/client/cursor.py | 9 ++++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/setup.py b/setup.py index b9548291..cdd445ef 100644 --- a/setup.py +++ b/setup.py @@ -26,7 +26,6 @@ requirements = [ 'urllib3>=1.9', - "pandas>=1.2.2, <1.3", ] diff --git a/src/crate/client/cursor.py b/src/crate/client/cursor.py index 49bdef76..be38a26a 100644 --- a/src/crate/client/cursor.py +++ b/src/crate/client/cursor.py @@ -22,7 +22,7 @@ from .exceptions import ProgrammingError from distutils.version import StrictVersion import warnings -import pandas as pd +from datetime import datetime BULK_INSERT_MIN_VERSION = StrictVersion("0.42.0") @@ -75,7 +75,10 @@ def _transform_date_columns(row, gen_flags): Generates iterates over each value from a row and converts timestamps to pandas TIMESTAMP """ for value in row: - flag = next(gen_flags) + try: + flag = next(gen_flags) + except StopIteration: + break if not flag or value is None: yield value @@ -83,7 +86,7 @@ def _transform_date_columns(row, gen_flags): if value < 0: yield None else: - value = pd.Timestamp(float(str(value)[0:13]), unit='ms') + value = datetime.fromtimestamp(value/1000) yield value def executemany(self, sql, seq_of_parameters): From e9788ea702aa79c1ecc215dbc1f6dc8af64fed65 Mon Sep 17 00:00:00 2001 From: Aymaru Date: Tue, 30 Mar 2021 16:03:02 -0600 Subject: [PATCH 17/25] fixed - E226 missing whitespace around arithmetic operator --- src/crate/client/cursor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/crate/client/cursor.py b/src/crate/client/cursor.py index be38a26a..26255088 100644 --- a/src/crate/client/cursor.py +++ b/src/crate/client/cursor.py @@ -86,7 +86,7 @@ def _transform_date_columns(row, gen_flags): if value < 0: yield None else: - value = datetime.fromtimestamp(value/1000) + value = datetime.fromtimestamp(value / 1000) yield value def executemany(self, sql, seq_of_parameters): From eebd67094e54ac637496fa8d56b36c37ac64d031 Mon Sep 17 00:00:00 2001 From: Aymaru Date: Wed, 31 Mar 2021 10:44:18 -0600 Subject: [PATCH 18/25] Changed yield value --- src/crate/client/cursor.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/crate/client/cursor.py b/src/crate/client/cursor.py index 26255088..997fc370 100644 --- a/src/crate/client/cursor.py +++ b/src/crate/client/cursor.py @@ -53,10 +53,14 @@ def execute(self, sql, parameters=None, bulk_parameters=None): self._result = self.connection.client.sql(sql, parameters, bulk_parameters) + if "rows" in self._result: + transformed_result = False if "col_types" in self._result: + transformed_result = True self.rows = self.result_set_transformed() - if self.rows is None: + + if not transformed_result: self.rows = iter(self._result["rows"]) def result_set_transformed(self): @@ -67,7 +71,7 @@ def result_set_transformed(self): self._result["col_types"]] for row in self._result["rows"]: gen_flags = (flag for flag in rows_to_convert) - yield self._transform_date_columns(row, gen_flags) + yield [t_row for t_row in self._transform_date_columns(row, gen_flags)] @staticmethod def _transform_date_columns(row, gen_flags): From 187ed3f5c7abcdccda453b5c590ecc7e2954ba85 Mon Sep 17 00:00:00 2001 From: Andreas Motl Date: Tue, 23 Feb 2021 18:17:40 +0100 Subject: [PATCH 19/25] Adjust badges, bump SQLAlchemy test version and add "workflow_dispatch" - Adjust and add some badges to README.rst. - For covering the most recent SQLAlchemy 1.3 release, bump to SQLAlchemy version 1.3.23. - Also, enable listening to "workflow_dispatch" events in order to be able to trigger workflows manually from the UI. --- .github/workflows/main.yml | 16 ++++++++-------- README.rst | 37 +++++++++++++++++++++++++------------ 2 files changed, 33 insertions(+), 20 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 50251dc7..c01c1160 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -1,19 +1,19 @@ ---- -name: test +name: Tests on: - pull_request: ~ push: - branches: - - master + branches: [ master ] + pull_request: + branches: [ master ] + workflow_dispatch: schedule: - cron: '0 2 * * *' jobs: test: - name: "Test python-${{ matrix.python-version }} - sqla: ${{ matrix.sqla-version }} + name: "Python: ${{ matrix.python-version }} + SQLA: ${{ matrix.sqla-version }} CrateDB: ${{ matrix.crate-version }} on ${{ matrix.os }}" runs-on: ${{ matrix.os }} @@ -21,7 +21,7 @@ jobs: matrix: crate-version: [nightly] os: [ubuntu-latest] - sqla-version: ['1.1.18', '1.2.19', '1.3.20'] + sqla-version: ['1.1.18', '1.2.19', '1.3.23'] python-version: [3.5, 3.6, 3.7, 3.8, 3.9] steps: diff --git a/README.rst b/README.rst index 673883f1..ebaeda27 100644 --- a/README.rst +++ b/README.rst @@ -2,29 +2,42 @@ CrateDB Python Client ===================== -.. image:: https://dev.azure.com/cratedb/crate-python/_apis/build/status/crate.crate-python?branchName=master - :target: https://dev.azure.com/cratedb/crate-python/_build/latest?definitionId=2&branchName=master - :alt: Azure Pipeline +.. image:: https://github.com/crate/crate-python/workflows/Tests/badge.svg + :target: https://github.com/crate/crate-python/actions?workflow=Tests + :alt: Build status + +.. image:: https://coveralls.io/repos/github/crate/crate-python/badge.svg?branch=master + :target: https://coveralls.io/github/crate/crate-python?branch=master + :alt: Coverage + +.. image:: https://readthedocs.org/projects/crate-python/badge/ + :target: https://crate.io/docs/python/ + :alt: Build status (documentation) .. image:: https://img.shields.io/pypi/v/crate.svg - :target: https://pypi.python.org/pypi/crate/ - :alt: PyPI Version + :target: https://pypi.org/project/crate/ + :alt: PyPI Version .. image:: https://img.shields.io/pypi/pyversions/crate.svg - :target: https://pypi.python.org/pypi/crate/ - :alt: Python Version + :target: https://pypi.org/project/crate/ + :alt: Python Version .. image:: https://img.shields.io/pypi/dw/crate.svg - :target: https://pypi.python.org/pypi/crate/ + :target: https://pypi.org/project/crate/ :alt: PyPI Downloads .. image:: https://img.shields.io/pypi/wheel/crate.svg - :target: https://pypi.python.org/pypi/crate/ + :target: https://pypi.org/project/crate/ :alt: Wheel -.. image:: https://coveralls.io/repos/github/crate/crate-python/badge.svg?branch=master - :target: https://coveralls.io/github/crate/crate-python?branch=master - :alt: Coverage +.. image:: https://img.shields.io/pypi/status/crate.svg + :target: https://pypi.org/project/crate/ + :alt: Status + +.. image:: https://img.shields.io/pypi/l/crate.svg + :target: https://pypi.org/project/crate/ + :alt: License + | From e3b8d50ad6ecd145d04afaf36401e669126c54cb Mon Sep 17 00:00:00 2001 From: Aymaru Date: Tue, 30 Mar 2021 12:34:34 -0600 Subject: [PATCH 20/25] updated conversion of timestamps --- src/crate/client/cursor.py | 41 +++++++++++++++++++------------------- 1 file changed, 21 insertions(+), 20 deletions(-) diff --git a/src/crate/client/cursor.py b/src/crate/client/cursor.py index aae621fe..49bdef76 100644 --- a/src/crate/client/cursor.py +++ b/src/crate/client/cursor.py @@ -22,7 +22,7 @@ from .exceptions import ProgrammingError from distutils.version import StrictVersion import warnings -from datetime import datetime +import pandas as pd BULK_INSERT_MIN_VERSION = StrictVersion("0.42.0") @@ -55,35 +55,36 @@ def execute(self, sql, parameters=None, bulk_parameters=None): bulk_parameters) if "rows" in self._result: if "col_types" in self._result: - rows_to_convert = self._get_rows_to_convert_to_date(self._result["col_types"]) - for flag in rows_to_convert: - if flag: - t_rows = (row for row in self._result["rows"]) - t_values = (self._transform_date_columns(row, (flag for flag in rows_to_convert)) - for row in t_rows) - self._result["rows"] = [[value for value in row] for row in t_values] - break - self.rows = iter(self._result["rows"]) + self.rows = self.result_set_transformed() + if self.rows is None: + self.rows = iter(self._result["rows"]) + + def result_set_transformed(self): + """ + Generator that iterates over each row from the result set + """ + rows_to_convert = [True if col_type == 11 or col_type == 15 else False for col_type in + self._result["col_types"]] + for row in self._result["rows"]: + gen_flags = (flag for flag in rows_to_convert) + yield self._transform_date_columns(row, gen_flags) @staticmethod def _transform_date_columns(row, gen_flags): """ - Generates a list of boolean. True if the column is type timestamp (11 - 15) + Generates iterates over each value from a row and converts timestamps to pandas TIMESTAMP """ for value in row: flag = next(gen_flags) + if not flag or value is None: yield value else: - value = datetime.fromtimestamp(float(str(value)[0:10])) - yield value - - @staticmethod - def _get_rows_to_convert_to_date(col_types): - """ - Generates a list of boolean. True if the column is type timestamp (11 - 15) - """ - return [True if col_type == 11 or col_type == 15 else False for col_type in col_types] + if value < 0: + yield None + else: + value = pd.Timestamp(float(str(value)[0:13]), unit='ms') + yield value def executemany(self, sql, seq_of_parameters): """ From 32a4826fac897051cce3ac4be91911f6532b4e1c Mon Sep 17 00:00:00 2001 From: Aymaru Date: Tue, 30 Mar 2021 12:53:54 -0600 Subject: [PATCH 21/25] Added pandas dependency --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index cdd445ef..b9548291 100644 --- a/setup.py +++ b/setup.py @@ -26,6 +26,7 @@ requirements = [ 'urllib3>=1.9', + "pandas>=1.2.2, <1.3", ] From 2c9e3aa10abe06d8258c2e298776d486c7dc148c Mon Sep 17 00:00:00 2001 From: Aymaru Date: Tue, 30 Mar 2021 16:00:29 -0600 Subject: [PATCH 22/25] Changed pandas timestamp to python datetime && deleted pandas dependecy --- setup.py | 1 - src/crate/client/cursor.py | 9 ++++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/setup.py b/setup.py index b9548291..cdd445ef 100644 --- a/setup.py +++ b/setup.py @@ -26,7 +26,6 @@ requirements = [ 'urllib3>=1.9', - "pandas>=1.2.2, <1.3", ] diff --git a/src/crate/client/cursor.py b/src/crate/client/cursor.py index 49bdef76..be38a26a 100644 --- a/src/crate/client/cursor.py +++ b/src/crate/client/cursor.py @@ -22,7 +22,7 @@ from .exceptions import ProgrammingError from distutils.version import StrictVersion import warnings -import pandas as pd +from datetime import datetime BULK_INSERT_MIN_VERSION = StrictVersion("0.42.0") @@ -75,7 +75,10 @@ def _transform_date_columns(row, gen_flags): Generates iterates over each value from a row and converts timestamps to pandas TIMESTAMP """ for value in row: - flag = next(gen_flags) + try: + flag = next(gen_flags) + except StopIteration: + break if not flag or value is None: yield value @@ -83,7 +86,7 @@ def _transform_date_columns(row, gen_flags): if value < 0: yield None else: - value = pd.Timestamp(float(str(value)[0:13]), unit='ms') + value = datetime.fromtimestamp(value/1000) yield value def executemany(self, sql, seq_of_parameters): From 2adf8333ec70e74fd6dbd8c49b8bb863e3f74703 Mon Sep 17 00:00:00 2001 From: Aymaru Date: Tue, 30 Mar 2021 16:03:02 -0600 Subject: [PATCH 23/25] fixed - E226 missing whitespace around arithmetic operator --- src/crate/client/cursor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/crate/client/cursor.py b/src/crate/client/cursor.py index be38a26a..26255088 100644 --- a/src/crate/client/cursor.py +++ b/src/crate/client/cursor.py @@ -86,7 +86,7 @@ def _transform_date_columns(row, gen_flags): if value < 0: yield None else: - value = datetime.fromtimestamp(value/1000) + value = datetime.fromtimestamp(value / 1000) yield value def executemany(self, sql, seq_of_parameters): From 9bc5f5159512d75fc29e35f7282fec1cf975a802 Mon Sep 17 00:00:00 2001 From: Aymaru Date: Wed, 31 Mar 2021 10:44:18 -0600 Subject: [PATCH 24/25] Changed yield value --- src/crate/client/cursor.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/crate/client/cursor.py b/src/crate/client/cursor.py index 26255088..997fc370 100644 --- a/src/crate/client/cursor.py +++ b/src/crate/client/cursor.py @@ -53,10 +53,14 @@ def execute(self, sql, parameters=None, bulk_parameters=None): self._result = self.connection.client.sql(sql, parameters, bulk_parameters) + if "rows" in self._result: + transformed_result = False if "col_types" in self._result: + transformed_result = True self.rows = self.result_set_transformed() - if self.rows is None: + + if not transformed_result: self.rows = iter(self._result["rows"]) def result_set_transformed(self): @@ -67,7 +71,7 @@ def result_set_transformed(self): self._result["col_types"]] for row in self._result["rows"]: gen_flags = (flag for flag in rows_to_convert) - yield self._transform_date_columns(row, gen_flags) + yield [t_row for t_row in self._transform_date_columns(row, gen_flags)] @staticmethod def _transform_date_columns(row, gen_flags): From aaaf7eeac59ba120c7e579d21e46e9474c3a88a9 Mon Sep 17 00:00:00 2001 From: Aymaru Date: Wed, 31 Mar 2021 14:08:00 -0600 Subject: [PATCH 25/25] Validate date type in processors --- src/crate/client/sqlalchemy/dialect.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/crate/client/sqlalchemy/dialect.py b/src/crate/client/sqlalchemy/dialect.py index d680e4b0..dde8f59a 100644 --- a/src/crate/client/sqlalchemy/dialect.py +++ b/src/crate/client/sqlalchemy/dialect.py @@ -92,6 +92,8 @@ def result_processor(self, dialect, coltype): def process(value): if not value: return + if isinstance(value, datetime): + return value.date() try: return datetime.utcfromtimestamp(value / 1e3).date() except TypeError: @@ -131,6 +133,8 @@ def result_processor(self, dialect, coltype): def process(value): if not value: return + if isinstance(value, datetime): + return value try: return datetime.utcfromtimestamp(value / 1e3) except TypeError: