From a7d5d5a6391fd82418c961095f87798a54ad1440 Mon Sep 17 00:00:00 2001 From: Michelle Ho Date: Thu, 11 May 2017 17:50:31 -0400 Subject: [PATCH 1/3] invalid timestamps are read as object type --- cartoframes/context.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/cartoframes/context.py b/cartoframes/context.py index c216e686b..f7b1466de 100644 --- a/cartoframes/context.py +++ b/cartoframes/context.py @@ -268,6 +268,8 @@ def query(self, query, table_name=None): query. Defaults to None (no table created). Returns: pandas.DataFrame: DataFrame representation of query supplied. + Pandas data types are inferred from PostgreSQL data types. + In the case of invalid timestamps, the data type 'object' is used. """ self._debug_print(query=query) if table_name: @@ -311,10 +313,19 @@ def query(self, query, table_name=None): if not schema.keys(): return None self._debug_print(fields=fields, schema=schema) - - df = pd.DataFrame( - data=select_res['rows'], - columns=[k for k in fields]).astype(schema) + try: + df = pd.DataFrame( + data=select_res['rows'], + columns=[k for k in fields]).astype(schema) + except pd.tslib.OutOfBoundsDatetime: + df = pd.DataFrame( + data=select_res['rows'], + columns=[k for k in fields]) + for field, dtype in schema.iteritems(): + try: + df.astype({field:dtype}) + except pd.tslib.OutOfBoundsDatetime: + pass if 'cartodb_id' in fields: df.set_index('cartodb_id', inplace=True) return df From a6efa770c5e37a3113de2492280b2ceb6c83076c Mon Sep 17 00:00:00 2001 From: Michelle Ho Date: Tue, 20 Jun 2017 15:04:49 -0400 Subject: [PATCH 2/3] all date types read as object --- cartoframes/context.py | 22 +++++++--------------- 1 file changed, 7 insertions(+), 15 deletions(-) diff --git a/cartoframes/context.py b/cartoframes/context.py index f7b1466de..fb8452ee8 100644 --- a/cartoframes/context.py +++ b/cartoframes/context.py @@ -269,7 +269,7 @@ def query(self, query, table_name=None): Returns: pandas.DataFrame: DataFrame representation of query supplied. Pandas data types are inferred from PostgreSQL data types. - In the case of invalid timestamps, the data type 'object' is used. + In the case of timestamps, the data type 'object' is used. """ self._debug_print(query=query) if table_name: @@ -297,7 +297,7 @@ def query(self, query, table_name=None): self._debug_print(select_res=select_res) pg2dtypes = { - 'date': 'datetime64', + 'date': 'object', 'number': 'float64', 'string': 'object', 'boolean': 'bool', @@ -313,19 +313,11 @@ def query(self, query, table_name=None): if not schema.keys(): return None self._debug_print(fields=fields, schema=schema) - try: - df = pd.DataFrame( - data=select_res['rows'], - columns=[k for k in fields]).astype(schema) - except pd.tslib.OutOfBoundsDatetime: - df = pd.DataFrame( - data=select_res['rows'], - columns=[k for k in fields]) - for field, dtype in schema.iteritems(): - try: - df.astype({field:dtype}) - except pd.tslib.OutOfBoundsDatetime: - pass + + df = pd.DataFrame( + data=select_res['rows'], + columns=[k for k in fields]).astype(schema) + if 'cartodb_id' in fields: df.set_index('cartodb_id', inplace=True) return df From 408b7ddc9af5194eec20b221086bc7e884382f3f Mon Sep 17 00:00:00 2001 From: Michelle Ho Date: Tue, 20 Jun 2017 15:07:35 -0400 Subject: [PATCH 3/3] changing doc string of query --- cartoframes/context.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cartoframes/context.py b/cartoframes/context.py index fb8452ee8..b51ba638d 100644 --- a/cartoframes/context.py +++ b/cartoframes/context.py @@ -269,7 +269,7 @@ def query(self, query, table_name=None): Returns: pandas.DataFrame: DataFrame representation of query supplied. Pandas data types are inferred from PostgreSQL data types. - In the case of timestamps, the data type 'object' is used. + In the case of PostgreSQL date types, the data type 'object' is used. """ self._debug_print(query=query) if table_name: