fix: try to parse record time (#186)

* fix: try to parse record time * fix: lock cdk version
rudderlabs · Apr 29, 2024 · cca3711 · cca3711
1 parent 762d0de
commit cca3711
Show file tree

Hide file tree

Showing 2 changed files with 17 additions and 4 deletions.
diff --git a/airbyte-integrations/connectors/source-mixpanel/setup.py b/airbyte-integrations/connectors/source-mixpanel/setup.py
@@ -6,7 +6,7 @@
 from setuptools import find_packages, setup
 
 MAIN_REQUIREMENTS = [
-    "airbyte-cdk~=0.2",
+    "airbyte-cdk==0.67",
 ]
 
 TEST_REQUIREMENTS = ["pytest~=6.1", "connector-acceptance-test", "pytest-mock~=3.6", "requests_mock~=1.8"]

diff --git a/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/streams/export.py b/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/streams/export.py
@@ -157,9 +157,22 @@ def process_response(self, response: requests.Response, **kwargs) -> Iterable[Ma
 
             # convert timestamp to datetime string
             if "time" in item: # time is not always present in the response
-                item["time"] = pendulum.from_timestamp(int(item["time"]), tz="UTC").to_iso8601_string()
-                if cursor_date and item["time"] < cursor_date:
-                    continue
+                timestamp = None
+                try:
+                    timestamp = int(item["time"])
+                except ValueError as e:
+                    self.logger.warning(f"couldn't parse record time {e}")
+
+                if not timestamp and "_time" in item:
+                    try:
+                        timestamp = int(item["_time"])
+                    except ValueError as e:
+                        self.logger.warning(f"couldn't parse record time {e}")
+
+                if timestamp:
+                    item["time"] = pendulum.from_timestamp(timestamp, tz="UTC").to_iso8601_string()
+                    if cursor_date and item["time"] < cursor_date:
+                        continue
 
             yield item