From f41d3239b56cec631fc4084440c5b8d2095f8011 Mon Sep 17 00:00:00 2001 From: Mohammad Amin Date: Tue, 22 Oct 2024 10:59:42 +0330 Subject: [PATCH 1/3] fix: Added timestamp conversion support! --- dags/hivemind_etl_helpers/ingestion_pipeline.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/dags/hivemind_etl_helpers/ingestion_pipeline.py b/dags/hivemind_etl_helpers/ingestion_pipeline.py index 93f6c8b1..0ab3693d 100644 --- a/dags/hivemind_etl_helpers/ingestion_pipeline.py +++ b/dags/hivemind_etl_helpers/ingestion_pipeline.py @@ -165,7 +165,15 @@ def get_latest_document_date( logging.info("No documents found in the collection.") latest_date = None else: - latest_date = parse(latest_document[0][0].payload[field_name]) + date_field = latest_document[0][0].payload[field_name] + + # if it was float timestamp + if field_schema == models.PayloadSchemaType.FLOAT: + latest_date = datetime.fromtimestamp(date_field) + + # it should be datetime in any other case + else: + latest_date = parse(latest_document[0][0].payload[field_name]) else: raise ValueError( From e9205d6c1f4e8d912505a67f0deb0d9a1d9674f0 Mon Sep 17 00:00:00 2001 From: Mohammad Amin Date: Tue, 22 Oct 2024 11:01:24 +0330 Subject: [PATCH 2/3] fix: clean code! --- dags/hivemind_etl_helpers/ingestion_pipeline.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dags/hivemind_etl_helpers/ingestion_pipeline.py b/dags/hivemind_etl_helpers/ingestion_pipeline.py index 0ab3693d..5de2b933 100644 --- a/dags/hivemind_etl_helpers/ingestion_pipeline.py +++ b/dags/hivemind_etl_helpers/ingestion_pipeline.py @@ -173,7 +173,7 @@ def get_latest_document_date( # it should be datetime in any other case else: - latest_date = parse(latest_document[0][0].payload[field_name]) + latest_date = parse(date_field) else: raise ValueError( From 439ca37353397638ba380d79563bac6519e03cc1 Mon Sep 17 00:00:00 2001 From: Mohammad Amin Date: Tue, 22 Oct 2024 11:08:17 +0330 Subject: [PATCH 3/3] fix: updated test case in order not to follow the results order! --- .../tests/integration/test_telegram_chats.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/dags/hivemind_etl_helpers/tests/integration/test_telegram_chats.py b/dags/hivemind_etl_helpers/tests/integration/test_telegram_chats.py index 6f179653..1275988d 100644 --- a/dags/hivemind_etl_helpers/tests/integration/test_telegram_chats.py +++ b/dags/hivemind_etl_helpers/tests/integration/test_telegram_chats.py @@ -106,11 +106,12 @@ def test_extract_chats_multiple_chats(self): ) chat_ids = self.tc_chats.extract_chats() - self.assertEqual( - chat_ids, - [ - (100001, "test chat"), - (100002, "test chat 2"), - (100003, "test chat 3"), - ], - ) + + self.assertEqual(len(chat_ids), 3) + expected_chats = [ + (100001, "test chat"), + (100002, "test chat 2"), + (100003, "test chat 3"), + ] + for id in chat_ids: + self.assertIn(id, expected_chats)