From f42651e422141d806b390d2e64e3ee847b3f0fe3 Mon Sep 17 00:00:00 2001 From: Mohammad Amin Date: Wed, 17 Jan 2024 18:49:25 +0330 Subject: [PATCH 1/2] update: hivemind etl dags at 02:00 AM UTC! --- dags/hivemind_etl.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/dags/hivemind_etl.py b/dags/hivemind_etl.py index d7057d86..3cef69f7 100644 --- a/dags/hivemind_etl.py +++ b/dags/hivemind_etl.py @@ -44,8 +44,8 @@ with DAG( dag_id="discord_vector_store_update", - start_date=datetime(2022, 11, 10, 12), - schedule_interval=timedelta(days=1), + start_date=datetime(2024, 1, 1), + schedule_interval="0 2 * * *", catchup=False, ) as dag: @@ -69,7 +69,11 @@ def start_discord_vectorstore(community_id: str): # with the length of the list start_discord_vectorstore.expand(community_id=communities) -with DAG(dag_id="discord_summary_vector_store", start_date=datetime(2023, 1, 1)) as dag: +with DAG( + dag_id="discord_summary_vector_store", + start_date=datetime(2024, 1, 1), + schedule_interval="0 2 * * *", +) as dag: @task def get_mongo_discord_communities() -> list[str]: From 86905cd59fd01e03bb0968bdd6b4d933fd0461a8 Mon Sep 17 00:00:00 2001 From: Mohammad Amin Date: Wed, 17 Jan 2024 19:09:07 +0330 Subject: [PATCH 2/2] fix linter issue! --- dags/hivemind_etl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dags/hivemind_etl.py b/dags/hivemind_etl.py index 3cef69f7..46e39560 100644 --- a/dags/hivemind_etl.py +++ b/dags/hivemind_etl.py @@ -19,7 +19,7 @@ from __future__ import annotations import logging -from datetime import datetime, timedelta +from datetime import datetime # import phoenix as px from airflow import DAG