From ef7e871566f736e47aaa62479d9ba3f5b2d8b9b0 Mon Sep 17 00:00:00 2001 From: Amisha Singla Date: Fri, 6 Dec 2024 17:13:03 -0600 Subject: [PATCH] udpate image udpate image udpate image udpate image udpate image --- airflow_variables_dev.json | 2 +- dags/external_data_dag.py | 7 +++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/airflow_variables_dev.json b/airflow_variables_dev.json index c6b1eb5c..32dd10b6 100644 --- a/airflow_variables_dev.json +++ b/airflow_variables_dev.json @@ -315,7 +315,7 @@ "schema_filepath": "/home/airflow/gcs/dags/schemas/", "sentry_dsn": "https://9e0a056541c3445083329b072f2df690@o14203.ingest.us.sentry.io/6190849", "sentry_environment": "development", - "stellar_etl_internal_image_name": "stellar/stellar-etl:204d343fa", + "stellar_etl_internal_image_name": " amishastellar/stellar-etl-internal:ab0a13897", "table_ids": { "accounts": "accounts", "assets": "history_assets", diff --git a/dags/external_data_dag.py b/dags/external_data_dag.py index bea102f1..96266482 100644 --- a/dags/external_data_dag.py +++ b/dags/external_data_dag.py @@ -4,10 +4,11 @@ """ from ast import literal_eval -from datetime import datetime +from datetime import datetime, timedelta from json import loads from airflow import DAG +from airflow.configuration import conf from airflow.models.variable import Variable from airflow.providers.cncf.kubernetes.operators.pod import KubernetesPodOperator from kubernetes.client import models as k8s @@ -56,8 +57,6 @@ def stellar_etl_internal_task( image = "{{ var.value.stellar_etl_internal_image_name }}" - logging.info(f"sh commands to run in pod: {args}") - return KubernetesPodOperator( task_id=task_name, name=task_name, @@ -80,7 +79,7 @@ def stellar_etl_internal_task( image_pull_policy="IfNotPresent", image_pull_secrets=[k8s.V1LocalObjectReference("private-docker-auth")], sla=timedelta( - seconds=Variable.get("task_sla", deserialize_json=True)[f"task_name"] + seconds=Variable.get("task_sla", deserialize_json=True)[task_name] ), trigger_rule="all_done", )