moar refactor

airflow_variables_dev.json

@@ -331,6 +331,7 @@
     "liquidity_pools": "liquidity_pools",
     "offers": "offers",
     "operations": "history_operations",
+    "retool_entity_data": "retool_entity_data",
     "signers": "account_signers",
     "trades": "history_trades",
     "transactions": "history_transactions",

airflow_variables_prod.json

@@ -309,10 +309,12 @@
       }
     }
   },
+  "retool_api_key": "test-api-key",
   "sandbox_dataset": "crypto_stellar_internal_sandbox",
   "schema_filepath": "/home/airflow/gcs/dags/schemas/",
   "sentry_dsn": "https://[email protected]/5806618",
   "sentry_environment": "production",
+  "stellar_etl_internal_image_name": "amishastellar/stellar-etl-internal:e3b9a2ea7",
   "table_ids": {
     "accounts": "accounts",
     "assets": "history_assets",
@@ -327,6 +329,7 @@
     "liquidity_pools": "liquidity_pools",
     "offers": "offers",
     "operations": "history_operations",
+    "retool_entity_data": "retool_entity_data",
     "signers": "account_signers",
     "trades": "history_trades",
     "transactions": "history_transactions",
@@ -347,9 +350,11 @@
     "create_sandbox": 1020,
     "current_state": 1200,
     "default": 60,
+    "del_ins_retool_entity_data_task": 720,
     "elementary_dbt_data_quality": 2100,
     "elementary_generate_report": 1200,
     "enriched_history_operations": 1800,
+    "export_retool_data": 720,
     "fee_stats": 360,
     "history_assets": 360,
     "liquidity_pool_trade_volume": 1200,

dags/external_data_dag.py

@@ -25,6 +25,12 @@
 
 init_sentry()
 
+EXTERNAL_DATA_TABLE_NAMES = Variable.get("table_ids", deserialize_json=True)
+EXTERNAL_DATA_PROJECT_NAME = Variable.get("bq_project")
+EXTERNAL_DATA_DATASET_NAME = Variable.get("bq_dataset")
+RETOOL_TABLE_NAME = EXTERNAL_DATA_TABLE_NAMES["retool_entity_data"]
+RETOOL_EXPORT_TASK_ID = "export_retool_data"
+
 # Initialize the DAG
 dag = DAG(
     "external_data_dag",
@@ -50,7 +56,7 @@
 
 retool_export_task = build_export_task(
     dag,
-    "export_retool_data",
+    RETOOL_EXPORT_TASK_ID,
     command="export-retool",
     cmd_args=[
         "--start-time",
@@ -101,14 +107,14 @@ def get_insert_to_bq_task(
 
 
 retool_insert_to_bq_task = get_insert_to_bq_task(
-    table_name="retool_entity_data",
-    project="test-hubble-319619",
-    dataset="test_crypto_stellar_internal",
-    export_task_id="export_retool_data",
+    table_name=RETOOL_TABLE_NAME,
+    project=EXTERNAL_DATA_PROJECT_NAME,
+    dataset=EXTERNAL_DATA_DATASET_NAME,
+    export_task_id=RETOOL_EXPORT_TASK_ID,
     source_object_suffix="",
     partition=False,
     cluster=False,
-    table_id="test-hubble-319619.test_crypto_stellar_internal.retool_entity_data",
+    table_id=f"{EXTERNAL_DATA_PROJECT_NAME}.{EXTERNAL_DATA_DATASET_NAME}.{RETOOL_TABLE_NAME}",
 )
 
 retool_export_task >> retool_insert_to_bq_task