From abde0d40ddabd8ee929bded0386cd55fee186572 Mon Sep 17 00:00:00 2001 From: Uche Madu Date: Thu, 19 Oct 2023 11:24:21 +0100 Subject: [PATCH] add test dag --- dags/test_gcslist.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 dags/test_gcslist.py diff --git a/dags/test_gcslist.py b/dags/test_gcslist.py new file mode 100644 index 0000000..898d7d2 --- /dev/null +++ b/dags/test_gcslist.py @@ -0,0 +1,19 @@ +from datetime import datetime, timedelta +from airflow.decorators import dag +from airflow.providers.google.cloud.operators.gcs import GCSListObjectsOperator + +@dag(schedule=timedelta(days=1), start_date=datetime(2023, 10, 19), catchup=False, default_args={ + 'owner': 'airflow', + 'retries': 1, + 'retry_delay': timedelta(minutes=5), +}) +def gcs_list_buckets(): + + GCSListObjectsOperator( + task_id='list_buckets', + bucket='deb-capstone', + match_glob='**/*/.csv', + gcp_conn_id='gcp' + ) + +gcs_list_buckets_dag = gcs_list_buckets()