diff --git a/dags/vz_google_sheets.py b/dags/vz_google_sheets.py index 4d57cbe82..a7c588a1c 100644 --- a/dags/vz_google_sheets.py +++ b/dags/vz_google_sheets.py @@ -62,7 +62,7 @@ items={"type": "number"}, ) }, - tags=["Vision Zero"] + tags=["Vision Zero", "google_sheets"] ) def get_vz_data(): """The main function of the SSZ DAG.""" @@ -147,7 +147,7 @@ def filter_spreadsheets(ds = None, **context): return sheets #to get credentials to access google sheets - google_cred = GoogleBaseHook('vz_api_google').get_credentials() + google_cred = GoogleBaseHook('google_sheets_api').get_credentials() pull_data.partial( engine=PostgresHook("vz_api_bot").get_sqlalchemy_engine(), diff --git a/dags/wys_pull.py b/dags/wys_pull.py index 5d609901c..2b159db2b 100644 --- a/dags/wys_pull.py +++ b/dags/wys_pull.py @@ -58,7 +58,7 @@ max_active_runs=5, template_searchpath=os.path.join(repo_path,'dags/sql'), schedule='0 17 * * *', # Run at 5:00 PM local time every day - tags=["wys", "data_pull", "partition_create", "data_checks"], + tags=["wys", "data_pull", "partition_create", "data_checks", "google_sheets"], doc_md=DOC_MD ) def pull_wys_dag(): @@ -164,7 +164,7 @@ def read_google_sheets(**context): wys_postgres = PostgresHook("wys_bot") #to get credentials to access google sheets - wys_api_hook = GoogleBaseHook('vz_api_google') + wys_api_hook = GoogleBaseHook('google_sheets_api') cred = wys_api_hook.get_credentials() service = build('sheets', 'v4', credentials=cred, cache_discovery=False) diff --git a/gis/school_safety_zones/README.md b/gis/school_safety_zones/README.md index 1e719c7e7..9e2de7992 100644 --- a/gis/school_safety_zones/README.md +++ b/gis/school_safety_zones/README.md @@ -47,7 +47,7 @@ The DAG consists of two main tasks as shown in the below figure: ## 3. Sheets Credentials -A credential file (named `key.json` in the script) is required to connect to the Google Sheets to pull data, the contents of this file can be downloaded from [the google console](https://console.cloud.google.com/iam-admin/serviceaccounts/details/) if you're logged in to the right google account. This is currently stored in an encrypted Airflow connection: `vz_api_google`. +A credential file (named `key.json` in the script) is required to connect to the Google Sheets to pull data, the contents of this file can be downloaded from [the google console](https://console.cloud.google.com/iam-admin/serviceaccounts/details/) if you're logged in to the right google account. This is currently stored in an encrypted Airflow connection: `google_sheets_api`. ## 4. Adding a new year @@ -66,7 +66,7 @@ CREATE TABLE vz_safety_programs_staging.school_safety_zone_yyyy_raw ( ### 4.2 Request sharing permission to the new sheet -The sheet must be shared with `vz-sheets@quickstart-1568664221624.iam.gserviceaccount.com`. This ought to be View-only. This email is saved in the Airflow credentials as `wys_cred.service_account_email`. +The sheet must be shared with `sheets-puller@bubbly-fuze-182523.iam.gserviceaccount.com`. This ought to be View-only. This email is associated with the `terrestrial.wherever@gmail.com` email and saved in the `google_sheets_api` Airflow connection. ### 4.3 Add the New Google Sheet to Airflow diff --git a/gis/school_safety_zones/schools.py b/gis/school_safety_zones/schools.py index 3c5efc3c7..4ae432f56 100755 --- a/gis/school_safety_zones/schools.py +++ b/gis/school_safety_zones/schools.py @@ -327,7 +327,7 @@ def pull_from_sheet_cli( connect_url = URL.create("postgresql+psycopg2", **config['DBSETTINGS']) engine = create_engine(connect_url) - google_cred = GoogleBaseHook('vz_api_google').get_credentials() + google_cred = GoogleBaseHook('google_sheets_api').get_credentials() if table is None: table = f"school_safety_zone_{year}_raw" diff --git a/test/integration/test_dags.py b/test/integration/test_dags.py index 2cdbeae20..2a995aabd 100644 --- a/test/integration/test_dags.py +++ b/test/integration/test_dags.py @@ -41,7 +41,7 @@ 'AIRFLOW_CONN_HERE_BOT': SAMPLE_CONN.get_uri(), 'AIRFLOW_CONN_RESCU_BOT': SAMPLE_CONN.get_uri(), 'AIRFLOW_CONN_VZ_API_BOT': SAMPLE_CONN.get_uri(), - 'AIRFLOW_CONN_VZ_API_GOOGLE': SAMPLE_CONN.get_uri(), + 'AIRFLOW_CONN_GOOGLE_SHEETS_API': SAMPLE_CONN.get_uri(), }