From 84750821635603adfe0c3d3f2ffac5706af97f2c Mon Sep 17 00:00:00 2001 From: Lennart Kats Date: Sat, 24 Aug 2024 21:12:05 +0200 Subject: [PATCH 1/4] Use periodic triggers in all templates --- .../resources/{{.project_name}}_job.yml.tmpl | 8 ++++---- .../resources/{{.project_name}}_job.yml.tmpl | 8 ++++---- .../resources/{{.project_name}}_sql_job.yml.tmpl | 8 ++++---- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/libs/template/templates/dbt-sql/template/{{.project_name}}/resources/{{.project_name}}_job.yml.tmpl b/libs/template/templates/dbt-sql/template/{{.project_name}}/resources/{{.project_name}}_job.yml.tmpl index bad12c755c..43fd2e5c82 100644 --- a/libs/template/templates/dbt-sql/template/{{.project_name}}/resources/{{.project_name}}_job.yml.tmpl +++ b/libs/template/templates/dbt-sql/template/{{.project_name}}/resources/{{.project_name}}_job.yml.tmpl @@ -3,10 +3,10 @@ resources: {{.project_name}}_job: name: {{.project_name}}_job - schedule: - # Run every day at 9:27 AM - quartz_cron_expression: 21 27 9 * * ? - timezone_id: UTC + trigger: + periodic: + interval: 1 + unit: DAYS email_notifications: on_failure: diff --git a/libs/template/templates/default-python/template/{{.project_name}}/resources/{{.project_name}}_job.yml.tmpl b/libs/template/templates/default-python/template/{{.project_name}}/resources/{{.project_name}}_job.yml.tmpl index dc79e3a171..0350d42cfb 100644 --- a/libs/template/templates/default-python/template/{{.project_name}}/resources/{{.project_name}}_job.yml.tmpl +++ b/libs/template/templates/default-python/template/{{.project_name}}/resources/{{.project_name}}_job.yml.tmpl @@ -10,10 +10,10 @@ resources: {{.project_name}}_job: name: {{.project_name}}_job - schedule: - # Run every day at 8:37 AM - quartz_cron_expression: '44 37 8 * * ?' - timezone_id: Europe/Amsterdam + trigger: + periodic: + interval: 1 + unit: DAYS {{- if not is_service_principal}} diff --git a/libs/template/templates/default-sql/template/{{.project_name}}/resources/{{.project_name}}_sql_job.yml.tmpl b/libs/template/templates/default-sql/template/{{.project_name}}/resources/{{.project_name}}_sql_job.yml.tmpl index 31d2d21a9f..2e6004186c 100644 --- a/libs/template/templates/default-sql/template/{{.project_name}}/resources/{{.project_name}}_sql_job.yml.tmpl +++ b/libs/template/templates/default-sql/template/{{.project_name}}/resources/{{.project_name}}_sql_job.yml.tmpl @@ -4,10 +4,10 @@ resources: {{.project_name}}_sql_job: name: {{.project_name}}_sql_job - schedule: - # Run every day at 7:17 AM - quartz_cron_expression: '44 17 7 * * ?' - timezone_id: Europe/Amsterdam + trigger: + periodic: + interval: 1 + unit: DAYS {{- if not is_service_principal}} From 1c34dc92a43924287cd6a6f94fd1b1ab0c5c3ec0 Mon Sep 17 00:00:00 2001 From: Lennart Kats Date: Wed, 4 Sep 2024 16:07:21 +0200 Subject: [PATCH 2/4] Add comments --- .../{{.project_name}}/resources/{{.project_name}}_job.yml.tmpl | 1 + .../{{.project_name}}/resources/{{.project_name}}_job.yml.tmpl | 1 + .../resources/{{.project_name}}_sql_job.yml.tmpl | 1 + 3 files changed, 3 insertions(+) diff --git a/libs/template/templates/dbt-sql/template/{{.project_name}}/resources/{{.project_name}}_job.yml.tmpl b/libs/template/templates/dbt-sql/template/{{.project_name}}/resources/{{.project_name}}_job.yml.tmpl index 43fd2e5c82..edeafd9b1d 100644 --- a/libs/template/templates/dbt-sql/template/{{.project_name}}/resources/{{.project_name}}_job.yml.tmpl +++ b/libs/template/templates/dbt-sql/template/{{.project_name}}/resources/{{.project_name}}_job.yml.tmpl @@ -4,6 +4,7 @@ resources: name: {{.project_name}}_job trigger: + # Run this job every day; see https://docs.databricks.com/api/workspace/jobs/create periodic: interval: 1 unit: DAYS diff --git a/libs/template/templates/default-python/template/{{.project_name}}/resources/{{.project_name}}_job.yml.tmpl b/libs/template/templates/default-python/template/{{.project_name}}/resources/{{.project_name}}_job.yml.tmpl index 0350d42cfb..29a9325824 100644 --- a/libs/template/templates/default-python/template/{{.project_name}}/resources/{{.project_name}}_job.yml.tmpl +++ b/libs/template/templates/default-python/template/{{.project_name}}/resources/{{.project_name}}_job.yml.tmpl @@ -11,6 +11,7 @@ resources: name: {{.project_name}}_job trigger: + # Run this job every day; see https://docs.databricks.com/api/workspace/jobs/create periodic: interval: 1 unit: DAYS diff --git a/libs/template/templates/default-sql/template/{{.project_name}}/resources/{{.project_name}}_sql_job.yml.tmpl b/libs/template/templates/default-sql/template/{{.project_name}}/resources/{{.project_name}}_sql_job.yml.tmpl index 2e6004186c..31bdf72d54 100644 --- a/libs/template/templates/default-sql/template/{{.project_name}}/resources/{{.project_name}}_sql_job.yml.tmpl +++ b/libs/template/templates/default-sql/template/{{.project_name}}/resources/{{.project_name}}_sql_job.yml.tmpl @@ -5,6 +5,7 @@ resources: name: {{.project_name}}_sql_job trigger: + # Run this job every day; see https://docs.databricks.com/api/workspace/jobs/create periodic: interval: 1 unit: DAYS From baaa234b43104d5ce893737c0e63fcef74eec2b5 Mon Sep 17 00:00:00 2001 From: Lennart Kats Date: Mon, 9 Sep 2024 13:49:57 +0200 Subject: [PATCH 3/4] Extend URL with an anchor --- .../{{.project_name}}/resources/{{.project_name}}_job.yml.tmpl | 2 +- .../{{.project_name}}/resources/{{.project_name}}_job.yml.tmpl | 2 +- .../resources/{{.project_name}}_sql_job.yml.tmpl | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/libs/template/templates/dbt-sql/template/{{.project_name}}/resources/{{.project_name}}_job.yml.tmpl b/libs/template/templates/dbt-sql/template/{{.project_name}}/resources/{{.project_name}}_job.yml.tmpl index edeafd9b1d..c4520f769d 100644 --- a/libs/template/templates/dbt-sql/template/{{.project_name}}/resources/{{.project_name}}_job.yml.tmpl +++ b/libs/template/templates/dbt-sql/template/{{.project_name}}/resources/{{.project_name}}_job.yml.tmpl @@ -4,7 +4,7 @@ resources: name: {{.project_name}}_job trigger: - # Run this job every day; see https://docs.databricks.com/api/workspace/jobs/create + # Run this job every day; see https://docs.databricks.com/api/workspace/jobs/create#trigger periodic: interval: 1 unit: DAYS diff --git a/libs/template/templates/default-python/template/{{.project_name}}/resources/{{.project_name}}_job.yml.tmpl b/libs/template/templates/default-python/template/{{.project_name}}/resources/{{.project_name}}_job.yml.tmpl index 29a9325824..416e9efcb7 100644 --- a/libs/template/templates/default-python/template/{{.project_name}}/resources/{{.project_name}}_job.yml.tmpl +++ b/libs/template/templates/default-python/template/{{.project_name}}/resources/{{.project_name}}_job.yml.tmpl @@ -11,7 +11,7 @@ resources: name: {{.project_name}}_job trigger: - # Run this job every day; see https://docs.databricks.com/api/workspace/jobs/create + # Run this job every day; see https://docs.databricks.com/api/workspace/jobs/create#trigger periodic: interval: 1 unit: DAYS diff --git a/libs/template/templates/default-sql/template/{{.project_name}}/resources/{{.project_name}}_sql_job.yml.tmpl b/libs/template/templates/default-sql/template/{{.project_name}}/resources/{{.project_name}}_sql_job.yml.tmpl index 31bdf72d54..54c995ea51 100644 --- a/libs/template/templates/default-sql/template/{{.project_name}}/resources/{{.project_name}}_sql_job.yml.tmpl +++ b/libs/template/templates/default-sql/template/{{.project_name}}/resources/{{.project_name}}_sql_job.yml.tmpl @@ -5,7 +5,7 @@ resources: name: {{.project_name}}_sql_job trigger: - # Run this job every day; see https://docs.databricks.com/api/workspace/jobs/create + # Run this job every day; see https://docs.databricks.com/api/workspace/jobs/create#trigger periodic: interval: 1 unit: DAYS From a3c2c5b73591ffaf08228c91e656a356831e30bc Mon Sep 17 00:00:00 2001 From: Lennart Kats Date: Tue, 10 Sep 2024 14:19:47 +0200 Subject: [PATCH 4/4] Extend comment --- .../{{.project_name}}/resources/{{.project_name}}_job.yml.tmpl | 2 +- .../{{.project_name}}/resources/{{.project_name}}_job.yml.tmpl | 2 +- .../resources/{{.project_name}}_sql_job.yml.tmpl | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/libs/template/templates/dbt-sql/template/{{.project_name}}/resources/{{.project_name}}_job.yml.tmpl b/libs/template/templates/dbt-sql/template/{{.project_name}}/resources/{{.project_name}}_job.yml.tmpl index c4520f769d..e23c8dbced 100644 --- a/libs/template/templates/dbt-sql/template/{{.project_name}}/resources/{{.project_name}}_job.yml.tmpl +++ b/libs/template/templates/dbt-sql/template/{{.project_name}}/resources/{{.project_name}}_job.yml.tmpl @@ -4,7 +4,7 @@ resources: name: {{.project_name}}_job trigger: - # Run this job every day; see https://docs.databricks.com/api/workspace/jobs/create#trigger + # Run this job every day, exactly one day from the last run; see https://docs.databricks.com/api/workspace/jobs/create#trigger periodic: interval: 1 unit: DAYS diff --git a/libs/template/templates/default-python/template/{{.project_name}}/resources/{{.project_name}}_job.yml.tmpl b/libs/template/templates/default-python/template/{{.project_name}}/resources/{{.project_name}}_job.yml.tmpl index 416e9efcb7..d2100e9084 100644 --- a/libs/template/templates/default-python/template/{{.project_name}}/resources/{{.project_name}}_job.yml.tmpl +++ b/libs/template/templates/default-python/template/{{.project_name}}/resources/{{.project_name}}_job.yml.tmpl @@ -11,7 +11,7 @@ resources: name: {{.project_name}}_job trigger: - # Run this job every day; see https://docs.databricks.com/api/workspace/jobs/create#trigger + # Run this job every day, exactly one day from the last run; see https://docs.databricks.com/api/workspace/jobs/create#trigger periodic: interval: 1 unit: DAYS diff --git a/libs/template/templates/default-sql/template/{{.project_name}}/resources/{{.project_name}}_sql_job.yml.tmpl b/libs/template/templates/default-sql/template/{{.project_name}}/resources/{{.project_name}}_sql_job.yml.tmpl index 54c995ea51..4e6803da97 100644 --- a/libs/template/templates/default-sql/template/{{.project_name}}/resources/{{.project_name}}_sql_job.yml.tmpl +++ b/libs/template/templates/default-sql/template/{{.project_name}}/resources/{{.project_name}}_sql_job.yml.tmpl @@ -5,7 +5,7 @@ resources: name: {{.project_name}}_sql_job trigger: - # Run this job every day; see https://docs.databricks.com/api/workspace/jobs/create#trigger + # Run this job every day, exactly one day from the last run; see https://docs.databricks.com/api/workspace/jobs/create#trigger periodic: interval: 1 unit: DAYS