From deb3e365cdc4b3345542f8a9c4030eb0163c6bff Mon Sep 17 00:00:00 2001 From: "Lennart Kats (databricks)" Date: Wed, 19 Jun 2024 15:54:35 +0200 Subject: [PATCH] Pause quality monitors when "mode: development" is used (#1481) ## Changes Similar to scheduled jobs, quality monitors should be paused when in development mode (in line with the [behavior for scheduled jobs](https://docs.databricks.com/en/dev-tools/bundles/deployment-modes.html)). @aravind-segu @arpitjasa-db please take a look and verify this behavior. - [x] Followup: documentation changes. If we make this change we should update https://docs.databricks.com/dev-tools/bundles/deployment-modes.html. ## Tests Unit tests --- bundle/config/mutator/process_target_mode.go | 10 ++++++++++ .../config/mutator/process_target_mode_test.go | 16 ++++++++++++++++ bundle/tests/quality_monitor/databricks.yml | 15 +++++++++++---- bundle/tests/quality_monitor_test.go | 6 +++--- 4 files changed, 40 insertions(+), 7 deletions(-) diff --git a/bundle/config/mutator/process_target_mode.go b/bundle/config/mutator/process_target_mode.go index 8e70fab73b..53d97a5b40 100644 --- a/bundle/config/mutator/process_target_mode.go +++ b/bundle/config/mutator/process_target_mode.go @@ -11,6 +11,7 @@ import ( "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/log" + "github.com/databricks/databricks-sdk-go/service/catalog" "github.com/databricks/databricks-sdk-go/service/jobs" "github.com/databricks/databricks-sdk-go/service/ml" ) @@ -105,6 +106,15 @@ func transformDevelopmentMode(ctx context.Context, b *bundle.Bundle) diag.Diagno // (registered models in Unity Catalog don't yet support tags) } + for i := range r.QualityMonitors { + // Remove all schedules from monitors, since they don't support pausing/unpausing. + // Quality monitors might support the "pause" property in the future, so at the + // CLI level we do respect that property if it is set to "unpaused". + if r.QualityMonitors[i].Schedule != nil && r.QualityMonitors[i].Schedule.PauseStatus != catalog.MonitorCronSchedulePauseStatusUnpaused { + r.QualityMonitors[i].Schedule = nil + } + } + return nil } diff --git a/bundle/config/mutator/process_target_mode_test.go b/bundle/config/mutator/process_target_mode_test.go index cf8229bfeb..6a38f274fc 100644 --- a/bundle/config/mutator/process_target_mode_test.go +++ b/bundle/config/mutator/process_target_mode_test.go @@ -99,6 +99,20 @@ func mockBundle(mode config.Mode) *bundle.Bundle { }, QualityMonitors: map[string]*resources.QualityMonitor{ "qualityMonitor1": {CreateMonitor: &catalog.CreateMonitor{TableName: "qualityMonitor1"}}, + "qualityMonitor2": { + CreateMonitor: &catalog.CreateMonitor{ + TableName: "qualityMonitor2", + Schedule: &catalog.MonitorCronSchedule{}, + }, + }, + "qualityMonitor3": { + CreateMonitor: &catalog.CreateMonitor{ + TableName: "qualityMonitor3", + Schedule: &catalog.MonitorCronSchedule{ + PauseStatus: catalog.MonitorCronSchedulePauseStatusUnpaused, + }, + }, + }, }, }, }, @@ -151,6 +165,8 @@ func TestProcessTargetModeDevelopment(t *testing.T) { // Quality Monitor 1 assert.Equal(t, "qualityMonitor1", b.Config.Resources.QualityMonitors["qualityMonitor1"].TableName) + assert.Nil(t, b.Config.Resources.QualityMonitors["qualityMonitor2"].Schedule) + assert.Equal(t, catalog.MonitorCronSchedulePauseStatusUnpaused, b.Config.Resources.QualityMonitors["qualityMonitor3"].Schedule.PauseStatus) } func TestProcessTargetModeDevelopmentTagNormalizationForAws(t *testing.T) { diff --git a/bundle/tests/quality_monitor/databricks.yml b/bundle/tests/quality_monitor/databricks.yml index 3abcdfddaa..6138b9357c 100644 --- a/bundle/tests/quality_monitor/databricks.yml +++ b/bundle/tests/quality_monitor/databricks.yml @@ -1,19 +1,26 @@ +bundle: + name: quality_monitors + resources: quality_monitors: my_monitor: - table_name: "main.test.thing1" + table_name: "main.test.dev" assets_dir: "/Shared/provider-test/databricks_monitoring/main.test.thing1" - output_schema_name: "test" + output_schema_name: "main.dev" inference_log: granularities: ["1 day"] timestamp_col: "timestamp" prediction_col: "prediction" model_id_col: "model_id" problem_type: "PROBLEM_TYPE_REGRESSION" + schedule: + quartz_cron_expression: "0 0 12 * * ?" # every day at noon + timezone_id: UTC targets: development: mode: development + default: true resources: quality_monitors: my_monitor: @@ -24,14 +31,14 @@ targets: quality_monitors: my_monitor: table_name: "main.test.staging" - output_schema_name: "staging" + output_schema_name: "main.staging" production: resources: quality_monitors: my_monitor: table_name: "main.test.prod" - output_schema_name: "prod" + output_schema_name: "main.prod" inference_log: granularities: ["1 hour"] timestamp_col: "timestamp_prod" diff --git a/bundle/tests/quality_monitor_test.go b/bundle/tests/quality_monitor_test.go index d5db05196c..9b91052f59 100644 --- a/bundle/tests/quality_monitor_test.go +++ b/bundle/tests/quality_monitor_test.go @@ -24,7 +24,7 @@ func TestMonitorTableNames(t *testing.T) { p := b.Config.Resources.QualityMonitors["my_monitor"] assert.Equal(t, "main.test.dev", p.TableName) assert.Equal(t, "/Shared/provider-test/databricks_monitoring/main.test.thing1", p.AssetsDir) - assert.Equal(t, "test", p.OutputSchemaName) + assert.Equal(t, "main.dev", p.OutputSchemaName) assertExpectedMonitor(t, p) } @@ -36,7 +36,7 @@ func TestMonitorStaging(t *testing.T) { p := b.Config.Resources.QualityMonitors["my_monitor"] assert.Equal(t, "main.test.staging", p.TableName) assert.Equal(t, "/Shared/provider-test/databricks_monitoring/main.test.thing1", p.AssetsDir) - assert.Equal(t, "staging", p.OutputSchemaName) + assert.Equal(t, "main.staging", p.OutputSchemaName) assertExpectedMonitor(t, p) } @@ -48,7 +48,7 @@ func TestMonitorProduction(t *testing.T) { p := b.Config.Resources.QualityMonitors["my_monitor"] assert.Equal(t, "main.test.prod", p.TableName) assert.Equal(t, "/Shared/provider-test/databricks_monitoring/main.test.thing1", p.AssetsDir) - assert.Equal(t, "prod", p.OutputSchemaName) + assert.Equal(t, "main.prod", p.OutputSchemaName) inferenceLog := p.InferenceLog assert.Equal(t, []string{"1 day", "1 hour"}, inferenceLog.Granularities)