Include a permissions section in all templates (#1713)

## Changes

This updates the templates to include a `permissions` section. Having a
permissions section is a best practice, is helpful to understand the
notion of permissions, and helps diagnose permission errors
(#1386).

This is a cherry-pick from #1387.

This change was verified to work both in dev and prod. Existing unit
tests validate the validity of the templates in these modes.

libs/template/templates/dbt-sql/template/{{.project_name}}/databricks.yml.tmpl

@@ -12,8 +12,10 @@ include:
 targets:
   dev:
     default: true
-    # We use 'mode: development' to indicate this is a personal development copy.
-    # Any job schedules and triggers are paused by default.
+    # The default target uses 'mode: development' to create a development copy.
+    # - Deployed resources get prefixed with '[dev my_user_name]'
+    # - Any job schedules and triggers are paused by default.
+    # See also https://docs.databricks.com/dev-tools/bundles/deployment-modes.html.
     mode: development
     workspace:
       host: {{workspace_host}}
@@ -22,11 +24,10 @@ targets:
     mode: production
     workspace:
       host: {{workspace_host}}
-      # We always use /Users/{{user_name}} for all resources to make sure we only have a single copy.
+      # We explicitly specify /Users/{{user_name}} to make sure we only have a single copy.
       root_path: /Users/{{user_name}}/.bundle/${bundle.name}/${bundle.target}
-    {{- if not is_service_principal}}
+    permissions:
+      - {{if is_service_principal}}service_principal{{else}}user{{end}}_name: {{user_name}}
+        level: CAN_MANAGE
     run_as:
-      # This runs as {{user_name}} in production. We could also use a service principal here
-      # using service_principal_name (see the Databricks documentation).
-      user_name: {{user_name}}
-    {{- end}}
+      {{if is_service_principal}}service_principal{{else}}user{{end}}_name: {{user_name}}

libs/template/templates/default-python/template/{{.project_name}}/databricks.yml.tmpl

@@ -7,44 +7,24 @@ include:
   - resources/*.yml
 
 targets:
-  # The 'dev' target, for development purposes. This target is the default.
   dev:
-    # We use 'mode: development' to indicate this is a personal development copy:
+    # The default target uses 'mode: development' to create a development copy.
     # - Deployed resources get prefixed with '[dev my_user_name]'
-    # - Any job schedules and triggers are paused by default
-    # - The 'development' mode is used for Delta Live Tables pipelines
+    # - Any job schedules and triggers are paused by default.
+    # See also https://docs.databricks.com/dev-tools/bundles/deployment-modes.html.
     mode: development
     default: true
     workspace:
       host: {{workspace_host}}
 
-  ## Optionally, there could be a 'staging' target here.
-  ## (See Databricks docs on CI/CD at https://docs.databricks.com/dev-tools/bundles/ci-cd.html.)
-  #
-  # staging:
-  #   workspace:
-  #     host: {{workspace_host}}
-
-  # The 'prod' target, used for production deployment.
   prod:
-    # We use 'mode: production' to indicate this is a production deployment.
-    # Doing so enables strict verification of the settings below.
     mode: production
     workspace:
       host: {{workspace_host}}
-      # We always use /Users/{{user_name}} for all resources to make sure we only have a single copy.
-      {{- /*
-      Internal note 2023-12: CLI versions v0.211.0 and before would show an error when using `mode: production`
-      with a path that doesn't say "/Shared". For now, we'll include an extra comment in the template
-      to explain that customers should update if they see this.
-      */}}
-      # If this path results in an error, please make sure you have a recent version of the CLI installed.
+      # We explicitly specify /Users/{{user_name}} to make sure we only have a single copy.
       root_path: /Users/{{user_name}}/.bundle/${bundle.name}/${bundle.target}
+    permissions:
+      - {{if is_service_principal}}service_principal{{else}}user{{end}}_name: {{user_name}}
+        level: CAN_MANAGE
     run_as:
-      {{- if is_service_principal}}
-      service_principal_name: {{user_name}}
-      {{- else}}
-      # This runs as {{user_name}} in production. We could also use a service principal here,
-      # see https://docs.databricks.com/dev-tools/bundles/permissions.html.
-      user_name: {{user_name}}
-      {{- end}}
+      {{if is_service_principal}}service_principal{{else}}user{{end}}_name: {{user_name}}

libs/template/templates/default-sql/template/{{.project_name}}/databricks.yml.tmpl

@@ -18,16 +18,16 @@ variables:
 {{- $dev_schema := .shared_schema }}
 {{- $prod_schema := .shared_schema }}
 {{- if (regexp "^yes").MatchString .personal_schemas}}
-{{- $dev_schema = "${workspace.current_user.short_name}"}}
-{{- $prod_schema = "default"}}
+  {{- $dev_schema = "${workspace.current_user.short_name}"}}
+  {{- $prod_schema = "default"}}
 {{- end}}
 
-# Deployment targets.
 targets:
-  # The 'dev' target, for development purposes. This target is the default.
   dev:
-    # We use 'mode: development' to indicate this is a personal development copy.
-    # Any job schedules and triggers are paused by default
+    # The default target uses 'mode: development' to create a development copy.
+    # - Deployed resources get prefixed with '[dev my_user_name]'
+    # - Any job schedules and triggers are paused by default.
+    # See also https://docs.databricks.com/dev-tools/bundles/deployment-modes.html.
     mode: development
     default: true
     workspace:
@@ -37,35 +37,18 @@ targets:
       catalog: {{.default_catalog}}
       schema: {{$dev_schema}}
 
-  ## Optionally, there could be a 'staging' target here.
-  ## (See Databricks docs on CI/CD at https://docs.databricks.com/dev-tools/bundles/ci-cd.html.)
-  #
-  # staging:
-  #   workspace:
-  #     host: {{workspace_host}}
-
-  # The 'prod' target, used for production deployment.
   prod:
-    # We use 'mode: production' to indicate this is a production deployment.
-    # Doing so enables strict verification of the settings below.
     mode: production
     workspace:
       host: {{workspace_host}}
-      # We always use /Users/{{user_name}} for all resources to make sure we only have a single copy.
-      {{- /*
-      Internal note 2023-12: CLI versions v0.211.0 and before would show an error when using `mode: production`
-      with a path that doesn't say "/Shared". For now, we'll include an extra comment in the template
-      to explain that customers should update if they see this.
-      */}}
-      # If this path results in an error, please make sure you have a recent version of the CLI installed.
+      # We explicitly specify /Users/{{user_name}} to make sure we only have a single copy.
       root_path: /Users/{{user_name}}/.bundle/${bundle.name}/${bundle.target}
     variables:
       warehouse_id: {{index ((regexp "[^/]+$").FindStringSubmatch .http_path) 0}}
       catalog: {{.default_catalog}}
       schema: {{$prod_schema}}
-    {{- if not is_service_principal}}
+    permissions:
+      - {{if is_service_principal}}service_principal{{else}}user{{end}}_name: {{user_name}}
+        level: CAN_MANAGE
     run_as:
-      # This runs as {{user_name}} in production. We could also use a service principal here
-      # using service_principal_name (see https://docs.databricks.com/en/dev-tools/bundles/permissions.html).
-      user_name: {{user_name}}
-    {{end -}}
+      {{if is_service_principal}}service_principal{{else}}user{{end}}_name: {{user_name}}