Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Encourage the use of root_path in production to ensure single deployment #1712

Merged
merged 12 commits into from
Jan 13, 2025
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions bundle/bundle.go
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,9 @@ type Bundle struct {
// It is loaded from the bundle configuration files and mutators may update it.
Config config.Root

// Target stores a snapshot of the Root.Bundle.Target configuration when it was selected by SelectTarget.
Target *config.Target `json:"target_config,omitempty" bundle:"internal"`

// Metadata about the bundle deployment. This is the interface Databricks services
// rely on to integrate with bundles when they need additional information about
// a bundle deployment.
Expand Down
22 changes: 20 additions & 2 deletions bundle/config/mutator/process_target_mode.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package mutator

import (
"context"
"fmt"
"strings"

"github.com/databricks/cli/bundle"
Expand Down Expand Up @@ -146,8 +147,21 @@ func validateProductionMode(ctx context.Context, b *bundle.Bundle, isPrincipalUs
}
}

if !isPrincipalUsed && !isRunAsSet(r) {
return diag.Errorf("'run_as' must be set for all jobs when using 'mode: production'")
// We need to verify that there is only a single deployment of the current target.
// The best way to enforce this is to explicitly set root_path.
advice := fmt.Sprintf(
"set 'workspace.root_path' to make sure only one copy is deployed. A common practice is to use a username or principal name in this path, i.e. root_path: /Workspace/Users/%s/.bundle/${bundle.name}/${bundle.target}",
b.Config.Workspace.CurrentUser.UserName,
)
if !isExplicitRootSet(b) {
if isRunAsSet(r) || isPrincipalUsed {
// Just setting run_as is not enough to guarantee a single deployment,
// and neither is setting a principal.
// We only show a warning for these cases since we didn't historically
// report an error for them.
lennartkats-db marked this conversation as resolved.
Show resolved Hide resolved
return diag.Recommendationf("target with 'mode: production' should %s", advice)
}
return diag.Errorf("target with 'mode: production' must %s", advice)
}
return nil
}
Expand All @@ -164,6 +178,10 @@ func isRunAsSet(r config.Resources) bool {
return true
}

func isExplicitRootSet(b *bundle.Bundle) bool {
lennartkats-db marked this conversation as resolved.
Show resolved Hide resolved
return b.Target != nil && b.Target.Workspace != nil && b.Target.Workspace.RootPath != ""
}

func (m *processTargetMode) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
switch b.Config.Bundle.Mode {
case config.Development:
Expand Down
21 changes: 19 additions & 2 deletions bundle/config/mutator/process_target_mode_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -321,15 +321,15 @@ func TestProcessTargetModeProduction(t *testing.T) {
b := mockBundle(config.Production)

diags := validateProductionMode(context.Background(), b, false)
require.ErrorContains(t, diags.Error(), "run_as")
require.ErrorContains(t, diags.Error(), "target with 'mode: production' must set 'workspace.root_path' to make sure only one copy is deployed. A common practice is to use a username or principal name in this path, i.e. root_path: /Workspace/Users/[email protected]/.bundle/${bundle.name}/${bundle.target}")

b.Config.Workspace.StatePath = "/Shared/.bundle/x/y/state"
b.Config.Workspace.ArtifactPath = "/Shared/.bundle/x/y/artifacts"
b.Config.Workspace.FilePath = "/Shared/.bundle/x/y/files"
b.Config.Workspace.ResourcePath = "/Shared/.bundle/x/y/resources"

diags = validateProductionMode(context.Background(), b, false)
require.ErrorContains(t, diags.Error(), "production")
require.ErrorContains(t, diags.Error(), "target with 'mode: production' must set 'workspace.root_path' to make sure only one copy is deployed. A common practice is to use a username or principal name in this path, i.e. root_path: /Workspace/Users/[email protected]/.bundle/${bundle.name}/${bundle.target}")

permissions := []resources.Permission{
{
Expand Down Expand Up @@ -375,6 +375,23 @@ func TestProcessTargetModeProductionOkForPrincipal(t *testing.T) {
require.NoError(t, diags.Error())
}

func TestProcessTargetModeProductionOkWithRootPath(t *testing.T) {
b := mockBundle(config.Production)

// Our target has all kinds of problems when not using service principals ...
diags := validateProductionMode(context.Background(), b, false)
require.Error(t, diags.Error())

// ... but we're okay if we specify a root path
b.Target = &config.Target{
Workspace: &config.Workspace{
RootPath: "some-root-path",
},
}
diags = validateProductionMode(context.Background(), b, false)
require.NoError(t, diags.Error())
}

// Make sure that we have test coverage for all resource types
func TestAllResourcesMocked(t *testing.T) {
b := mockBundle(config.Development)
Expand Down
7 changes: 5 additions & 2 deletions bundle/config/mutator/select_target.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ type selectTarget struct {
}

// SelectTarget merges the specified target into the root configuration.
// After merging, it removes the 'Targets' section from the configuration.
func SelectTarget(name string) bundle.Mutator {
return &selectTarget{
name: name,
Expand All @@ -31,7 +32,7 @@ func (m *selectTarget) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnosti
}

// Get specified target
_, ok := b.Config.Targets[m.name]
target, ok := b.Config.Targets[m.name]
if !ok {
return diag.Errorf("%s: no such target. Available targets: %s", m.name, strings.Join(maps.Keys(b.Config.Targets), ", "))
}
Expand All @@ -43,13 +44,15 @@ func (m *selectTarget) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnosti
}

// Store specified target in configuration for reference.
b.Target = target
b.Config.Bundle.Target = m.name

// We do this for backward compatibility.
// TODO: remove when Environments section is not supported anymore.
b.Config.Bundle.Environment = b.Config.Bundle.Target

// Clear targets after loading.
// Cleanup the original targets and environments sections since they
// show up in the JSON output of the 'summary' and 'validate' commands.
b.Config.Targets = nil
b.Config.Environments = nil

pietern marked this conversation as resolved.
Show resolved Hide resolved
Expand Down
4 changes: 2 additions & 2 deletions bundle/config/root.go
Original file line number Diff line number Diff line change
Expand Up @@ -47,8 +47,8 @@ type Root struct {

// Targets can be used to differentiate settings and resources between
// bundle deployment targets (e.g. development, staging, production).
// If not specified, the code below initializes this field with a
// single default-initialized target called "default".
// Note that this field is set to 'nil' by the SelectTarget mutator;
// use bundle.Bundle.Target to access the selected target configuration.
Targets map[string]*Target `json:"targets,omitempty"`

// DEPRECATED. Left for backward compatibility with Targets
Expand Down
10 changes: 10 additions & 0 deletions libs/diag/diagnostic.go
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,16 @@ func Infof(format string, args ...any) Diagnostics {
}
}

// Recommendationf creates a new recommendation diagnostic.
func Recommendationf(format string, args ...any) Diagnostics {
return []Diagnostic{
{
Severity: Recommendation,
Summary: fmt.Sprintf(format, args...),
},
}
}

// Diagnostics holds zero or more instances of [Diagnostic].
type Diagnostics []Diagnostic

Expand Down
Loading