Merge remote-tracking branch 'databricks/main' into target-mode-for-m…

…onitors

README.md

@@ -4,18 +4,21 @@
 
 This project is in Public Preview.
 
-Documentation about the full REST API coverage is available in the [docs folder](docs/commands.md).
-
 Documentation is available at https://docs.databricks.com/dev-tools/cli/databricks-cli.html.
 
 ## Installation
 
 This CLI is packaged as a dependency-free binary executable and may be located in any directory.
 See https://github.com/databricks/cli/releases for releases and
-[the docs pages](https://docs.databricks.com/dev-tools/cli/databricks-cli.html) for
-installation instructions.
+the [Databricks documentation](https://docs.databricks.com/en/dev-tools/cli/install.html) for detailed information about installing the CLI.
+
+------
+### Homebrew
+
+We maintain a [Homebrew tap](https://github.com/databricks/homebrew-tap) for installing the Databricks CLI. You can find instructions for how to install, upgrade and downgrade the CLI using Homebrew [here](https://github.com/databricks/homebrew-tap/blob/main/README.md).
 
 ------
+### Docker
 You can use the CLI via a Docker image by pulling the image from `ghcr.io`. You can find all available versions
 at: https://github.com/databricks/cli/pkgs/container/cli.
 ```

bundle/bundle.go

@@ -16,6 +16,7 @@ import (
 	"github.com/databricks/cli/bundle/config"
 	"github.com/databricks/cli/bundle/env"
 	"github.com/databricks/cli/bundle/metadata"
+	"github.com/databricks/cli/libs/fileset"
 	"github.com/databricks/cli/libs/folders"
 	"github.com/databricks/cli/libs/git"
 	"github.com/databricks/cli/libs/locker"
@@ -50,6 +51,9 @@ type Bundle struct {
 	clientOnce sync.Once
 	client     *databricks.WorkspaceClient
 
+	// Files that are synced to the workspace.file_path
+	Files []fileset.File
+
 	// Stores an initialized copy of this bundle's Terraform wrapper.
 	Terraform *tfexec.Terraform
 

bundle/config/mutator/mutator.go

@@ -26,10 +26,3 @@ func DefaultMutators() []bundle.Mutator {
 		LoadGitDetails(),
 	}
 }
-
-func DefaultMutatorsForTarget(target string) []bundle.Mutator {
-	return append(
-		DefaultMutators(),
-		SelectTarget(target),
-	)
-}

bundle/config/mutator/resolve_resource_references_test.go

@@ -8,6 +8,7 @@ import (
 	"github.com/databricks/cli/bundle"
 	"github.com/databricks/cli/bundle/config"
 	"github.com/databricks/cli/bundle/config/variable"
+	"github.com/databricks/cli/libs/env"
 	"github.com/stretchr/testify/mock"
 	"github.com/stretchr/testify/require"
 
@@ -194,3 +195,37 @@ func TestResolveLookupVariableReferencesInVariableLookups(t *testing.T) {
 	diags := bundle.Apply(context.Background(), b, bundle.Seq(ResolveVariableReferencesInLookup(), ResolveResourceReferences()))
 	require.ErrorContains(t, diags.Error(), "lookup variables cannot contain references to another lookup variables")
 }
+
+func TestNoResolveLookupIfVariableSetWithEnvVariable(t *testing.T) {
+	s := func(s string) *string {
+		return &s
+	}
+
+	b := &bundle.Bundle{
+		Config: config.Root{
+			Bundle: config.Bundle{
+				Target: "dev",
+			},
+			Variables: map[string]*variable.Variable{
+				"foo": {
+					Value: s("bar"),
+				},
+				"lookup": {
+					Lookup: &variable.Lookup{
+						Cluster: "cluster-${var.foo}-${bundle.target}",
+					},
+				},
+			},
+		},
+	}
+
+	m := mocks.NewMockWorkspaceClient(t)
+	b.SetWorkpaceClient(m.WorkspaceClient)
+
+	ctx := context.Background()
+	ctx = env.Set(ctx, "BUNDLE_VAR_lookup", "1234-5678-abcd")
+
+	diags := bundle.Apply(ctx, b, bundle.Seq(SetVariables(), ResolveVariableReferencesInLookup(), ResolveResourceReferences()))
+	require.NoError(t, diags.Error())
+	require.Equal(t, "1234-5678-abcd", *b.Config.Variables["lookup"].Value)
+}

bundle/config/mutator/set_variables.go

@@ -37,6 +37,12 @@ func setVariable(ctx context.Context, v *variable.Variable, name string) diag.Di
 		return nil
 	}
 
+	// case: Defined a variable for named lookup for a resource
+	// It will be resolved later in ResolveResourceReferences mutator
+	if v.Lookup != nil {
+		return nil
+	}
+
 	// case: Set the variable to its default value
 	if v.HasDefault() {
 		err := v.Set(*v.Default)
@@ -46,12 +52,6 @@ func setVariable(ctx context.Context, v *variable.Variable, name string) diag.Di
 		return nil
 	}
 
-	// case: Defined a variable for named lookup for a resource
-	// It will be resolved later in ResolveResourceReferences mutator
-	if v.Lookup != nil {
-		return nil
-	}
-
 	// We should have had a value to set for the variable at this point.
 	return diag.Errorf(`no value assigned to required variable %s. Assignment can be done through the "--var" flag or by setting the %s environment variable`, name, bundleVarPrefix+name)
 }

bundle/config/root.go

@@ -471,3 +471,9 @@ func (r Root) GetLocation(path string) dyn.Location {
 	}
 	return v.Location()
 }
+
+// Value returns the dynamic configuration value of the root object. This value
+// is the source of truth and is kept in sync with values in the typed configuration.
+func (r Root) Value() dyn.Value {
+	return r.value
+}

bundle/deploy/files/upload.go

@@ -23,7 +23,7 @@ func (m *upload) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
 		return diag.FromErr(err)
 	}
 
-	err = sync.RunOnce(ctx)
+	b.Files, err = sync.RunOnce(ctx)
 	if err != nil {
 		return diag.FromErr(err)
 	}

bundle/deploy/state_update.go

@@ -11,7 +11,6 @@ import (
 	"time"
 
 	"github.com/databricks/cli/bundle"
-	"github.com/databricks/cli/bundle/deploy/files"
 	"github.com/databricks/cli/internal/build"
 	"github.com/databricks/cli/libs/diag"
 	"github.com/databricks/cli/libs/log"
@@ -40,19 +39,8 @@ func (s *stateUpdate) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnost
 	state.CliVersion = build.GetInfo().Version
 	state.Version = DeploymentStateVersion
 
-	// Get the current file list.
-	sync, err := files.GetSync(ctx, bundle.ReadOnly(b))
-	if err != nil {
-		return diag.FromErr(err)
-	}
-
-	files, err := sync.GetFileList(ctx)
-	if err != nil {
-		return diag.FromErr(err)
-	}
-
-	// Update the state with the current file list.
-	fl, err := FromSlice(files)
+	// Update the state with the current list of synced files.
+	fl, err := FromSlice(b.Files)
 	if err != nil {
 		return diag.FromErr(err)
 	}

bundle/deploy/state_update_test.go

@@ -10,19 +10,23 @@ import (
 	"github.com/databricks/cli/bundle/config"
 	"github.com/databricks/cli/internal/build"
 	"github.com/databricks/cli/internal/testutil"
-	databrickscfg "github.com/databricks/databricks-sdk-go/config"
-	"github.com/databricks/databricks-sdk-go/experimental/mocks"
+	"github.com/databricks/cli/libs/fileset"
+	"github.com/databricks/cli/libs/vfs"
 	"github.com/databricks/databricks-sdk-go/service/iam"
-	"github.com/databricks/databricks-sdk-go/service/workspace"
-	"github.com/stretchr/testify/mock"
 	"github.com/stretchr/testify/require"
 )
 
-func TestStateUpdate(t *testing.T) {
-	s := &stateUpdate{}
+func setupBundleForStateUpdate(t *testing.T) *bundle.Bundle {
+	tmpDir := t.TempDir()
+
+	testutil.Touch(t, tmpDir, "test1.py")
+	testutil.TouchNotebook(t, tmpDir, "test2.py")
+
+	files, err := fileset.New(vfs.MustNew(tmpDir)).All()
+	require.NoError(t, err)
 
-	b := &bundle.Bundle{
-		RootPath: t.TempDir(),
+	return &bundle.Bundle{
+		RootPath: tmpDir,
 		Config: config.Root{
 			Bundle: config.Bundle{
 				Target: "default",
@@ -37,22 +41,14 @@ func TestStateUpdate(t *testing.T) {
 				},
 			},
 		},
+		Files: files,
 	}
+}
 
-	testutil.Touch(t, b.RootPath, "test1.py")
-	testutil.Touch(t, b.RootPath, "test2.py")
-
-	m := mocks.NewMockWorkspaceClient(t)
-	m.WorkspaceClient.Config = &databrickscfg.Config{
-		Host: "https://test.com",
-	}
-	b.SetWorkpaceClient(m.WorkspaceClient)
-
-	wsApi := m.GetMockWorkspaceAPI()
-	wsApi.EXPECT().GetStatusByPath(mock.Anything, "/files").Return(&workspace.ObjectInfo{
-		ObjectType: "DIRECTORY",
-	}, nil)
+func TestStateUpdate(t *testing.T) {
+	s := &stateUpdate{}
 
+	b := setupBundleForStateUpdate(t)
 	ctx := context.Background()
 
 	diags := bundle.Apply(ctx, b, s)
@@ -63,7 +59,15 @@ func TestStateUpdate(t *testing.T) {
 	require.NoError(t, err)
 
 	require.Equal(t, int64(1), state.Seq)
-	require.Len(t, state.Files, 3)
+	require.Equal(t, state.Files, Filelist{
+		{
+			LocalPath: "test1.py",
+		},
+		{
+			LocalPath:  "test2.py",
+			IsNotebook: true,
+		},
+	})
 	require.Equal(t, build.GetInfo().Version, state.CliVersion)
 
 	diags = bundle.Apply(ctx, b, s)
@@ -74,45 +78,22 @@ func TestStateUpdate(t *testing.T) {
 	require.NoError(t, err)
 
 	require.Equal(t, int64(2), state.Seq)
-	require.Len(t, state.Files, 3)
+	require.Equal(t, state.Files, Filelist{
+		{
+			LocalPath: "test1.py",
+		},
+		{
+			LocalPath:  "test2.py",
+			IsNotebook: true,
+		},
+	})
 	require.Equal(t, build.GetInfo().Version, state.CliVersion)
 }
 
 func TestStateUpdateWithExistingState(t *testing.T) {
 	s := &stateUpdate{}
 
-	b := &bundle.Bundle{
-		RootPath: t.TempDir(),
-		Config: config.Root{
-			Bundle: config.Bundle{
-				Target: "default",
-			},
-			Workspace: config.Workspace{
-				StatePath: "/state",
-				FilePath:  "/files",
-				CurrentUser: &config.User{
-					User: &iam.User{
-						UserName: "test-user",
-					},
-				},
-			},
-		},
-	}
-
-	testutil.Touch(t, b.RootPath, "test1.py")
-	testutil.Touch(t, b.RootPath, "test2.py")
-
-	m := mocks.NewMockWorkspaceClient(t)
-	m.WorkspaceClient.Config = &databrickscfg.Config{
-		Host: "https://test.com",
-	}
-	b.SetWorkpaceClient(m.WorkspaceClient)
-
-	wsApi := m.GetMockWorkspaceAPI()
-	wsApi.EXPECT().GetStatusByPath(mock.Anything, "/files").Return(&workspace.ObjectInfo{
-		ObjectType: "DIRECTORY",
-	}, nil)
-
+	b := setupBundleForStateUpdate(t)
 	ctx := context.Background()
 
 	// Create an existing state file.
@@ -144,6 +125,14 @@ func TestStateUpdateWithExistingState(t *testing.T) {
 	require.NoError(t, err)
 
 	require.Equal(t, int64(11), state.Seq)
-	require.Len(t, state.Files, 3)
+	require.Equal(t, state.Files, Filelist{
+		{
+			LocalPath: "test1.py",
+		},
+		{
+			LocalPath:  "test2.py",
+			IsNotebook: true,
+		},
+	})
 	require.Equal(t, build.GetInfo().Version, state.CliVersion)
 }

bundle/tests/variables/env_overrides/databricks.yml

@@ -8,14 +8,16 @@ variables:
 
   d:
     description: variable with lookup
-    lookup:
-      cluster: some-cluster
+    default: ""
 
   e:
     description: variable with lookup
-    lookup:
-      instance_pool: some-pool
+    default: "some-value"
 
+  f:
+    description: variable with lookup
+    lookup:
+      cluster_policy: wrong-cluster-policy
 bundle:
   name: test bundle
 
@@ -49,4 +51,7 @@ targets:
       e:
         lookup:
           instance_pool: some-test-instance-pool
+      f:
+        lookup:
+          cluster_policy: some-test-cluster-policy
       b: prod-b