Merge branch 'main' of github.com:databricks/cli into tmp/in-place-pr…

…ototype

bundle/config/mutator/translate_paths.go

@@ -177,7 +177,7 @@ func (t *translateContext) translateNoOp(literal, localFullPath, localRelPath, r
 }
 
 func (t *translateContext) retainLocalAbsoluteFilePath(literal, localFullPath, localRelPath, remotePath string) (string, error) {
-	info, err := t.b.SyncRoot.Stat(localRelPath)
+	info, err := t.b.SyncRoot.Stat(filepath.ToSlash(localRelPath))
 	if errors.Is(err, fs.ErrNotExist) {
 		return "", fmt.Errorf("file %s not found", literal)
 	}

bundle/config/mutator/translate_paths_dashboards_test.go

@@ -0,0 +1,54 @@
+package mutator_test
+
+import (
+	"context"
+	"path/filepath"
+	"testing"
+
+	"github.com/databricks/cli/bundle"
+	"github.com/databricks/cli/bundle/config"
+	"github.com/databricks/cli/bundle/config/mutator"
+	"github.com/databricks/cli/bundle/config/resources"
+	"github.com/databricks/cli/bundle/internal/bundletest"
+	"github.com/databricks/cli/libs/dyn"
+	"github.com/databricks/cli/libs/vfs"
+	"github.com/databricks/databricks-sdk-go/service/dashboards"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestTranslatePathsDashboards_FilePathRelativeSubDirectory(t *testing.T) {
+	dir := t.TempDir()
+	touchEmptyFile(t, filepath.Join(dir, "src", "my_dashboard.lvdash.json"))
+
+	b := &bundle.Bundle{
+		SyncRootPath: dir,
+		SyncRoot:     vfs.MustNew(dir),
+		Config: config.Root{
+			Resources: config.Resources{
+				Dashboards: map[string]*resources.Dashboard{
+					"dashboard": {
+						CreateDashboardRequest: &dashboards.CreateDashboardRequest{
+							DisplayName: "My Dashboard",
+						},
+						FilePath: "../src/my_dashboard.lvdash.json",
+					},
+				},
+			},
+		},
+	}
+
+	bundletest.SetLocation(b, "resources.dashboards", []dyn.Location{{
+		File: filepath.Join(dir, "resources/dashboard.yml"),
+	}})
+
+	diags := bundle.Apply(context.Background(), b, mutator.TranslatePaths())
+	require.NoError(t, diags.Error())
+
+	// Assert that the file path for the dashboard has been converted to its local absolute path.
+	assert.Equal(
+		t,
+		filepath.Join(dir, "src", "my_dashboard.lvdash.json"),
+		b.Config.Resources.Dashboards["dashboard"].FilePath,
+	)
+}

cmd/bundle/run.go

@@ -35,17 +35,23 @@ func promptRunArgument(ctx context.Context, b *bundle.Bundle) (string, error) {
 	return key, nil
 }
 
-func resolveRunArgument(ctx context.Context, b *bundle.Bundle, args []string) (string, error) {
+// resolveRunArgument resolves the resource key to run.
+// It returns the remaining arguments to pass to the runner, if applicable.
+func resolveRunArgument(ctx context.Context, b *bundle.Bundle, args []string) (string, []string, error) {
 	// If no arguments are specified, prompt the user to select something to run.
 	if len(args) == 0 && cmdio.IsPromptSupported(ctx) {
-		return promptRunArgument(ctx, b)
+		key, err := promptRunArgument(ctx, b)
+		if err != nil {
+			return "", nil, err
+		}
+		return key, args, nil
 	}
 
 	if len(args) < 1 {
-		return "", fmt.Errorf("expected a KEY of the resource to run")
+		return "", nil, fmt.Errorf("expected a KEY of the resource to run")
 	}
 
-	return args[0], nil
+	return args[0], args[1:], nil
 }
 
 func keyToRunner(b *bundle.Bundle, arg string) (run.Runner, error) {
@@ -109,7 +115,7 @@ task or a Python wheel task, the second example applies.
 			return err
 		}
 
-		arg, err := resolveRunArgument(ctx, b, args)
+		key, args, err := resolveRunArgument(ctx, b, args)
 		if err != nil {
 			return err
 		}
@@ -124,13 +130,13 @@ task or a Python wheel task, the second example applies.
 			return err
 		}
 
-		runner, err := keyToRunner(b, arg)
+		runner, err := keyToRunner(b, key)
 		if err != nil {
 			return err
 		}
 
 		// Parse additional positional arguments.
-		err = runner.ParseArgs(args[1:], &runOptions)
+		err = runner.ParseArgs(args, &runOptions)
 		if err != nil {
 			return err
 		}

cmd/root/root.go

@@ -75,6 +75,7 @@ func New(ctx context.Context) *cobra.Command {
 
 		// Configure our user agent with the command that's about to be executed.
 		ctx = withCommandInUserAgent(ctx, cmd)
+		ctx = withCommandExecIdInUserAgent(ctx)
 		ctx = withUpstreamInUserAgent(ctx)
 		cmd.SetContext(ctx)
 		return nil

cmd/root/user_agent_command_exec_id.go

@@ -0,0 +1,14 @@
+package root
+
+import (
+	"context"
+
+	"github.com/databricks/databricks-sdk-go/useragent"
+	"github.com/google/uuid"
+)
+
+func withCommandExecIdInUserAgent(ctx context.Context) context.Context {
+	// A UUID that will allow us to correlate multiple API requests made by
+	// the same CLI invocation.
+	return useragent.InContext(ctx, "cmd-exec-id", uuid.New().String())
+}

cmd/root/user_agent_command_exec_id_test.go

@@ -0,0 +1,26 @@
+package root
+
+import (
+	"context"
+	"regexp"
+	"testing"
+
+	"github.com/databricks/databricks-sdk-go/useragent"
+	"github.com/google/uuid"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestWithCommandExecIdInUserAgent(t *testing.T) {
+	ctx := withCommandExecIdInUserAgent(context.Background())
+
+	// Check that the command exec ID is in the user agent string.
+	ua := useragent.FromContext(ctx)
+	re := regexp.MustCompile(`cmd-exec-id/([a-f0-9-]+)`)
+	matches := re.FindAllStringSubmatch(ua, -1)
+
+	// Assert that we have exactly one match and that it's a valid UUID.
+	require.Len(t, matches, 1)
+	_, err := uuid.Parse(matches[0][1])
+	assert.NoError(t, err)
+}

cmd/root/user_agent_command_test.go

@@ -1,13 +1,15 @@
 package root
 
 import (
+	"context"
 	"testing"
 
+	"github.com/databricks/databricks-sdk-go/useragent"
 	"github.com/spf13/cobra"
 	"github.com/stretchr/testify/assert"
 )
 
-func TestCommandString(t *testing.T) {
+func TestWithCommandInUserAgent(t *testing.T) {
 	root := &cobra.Command{
 		Use: "root",
 	}
@@ -26,4 +28,9 @@ func TestCommandString(t *testing.T) {
 	assert.Equal(t, "root", commandString(root))
 	assert.Equal(t, "hello", commandString(hello))
 	assert.Equal(t, "hello_world", commandString(world))
+
+	ctx := withCommandInUserAgent(context.Background(), world)
+
+	ua := useragent.FromContext(ctx)
+	assert.Contains(t, ua, "cmd/hello_world")
 }

go.mod

@@ -14,7 +14,7 @@ require (
 	github.com/hashicorp/go-version v1.7.0 // MPL 2.0
 	github.com/hashicorp/hc-install v0.9.0 // MPL 2.0
 	github.com/hashicorp/terraform-exec v0.21.0 // MPL 2.0
-	github.com/hashicorp/terraform-json v0.22.1 // MPL 2.0
+	github.com/hashicorp/terraform-json v0.23.0 // MPL 2.0
 	github.com/manifoldco/promptui v0.9.0 // BSD-3-Clause
 	github.com/mattn/go-isatty v0.0.20 // MIT
 	github.com/nwidger/jsoncolor v0.3.2 // MIT
@@ -56,7 +56,7 @@ require (
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/stretchr/objx v0.5.2 // indirect
-	github.com/zclconf/go-cty v1.14.4 // indirect
+	github.com/zclconf/go-cty v1.15.0 // indirect
 	go.opencensus.io v0.24.0 // indirect
 	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect
 	go.opentelemetry.io/otel v1.24.0 // indirect

internal/bundle/bundles/python_wheel_task_with_cluster/databricks_template_schema.json

@@ -0,0 +1,25 @@
+{
+    "properties": {
+        "project_name": {
+            "type": "string",
+            "default": "my_test_code",
+            "description": "Unique name for this project"
+        },
+        "spark_version": {
+            "type": "string",
+            "description": "Spark version used for job cluster"
+        },
+        "node_type_id": {
+            "type": "string",
+            "description": "Node type id for job cluster"
+        },
+        "unique_id": {
+            "type": "string",
+            "description": "Unique ID for job name"
+        },
+        "instance_pool_id": {
+            "type": "string",
+            "description": "Instance pool id for job cluster"
+        }
+    }
+}

internal/bundle/bundles/python_wheel_task_with_cluster/template/databricks.yml.tmpl

@@ -0,0 +1,29 @@
+bundle:
+  name: wheel-task
+
+workspace:
+  root_path: "~/.bundle/{{.unique_id}}"
+
+resources:
+  clusters:
+    test_cluster:
+      cluster_name: "test-cluster-{{.unique_id}}"
+      spark_version: "{{.spark_version}}"
+      node_type_id: "{{.node_type_id}}"
+      num_workers: 1
+      data_security_mode: USER_ISOLATION
+
+  jobs:
+    some_other_job:
+      name: "[${bundle.target}] Test Wheel Job {{.unique_id}}"
+      tasks:
+        - task_key: TestTask
+          existing_cluster_id: "${resources.clusters.test_cluster.cluster_id}"
+          python_wheel_task:
+            package_name: my_test_code
+            entry_point: run
+            parameters:
+              - "one"
+              - "two"
+          libraries:
+            - whl: ./dist/*.whl

internal/bundle/bundles/python_wheel_task_with_cluster/template/setup.py.tmpl

@@ -0,0 +1,15 @@
+from setuptools import setup, find_packages
+
+import {{.project_name}}
+
+setup(
+    name="{{.project_name}}",
+    version={{.project_name}}.__version__,
+    author={{.project_name}}.__author__,
+    url="https://databricks.com",
+    author_email="[email protected]",
+    description="my example wheel",
+    packages=find_packages(include=["{{.project_name}}"]),
+    entry_points={"group1": "run={{.project_name}}.__main__:main"},
+    install_requires=["setuptools"],
+)

internal/bundle/bundles/python_wheel_task_with_cluster/template/{{.project_name}}/__init__.py

@@ -0,0 +1,2 @@
+__version__ = "0.0.1"
+__author__ = "Databricks"

internal/bundle/bundles/python_wheel_task_with_cluster/template/{{.project_name}}/__main__.py

@@ -0,0 +1,16 @@
+"""
+The entry point of the Python Wheel
+"""
+
+import sys
+
+
+def main():
+    # This method will print the provided arguments
+    print("Hello from my func")
+    print("Got arguments:")
+    print(sys.argv)
+
+
+if __name__ == "__main__":
+    main()

internal/bundle/python_wheel_test.go

@@ -5,17 +5,18 @@ import (
 
 	"github.com/databricks/cli/internal"
 	"github.com/databricks/cli/internal/acc"
+	"github.com/databricks/cli/internal/testutil"
 	"github.com/databricks/cli/libs/env"
 	"github.com/google/uuid"
 	"github.com/stretchr/testify/require"
 )
 
-func runPythonWheelTest(t *testing.T, sparkVersion string, pythonWheelWrapper bool) {
+func runPythonWheelTest(t *testing.T, templateName string, sparkVersion string, pythonWheelWrapper bool) {
 	ctx, _ := acc.WorkspaceTest(t)
 
 	nodeTypeId := internal.GetNodeTypeId(env.Get(ctx, "CLOUD_ENV"))
 	instancePoolId := env.Get(ctx, "TEST_INSTANCE_POOL_ID")
-	bundleRoot, err := initTestTemplate(t, ctx, "python_wheel_task", map[string]any{
+	bundleRoot, err := initTestTemplate(t, ctx, templateName, map[string]any{
 		"node_type_id":         nodeTypeId,
 		"unique_id":            uuid.New().String(),
 		"spark_version":        sparkVersion,
@@ -45,9 +46,19 @@ func runPythonWheelTest(t *testing.T, sparkVersion string, pythonWheelWrapper bo
 }
 
 func TestAccPythonWheelTaskDeployAndRunWithoutWrapper(t *testing.T) {
-	runPythonWheelTest(t, "13.3.x-snapshot-scala2.12", false)
+	runPythonWheelTest(t, "python_wheel_task", "13.3.x-snapshot-scala2.12", false)
 }
 
 func TestAccPythonWheelTaskDeployAndRunWithWrapper(t *testing.T) {
-	runPythonWheelTest(t, "12.2.x-scala2.12", true)
+	runPythonWheelTest(t, "python_wheel_task", "12.2.x-scala2.12", true)
+}
+
+func TestAccPythonWheelTaskDeployAndRunOnInteractiveCluster(t *testing.T) {
+	_, wt := acc.WorkspaceTest(t)
+
+	if testutil.IsAWSCloud(wt.T) {
+		t.Skip("Skipping test for AWS cloud because it is not permitted to create clusters")
+	}
+
+	runPythonWheelTest(t, "python_wheel_task_with_cluster", defaultSparkVersion, false)
 }