Added support for Databricks Apps in DABs

bundle/apps/interpolate_variables.go

@@ -0,0 +1,69 @@
+package apps
+
+import (
+	"context"
+
+	"github.com/databricks/cli/bundle"
+	"github.com/databricks/cli/libs/diag"
+	"github.com/databricks/cli/libs/dyn"
+	"github.com/databricks/cli/libs/dyn/dynvar"
+)
+
+type interpolateVariables struct{}
+
+func (i *interpolateVariables) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
+	pattern := dyn.NewPattern(
+		dyn.Key("resources"),
+		dyn.Key("apps"),
+		dyn.AnyKey(),
+		dyn.Key("config"),
+	)
+
+	err := b.Config.Mutate(func(root dyn.Value) (dyn.Value, error) {
+		return dyn.MapByPattern(root, pattern, func(p dyn.Path, v dyn.Value) (dyn.Value, error) {
+			return dynvar.Resolve(v, func(path dyn.Path) (dyn.Value, error) {
+				switch path[0] {
+				case dyn.Key("databricks_pipeline"):
+					path = dyn.NewPath(dyn.Key("resources"), dyn.Key("pipelines")).Append(path[1:]...)
+				case dyn.Key("databricks_job"):
+					path = dyn.NewPath(dyn.Key("resources"), dyn.Key("jobs")).Append(path[1:]...)
+				case dyn.Key("databricks_mlflow_model"):
+					path = dyn.NewPath(dyn.Key("resources"), dyn.Key("models")).Append(path[1:]...)
+				case dyn.Key("databricks_mlflow_experiment"):
+					path = dyn.NewPath(dyn.Key("resources"), dyn.Key("experiments")).Append(path[1:]...)
+				case dyn.Key("databricks_model_serving"):
+					path = dyn.NewPath(dyn.Key("resources"), dyn.Key("model_serving_endpoints")).Append(path[1:]...)
+				case dyn.Key("databricks_registered_model"):
+					path = dyn.NewPath(dyn.Key("resources"), dyn.Key("registered_models")).Append(path[1:]...)
+				case dyn.Key("databricks_quality_monitor"):
+					path = dyn.NewPath(dyn.Key("resources"), dyn.Key("quality_monitors")).Append(path[1:]...)
+				case dyn.Key("databricks_schema"):
+					path = dyn.NewPath(dyn.Key("resources"), dyn.Key("schemas")).Append(path[1:]...)
+				case dyn.Key("databricks_volume"):
+					path = dyn.NewPath(dyn.Key("resources"), dyn.Key("volumes")).Append(path[1:]...)
+				case dyn.Key("databricks_cluster"):
+					path = dyn.NewPath(dyn.Key("resources"), dyn.Key("clusters")).Append(path[1:]...)
+				case dyn.Key("databricks_dashboard"):
+					path = dyn.NewPath(dyn.Key("resources"), dyn.Key("dashboards")).Append(path[1:]...)
+				case dyn.Key("databricks_app"):
+					path = dyn.NewPath(dyn.Key("resources"), dyn.Key("apps")).Append(path[1:]...)
+				default:
+					// Trigger "key not found" for unknown resource types.
+					return dyn.GetByPath(root, path)
+				}
+
+				return dyn.GetByPath(root, path)
+			})
+		})
+	})
+
+	return diag.FromErr(err)
+}
+
+func (i *interpolateVariables) Name() string {
+	return "apps.InterpolateVariables"
+}
+
+func InterpolateVariables() bundle.Mutator {
+	return &interpolateVariables{}
+}

bundle/apps/interpolate_variables_test.go

@@ -0,0 +1,49 @@
+package apps
+
+import (
+	"context"
+	"testing"
+
+	"github.com/databricks/cli/bundle"
+	"github.com/databricks/cli/bundle/config"
+	"github.com/databricks/cli/bundle/config/resources"
+	"github.com/databricks/databricks-sdk-go/service/apps"
+	"github.com/stretchr/testify/require"
+)
+
+func TestAppInterpolateVariables(t *testing.T) {
+	b := &bundle.Bundle{
+		Config: config.Root{
+			Resources: config.Resources{
+				Apps: map[string]*resources.App{
+					"my_app_1": {
+						App: &apps.App{
+							Name: "my_app_1",
+						},
+						Config: map[string]any{
+							"command": []string{"echo", "hello"},
+							"env": []map[string]string{
+								{"name": "JOB_ID", "value": "${resources.jobs.my_job.id}"},
+							},
+						},
+					},
+					"my_app_2": {
+						App: &apps.App{
+							Name: "my_app_2",
+						},
+					},
+				},
+				Jobs: map[string]*resources.Job{
+					"my_job": {
+						ID: "123",
+					},
+				},
+			},
+		},
+	}
+
+	diags := bundle.Apply(context.Background(), b, InterpolateVariables())
+	require.Empty(t, diags)
+	require.Equal(t, []any([]any{map[string]any{"name": "JOB_ID", "value": "123"}}), b.Config.Resources.Apps["my_app_1"].Config["env"])
+	require.Nil(t, b.Config.Resources.Apps["my_app_2"].Config)
+}

bundle/apps/upload_config.go

@@ -0,0 +1,105 @@
+package apps
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"path"
+	"strings"
+
+	"github.com/databricks/cli/bundle"
+	"github.com/databricks/cli/bundle/config/resources"
+	"github.com/databricks/cli/bundle/deploy"
+	"github.com/databricks/cli/libs/diag"
+	"github.com/databricks/cli/libs/filer"
+	"golang.org/x/sync/errgroup"
+
+	"gopkg.in/yaml.v3"
+)
+
+type uploadConfig struct {
+	filerFactory deploy.FilerFactory
+}
+
+func (u *uploadConfig) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
+	var diags diag.Diagnostics
+	errGroup, ctx := errgroup.WithContext(ctx)
+
+	for key, app := range b.Config.Resources.Apps {
+		// If the app has a config, we need to deploy it first.
+		// It means we need to write app.yml file with the content of the config field
+		// to the remote source code path of the app.
+		if app.Config != nil {
+			if !strings.HasPrefix(app.SourceCodePath, b.Config.Workspace.FilePath) {
+				diags = append(diags, diag.Diagnostic{
+					Severity:  diag.Error,
+					Summary:   "App source code invalid",
+					Detail:    fmt.Sprintf("App source code path %s is not within file path %s", app.SourceCodePath, b.Config.Workspace.FilePath),
+					Locations: b.Config.GetLocations(fmt.Sprintf("resources.apps.%s.source_code_path", key)),
+				})
+
+				continue
+			}
+
+			appPath := strings.TrimPrefix(app.SourceCodePath, b.Config.Workspace.FilePath)
+
+			buf, err := configToYaml(app)
+			if err != nil {
+				return diag.FromErr(err)
+			}
+
+			// When the app is started, create a new app deployment and wait for it to complete.
+			f, err := u.filerFactory(b)
+			if err != nil {
+				return diag.FromErr(err)
+			}
+
+			errGroup.Go(func() error {
+				err = f.Write(ctx, path.Join(appPath, "app.yml"), buf, filer.OverwriteIfExists)
+				if err != nil {
+					diags = append(diags, diag.Diagnostic{
+						Severity:  diag.Error,
+						Summary:   "Failed to save config",
+						Detail:    fmt.Sprintf("Failed to write %s file: %s", path.Join(app.SourceCodePath, "app.yml"), err),
+						Locations: b.Config.GetLocations(fmt.Sprintf("resources.apps.%s", key)),
+					})
+				}
+				return nil
+			})
+		}
+	}
+
+	if err := errGroup.Wait(); err != nil {
+		return diag.FromErr(err)
+	}
+
+	return diags
+}
+
+// Name implements bundle.Mutator.
+func (u *uploadConfig) Name() string {
+	return "apps:UploadConfig"
+}
+
+func UploadConfig() bundle.Mutator {
+	return &uploadConfig{
+		filerFactory: func(b *bundle.Bundle) (filer.Filer, error) {
+			return filer.NewWorkspaceFilesClient(b.WorkspaceClient(), b.Config.Workspace.FilePath)
+		},
+	}
+}
+
+func configToYaml(app *resources.App) (*bytes.Buffer, error) {
+	buf := bytes.NewBuffer(nil)
+	enc := yaml.NewEncoder(buf)
+	enc.SetIndent(2)
+
+	err := enc.Encode(app.Config)
+	defer enc.Close()
+
+	if err != nil {
+		return nil, fmt.Errorf("failed to encode app config to yaml: %w", err)
+	}
+
+	return buf, nil
+}

bundle/apps/upload_config_test.go

@@ -0,0 +1,69 @@
+package apps
+
+import (
+	"bytes"
+	"context"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/databricks/cli/bundle"
+	"github.com/databricks/cli/bundle/config"
+	"github.com/databricks/cli/bundle/config/resources"
+	mockfiler "github.com/databricks/cli/internal/mocks/libs/filer"
+	"github.com/databricks/cli/libs/filer"
+	"github.com/databricks/cli/libs/vfs"
+	"github.com/databricks/databricks-sdk-go/service/apps"
+	"github.com/stretchr/testify/mock"
+	"github.com/stretchr/testify/require"
+)
+
+func TestAppUploadConfig(t *testing.T) {
+	root := t.TempDir()
+	err := os.MkdirAll(filepath.Join(root, "my_app"), 0700)
+	require.NoError(t, err)
+
+	b := &bundle.Bundle{
+		BundleRootPath: root,
+		SyncRoot:       vfs.MustNew(root),
+		Config: config.Root{
+			Workspace: config.Workspace{
+				RootPath: "/Workspace/Users/[email protected]/",
+			},
+			Resources: config.Resources{
+				Apps: map[string]*resources.App{
+					"my_app": {
+						App: &apps.App{
+							Name: "my_app",
+						},
+						SourceCodePath: "./my_app",
+						Config: map[string]any{
+							"command": []string{"echo", "hello"},
+							"env": []map[string]string{
+								{"name": "MY_APP", "value": "my value"},
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+
+	mockFiler := mockfiler.NewMockFiler(t)
+	mockFiler.EXPECT().Write(mock.Anything, "my_app/app.yml", bytes.NewBufferString(`command:
+  - echo
+  - hello
+env:
+  - name: MY_APP
+    value: my value
+`), filer.OverwriteIfExists).Return(nil)
+
+	u := uploadConfig{
+		filerFactory: func(b *bundle.Bundle) (filer.Filer, error) {
+			return mockFiler, nil
+		},
+	}
+
+	diags := bundle.Apply(context.Background(), b, &u)
+	require.NoError(t, diags.Error())
+}

bundle/apps/validate.go

@@ -0,0 +1,41 @@
+package apps
+
+import (
+	"context"
+	"fmt"
+	"path"
+
+	"github.com/databricks/cli/bundle"
+	"github.com/databricks/cli/libs/diag"
+)
+
+type validate struct {
+}
+
+func (v *validate) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
+	var diags diag.Diagnostics
+	possibleConfigFiles := []string{"app.yml", "app.yaml"}
+
+	for _, app := range b.Config.Resources.Apps {
+		for _, configFile := range possibleConfigFiles {
+			cf := path.Join(app.SourceCodePath, configFile)
+			if _, err := b.SyncRoot.Stat(cf); err == nil {
+				diags = append(diags, diag.Diagnostic{
+					Severity: diag.Error,
+					Summary:  fmt.Sprintf("%s detected", configFile),
+					Detail:   fmt.Sprintf("remove %s and use 'config' property for app resource '%s' instead", cf, app.Name),
+				})
+			}
+		}
+	}
+
+	return diags
+}
+
+func (v *validate) Name() string {
+	return "apps.Validate"
+}
+
+func Validate() bundle.Mutator {
+	return &validate{}
+}

bundle/apps/validate_test.go

@@ -0,0 +1,55 @@
+package apps
+
+import (
+	"context"
+	"path/filepath"
+	"testing"
+
+	"github.com/databricks/cli/bundle"
+	"github.com/databricks/cli/bundle/config"
+	"github.com/databricks/cli/bundle/config/mutator"
+	"github.com/databricks/cli/bundle/config/resources"
+	"github.com/databricks/cli/bundle/internal/bundletest"
+	"github.com/databricks/cli/internal/testutil"
+	"github.com/databricks/cli/libs/dyn"
+	"github.com/databricks/cli/libs/vfs"
+	"github.com/databricks/databricks-sdk-go/service/apps"
+	"github.com/stretchr/testify/require"
+)
+
+func TestAppsValidate(t *testing.T) {
+	tmpDir := t.TempDir()
+	testutil.Touch(t, tmpDir, "app1", "app.yml")
+	testutil.Touch(t, tmpDir, "app2", "app.py")
+
+	b := &bundle.Bundle{
+		BundleRootPath: tmpDir,
+		SyncRootPath:   tmpDir,
+		SyncRoot:       vfs.MustNew(tmpDir),
+		Config: config.Root{
+			Resources: config.Resources{
+				Apps: map[string]*resources.App{
+					"app1": {
+						App: &apps.App{
+							Name: "app1",
+						},
+						SourceCodePath: "./app1",
+					},
+					"app2": {
+						App: &apps.App{
+							Name: "app2",
+						},
+						SourceCodePath: "./app2",
+					},
+				},
+			},
+		},
+	}
+
+	bundletest.SetLocation(b, ".", []dyn.Location{{File: filepath.Join(tmpDir, "databricks.yml")}})
+
+	diags := bundle.Apply(context.Background(), b, bundle.Seq(mutator.TranslatePaths(), Validate()))
+	require.Len(t, diags, 1)
+	require.Equal(t, "app.yml detected", diags[0].Summary)
+	require.Contains(t, diags[0].Detail, "app.yml and use 'config' property for app resource")
+}