diff --git a/.codegen/lookup.go.tmpl b/.codegen/lookup.go.tmpl index 7e643a90c3..431709f901 100644 --- a/.codegen/lookup.go.tmpl +++ b/.codegen/lookup.go.tmpl @@ -116,12 +116,12 @@ func allResolvers() *resolvers { {{range .Services -}} {{- if in $allowlist .KebabName -}} r.{{.Singular.PascalName}} = func(ctx context.Context, w *databricks.WorkspaceClient, name string) (string, error) { - entity, err := w.{{.PascalName}}.GetBy{{range .List.NamedIdMap.NamePath}}{{.PascalName}}{{end}}(ctx, name) + entity, err := w.{{.PascalName}}.GetBy{{range .NamedIdMap.NamePath}}{{.PascalName}}{{end}}(ctx, name) if err != nil { return "", err } - return fmt.Sprint(entity.{{ getOrDefault $customField .KebabName ((index .List.NamedIdMap.IdPath 0).PascalName) }}), nil + return fmt.Sprint(entity.{{ getOrDefault $customField .KebabName ((index .NamedIdMap.IdPath 0).PascalName) }}), nil } {{end -}} {{- end}} diff --git a/CHANGELOG.md b/CHANGELOG.md index 622519f230..d1e0b9a5a2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,33 @@ # Version changelog +## 0.225.0 + +Bundles: + * Add resource for UC schemas to DABs ([#1413](https://github.com/databricks/cli/pull/1413)). + +Internal: + * Use dynamic walking to validate unique resource keys ([#1614](https://github.com/databricks/cli/pull/1614)). + * Regenerate TF schema ([#1635](https://github.com/databricks/cli/pull/1635)). + * Add upgrade and upgrade eager flags to pip install call ([#1636](https://github.com/databricks/cli/pull/1636)). + * Added test for negation pattern in sync include exclude section ([#1637](https://github.com/databricks/cli/pull/1637)). + * Use precomputed terraform plan for `bundle deploy` ([#1640](https://github.com/databricks/cli/pull/1640)). + +## 0.224.1 + +Bundles: + * Add UUID function to bundle template functions ([#1612](https://github.com/databricks/cli/pull/1612)). + * Upgrade TF provider to 1.49.0 ([#1617](https://github.com/databricks/cli/pull/1617)). + * Upgrade TF provider to 1.49.1 ([#1626](https://github.com/databricks/cli/pull/1626)). + * Support multiple locations for diagnostics ([#1610](https://github.com/databricks/cli/pull/1610)). + * Split artifact cleanup into prepare step before build ([#1618](https://github.com/databricks/cli/pull/1618)). + * Move to a single prompt during bundle destroy ([#1583](https://github.com/databricks/cli/pull/1583)). + +Internal: + * Add tests for the Workspace API readahead cache ([#1605](https://github.com/databricks/cli/pull/1605)). + * Update Python dependencies before install when upgrading a labs project ([#1624](https://github.com/databricks/cli/pull/1624)). + + + ## 0.224.0 CLI: diff --git a/bundle/artifacts/artifacts.go b/bundle/artifacts/artifacts.go index 15565cd60b..e5e55a14d5 100644 --- a/bundle/artifacts/artifacts.go +++ b/bundle/artifacts/artifacts.go @@ -1,24 +1,16 @@ package artifacts import ( - "bytes" "context" - "errors" "fmt" - "os" - "path" - "path/filepath" - "strings" "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/artifacts/whl" "github.com/databricks/cli/bundle/config" - "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/bundle/config/mutator" "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/filer" "github.com/databricks/cli/libs/log" - "github.com/databricks/databricks-sdk-go" ) type mutatorFactory = func(name string) bundle.Mutator @@ -27,7 +19,9 @@ var buildMutators map[config.ArtifactType]mutatorFactory = map[config.ArtifactTy config.ArtifactPythonWheel: whl.Build, } -var uploadMutators map[config.ArtifactType]mutatorFactory = map[config.ArtifactType]mutatorFactory{} +var prepareMutators map[config.ArtifactType]mutatorFactory = map[config.ArtifactType]mutatorFactory{ + config.ArtifactPythonWheel: whl.Prepare, +} func getBuildMutator(t config.ArtifactType, name string) bundle.Mutator { mutatorFactory, ok := buildMutators[t] @@ -38,10 +32,12 @@ func getBuildMutator(t config.ArtifactType, name string) bundle.Mutator { return mutatorFactory(name) } -func getUploadMutator(t config.ArtifactType, name string) bundle.Mutator { - mutatorFactory, ok := uploadMutators[t] +func getPrepareMutator(t config.ArtifactType, name string) bundle.Mutator { + mutatorFactory, ok := prepareMutators[t] if !ok { - mutatorFactory = BasicUpload + mutatorFactory = func(_ string) bundle.Mutator { + return mutator.NoOp() + } } return mutatorFactory(name) @@ -76,174 +72,3 @@ func (m *basicBuild) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnosti return nil } - -// Basic Upload defines a general upload mutator which uploads artifact as a library to workspace -type basicUpload struct { - name string -} - -func BasicUpload(name string) bundle.Mutator { - return &basicUpload{name: name} -} - -func (m *basicUpload) Name() string { - return fmt.Sprintf("artifacts.Upload(%s)", m.name) -} - -func (m *basicUpload) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { - artifact, ok := b.Config.Artifacts[m.name] - if !ok { - return diag.Errorf("artifact doesn't exist: %s", m.name) - } - - if len(artifact.Files) == 0 { - return diag.Errorf("artifact source is not configured: %s", m.name) - } - - uploadPath, err := getUploadBasePath(b) - if err != nil { - return diag.FromErr(err) - } - - client, err := getFilerForArtifacts(b.WorkspaceClient(), uploadPath) - if err != nil { - return diag.FromErr(err) - } - - err = uploadArtifact(ctx, b, artifact, uploadPath, client) - if err != nil { - return diag.Errorf("upload for %s failed, error: %v", m.name, err) - } - - return nil -} - -func getFilerForArtifacts(w *databricks.WorkspaceClient, uploadPath string) (filer.Filer, error) { - if isVolumesPath(uploadPath) { - return filer.NewFilesClient(w, uploadPath) - } - return filer.NewWorkspaceFilesClient(w, uploadPath) -} - -func isVolumesPath(path string) bool { - return strings.HasPrefix(path, "/Volumes/") -} - -func uploadArtifact(ctx context.Context, b *bundle.Bundle, a *config.Artifact, uploadPath string, client filer.Filer) error { - for i := range a.Files { - f := &a.Files[i] - - filename := filepath.Base(f.Source) - cmdio.LogString(ctx, fmt.Sprintf("Uploading %s...", filename)) - - err := uploadArtifactFile(ctx, f.Source, client) - if err != nil { - return err - } - - log.Infof(ctx, "Upload succeeded") - f.RemotePath = path.Join(uploadPath, filepath.Base(f.Source)) - remotePath := f.RemotePath - - if !strings.HasPrefix(f.RemotePath, "/Workspace/") && !strings.HasPrefix(f.RemotePath, "/Volumes/") { - wsfsBase := "/Workspace" - remotePath = path.Join(wsfsBase, f.RemotePath) - } - - for _, job := range b.Config.Resources.Jobs { - rewriteArtifactPath(b, f, job, remotePath) - } - } - - return nil -} - -func rewriteArtifactPath(b *bundle.Bundle, f *config.ArtifactFile, job *resources.Job, remotePath string) { - // Rewrite artifact path in job task libraries - for i := range job.Tasks { - task := &job.Tasks[i] - for j := range task.Libraries { - lib := &task.Libraries[j] - if lib.Whl != "" && isArtifactMatchLibrary(f, lib.Whl, b) { - lib.Whl = remotePath - } - if lib.Jar != "" && isArtifactMatchLibrary(f, lib.Jar, b) { - lib.Jar = remotePath - } - } - - // Rewrite artifact path in job task libraries for ForEachTask - if task.ForEachTask != nil { - forEachTask := task.ForEachTask - for j := range forEachTask.Task.Libraries { - lib := &forEachTask.Task.Libraries[j] - if lib.Whl != "" && isArtifactMatchLibrary(f, lib.Whl, b) { - lib.Whl = remotePath - } - if lib.Jar != "" && isArtifactMatchLibrary(f, lib.Jar, b) { - lib.Jar = remotePath - } - } - } - } - - // Rewrite artifact path in job environments - for i := range job.Environments { - env := &job.Environments[i] - if env.Spec == nil { - continue - } - - for j := range env.Spec.Dependencies { - lib := env.Spec.Dependencies[j] - if isArtifactMatchLibrary(f, lib, b) { - env.Spec.Dependencies[j] = remotePath - } - } - } -} - -func isArtifactMatchLibrary(f *config.ArtifactFile, libPath string, b *bundle.Bundle) bool { - if !filepath.IsAbs(libPath) { - libPath = filepath.Join(b.RootPath, libPath) - } - - // libPath can be a glob pattern, so do the match first - matches, err := filepath.Glob(libPath) - if err != nil { - return false - } - - for _, m := range matches { - if m == f.Source { - return true - } - } - - return false -} - -// Function to upload artifact file to Workspace -func uploadArtifactFile(ctx context.Context, file string, client filer.Filer) error { - raw, err := os.ReadFile(file) - if err != nil { - return fmt.Errorf("unable to read %s: %w", file, errors.Unwrap(err)) - } - - filename := filepath.Base(file) - err = client.Write(ctx, filename, bytes.NewReader(raw), filer.OverwriteIfExists, filer.CreateParentDirectories) - if err != nil { - return fmt.Errorf("unable to import %s: %w", filename, err) - } - - return nil -} - -func getUploadBasePath(b *bundle.Bundle) (string, error) { - artifactPath := b.Config.Workspace.ArtifactPath - if artifactPath == "" { - return "", fmt.Errorf("remote artifact path not configured") - } - - return path.Join(artifactPath, ".internal"), nil -} diff --git a/bundle/artifacts/artifacts_test.go b/bundle/artifacts/artifacts_test.go deleted file mode 100644 index 6d85f3af90..0000000000 --- a/bundle/artifacts/artifacts_test.go +++ /dev/null @@ -1,196 +0,0 @@ -package artifacts - -import ( - "context" - "path/filepath" - "testing" - - "github.com/databricks/cli/bundle" - "github.com/databricks/cli/bundle/config" - "github.com/databricks/cli/bundle/config/resources" - mockfiler "github.com/databricks/cli/internal/mocks/libs/filer" - "github.com/databricks/cli/internal/testutil" - "github.com/databricks/cli/libs/filer" - "github.com/databricks/databricks-sdk-go/service/compute" - "github.com/databricks/databricks-sdk-go/service/jobs" - "github.com/stretchr/testify/mock" - "github.com/stretchr/testify/require" -) - -func TestArtifactUploadForWorkspace(t *testing.T) { - tmpDir := t.TempDir() - whlFolder := filepath.Join(tmpDir, "whl") - testutil.Touch(t, whlFolder, "source.whl") - whlLocalPath := filepath.Join(whlFolder, "source.whl") - - b := &bundle.Bundle{ - RootPath: tmpDir, - Config: config.Root{ - Workspace: config.Workspace{ - ArtifactPath: "/foo/bar/artifacts", - }, - Artifacts: config.Artifacts{ - "whl": { - Type: config.ArtifactPythonWheel, - Files: []config.ArtifactFile{ - {Source: whlLocalPath}, - }, - }, - }, - Resources: config.Resources{ - Jobs: map[string]*resources.Job{ - "job": { - JobSettings: &jobs.JobSettings{ - Tasks: []jobs.Task{ - { - Libraries: []compute.Library{ - { - Whl: filepath.Join("whl", "*.whl"), - }, - { - Whl: "/Workspace/Users/foo@bar.com/mywheel.whl", - }, - }, - }, - { - ForEachTask: &jobs.ForEachTask{ - Task: jobs.Task{ - Libraries: []compute.Library{ - { - Whl: filepath.Join("whl", "*.whl"), - }, - { - Whl: "/Workspace/Users/foo@bar.com/mywheel.whl", - }, - }, - }, - }, - }, - }, - Environments: []jobs.JobEnvironment{ - { - Spec: &compute.Environment{ - Dependencies: []string{ - filepath.Join("whl", "source.whl"), - "/Workspace/Users/foo@bar.com/mywheel.whl", - }, - }, - }, - }, - }, - }, - }, - }, - }, - } - - artifact := b.Config.Artifacts["whl"] - mockFiler := mockfiler.NewMockFiler(t) - mockFiler.EXPECT().Write( - mock.Anything, - filepath.Join("source.whl"), - mock.AnythingOfType("*bytes.Reader"), - filer.OverwriteIfExists, - filer.CreateParentDirectories, - ).Return(nil) - - err := uploadArtifact(context.Background(), b, artifact, "/foo/bar/artifacts", mockFiler) - require.NoError(t, err) - - // Test that libraries path is updated - require.Equal(t, "/Workspace/foo/bar/artifacts/source.whl", b.Config.Resources.Jobs["job"].JobSettings.Tasks[0].Libraries[0].Whl) - require.Equal(t, "/Workspace/Users/foo@bar.com/mywheel.whl", b.Config.Resources.Jobs["job"].JobSettings.Tasks[0].Libraries[1].Whl) - require.Equal(t, "/Workspace/foo/bar/artifacts/source.whl", b.Config.Resources.Jobs["job"].JobSettings.Environments[0].Spec.Dependencies[0]) - require.Equal(t, "/Workspace/Users/foo@bar.com/mywheel.whl", b.Config.Resources.Jobs["job"].JobSettings.Environments[0].Spec.Dependencies[1]) - require.Equal(t, "/Workspace/foo/bar/artifacts/source.whl", b.Config.Resources.Jobs["job"].JobSettings.Tasks[1].ForEachTask.Task.Libraries[0].Whl) - require.Equal(t, "/Workspace/Users/foo@bar.com/mywheel.whl", b.Config.Resources.Jobs["job"].JobSettings.Tasks[1].ForEachTask.Task.Libraries[1].Whl) -} - -func TestArtifactUploadForVolumes(t *testing.T) { - tmpDir := t.TempDir() - whlFolder := filepath.Join(tmpDir, "whl") - testutil.Touch(t, whlFolder, "source.whl") - whlLocalPath := filepath.Join(whlFolder, "source.whl") - - b := &bundle.Bundle{ - RootPath: tmpDir, - Config: config.Root{ - Workspace: config.Workspace{ - ArtifactPath: "/Volumes/foo/bar/artifacts", - }, - Artifacts: config.Artifacts{ - "whl": { - Type: config.ArtifactPythonWheel, - Files: []config.ArtifactFile{ - {Source: whlLocalPath}, - }, - }, - }, - Resources: config.Resources{ - Jobs: map[string]*resources.Job{ - "job": { - JobSettings: &jobs.JobSettings{ - Tasks: []jobs.Task{ - { - Libraries: []compute.Library{ - { - Whl: filepath.Join("whl", "*.whl"), - }, - { - Whl: "/Volumes/some/path/mywheel.whl", - }, - }, - }, - { - ForEachTask: &jobs.ForEachTask{ - Task: jobs.Task{ - Libraries: []compute.Library{ - { - Whl: filepath.Join("whl", "*.whl"), - }, - { - Whl: "/Volumes/some/path/mywheel.whl", - }, - }, - }, - }, - }, - }, - Environments: []jobs.JobEnvironment{ - { - Spec: &compute.Environment{ - Dependencies: []string{ - filepath.Join("whl", "source.whl"), - "/Volumes/some/path/mywheel.whl", - }, - }, - }, - }, - }, - }, - }, - }, - }, - } - - artifact := b.Config.Artifacts["whl"] - mockFiler := mockfiler.NewMockFiler(t) - mockFiler.EXPECT().Write( - mock.Anything, - filepath.Join("source.whl"), - mock.AnythingOfType("*bytes.Reader"), - filer.OverwriteIfExists, - filer.CreateParentDirectories, - ).Return(nil) - - err := uploadArtifact(context.Background(), b, artifact, "/Volumes/foo/bar/artifacts", mockFiler) - require.NoError(t, err) - - // Test that libraries path is updated - require.Equal(t, "/Volumes/foo/bar/artifacts/source.whl", b.Config.Resources.Jobs["job"].JobSettings.Tasks[0].Libraries[0].Whl) - require.Equal(t, "/Volumes/some/path/mywheel.whl", b.Config.Resources.Jobs["job"].JobSettings.Tasks[0].Libraries[1].Whl) - require.Equal(t, "/Volumes/foo/bar/artifacts/source.whl", b.Config.Resources.Jobs["job"].JobSettings.Environments[0].Spec.Dependencies[0]) - require.Equal(t, "/Volumes/some/path/mywheel.whl", b.Config.Resources.Jobs["job"].JobSettings.Environments[0].Spec.Dependencies[1]) - require.Equal(t, "/Volumes/foo/bar/artifacts/source.whl", b.Config.Resources.Jobs["job"].JobSettings.Tasks[1].ForEachTask.Task.Libraries[0].Whl) - require.Equal(t, "/Volumes/some/path/mywheel.whl", b.Config.Resources.Jobs["job"].JobSettings.Tasks[1].ForEachTask.Task.Libraries[1].Whl) -} diff --git a/bundle/artifacts/autodetect.go b/bundle/artifacts/autodetect.go index 0e94edd820..569a480f00 100644 --- a/bundle/artifacts/autodetect.go +++ b/bundle/artifacts/autodetect.go @@ -29,6 +29,5 @@ func (m *autodetect) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnosti return bundle.Apply(ctx, b, bundle.Seq( whl.DetectPackage(), - whl.DefineArtifactsFromLibraries(), )) } diff --git a/bundle/artifacts/build.go b/bundle/artifacts/build.go index c8c3bf67ca..0446135b6a 100644 --- a/bundle/artifacts/build.go +++ b/bundle/artifacts/build.go @@ -3,10 +3,8 @@ package artifacts import ( "context" "fmt" - "path/filepath" "github.com/databricks/cli/bundle" - "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/libs/diag" ) @@ -35,14 +33,7 @@ func (m *build) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { return diag.Errorf("artifact doesn't exist: %s", m.name) } - // Check if source paths are absolute, if not, make them absolute - for k := range artifact.Files { - f := &artifact.Files[k] - if !filepath.IsAbs(f.Source) { - dirPath := filepath.Dir(artifact.ConfigFilePath) - f.Source = filepath.Join(dirPath, f.Source) - } - } + var mutators []bundle.Mutator // Skip building if build command is not specified or infered if artifact.BuildCommand == "" { @@ -54,54 +45,13 @@ func (m *build) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { // We can skip calling build mutator if there is no build command // But we still need to expand glob references in files source path. - diags := expandGlobReference(artifact) - return diags - } - - // If artifact path is not provided, use bundle root dir - if artifact.Path == "" { - artifact.Path = b.RootPath - } - - if !filepath.IsAbs(artifact.Path) { - dirPath := filepath.Dir(artifact.ConfigFilePath) - artifact.Path = filepath.Join(dirPath, artifact.Path) - } - - diags := bundle.Apply(ctx, b, getBuildMutator(artifact.Type, m.name)) - if diags.HasError() { - return diags + } else { + mutators = append(mutators, getBuildMutator(artifact.Type, m.name)) } // We need to expand glob reference after build mutator is applied because // if we do it before, any files that are generated by build command will // not be included into artifact.Files and thus will not be uploaded. - d := expandGlobReference(artifact) - return diags.Extend(d) -} - -func expandGlobReference(artifact *config.Artifact) diag.Diagnostics { - var diags diag.Diagnostics - - // Expand any glob reference in files source path - files := make([]config.ArtifactFile, 0, len(artifact.Files)) - for _, f := range artifact.Files { - matches, err := filepath.Glob(f.Source) - if err != nil { - return diags.Extend(diag.Errorf("unable to find files for %s: %v", f.Source, err)) - } - - if len(matches) == 0 { - return diags.Extend(diag.Errorf("no files found for %s", f.Source)) - } - - for _, match := range matches { - files = append(files, config.ArtifactFile{ - Source: match, - }) - } - } - - artifact.Files = files - return diags + mutators = append(mutators, &expandGlobs{name: m.name}) + return bundle.Apply(ctx, b, bundle.Seq(mutators...)) } diff --git a/bundle/artifacts/expand_globs.go b/bundle/artifacts/expand_globs.go new file mode 100644 index 0000000000..617444054d --- /dev/null +++ b/bundle/artifacts/expand_globs.go @@ -0,0 +1,110 @@ +package artifacts + +import ( + "context" + "fmt" + "path/filepath" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dyn" +) + +type expandGlobs struct { + name string +} + +func (m *expandGlobs) Name() string { + return fmt.Sprintf("artifacts.ExpandGlobs(%s)", m.name) +} + +func createGlobError(v dyn.Value, p dyn.Path, message string) diag.Diagnostic { + // The pattern contained in v is an absolute path. + // Make it relative to the value's location to make it more readable. + source := v.MustString() + if l := v.Location(); l.File != "" { + rel, err := filepath.Rel(filepath.Dir(l.File), source) + if err == nil { + source = rel + } + } + + return diag.Diagnostic{ + Severity: diag.Error, + Summary: fmt.Sprintf("%s: %s", source, message), + Locations: []dyn.Location{v.Location()}, + + Paths: []dyn.Path{ + // Hack to clone the path. This path copy is mutable. + // To be addressed in a later PR. + p.Append(), + }, + } +} + +func (m *expandGlobs) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { + // Base path for this mutator. + // This path is set with the list of expanded globs when done. + base := dyn.NewPath( + dyn.Key("artifacts"), + dyn.Key(m.name), + dyn.Key("files"), + ) + + // Pattern to match the source key in the files sequence. + pattern := dyn.NewPatternFromPath(base).Append( + dyn.AnyIndex(), + dyn.Key("source"), + ) + + var diags diag.Diagnostics + err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { + var output []dyn.Value + _, err := dyn.MapByPattern(v, pattern, func(p dyn.Path, v dyn.Value) (dyn.Value, error) { + if v.Kind() != dyn.KindString { + return v, nil + } + + source := v.MustString() + + // Expand any glob reference in files source path + matches, err := filepath.Glob(source) + if err != nil { + diags = diags.Append(createGlobError(v, p, err.Error())) + + // Continue processing and leave this value unchanged. + return v, nil + } + + if len(matches) == 0 { + diags = diags.Append(createGlobError(v, p, "no matching files")) + + // Continue processing and leave this value unchanged. + return v, nil + } + + for _, match := range matches { + output = append(output, dyn.V( + map[string]dyn.Value{ + "source": dyn.NewValue(match, v.Locations()), + }, + )) + } + + return v, nil + }) + + if err != nil || diags.HasError() { + return v, err + } + + // Set the expanded globs back into the configuration. + return dyn.SetByPath(v, base, dyn.V(output)) + }) + + if err != nil { + return diag.FromErr(err) + } + + return diags +} diff --git a/bundle/artifacts/expand_globs_test.go b/bundle/artifacts/expand_globs_test.go new file mode 100644 index 0000000000..c9c478448f --- /dev/null +++ b/bundle/artifacts/expand_globs_test.go @@ -0,0 +1,156 @@ +package artifacts + +import ( + "context" + "fmt" + "path/filepath" + "testing" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/internal/bundletest" + "github.com/databricks/cli/internal/testutil" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestExpandGlobs_Nominal(t *testing.T) { + tmpDir := t.TempDir() + + testutil.Touch(t, tmpDir, "aa1.txt") + testutil.Touch(t, tmpDir, "aa2.txt") + testutil.Touch(t, tmpDir, "bb.txt") + testutil.Touch(t, tmpDir, "bc.txt") + + b := &bundle.Bundle{ + RootPath: tmpDir, + Config: config.Root{ + Artifacts: config.Artifacts{ + "test": { + Files: []config.ArtifactFile{ + {Source: "./aa*.txt"}, + {Source: "./b[bc].txt"}, + }, + }, + }, + }, + } + + bundletest.SetLocation(b, "artifacts", filepath.Join(tmpDir, "databricks.yml")) + + ctx := context.Background() + diags := bundle.Apply(ctx, b, bundle.Seq( + // Run prepare first to make paths absolute. + &prepare{"test"}, + &expandGlobs{"test"}, + )) + require.NoError(t, diags.Error()) + + // Assert that the expanded paths are correct. + a, ok := b.Config.Artifacts["test"] + if !assert.True(t, ok) { + return + } + assert.Len(t, a.Files, 4) + assert.Equal(t, filepath.Join(tmpDir, "aa1.txt"), a.Files[0].Source) + assert.Equal(t, filepath.Join(tmpDir, "aa2.txt"), a.Files[1].Source) + assert.Equal(t, filepath.Join(tmpDir, "bb.txt"), a.Files[2].Source) + assert.Equal(t, filepath.Join(tmpDir, "bc.txt"), a.Files[3].Source) +} + +func TestExpandGlobs_InvalidPattern(t *testing.T) { + tmpDir := t.TempDir() + + b := &bundle.Bundle{ + RootPath: tmpDir, + Config: config.Root{ + Artifacts: config.Artifacts{ + "test": { + Files: []config.ArtifactFile{ + {Source: "a[.txt"}, + {Source: "./a[.txt"}, + {Source: "../a[.txt"}, + {Source: "subdir/a[.txt"}, + }, + }, + }, + }, + } + + bundletest.SetLocation(b, "artifacts", filepath.Join(tmpDir, "databricks.yml")) + + ctx := context.Background() + diags := bundle.Apply(ctx, b, bundle.Seq( + // Run prepare first to make paths absolute. + &prepare{"test"}, + &expandGlobs{"test"}, + )) + + assert.Len(t, diags, 4) + assert.Equal(t, fmt.Sprintf("%s: syntax error in pattern", filepath.Clean("a[.txt")), diags[0].Summary) + assert.Equal(t, filepath.Join(tmpDir, "databricks.yml"), diags[0].Locations[0].File) + assert.Equal(t, "artifacts.test.files[0].source", diags[0].Paths[0].String()) + assert.Equal(t, fmt.Sprintf("%s: syntax error in pattern", filepath.Clean("a[.txt")), diags[1].Summary) + assert.Equal(t, filepath.Join(tmpDir, "databricks.yml"), diags[1].Locations[0].File) + assert.Equal(t, "artifacts.test.files[1].source", diags[1].Paths[0].String()) + assert.Equal(t, fmt.Sprintf("%s: syntax error in pattern", filepath.Clean("../a[.txt")), diags[2].Summary) + assert.Equal(t, filepath.Join(tmpDir, "databricks.yml"), diags[2].Locations[0].File) + assert.Equal(t, "artifacts.test.files[2].source", diags[2].Paths[0].String()) + assert.Equal(t, fmt.Sprintf("%s: syntax error in pattern", filepath.Clean("subdir/a[.txt")), diags[3].Summary) + assert.Equal(t, filepath.Join(tmpDir, "databricks.yml"), diags[3].Locations[0].File) + assert.Equal(t, "artifacts.test.files[3].source", diags[3].Paths[0].String()) +} + +func TestExpandGlobs_NoMatches(t *testing.T) { + tmpDir := t.TempDir() + + testutil.Touch(t, tmpDir, "a1.txt") + testutil.Touch(t, tmpDir, "a2.txt") + testutil.Touch(t, tmpDir, "b1.txt") + testutil.Touch(t, tmpDir, "b2.txt") + + b := &bundle.Bundle{ + RootPath: tmpDir, + Config: config.Root{ + Artifacts: config.Artifacts{ + "test": { + Files: []config.ArtifactFile{ + {Source: "a*.txt"}, + {Source: "b*.txt"}, + {Source: "c*.txt"}, + {Source: "d*.txt"}, + }, + }, + }, + }, + } + + bundletest.SetLocation(b, "artifacts", filepath.Join(tmpDir, "databricks.yml")) + + ctx := context.Background() + diags := bundle.Apply(ctx, b, bundle.Seq( + // Run prepare first to make paths absolute. + &prepare{"test"}, + &expandGlobs{"test"}, + )) + + assert.Len(t, diags, 2) + assert.Equal(t, "c*.txt: no matching files", diags[0].Summary) + assert.Equal(t, filepath.Join(tmpDir, "databricks.yml"), diags[0].Locations[0].File) + assert.Equal(t, "artifacts.test.files[2].source", diags[0].Paths[0].String()) + assert.Equal(t, "d*.txt: no matching files", diags[1].Summary) + assert.Equal(t, filepath.Join(tmpDir, "databricks.yml"), diags[1].Locations[0].File) + assert.Equal(t, "artifacts.test.files[3].source", diags[1].Paths[0].String()) + + // Assert that the original paths are unchanged. + a, ok := b.Config.Artifacts["test"] + if !assert.True(t, ok) { + return + } + + assert.Len(t, a.Files, 4) + assert.Equal(t, "a*.txt", filepath.Base(a.Files[0].Source)) + assert.Equal(t, "b*.txt", filepath.Base(a.Files[1].Source)) + assert.Equal(t, "c*.txt", filepath.Base(a.Files[2].Source)) + assert.Equal(t, "d*.txt", filepath.Base(a.Files[3].Source)) +} diff --git a/bundle/artifacts/prepare.go b/bundle/artifacts/prepare.go new file mode 100644 index 0000000000..fb61ed9e28 --- /dev/null +++ b/bundle/artifacts/prepare.go @@ -0,0 +1,58 @@ +package artifacts + +import ( + "context" + "fmt" + "path/filepath" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/libs/diag" +) + +func PrepareAll() bundle.Mutator { + return &all{ + name: "Prepare", + fn: prepareArtifactByName, + } +} + +type prepare struct { + name string +} + +func prepareArtifactByName(name string) (bundle.Mutator, error) { + return &prepare{name}, nil +} + +func (m *prepare) Name() string { + return fmt.Sprintf("artifacts.Prepare(%s)", m.name) +} + +func (m *prepare) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { + artifact, ok := b.Config.Artifacts[m.name] + if !ok { + return diag.Errorf("artifact doesn't exist: %s", m.name) + } + + l := b.Config.GetLocation("artifacts." + m.name) + dirPath := filepath.Dir(l.File) + + // Check if source paths are absolute, if not, make them absolute + for k := range artifact.Files { + f := &artifact.Files[k] + if !filepath.IsAbs(f.Source) { + f.Source = filepath.Join(dirPath, f.Source) + } + } + + // If artifact path is not provided, use bundle root dir + if artifact.Path == "" { + artifact.Path = b.RootPath + } + + if !filepath.IsAbs(artifact.Path) { + artifact.Path = filepath.Join(dirPath, artifact.Path) + } + + return bundle.Apply(ctx, b, getPrepareMutator(artifact.Type, m.name)) +} diff --git a/bundle/artifacts/upload.go b/bundle/artifacts/upload.go index 3af50021e8..58c006dc18 100644 --- a/bundle/artifacts/upload.go +++ b/bundle/artifacts/upload.go @@ -2,50 +2,18 @@ package artifacts import ( "context" - "fmt" "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/libraries" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/filer" "github.com/databricks/cli/libs/log" ) -func UploadAll() bundle.Mutator { - return &all{ - name: "Upload", - fn: uploadArtifactByName, - } -} - func CleanUp() bundle.Mutator { return &cleanUp{} } -type upload struct { - name string -} - -func uploadArtifactByName(name string) (bundle.Mutator, error) { - return &upload{name}, nil -} - -func (m *upload) Name() string { - return fmt.Sprintf("artifacts.Upload(%s)", m.name) -} - -func (m *upload) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { - artifact, ok := b.Config.Artifacts[m.name] - if !ok { - return diag.Errorf("artifact doesn't exist: %s", m.name) - } - - if len(artifact.Files) == 0 { - return diag.Errorf("artifact source is not configured: %s", m.name) - } - - return bundle.Apply(ctx, b, getUploadMutator(artifact.Type, m.name)) -} - type cleanUp struct{} func (m *cleanUp) Name() string { @@ -53,12 +21,12 @@ func (m *cleanUp) Name() string { } func (m *cleanUp) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { - uploadPath, err := getUploadBasePath(b) + uploadPath, err := libraries.GetUploadBasePath(b) if err != nil { return diag.FromErr(err) } - client, err := getFilerForArtifacts(b.WorkspaceClient(), uploadPath) + client, err := libraries.GetFilerForLibraries(b.WorkspaceClient(), uploadPath) if err != nil { return diag.FromErr(err) } diff --git a/bundle/artifacts/upload_test.go b/bundle/artifacts/upload_test.go deleted file mode 100644 index cf08843a70..0000000000 --- a/bundle/artifacts/upload_test.go +++ /dev/null @@ -1,109 +0,0 @@ -package artifacts - -import ( - "context" - "os" - "path/filepath" - "testing" - - "github.com/databricks/cli/bundle" - "github.com/databricks/cli/bundle/config" - "github.com/databricks/cli/bundle/internal/bundletest" - "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/testfile" - "github.com/stretchr/testify/require" -) - -type noop struct{} - -func (n *noop) Apply(context.Context, *bundle.Bundle) diag.Diagnostics { - return nil -} - -func (n *noop) Name() string { - return "noop" -} - -func TestExpandGlobFilesSource(t *testing.T) { - rootPath := t.TempDir() - err := os.Mkdir(filepath.Join(rootPath, "test"), 0755) - require.NoError(t, err) - - t1 := testfile.CreateFile(t, filepath.Join(rootPath, "test", "myjar1.jar")) - t1.Close(t) - - t2 := testfile.CreateFile(t, filepath.Join(rootPath, "test", "myjar2.jar")) - t2.Close(t) - - b := &bundle.Bundle{ - RootPath: rootPath, - Config: config.Root{ - Artifacts: map[string]*config.Artifact{ - "test": { - Type: "custom", - Files: []config.ArtifactFile{ - { - Source: filepath.Join("..", "test", "*.jar"), - }, - }, - }, - }, - }, - } - - bundletest.SetLocation(b, ".", filepath.Join(rootPath, "resources", "artifacts.yml")) - - u := &upload{"test"} - uploadMutators[config.ArtifactType("custom")] = func(name string) bundle.Mutator { - return &noop{} - } - - bm := &build{"test"} - buildMutators[config.ArtifactType("custom")] = func(name string) bundle.Mutator { - return &noop{} - } - - diags := bundle.Apply(context.Background(), b, bundle.Seq(bm, u)) - require.NoError(t, diags.Error()) - - require.Equal(t, 2, len(b.Config.Artifacts["test"].Files)) - require.Equal(t, filepath.Join(rootPath, "test", "myjar1.jar"), b.Config.Artifacts["test"].Files[0].Source) - require.Equal(t, filepath.Join(rootPath, "test", "myjar2.jar"), b.Config.Artifacts["test"].Files[1].Source) -} - -func TestExpandGlobFilesSourceWithNoMatches(t *testing.T) { - rootPath := t.TempDir() - err := os.Mkdir(filepath.Join(rootPath, "test"), 0755) - require.NoError(t, err) - - b := &bundle.Bundle{ - RootPath: rootPath, - Config: config.Root{ - Artifacts: map[string]*config.Artifact{ - "test": { - Type: "custom", - Files: []config.ArtifactFile{ - { - Source: filepath.Join("..", "test", "myjar.jar"), - }, - }, - }, - }, - }, - } - - bundletest.SetLocation(b, ".", filepath.Join(rootPath, "resources", "artifacts.yml")) - - u := &upload{"test"} - uploadMutators[config.ArtifactType("custom")] = func(name string) bundle.Mutator { - return &noop{} - } - - bm := &build{"test"} - buildMutators[config.ArtifactType("custom")] = func(name string) bundle.Mutator { - return &noop{} - } - - diags := bundle.Apply(context.Background(), b, bundle.Seq(bm, u)) - require.ErrorContains(t, diags.Error(), "no files found for") -} diff --git a/bundle/artifacts/whl/autodetect.go b/bundle/artifacts/whl/autodetect.go index ee77fff01b..1601767f69 100644 --- a/bundle/artifacts/whl/autodetect.go +++ b/bundle/artifacts/whl/autodetect.go @@ -27,9 +27,9 @@ func (m *detectPkg) Name() string { } func (m *detectPkg) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { - wheelTasks := libraries.FindAllWheelTasksWithLocalLibraries(b) - if len(wheelTasks) == 0 { - log.Infof(ctx, "No local wheel tasks in databricks.yml config, skipping auto detect") + tasks := libraries.FindTasksWithLocalLibraries(b) + if len(tasks) == 0 { + log.Infof(ctx, "No local tasks in databricks.yml config, skipping auto detect") return nil } log.Infof(ctx, "Detecting Python wheel project...") diff --git a/bundle/artifacts/whl/build.go b/bundle/artifacts/whl/build.go index 992ade297b..18d4b8ede6 100644 --- a/bundle/artifacts/whl/build.go +++ b/bundle/artifacts/whl/build.go @@ -3,7 +3,6 @@ package whl import ( "context" "fmt" - "os" "path/filepath" "github.com/databricks/cli/bundle" @@ -36,18 +35,14 @@ func (m *build) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { cmdio.LogString(ctx, fmt.Sprintf("Building %s...", m.name)) - dir := artifact.Path - - distPath := filepath.Join(dir, "dist") - os.RemoveAll(distPath) - python.CleanupWheelFolder(dir) - out, err := artifact.Build(ctx) if err != nil { return diag.Errorf("build failed %s, error: %v, output: %s", m.name, err, out) } log.Infof(ctx, "Build succeeded") + dir := artifact.Path + distPath := filepath.Join(artifact.Path, "dist") wheels := python.FindFilesWithSuffixInPath(distPath, ".whl") if len(wheels) == 0 { return diag.Errorf("cannot find built wheel in %s for package %s", dir, m.name) diff --git a/bundle/artifacts/whl/from_libraries.go b/bundle/artifacts/whl/from_libraries.go deleted file mode 100644 index ad321557cb..0000000000 --- a/bundle/artifacts/whl/from_libraries.go +++ /dev/null @@ -1,74 +0,0 @@ -package whl - -import ( - "context" - "path/filepath" - - "github.com/databricks/cli/bundle" - "github.com/databricks/cli/bundle/config" - "github.com/databricks/cli/bundle/libraries" - "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/log" -) - -type fromLibraries struct{} - -func DefineArtifactsFromLibraries() bundle.Mutator { - return &fromLibraries{} -} - -func (m *fromLibraries) Name() string { - return "artifacts.whl.DefineArtifactsFromLibraries" -} - -func (*fromLibraries) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { - if len(b.Config.Artifacts) != 0 { - log.Debugf(ctx, "Skipping defining artifacts from libraries because artifacts section is explicitly defined") - return nil - } - - tasks := libraries.FindAllWheelTasksWithLocalLibraries(b) - for _, task := range tasks { - for _, lib := range task.Libraries { - matchAndAdd(ctx, lib.Whl, b) - } - } - - envs := libraries.FindAllEnvironments(b) - for _, jobEnvs := range envs { - for _, env := range jobEnvs { - if env.Spec != nil { - for _, dep := range env.Spec.Dependencies { - if libraries.IsEnvironmentDependencyLocal(dep) { - matchAndAdd(ctx, dep, b) - } - } - } - } - } - - return nil -} - -func matchAndAdd(ctx context.Context, lib string, b *bundle.Bundle) { - matches, err := filepath.Glob(filepath.Join(b.RootPath, lib)) - // File referenced from libraries section does not exists, skipping - if err != nil { - return - } - - for _, match := range matches { - name := filepath.Base(match) - if b.Config.Artifacts == nil { - b.Config.Artifacts = make(map[string]*config.Artifact) - } - - log.Debugf(ctx, "Adding an artifact block for %s", match) - b.Config.Artifacts[name] = &config.Artifact{ - Files: []config.ArtifactFile{ - {Source: match}, - }, - Type: config.ArtifactPythonWheel, - } - } -} diff --git a/bundle/artifacts/whl/prepare.go b/bundle/artifacts/whl/prepare.go new file mode 100644 index 0000000000..0fbb2080af --- /dev/null +++ b/bundle/artifacts/whl/prepare.go @@ -0,0 +1,53 @@ +package whl + +import ( + "context" + "fmt" + "os" + "path/filepath" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/log" + "github.com/databricks/cli/libs/python" +) + +type prepare struct { + name string +} + +func Prepare(name string) bundle.Mutator { + return &prepare{ + name: name, + } +} + +func (m *prepare) Name() string { + return fmt.Sprintf("artifacts.whl.Prepare(%s)", m.name) +} + +func (m *prepare) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { + artifact, ok := b.Config.Artifacts[m.name] + if !ok { + return diag.Errorf("artifact doesn't exist: %s", m.name) + } + + // If there is no build command for the artifact, we don't need to cleanup the dist folder before + if artifact.BuildCommand == "" { + return nil + } + + dir := artifact.Path + + distPath := filepath.Join(dir, "dist") + + // If we have multiple artifacts con figured, prepare will be called multiple times + // The first time we will remove the folders, other times will be no-op. + err := os.RemoveAll(distPath) + if err != nil { + log.Infof(ctx, "Failed to remove dist folder: %v", err) + } + python.CleanupWheelFolder(dir) + + return nil +} diff --git a/bundle/config/artifact.go b/bundle/config/artifact.go index 219def5714..9a5690f579 100644 --- a/bundle/config/artifact.go +++ b/bundle/config/artifact.go @@ -4,18 +4,11 @@ import ( "context" "fmt" - "github.com/databricks/cli/bundle/config/paths" "github.com/databricks/cli/libs/exec" ) type Artifacts map[string]*Artifact -func (artifacts Artifacts) ConfigureConfigFilePath() { - for _, artifact := range artifacts { - artifact.ConfigureConfigFilePath() - } -} - type ArtifactType string const ArtifactPythonWheel ArtifactType = `whl` @@ -40,8 +33,6 @@ type Artifact struct { BuildCommand string `json:"build,omitempty"` Executable exec.ExecutableType `json:"executable,omitempty"` - - paths.Paths } func (a *Artifact) Build(ctx context.Context) ([]byte, error) { diff --git a/bundle/config/mutator/merge_job_parameters.go b/bundle/config/mutator/merge_job_parameters.go new file mode 100644 index 0000000000..51a919d989 --- /dev/null +++ b/bundle/config/mutator/merge_job_parameters.go @@ -0,0 +1,45 @@ +package mutator + +import ( + "context" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/dyn/merge" +) + +type mergeJobParameters struct{} + +func MergeJobParameters() bundle.Mutator { + return &mergeJobParameters{} +} + +func (m *mergeJobParameters) Name() string { + return "MergeJobParameters" +} + +func (m *mergeJobParameters) parameterNameString(v dyn.Value) string { + switch v.Kind() { + case dyn.KindInvalid, dyn.KindNil: + return "" + case dyn.KindString: + return v.MustString() + default: + panic("task key must be a string") + } +} + +func (m *mergeJobParameters) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { + err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { + if v.Kind() == dyn.KindNil { + return v, nil + } + + return dyn.Map(v, "resources.jobs", dyn.Foreach(func(_ dyn.Path, job dyn.Value) (dyn.Value, error) { + return dyn.Map(job, "parameters", merge.ElementsByKey("name", m.parameterNameString)) + })) + }) + + return diag.FromErr(err) +} diff --git a/bundle/config/mutator/merge_job_parameters_test.go b/bundle/config/mutator/merge_job_parameters_test.go new file mode 100644 index 0000000000..f03dea734e --- /dev/null +++ b/bundle/config/mutator/merge_job_parameters_test.go @@ -0,0 +1,80 @@ +package mutator_test + +import ( + "context" + "testing" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/config/mutator" + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/databricks-sdk-go/service/jobs" + "github.com/stretchr/testify/assert" +) + +func TestMergeJobParameters(t *testing.T) { + b := &bundle.Bundle{ + Config: config.Root{ + Resources: config.Resources{ + Jobs: map[string]*resources.Job{ + "foo": { + JobSettings: &jobs.JobSettings{ + Parameters: []jobs.JobParameterDefinition{ + { + Name: "foo", + Default: "v1", + }, + { + Name: "bar", + Default: "v1", + }, + { + Name: "foo", + Default: "v2", + }, + }, + }, + }, + }, + }, + }, + } + + diags := bundle.Apply(context.Background(), b, mutator.MergeJobParameters()) + assert.NoError(t, diags.Error()) + + j := b.Config.Resources.Jobs["foo"] + + assert.Len(t, j.Parameters, 2) + assert.Equal(t, "foo", j.Parameters[0].Name) + assert.Equal(t, "v2", j.Parameters[0].Default) + assert.Equal(t, "bar", j.Parameters[1].Name) + assert.Equal(t, "v1", j.Parameters[1].Default) +} + +func TestMergeJobParametersWithNilKey(t *testing.T) { + b := &bundle.Bundle{ + Config: config.Root{ + Resources: config.Resources{ + Jobs: map[string]*resources.Job{ + "foo": { + JobSettings: &jobs.JobSettings{ + Parameters: []jobs.JobParameterDefinition{ + { + Default: "v1", + }, + { + Default: "v2", + }, + }, + }, + }, + }, + }, + }, + } + + diags := bundle.Apply(context.Background(), b, mutator.MergeJobParameters()) + assert.NoError(t, diags.Error()) + assert.Len(t, b.Config.Resources.Jobs["foo"].Parameters, 1) +} diff --git a/bundle/config/mutator/mutator.go b/bundle/config/mutator/mutator.go index 52f85eeb8e..0458beff44 100644 --- a/bundle/config/mutator/mutator.go +++ b/bundle/config/mutator/mutator.go @@ -5,6 +5,7 @@ import ( "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/loader" pythonmutator "github.com/databricks/cli/bundle/config/mutator/python" + "github.com/databricks/cli/bundle/config/validate" "github.com/databricks/cli/bundle/scripts" ) @@ -26,5 +27,9 @@ func DefaultMutators() []bundle.Mutator { DefineDefaultTarget(), LoadGitDetails(), pythonmutator.PythonMutator(pythonmutator.PythonMutatorPhaseLoad), + + // Note: This mutator must run before the target overrides are merged. + // See the mutator for more details. + validate.UniqueResourceKeys(), } } diff --git a/bundle/config/mutator/process_target_mode_test.go b/bundle/config/mutator/process_target_mode_test.go index 351d3c3603..1c8671b4c5 100644 --- a/bundle/config/mutator/process_target_mode_test.go +++ b/bundle/config/mutator/process_target_mode_test.go @@ -116,6 +116,9 @@ func mockBundle(mode config.Mode) *bundle.Bundle { }, }, }, + Schemas: map[string]*resources.Schema{ + "schema1": {CreateSchema: &catalog.CreateSchema{Name: "schema1"}}, + }, }, }, // Use AWS implementation for testing. @@ -171,6 +174,9 @@ func TestProcessTargetModeDevelopment(t *testing.T) { assert.Equal(t, "qualityMonitor1", b.Config.Resources.QualityMonitors["qualityMonitor1"].TableName) assert.Nil(t, b.Config.Resources.QualityMonitors["qualityMonitor2"].Schedule) assert.Equal(t, catalog.MonitorCronSchedulePauseStatusUnpaused, b.Config.Resources.QualityMonitors["qualityMonitor3"].Schedule.PauseStatus) + + // Schema 1 + assert.Equal(t, "dev_lennart_schema1", b.Config.Resources.Schemas["schema1"].Name) } func TestProcessTargetModeDevelopmentTagNormalizationForAws(t *testing.T) { diff --git a/bundle/config/mutator/python/python_diagnostics.go b/bundle/config/mutator/python/python_diagnostics.go index b8efc9ef73..12822065bb 100644 --- a/bundle/config/mutator/python/python_diagnostics.go +++ b/bundle/config/mutator/python/python_diagnostics.go @@ -54,13 +54,23 @@ func parsePythonDiagnostics(input io.Reader) (diag.Diagnostics, error) { if err != nil { return nil, fmt.Errorf("failed to parse path: %s", err) } + var paths []dyn.Path + if path != nil { + paths = []dyn.Path{path} + } + + var locations []dyn.Location + location := convertPythonLocation(parsedLine.Location) + if location != (dyn.Location{}) { + locations = append(locations, location) + } diag := diag.Diagnostic{ - Severity: severity, - Summary: parsedLine.Summary, - Detail: parsedLine.Detail, - Location: convertPythonLocation(parsedLine.Location), - Path: path, + Severity: severity, + Summary: parsedLine.Summary, + Detail: parsedLine.Detail, + Locations: locations, + Paths: paths, } diags = diags.Append(diag) diff --git a/bundle/config/mutator/python/python_diagnostics_test.go b/bundle/config/mutator/python/python_diagnostics_test.go index 7b66e2537b..b73b0f73cd 100644 --- a/bundle/config/mutator/python/python_diagnostics_test.go +++ b/bundle/config/mutator/python/python_diagnostics_test.go @@ -39,10 +39,12 @@ func TestParsePythonDiagnostics(t *testing.T) { { Severity: diag.Error, Summary: "error summary", - Location: dyn.Location{ - File: "src/examples/file.py", - Line: 1, - Column: 2, + Locations: []dyn.Location{ + { + File: "src/examples/file.py", + Line: 1, + Column: 2, + }, }, }, }, @@ -54,7 +56,7 @@ func TestParsePythonDiagnostics(t *testing.T) { { Severity: diag.Error, Summary: "error summary", - Path: dyn.MustPathFromString("resources.jobs.job0.name"), + Paths: []dyn.Path{dyn.MustPathFromString("resources.jobs.job0.name")}, }, }, }, diff --git a/bundle/config/mutator/python/python_mutator_test.go b/bundle/config/mutator/python/python_mutator_test.go index 588589831b..fbe835f928 100644 --- a/bundle/config/mutator/python/python_mutator_test.go +++ b/bundle/config/mutator/python/python_mutator_test.go @@ -97,11 +97,14 @@ func TestPythonMutator_load(t *testing.T) { assert.Equal(t, 1, len(diags)) assert.Equal(t, "job doesn't have any tasks", diags[0].Summary) - assert.Equal(t, dyn.Location{ - File: "src/examples/file.py", - Line: 10, - Column: 5, - }, diags[0].Location) + assert.Equal(t, []dyn.Location{ + { + File: "src/examples/file.py", + Line: 10, + Column: 5, + }, + }, diags[0].Locations) + } func TestPythonMutator_load_disallowed(t *testing.T) { diff --git a/bundle/config/mutator/run_as.go b/bundle/config/mutator/run_as.go index d344a988ae..423bc38e2d 100644 --- a/bundle/config/mutator/run_as.go +++ b/bundle/config/mutator/run_as.go @@ -178,10 +178,10 @@ func (m *setRunAs) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { setRunAsForJobs(b) return diag.Diagnostics{ { - Severity: diag.Warning, - Summary: "You are using the legacy mode of run_as. The support for this mode is experimental and might be removed in a future release of the CLI. In order to run the DLT pipelines in your DAB as the run_as user this mode changes the owners of the pipelines to the run_as identity, which requires the user deploying the bundle to be a workspace admin, and also a Metastore admin if the pipeline target is in UC.", - Path: dyn.MustPathFromString("experimental.use_legacy_run_as"), - Location: b.Config.GetLocation("experimental.use_legacy_run_as"), + Severity: diag.Warning, + Summary: "You are using the legacy mode of run_as. The support for this mode is experimental and might be removed in a future release of the CLI. In order to run the DLT pipelines in your DAB as the run_as user this mode changes the owners of the pipelines to the run_as identity, which requires the user deploying the bundle to be a workspace admin, and also a Metastore admin if the pipeline target is in UC.", + Paths: []dyn.Path{dyn.MustPathFromString("experimental.use_legacy_run_as")}, + Locations: b.Config.GetLocations("experimental.use_legacy_run_as"), }, } } diff --git a/bundle/config/mutator/run_as_test.go b/bundle/config/mutator/run_as_test.go index 67bf7bcc2a..e6cef9ba45 100644 --- a/bundle/config/mutator/run_as_test.go +++ b/bundle/config/mutator/run_as_test.go @@ -39,6 +39,7 @@ func allResourceTypes(t *testing.T) []string { "pipelines", "quality_monitors", "registered_models", + "schemas", }, resourceTypes, ) @@ -136,6 +137,7 @@ func TestRunAsErrorForUnsupportedResources(t *testing.T) { "models", "registered_models", "experiments", + "schemas", } base := config.Root{ diff --git a/bundle/config/mutator/trampoline_test.go b/bundle/config/mutator/trampoline_test.go index e39076647f..de395c1659 100644 --- a/bundle/config/mutator/trampoline_test.go +++ b/bundle/config/mutator/trampoline_test.go @@ -9,7 +9,6 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" - "github.com/databricks/cli/bundle/config/paths" "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/databricks-sdk-go/service/jobs" "github.com/stretchr/testify/require" @@ -65,9 +64,6 @@ func TestGenerateTrampoline(t *testing.T) { Resources: config.Resources{ Jobs: map[string]*resources.Job{ "test": { - Paths: paths.Paths{ - ConfigFilePath: tmpDir, - }, JobSettings: &jobs.JobSettings{ Tasks: tasks, }, diff --git a/bundle/config/mutator/translate_paths_jobs.go b/bundle/config/mutator/translate_paths_jobs.go index 60cc8bb9aa..6febf4f8f7 100644 --- a/bundle/config/mutator/translate_paths_jobs.go +++ b/bundle/config/mutator/translate_paths_jobs.go @@ -78,7 +78,7 @@ func (t *translateContext) jobRewritePatterns() []jobRewritePattern { ), t.translateNoOpWithPrefix, func(s string) bool { - return !libraries.IsEnvironmentDependencyLocal(s) + return !libraries.IsLibraryLocal(s) }, }, } diff --git a/bundle/config/paths/paths.go b/bundle/config/paths/paths.go deleted file mode 100644 index 95977ee373..0000000000 --- a/bundle/config/paths/paths.go +++ /dev/null @@ -1,22 +0,0 @@ -package paths - -import ( - "github.com/databricks/cli/libs/dyn" -) - -type Paths struct { - // Absolute path on the local file system to the configuration file that holds - // the definition of this resource. - ConfigFilePath string `json:"-" bundle:"readonly"` - - // DynamicValue stores the [dyn.Value] of the containing struct. - // This assumes that this struct is always embedded. - DynamicValue dyn.Value `json:"-"` -} - -func (p *Paths) ConfigureConfigFilePath() { - if !p.DynamicValue.IsValid() { - panic("DynamicValue not set") - } - p.ConfigFilePath = p.DynamicValue.Location().File -} diff --git a/bundle/config/resources.go b/bundle/config/resources.go index f70052ec02..22d69ffb53 100644 --- a/bundle/config/resources.go +++ b/bundle/config/resources.go @@ -18,203 +18,17 @@ type Resources struct { ModelServingEndpoints map[string]*resources.ModelServingEndpoint `json:"model_serving_endpoints,omitempty"` RegisteredModels map[string]*resources.RegisteredModel `json:"registered_models,omitempty"` QualityMonitors map[string]*resources.QualityMonitor `json:"quality_monitors,omitempty"` -} - -type UniqueResourceIdTracker struct { - Type map[string]string - ConfigPath map[string]string -} - -// verifies merging is safe by checking no duplicate identifiers exist -func (r *Resources) VerifySafeMerge(other *Resources) error { - rootTracker, err := r.VerifyUniqueResourceIdentifiers() - if err != nil { - return err - } - otherTracker, err := other.VerifyUniqueResourceIdentifiers() - if err != nil { - return err - } - for k := range otherTracker.Type { - if _, ok := rootTracker.Type[k]; ok { - return fmt.Errorf("multiple resources named %s (%s at %s, %s at %s)", - k, - rootTracker.Type[k], - rootTracker.ConfigPath[k], - otherTracker.Type[k], - otherTracker.ConfigPath[k], - ) - } - } - return nil -} - -// This function verifies there are no duplicate names used for the resource definations -func (r *Resources) VerifyUniqueResourceIdentifiers() (*UniqueResourceIdTracker, error) { - tracker := &UniqueResourceIdTracker{ - Type: make(map[string]string), - ConfigPath: make(map[string]string), - } - for k := range r.Jobs { - tracker.Type[k] = "job" - tracker.ConfigPath[k] = r.Jobs[k].ConfigFilePath - } - for k := range r.Pipelines { - if _, ok := tracker.Type[k]; ok { - return tracker, fmt.Errorf("multiple resources named %s (%s at %s, %s at %s)", - k, - tracker.Type[k], - tracker.ConfigPath[k], - "pipeline", - r.Pipelines[k].ConfigFilePath, - ) - } - tracker.Type[k] = "pipeline" - tracker.ConfigPath[k] = r.Pipelines[k].ConfigFilePath - } - for k := range r.Models { - if _, ok := tracker.Type[k]; ok { - return tracker, fmt.Errorf("multiple resources named %s (%s at %s, %s at %s)", - k, - tracker.Type[k], - tracker.ConfigPath[k], - "mlflow_model", - r.Models[k].ConfigFilePath, - ) - } - tracker.Type[k] = "mlflow_model" - tracker.ConfigPath[k] = r.Models[k].ConfigFilePath - } - for k := range r.Experiments { - if _, ok := tracker.Type[k]; ok { - return tracker, fmt.Errorf("multiple resources named %s (%s at %s, %s at %s)", - k, - tracker.Type[k], - tracker.ConfigPath[k], - "mlflow_experiment", - r.Experiments[k].ConfigFilePath, - ) - } - tracker.Type[k] = "mlflow_experiment" - tracker.ConfigPath[k] = r.Experiments[k].ConfigFilePath - } - for k := range r.ModelServingEndpoints { - if _, ok := tracker.Type[k]; ok { - return tracker, fmt.Errorf("multiple resources named %s (%s at %s, %s at %s)", - k, - tracker.Type[k], - tracker.ConfigPath[k], - "model_serving_endpoint", - r.ModelServingEndpoints[k].ConfigFilePath, - ) - } - tracker.Type[k] = "model_serving_endpoint" - tracker.ConfigPath[k] = r.ModelServingEndpoints[k].ConfigFilePath - } - for k := range r.RegisteredModels { - if _, ok := tracker.Type[k]; ok { - return tracker, fmt.Errorf("multiple resources named %s (%s at %s, %s at %s)", - k, - tracker.Type[k], - tracker.ConfigPath[k], - "registered_model", - r.RegisteredModels[k].ConfigFilePath, - ) - } - tracker.Type[k] = "registered_model" - tracker.ConfigPath[k] = r.RegisteredModels[k].ConfigFilePath - } - for k := range r.QualityMonitors { - if _, ok := tracker.Type[k]; ok { - return tracker, fmt.Errorf("multiple resources named %s (%s at %s, %s at %s)", - k, - tracker.Type[k], - tracker.ConfigPath[k], - "quality_monitor", - r.QualityMonitors[k].ConfigFilePath, - ) - } - tracker.Type[k] = "quality_monitor" - tracker.ConfigPath[k] = r.QualityMonitors[k].ConfigFilePath - } - return tracker, nil -} - -type resource struct { - resource ConfigResource - resource_type string - key string -} - -func (r *Resources) allResources() []resource { - all := make([]resource, 0) - for k, e := range r.Jobs { - all = append(all, resource{resource_type: "job", resource: e, key: k}) - } - for k, e := range r.Pipelines { - all = append(all, resource{resource_type: "pipeline", resource: e, key: k}) - } - for k, e := range r.Models { - all = append(all, resource{resource_type: "model", resource: e, key: k}) - } - for k, e := range r.Experiments { - all = append(all, resource{resource_type: "experiment", resource: e, key: k}) - } - for k, e := range r.ModelServingEndpoints { - all = append(all, resource{resource_type: "serving endpoint", resource: e, key: k}) - } - for k, e := range r.RegisteredModels { - all = append(all, resource{resource_type: "registered model", resource: e, key: k}) - } - for k, e := range r.QualityMonitors { - all = append(all, resource{resource_type: "quality monitor", resource: e, key: k}) - } - return all -} - -func (r *Resources) VerifyAllResourcesDefined() error { - all := r.allResources() - for _, e := range all { - err := e.resource.Validate() - if err != nil { - return fmt.Errorf("%s %s is not defined", e.resource_type, e.key) - } - } - - return nil -} - -// ConfigureConfigFilePath sets the specified path for all resources contained in this instance. -// This property is used to correctly resolve paths relative to the path -// of the configuration file they were defined in. -func (r *Resources) ConfigureConfigFilePath() { - for _, e := range r.Jobs { - e.ConfigureConfigFilePath() - } - for _, e := range r.Pipelines { - e.ConfigureConfigFilePath() - } - for _, e := range r.Models { - e.ConfigureConfigFilePath() - } - for _, e := range r.Experiments { - e.ConfigureConfigFilePath() - } - for _, e := range r.ModelServingEndpoints { - e.ConfigureConfigFilePath() - } - for _, e := range r.RegisteredModels { - e.ConfigureConfigFilePath() - } - for _, e := range r.QualityMonitors { - e.ConfigureConfigFilePath() - } + Schemas map[string]*resources.Schema `json:"schemas,omitempty"` } type ConfigResource interface { + // Function to assert if the resource exists in the workspace configured in + // the input workspace client. Exists(ctx context.Context, w *databricks.WorkspaceClient, id string) (bool, error) + + // Terraform equivalent name of the resource. For example "databricks_job" + // for jobs and "databricks_pipeline" for pipelines. TerraformResourceName() string - Validate() error } func (r *Resources) FindResourceByConfigKey(key string) (ConfigResource, error) { diff --git a/bundle/config/resources/job.go b/bundle/config/resources/job.go index dde5d5663d..d8f97a2db6 100644 --- a/bundle/config/resources/job.go +++ b/bundle/config/resources/job.go @@ -2,10 +2,8 @@ package resources import ( "context" - "fmt" "strconv" - "github.com/databricks/cli/bundle/config/paths" "github.com/databricks/cli/libs/log" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/marshal" @@ -17,8 +15,6 @@ type Job struct { Permissions []Permission `json:"permissions,omitempty"` ModifiedStatus ModifiedStatus `json:"modified_status,omitempty" bundle:"internal"` - paths.Paths - *jobs.JobSettings } @@ -48,11 +44,3 @@ func (j *Job) Exists(ctx context.Context, w *databricks.WorkspaceClient, id stri func (j *Job) TerraformResourceName() string { return "databricks_job" } - -func (j *Job) Validate() error { - if j == nil || !j.DynamicValue.IsValid() || j.JobSettings == nil { - return fmt.Errorf("job is not defined") - } - - return nil -} diff --git a/bundle/config/resources/mlflow_experiment.go b/bundle/config/resources/mlflow_experiment.go index 7854ee7e81..0ab4864360 100644 --- a/bundle/config/resources/mlflow_experiment.go +++ b/bundle/config/resources/mlflow_experiment.go @@ -2,9 +2,7 @@ package resources import ( "context" - "fmt" - "github.com/databricks/cli/bundle/config/paths" "github.com/databricks/cli/libs/log" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/marshal" @@ -16,8 +14,6 @@ type MlflowExperiment struct { Permissions []Permission `json:"permissions,omitempty"` ModifiedStatus ModifiedStatus `json:"modified_status,omitempty" bundle:"internal"` - paths.Paths - *ml.Experiment } @@ -43,11 +39,3 @@ func (s *MlflowExperiment) Exists(ctx context.Context, w *databricks.WorkspaceCl func (s *MlflowExperiment) TerraformResourceName() string { return "databricks_mlflow_experiment" } - -func (s *MlflowExperiment) Validate() error { - if s == nil || !s.DynamicValue.IsValid() { - return fmt.Errorf("experiment is not defined") - } - - return nil -} diff --git a/bundle/config/resources/mlflow_model.go b/bundle/config/resources/mlflow_model.go index 40da9f87df..300474e359 100644 --- a/bundle/config/resources/mlflow_model.go +++ b/bundle/config/resources/mlflow_model.go @@ -2,9 +2,7 @@ package resources import ( "context" - "fmt" - "github.com/databricks/cli/bundle/config/paths" "github.com/databricks/cli/libs/log" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/marshal" @@ -16,8 +14,6 @@ type MlflowModel struct { Permissions []Permission `json:"permissions,omitempty"` ModifiedStatus ModifiedStatus `json:"modified_status,omitempty" bundle:"internal"` - paths.Paths - *ml.Model } @@ -43,11 +39,3 @@ func (s *MlflowModel) Exists(ctx context.Context, w *databricks.WorkspaceClient, func (s *MlflowModel) TerraformResourceName() string { return "databricks_mlflow_model" } - -func (s *MlflowModel) Validate() error { - if s == nil || !s.DynamicValue.IsValid() { - return fmt.Errorf("model is not defined") - } - - return nil -} diff --git a/bundle/config/resources/model_serving_endpoint.go b/bundle/config/resources/model_serving_endpoint.go index 503cfbbb74..5efb7ea267 100644 --- a/bundle/config/resources/model_serving_endpoint.go +++ b/bundle/config/resources/model_serving_endpoint.go @@ -2,9 +2,7 @@ package resources import ( "context" - "fmt" - "github.com/databricks/cli/bundle/config/paths" "github.com/databricks/cli/libs/log" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/marshal" @@ -20,10 +18,6 @@ type ModelServingEndpoint struct { // as a reference in other resources. This value is returned by terraform. ID string `json:"id,omitempty" bundle:"readonly"` - // Path to config file where the resource is defined. All bundle resources - // include this for interpolation purposes. - paths.Paths - // This is a resource agnostic implementation of permissions for ACLs. // Implementation could be different based on the resource type. Permissions []Permission `json:"permissions,omitempty"` @@ -53,11 +47,3 @@ func (s *ModelServingEndpoint) Exists(ctx context.Context, w *databricks.Workspa func (s *ModelServingEndpoint) TerraformResourceName() string { return "databricks_model_serving" } - -func (s *ModelServingEndpoint) Validate() error { - if s == nil || !s.DynamicValue.IsValid() { - return fmt.Errorf("serving endpoint is not defined") - } - - return nil -} diff --git a/bundle/config/resources/pipeline.go b/bundle/config/resources/pipeline.go index 7e914b9096..55270be654 100644 --- a/bundle/config/resources/pipeline.go +++ b/bundle/config/resources/pipeline.go @@ -2,9 +2,7 @@ package resources import ( "context" - "fmt" - "github.com/databricks/cli/bundle/config/paths" "github.com/databricks/cli/libs/log" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/marshal" @@ -16,8 +14,6 @@ type Pipeline struct { Permissions []Permission `json:"permissions,omitempty"` ModifiedStatus ModifiedStatus `json:"modified_status,omitempty" bundle:"internal"` - paths.Paths - *pipelines.PipelineSpec } @@ -43,11 +39,3 @@ func (p *Pipeline) Exists(ctx context.Context, w *databricks.WorkspaceClient, id func (p *Pipeline) TerraformResourceName() string { return "databricks_pipeline" } - -func (p *Pipeline) Validate() error { - if p == nil || !p.DynamicValue.IsValid() { - return fmt.Errorf("pipeline is not defined") - } - - return nil -} diff --git a/bundle/config/resources/quality_monitor.go b/bundle/config/resources/quality_monitor.go index 0d13e58fa1..9160782cd0 100644 --- a/bundle/config/resources/quality_monitor.go +++ b/bundle/config/resources/quality_monitor.go @@ -2,9 +2,7 @@ package resources import ( "context" - "fmt" - "github.com/databricks/cli/bundle/config/paths" "github.com/databricks/cli/libs/log" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/marshal" @@ -21,10 +19,6 @@ type QualityMonitor struct { // as a reference in other resources. This value is returned by terraform. ID string `json:"id,omitempty" bundle:"readonly"` - // Path to config file where the resource is defined. All bundle resources - // include this for interpolation purposes. - paths.Paths - ModifiedStatus ModifiedStatus `json:"modified_status,omitempty" bundle:"internal"` } @@ -50,11 +44,3 @@ func (s *QualityMonitor) Exists(ctx context.Context, w *databricks.WorkspaceClie func (s *QualityMonitor) TerraformResourceName() string { return "databricks_quality_monitor" } - -func (s *QualityMonitor) Validate() error { - if s == nil || !s.DynamicValue.IsValid() { - return fmt.Errorf("quality monitor is not defined") - } - - return nil -} diff --git a/bundle/config/resources/registered_model.go b/bundle/config/resources/registered_model.go index fba643c69b..6033ffdf2b 100644 --- a/bundle/config/resources/registered_model.go +++ b/bundle/config/resources/registered_model.go @@ -2,9 +2,7 @@ package resources import ( "context" - "fmt" - "github.com/databricks/cli/bundle/config/paths" "github.com/databricks/cli/libs/log" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/marshal" @@ -21,10 +19,6 @@ type RegisteredModel struct { // as a reference in other resources. This value is returned by terraform. ID string `json:"id,omitempty" bundle:"readonly"` - // Path to config file where the resource is defined. All bundle resources - // include this for interpolation purposes. - paths.Paths - // This represents the input args for terraform, and will get converted // to a HCL representation for CRUD *catalog.CreateRegisteredModelRequest @@ -54,11 +48,3 @@ func (s *RegisteredModel) Exists(ctx context.Context, w *databricks.WorkspaceCli func (s *RegisteredModel) TerraformResourceName() string { return "databricks_registered_model" } - -func (s *RegisteredModel) Validate() error { - if s == nil || !s.DynamicValue.IsValid() { - return fmt.Errorf("registered model is not defined") - } - - return nil -} diff --git a/bundle/config/resources/schema.go b/bundle/config/resources/schema.go new file mode 100644 index 0000000000..7ab00495a8 --- /dev/null +++ b/bundle/config/resources/schema.go @@ -0,0 +1,27 @@ +package resources + +import ( + "github.com/databricks/databricks-sdk-go/marshal" + "github.com/databricks/databricks-sdk-go/service/catalog" +) + +type Schema struct { + // List of grants to apply on this schema. + Grants []Grant `json:"grants,omitempty"` + + // Full name of the schema (catalog_name.schema_name). This value is read from + // the terraform state after deployment succeeds. + ID string `json:"id,omitempty" bundle:"readonly"` + + *catalog.CreateSchema + + ModifiedStatus ModifiedStatus `json:"modified_status,omitempty" bundle:"internal"` +} + +func (s *Schema) UnmarshalJSON(b []byte) error { + return marshal.Unmarshal(b, s) +} + +func (s Schema) MarshalJSON() ([]byte, error) { + return marshal.Marshal(s) +} diff --git a/bundle/config/resources_test.go b/bundle/config/resources_test.go index 7415029b13..6860d73daa 100644 --- a/bundle/config/resources_test.go +++ b/bundle/config/resources_test.go @@ -5,129 +5,9 @@ import ( "reflect" "testing" - "github.com/databricks/cli/bundle/config/paths" - "github.com/databricks/cli/bundle/config/resources" "github.com/stretchr/testify/assert" ) -func TestVerifyUniqueResourceIdentifiers(t *testing.T) { - r := Resources{ - Jobs: map[string]*resources.Job{ - "foo": { - Paths: paths.Paths{ - ConfigFilePath: "foo.yml", - }, - }, - }, - Models: map[string]*resources.MlflowModel{ - "bar": { - Paths: paths.Paths{ - ConfigFilePath: "bar.yml", - }, - }, - }, - Experiments: map[string]*resources.MlflowExperiment{ - "foo": { - Paths: paths.Paths{ - ConfigFilePath: "foo2.yml", - }, - }, - }, - } - _, err := r.VerifyUniqueResourceIdentifiers() - assert.ErrorContains(t, err, "multiple resources named foo (job at foo.yml, mlflow_experiment at foo2.yml)") -} - -func TestVerifySafeMerge(t *testing.T) { - r := Resources{ - Jobs: map[string]*resources.Job{ - "foo": { - Paths: paths.Paths{ - ConfigFilePath: "foo.yml", - }, - }, - }, - Models: map[string]*resources.MlflowModel{ - "bar": { - Paths: paths.Paths{ - ConfigFilePath: "bar.yml", - }, - }, - }, - } - other := Resources{ - Pipelines: map[string]*resources.Pipeline{ - "foo": { - Paths: paths.Paths{ - ConfigFilePath: "foo2.yml", - }, - }, - }, - } - err := r.VerifySafeMerge(&other) - assert.ErrorContains(t, err, "multiple resources named foo (job at foo.yml, pipeline at foo2.yml)") -} - -func TestVerifySafeMergeForSameResourceType(t *testing.T) { - r := Resources{ - Jobs: map[string]*resources.Job{ - "foo": { - Paths: paths.Paths{ - ConfigFilePath: "foo.yml", - }, - }, - }, - Models: map[string]*resources.MlflowModel{ - "bar": { - Paths: paths.Paths{ - ConfigFilePath: "bar.yml", - }, - }, - }, - } - other := Resources{ - Jobs: map[string]*resources.Job{ - "foo": { - Paths: paths.Paths{ - ConfigFilePath: "foo2.yml", - }, - }, - }, - } - err := r.VerifySafeMerge(&other) - assert.ErrorContains(t, err, "multiple resources named foo (job at foo.yml, job at foo2.yml)") -} - -func TestVerifySafeMergeForRegisteredModels(t *testing.T) { - r := Resources{ - Jobs: map[string]*resources.Job{ - "foo": { - Paths: paths.Paths{ - ConfigFilePath: "foo.yml", - }, - }, - }, - RegisteredModels: map[string]*resources.RegisteredModel{ - "bar": { - Paths: paths.Paths{ - ConfigFilePath: "bar.yml", - }, - }, - }, - } - other := Resources{ - RegisteredModels: map[string]*resources.RegisteredModel{ - "bar": { - Paths: paths.Paths{ - ConfigFilePath: "bar2.yml", - }, - }, - }, - } - err := r.VerifySafeMerge(&other) - assert.ErrorContains(t, err, "multiple resources named bar (registered_model at bar.yml, registered_model at bar2.yml)") -} - // This test ensures that all resources have a custom marshaller and unmarshaller. // This is required because DABs resources map to Databricks APIs, and they do so // by embedding the corresponding Go SDK structs. diff --git a/bundle/config/root.go b/bundle/config/root.go index 694f92d23a..86dc33921d 100644 --- a/bundle/config/root.go +++ b/bundle/config/root.go @@ -104,11 +104,6 @@ func LoadFromBytes(path string, raw []byte) (*Root, diag.Diagnostics) { if err != nil { return nil, diag.Errorf("failed to load %s: %v", path, err) } - - _, err = r.Resources.VerifyUniqueResourceIdentifiers() - if err != nil { - diags = diags.Extend(diag.FromErr(err)) - } return &r, diags } @@ -145,17 +140,6 @@ func (r *Root) updateWithDynamicValue(nv dyn.Value) error { // Assign the normalized configuration tree. r.value = nv - - // At the moment the check has to be done as part of updateWithDynamicValue - // because otherwise ConfigureConfigFilePath will fail with a panic. - // In the future, we should move this check to a separate mutator in initialise phase. - err = r.Resources.VerifyAllResourcesDefined() - if err != nil { - return err - } - - // Assign config file paths after converting to typed configuration. - r.ConfigureConfigFilePath() return nil } @@ -247,15 +231,6 @@ func (r *Root) MarkMutatorExit(ctx context.Context) error { return nil } -// SetConfigFilePath configures the path that its configuration -// was loaded from in configuration leafs that require it. -func (r *Root) ConfigureConfigFilePath() { - r.Resources.ConfigureConfigFilePath() - if r.Artifacts != nil { - r.Artifacts.ConfigureConfigFilePath() - } -} - // Initializes variables using values passed from the command line flag // Input has to be a string of the form `foo=bar`. In this case the variable with // name `foo` is assigned the value `bar` @@ -285,12 +260,6 @@ func (r *Root) InitializeVariables(vars []string) error { } func (r *Root) Merge(other *Root) error { - // Check for safe merge, protecting against duplicate resource identifiers - err := r.Resources.VerifySafeMerge(&other.Resources) - if err != nil { - return err - } - // Merge dynamic configuration values. return r.Mutate(func(root dyn.Value) (dyn.Value, error) { return merge.Merge(root, other.value) @@ -529,6 +498,17 @@ func (r Root) GetLocation(path string) dyn.Location { return v.Location() } +// Get all locations of the configuration value at the specified path. We need both +// this function and it's singular version (GetLocation) because some diagnostics just need +// the primary location and some need all locations associated with a configuration value. +func (r Root) GetLocations(path string) []dyn.Location { + v, err := dyn.Get(r.value, path) + if err != nil { + return []dyn.Location{} + } + return v.Locations() +} + // Value returns the dynamic configuration value of the root object. This value // is the source of truth and is kept in sync with values in the typed configuration. func (r Root) Value() dyn.Value { diff --git a/bundle/config/root_test.go b/bundle/config/root_test.go index aed670d6cd..c95e6e86cd 100644 --- a/bundle/config/root_test.go +++ b/bundle/config/root_test.go @@ -30,22 +30,6 @@ func TestRootLoad(t *testing.T) { assert.Equal(t, "basic", root.Bundle.Name) } -func TestDuplicateIdOnLoadReturnsError(t *testing.T) { - _, diags := Load("./testdata/duplicate_resource_names_in_root/databricks.yml") - assert.ErrorContains(t, diags.Error(), "multiple resources named foo (job at ./testdata/duplicate_resource_names_in_root/databricks.yml, pipeline at ./testdata/duplicate_resource_names_in_root/databricks.yml)") -} - -func TestDuplicateIdOnMergeReturnsError(t *testing.T) { - root, diags := Load("./testdata/duplicate_resource_name_in_subconfiguration/databricks.yml") - require.NoError(t, diags.Error()) - - other, diags := Load("./testdata/duplicate_resource_name_in_subconfiguration/resources.yml") - require.NoError(t, diags.Error()) - - err := root.Merge(other) - assert.ErrorContains(t, err, "multiple resources named foo (job at ./testdata/duplicate_resource_name_in_subconfiguration/databricks.yml, pipeline at ./testdata/duplicate_resource_name_in_subconfiguration/resources.yml)") -} - func TestInitializeVariables(t *testing.T) { fooDefault := "abc" root := &Root{ diff --git a/bundle/config/validate/all_resources_have_values.go b/bundle/config/validate/all_resources_have_values.go new file mode 100644 index 0000000000..019fe48a21 --- /dev/null +++ b/bundle/config/validate/all_resources_have_values.go @@ -0,0 +1,47 @@ +package validate + +import ( + "context" + "fmt" + "strings" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dyn" +) + +func AllResourcesHaveValues() bundle.Mutator { + return &allResourcesHaveValues{} +} + +type allResourcesHaveValues struct{} + +func (m *allResourcesHaveValues) Name() string { + return "validate:AllResourcesHaveValues" +} + +func (m *allResourcesHaveValues) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { + rv := b.Config.Value().Get("resources") + + // Skip if there are no resources block defined, or the resources block is empty. + if rv.Kind() == dyn.KindInvalid || rv.Kind() == dyn.KindNil { + return nil + } + + _, err := dyn.MapByPattern( + rv, + dyn.NewPattern(dyn.AnyKey(), dyn.AnyKey()), + func(p dyn.Path, v dyn.Value) (dyn.Value, error) { + if v.Kind() == dyn.KindInvalid || v.Kind() == dyn.KindNil { + // Type of the resource, stripped of the trailing 's' to make it + // singular. + rType := strings.TrimSuffix(p[0].Key(), "s") + + rName := p[1].Key() + return v, fmt.Errorf("%s %s is not defined", rType, rName) + } + return v, nil + }, + ) + return diag.FromErr(err) +} diff --git a/bundle/config/validate/files_to_sync.go b/bundle/config/validate/files_to_sync.go index d53e382432..7cdad772ac 100644 --- a/bundle/config/validate/files_to_sync.go +++ b/bundle/config/validate/files_to_sync.go @@ -6,6 +6,7 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/deploy/files" "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dyn" ) func FilesToSync() bundle.ReadOnlyMutator { @@ -45,8 +46,10 @@ func (v *filesToSync) Apply(ctx context.Context, rb bundle.ReadOnlyBundle) diag. diags = diags.Append(diag.Diagnostic{ Severity: diag.Warning, Summary: "There are no files to sync, please check your .gitignore and sync.exclude configuration", - Location: loc.Location(), - Path: loc.Path(), + // Show all locations where sync.exclude is defined, since merging + // sync.exclude is additive. + Locations: loc.Locations(), + Paths: []dyn.Path{loc.Path()}, }) } diff --git a/bundle/config/validate/job_cluster_key_defined.go b/bundle/config/validate/job_cluster_key_defined.go index 37ed3f417e..368c3edb13 100644 --- a/bundle/config/validate/job_cluster_key_defined.go +++ b/bundle/config/validate/job_cluster_key_defined.go @@ -6,6 +6,7 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dyn" ) func JobClusterKeyDefined() bundle.ReadOnlyMutator { @@ -41,8 +42,11 @@ func (v *jobClusterKeyDefined) Apply(ctx context.Context, rb bundle.ReadOnlyBund diags = diags.Append(diag.Diagnostic{ Severity: diag.Warning, Summary: fmt.Sprintf("job_cluster_key %s is not defined", task.JobClusterKey), - Location: loc.Location(), - Path: loc.Path(), + // Show only the location where the job_cluster_key is defined. + // Other associated locations are not relevant since they are + // overridden during merging. + Locations: []dyn.Location{loc.Location()}, + Paths: []dyn.Path{loc.Path()}, }) } } diff --git a/bundle/config/validate/unique_resource_keys.go b/bundle/config/validate/unique_resource_keys.go new file mode 100644 index 0000000000..d6212b0acf --- /dev/null +++ b/bundle/config/validate/unique_resource_keys.go @@ -0,0 +1,116 @@ +package validate + +import ( + "context" + "fmt" + "slices" + "sort" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dyn" +) + +// This mutator validates that: +// +// 1. Each resource key is unique across different resource types. No two resources +// of the same type can have the same key. This is because command like "bundle run" +// rely on the resource key to identify the resource to run. +// Eg: jobs.foo and pipelines.foo are not allowed simultaneously. +// +// 2. Each resource definition is contained within a single file, and is not spread +// across multiple files. Note: This is not applicable to resource configuration +// defined in a target override. That is why this mutator MUST run before the target +// overrides are merged. +func UniqueResourceKeys() bundle.Mutator { + return &uniqueResourceKeys{} +} + +type uniqueResourceKeys struct{} + +func (m *uniqueResourceKeys) Name() string { + return "validate:unique_resource_keys" +} + +func (m *uniqueResourceKeys) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { + diags := diag.Diagnostics{} + + type metadata struct { + locations []dyn.Location + paths []dyn.Path + } + + // Maps of resource key to the paths and locations the resource is defined at. + resourceMetadata := map[string]*metadata{} + + rv := b.Config.Value().Get("resources") + + // return early if no resources are defined or the resources block is empty. + if rv.Kind() == dyn.KindInvalid || rv.Kind() == dyn.KindNil { + return diags + } + + // Gather the paths and locations of all resources. + _, err := dyn.MapByPattern( + rv, + dyn.NewPattern(dyn.AnyKey(), dyn.AnyKey()), + func(p dyn.Path, v dyn.Value) (dyn.Value, error) { + // The key for the resource. Eg: "my_job" for jobs.my_job. + k := p[1].Key() + + m, ok := resourceMetadata[k] + if !ok { + m = &metadata{ + paths: []dyn.Path{}, + locations: []dyn.Location{}, + } + } + + // dyn.Path under the hood is a slice. The code that walks the configuration + // tree uses the same underlying slice to track the path as it walks + // the tree. So, we need to clone it here. + m.paths = append(m.paths, slices.Clone(p)) + m.locations = append(m.locations, v.Locations()...) + + resourceMetadata[k] = m + return v, nil + }, + ) + if err != nil { + return diag.FromErr(err) + } + + for k, v := range resourceMetadata { + if len(v.locations) <= 1 { + continue + } + + // Sort the locations and paths for consistent error messages. This helps + // with unit testing. + sort.Slice(v.locations, func(i, j int) bool { + l1 := v.locations[i] + l2 := v.locations[j] + + if l1.File != l2.File { + return l1.File < l2.File + } + if l1.Line != l2.Line { + return l1.Line < l2.Line + } + return l1.Column < l2.Column + }) + sort.Slice(v.paths, func(i, j int) bool { + return v.paths[i].String() < v.paths[j].String() + }) + + // If there are multiple resources with the same key, report an error. + diags = append(diags, diag.Diagnostic{ + Severity: diag.Error, + Summary: fmt.Sprintf("multiple resources have been defined with the same key: %s", k), + Locations: v.locations, + Paths: v.paths, + }) + } + + return diags +} diff --git a/bundle/config/validate/validate.go b/bundle/config/validate/validate.go index af7e984a11..b4da0bc053 100644 --- a/bundle/config/validate/validate.go +++ b/bundle/config/validate/validate.go @@ -20,6 +20,10 @@ func (l location) Location() dyn.Location { return l.rb.Config().GetLocation(l.path) } +func (l location) Locations() []dyn.Location { + return l.rb.Config().GetLocations(l.path) +} + func (l location) Path() dyn.Path { return dyn.MustPathFromString(l.path) } diff --git a/bundle/config/validate/validate_sync_patterns.go b/bundle/config/validate/validate_sync_patterns.go index a04c10776c..fd011bf780 100644 --- a/bundle/config/validate/validate_sync_patterns.go +++ b/bundle/config/validate/validate_sync_patterns.go @@ -3,10 +3,12 @@ package validate import ( "context" "fmt" + "strings" "sync" "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/fileset" "golang.org/x/sync/errgroup" ) @@ -48,7 +50,13 @@ func checkPatterns(patterns []string, path string, rb bundle.ReadOnlyBundle) (di for i, pattern := range patterns { index := i - p := pattern + fullPattern := pattern + // If the pattern is negated, strip the negation prefix + // and check if the pattern matches any files. + // Negation in gitignore syntax means "don't look at this path' + // So if p matches nothing it's useless negation, but if there are matches, + // it means: do not include these files into result set + p := strings.TrimPrefix(fullPattern, "!") errs.Go(func() error { fs, err := fileset.NewGlobSet(rb.BundleRoot(), []string{p}) if err != nil { @@ -64,10 +72,10 @@ func checkPatterns(patterns []string, path string, rb bundle.ReadOnlyBundle) (di loc := location{path: fmt.Sprintf("%s[%d]", path, index), rb: rb} mu.Lock() diags = diags.Append(diag.Diagnostic{ - Severity: diag.Warning, - Summary: fmt.Sprintf("Pattern %s does not match any files", p), - Location: loc.Location(), - Path: loc.Path(), + Severity: diag.Warning, + Summary: fmt.Sprintf("Pattern %s does not match any files", fullPattern), + Locations: []dyn.Location{loc.Location()}, + Paths: []dyn.Path{loc.Path()}, }) mu.Unlock() } diff --git a/bundle/deploy/files/delete.go b/bundle/deploy/files/delete.go index 1339714495..bb28c2722c 100644 --- a/bundle/deploy/files/delete.go +++ b/bundle/deploy/files/delete.go @@ -12,7 +12,6 @@ import ( "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/sync" "github.com/databricks/databricks-sdk-go/service/workspace" - "github.com/fatih/color" ) type delete struct{} @@ -22,24 +21,7 @@ func (m *delete) Name() string { } func (m *delete) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { - // Do not delete files if terraform destroy was not consented - if !b.Plan.IsEmpty && !b.Plan.ConfirmApply { - return nil - } - - cmdio.LogString(ctx, "Starting deletion of remote bundle files") - cmdio.LogString(ctx, fmt.Sprintf("Bundle remote directory is %s", b.Config.Workspace.RootPath)) - - red := color.New(color.FgRed).SprintFunc() - if !b.AutoApprove { - proceed, err := cmdio.AskYesOrNo(ctx, fmt.Sprintf("\n%s and all files in it will be %s Proceed?", b.Config.Workspace.RootPath, red("deleted permanently!"))) - if err != nil { - return diag.FromErr(err) - } - if !proceed { - return nil - } - } + cmdio.LogString(ctx, "Deleting files...") err := b.WorkspaceClient().Workspace.Delete(ctx, workspace.Delete{ Path: b.Config.Workspace.RootPath, @@ -54,8 +36,6 @@ func (m *delete) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { if err != nil { return diag.FromErr(err) } - - cmdio.LogString(ctx, "Successfully deleted files!") return nil } diff --git a/bundle/deploy/metadata/compute.go b/bundle/deploy/metadata/compute.go index 0347654848..6ab997e27a 100644 --- a/bundle/deploy/metadata/compute.go +++ b/bundle/deploy/metadata/compute.go @@ -39,7 +39,8 @@ func (m *compute) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { for name, job := range b.Config.Resources.Jobs { // Compute config file path the job is defined in, relative to the bundle // root - relativePath, err := filepath.Rel(b.RootPath, job.ConfigFilePath) + l := b.Config.GetLocation("resources.jobs." + name) + relativePath, err := filepath.Rel(b.RootPath, l.File) if err != nil { return diag.Errorf("failed to compute relative path for job %s: %v", name, err) } diff --git a/bundle/deploy/terraform/apply.go b/bundle/deploy/terraform/apply.go index e4acda852f..e52d0ca8f1 100644 --- a/bundle/deploy/terraform/apply.go +++ b/bundle/deploy/terraform/apply.go @@ -4,7 +4,6 @@ import ( "context" "github.com/databricks/cli/bundle" - "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/log" "github.com/hashicorp/terraform-exec/tfexec" @@ -17,28 +16,32 @@ func (w *apply) Name() string { } func (w *apply) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { + // return early if plan is empty + if b.Plan.IsEmpty { + log.Debugf(ctx, "No changes in plan. Skipping terraform apply.") + return nil + } + tf := b.Terraform if tf == nil { return diag.Errorf("terraform not initialized") } - cmdio.LogString(ctx, "Deploying resources...") - - err := tf.Init(ctx, tfexec.Upgrade(true)) - if err != nil { - return diag.Errorf("terraform init: %v", err) + if b.Plan.Path == "" { + return diag.Errorf("no plan found") } - err = tf.Apply(ctx) + // Apply terraform according to the computed plan + err := tf.Apply(ctx, tfexec.DirOrPlan(b.Plan.Path)) if err != nil { return diag.Errorf("terraform apply: %v", err) } - log.Infof(ctx, "Resource deployment completed") + log.Infof(ctx, "terraform apply completed") return nil } -// Apply returns a [bundle.Mutator] that runs the equivalent of `terraform apply` +// Apply returns a [bundle.Mutator] that runs the equivalent of `terraform apply ./plan` // from the bundle's ephemeral working directory for Terraform. func Apply() bundle.Mutator { return &apply{} diff --git a/bundle/deploy/terraform/convert.go b/bundle/deploy/terraform/convert.go index a6ec04d9a2..f13c241cee 100644 --- a/bundle/deploy/terraform/convert.go +++ b/bundle/deploy/terraform/convert.go @@ -66,8 +66,10 @@ func convGrants(acl []resources.Grant) *schema.ResourceGrants { // BundleToTerraform converts resources in a bundle configuration // to the equivalent Terraform JSON representation. // -// NOTE: THIS IS CURRENTLY A HACK. WE NEED A BETTER WAY TO -// CONVERT TO/FROM TERRAFORM COMPATIBLE FORMAT. +// Note: This function is an older implementation of the conversion logic. It is +// no longer used in any code paths. It is kept around to be used in tests. +// New resources do not need to modify this function and can instead can define +// the conversion login in the tfdyn package. func BundleToTerraform(config *config.Root) *schema.Root { tfroot := schema.NewRoot() tfroot.Provider = schema.NewProviders() @@ -382,6 +384,16 @@ func TerraformToBundle(state *resourcesState, config *config.Root) error { } cur.ID = instance.Attributes.ID config.Resources.QualityMonitors[resource.Name] = cur + case "databricks_schema": + if config.Resources.Schemas == nil { + config.Resources.Schemas = make(map[string]*resources.Schema) + } + cur := config.Resources.Schemas[resource.Name] + if cur == nil { + cur = &resources.Schema{ModifiedStatus: resources.ModifiedStatusDeleted} + } + cur.ID = instance.Attributes.ID + config.Resources.Schemas[resource.Name] = cur case "databricks_permissions": case "databricks_grants": // Ignore; no need to pull these back into the configuration. @@ -426,6 +438,11 @@ func TerraformToBundle(state *resourcesState, config *config.Root) error { src.ModifiedStatus = resources.ModifiedStatusCreated } } + for _, src := range config.Resources.Schemas { + if src.ModifiedStatus == "" && src.ID == "" { + src.ModifiedStatus = resources.ModifiedStatusCreated + } + } return nil } diff --git a/bundle/deploy/terraform/convert_test.go b/bundle/deploy/terraform/convert_test.go index 7ea4485388..e4ef6114a9 100644 --- a/bundle/deploy/terraform/convert_test.go +++ b/bundle/deploy/terraform/convert_test.go @@ -655,6 +655,14 @@ func TestTerraformToBundleEmptyLocalResources(t *testing.T) { {Attributes: stateInstanceAttributes{ID: "1"}}, }, }, + { + Type: "databricks_schema", + Mode: "managed", + Name: "test_schema", + Instances: []stateResourceInstance{ + {Attributes: stateInstanceAttributes{ID: "1"}}, + }, + }, }, } err := TerraformToBundle(&tfState, &config) @@ -681,6 +689,9 @@ func TestTerraformToBundleEmptyLocalResources(t *testing.T) { assert.Equal(t, "1", config.Resources.QualityMonitors["test_monitor"].ID) assert.Equal(t, resources.ModifiedStatusDeleted, config.Resources.QualityMonitors["test_monitor"].ModifiedStatus) + assert.Equal(t, "1", config.Resources.Schemas["test_schema"].ID) + assert.Equal(t, resources.ModifiedStatusDeleted, config.Resources.Schemas["test_schema"].ModifiedStatus) + AssertFullResourceCoverage(t, &config) } @@ -736,6 +747,13 @@ func TestTerraformToBundleEmptyRemoteResources(t *testing.T) { }, }, }, + Schemas: map[string]*resources.Schema{ + "test_schema": { + CreateSchema: &catalog.CreateSchema{ + Name: "test_schema", + }, + }, + }, }, } var tfState = resourcesState{ @@ -765,6 +783,9 @@ func TestTerraformToBundleEmptyRemoteResources(t *testing.T) { assert.Equal(t, "", config.Resources.QualityMonitors["test_monitor"].ID) assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.QualityMonitors["test_monitor"].ModifiedStatus) + assert.Equal(t, "", config.Resources.Schemas["test_schema"].ID) + assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.Schemas["test_schema"].ModifiedStatus) + AssertFullResourceCoverage(t, &config) } @@ -855,6 +876,18 @@ func TestTerraformToBundleModifiedResources(t *testing.T) { }, }, }, + Schemas: map[string]*resources.Schema{ + "test_schema": { + CreateSchema: &catalog.CreateSchema{ + Name: "test_schema", + }, + }, + "test_schema_new": { + CreateSchema: &catalog.CreateSchema{ + Name: "test_schema_new", + }, + }, + }, }, } var tfState = resourcesState{ @@ -971,6 +1004,22 @@ func TestTerraformToBundleModifiedResources(t *testing.T) { {Attributes: stateInstanceAttributes{ID: "test_monitor_old"}}, }, }, + { + Type: "databricks_schema", + Mode: "managed", + Name: "test_schema", + Instances: []stateResourceInstance{ + {Attributes: stateInstanceAttributes{ID: "1"}}, + }, + }, + { + Type: "databricks_schema", + Mode: "managed", + Name: "test_schema_old", + Instances: []stateResourceInstance{ + {Attributes: stateInstanceAttributes{ID: "2"}}, + }, + }, }, } err := TerraformToBundle(&tfState, &config) @@ -1024,6 +1073,14 @@ func TestTerraformToBundleModifiedResources(t *testing.T) { assert.Equal(t, resources.ModifiedStatusDeleted, config.Resources.QualityMonitors["test_monitor_old"].ModifiedStatus) assert.Equal(t, "", config.Resources.QualityMonitors["test_monitor_new"].ID) assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.QualityMonitors["test_monitor_new"].ModifiedStatus) + + assert.Equal(t, "1", config.Resources.Schemas["test_schema"].ID) + assert.Equal(t, "", config.Resources.Schemas["test_schema"].ModifiedStatus) + assert.Equal(t, "2", config.Resources.Schemas["test_schema_old"].ID) + assert.Equal(t, resources.ModifiedStatusDeleted, config.Resources.Schemas["test_schema_old"].ModifiedStatus) + assert.Equal(t, "", config.Resources.Schemas["test_schema_new"].ID) + assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.Schemas["test_schema_new"].ModifiedStatus) + AssertFullResourceCoverage(t, &config) } diff --git a/bundle/deploy/terraform/destroy.go b/bundle/deploy/terraform/destroy.go deleted file mode 100644 index 16f074a222..0000000000 --- a/bundle/deploy/terraform/destroy.go +++ /dev/null @@ -1,124 +0,0 @@ -package terraform - -import ( - "context" - "fmt" - "strings" - - "github.com/databricks/cli/bundle" - "github.com/databricks/cli/libs/cmdio" - "github.com/databricks/cli/libs/diag" - "github.com/fatih/color" - "github.com/hashicorp/terraform-exec/tfexec" - tfjson "github.com/hashicorp/terraform-json" -) - -type PlanResourceChange struct { - ResourceType string `json:"resource_type"` - Action string `json:"action"` - ResourceName string `json:"resource_name"` -} - -func (c *PlanResourceChange) String() string { - result := strings.Builder{} - switch c.Action { - case "delete": - result.WriteString(" delete ") - default: - result.WriteString(c.Action + " ") - } - switch c.ResourceType { - case "databricks_job": - result.WriteString("job ") - case "databricks_pipeline": - result.WriteString("pipeline ") - default: - result.WriteString(c.ResourceType + " ") - } - result.WriteString(c.ResourceName) - return result.String() -} - -func (c *PlanResourceChange) IsInplaceSupported() bool { - return false -} - -func logDestroyPlan(ctx context.Context, changes []*tfjson.ResourceChange) error { - cmdio.LogString(ctx, "The following resources will be removed:") - for _, c := range changes { - if c.Change.Actions.Delete() { - cmdio.Log(ctx, &PlanResourceChange{ - ResourceType: c.Type, - Action: "delete", - ResourceName: c.Name, - }) - } - } - return nil -} - -type destroy struct{} - -func (w *destroy) Name() string { - return "terraform.Destroy" -} - -func (w *destroy) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { - // return early if plan is empty - if b.Plan.IsEmpty { - cmdio.LogString(ctx, "No resources to destroy in plan. Skipping destroy!") - return nil - } - - tf := b.Terraform - if tf == nil { - return diag.Errorf("terraform not initialized") - } - - // read plan file - plan, err := tf.ShowPlanFile(ctx, b.Plan.Path) - if err != nil { - return diag.FromErr(err) - } - - // print the resources that will be destroyed - err = logDestroyPlan(ctx, plan.ResourceChanges) - if err != nil { - return diag.FromErr(err) - } - - // Ask for confirmation, if needed - if !b.Plan.ConfirmApply { - red := color.New(color.FgRed).SprintFunc() - b.Plan.ConfirmApply, err = cmdio.AskYesOrNo(ctx, fmt.Sprintf("\nThis will permanently %s resources! Proceed?", red("destroy"))) - if err != nil { - return diag.FromErr(err) - } - } - - // return if confirmation was not provided - if !b.Plan.ConfirmApply { - return nil - } - - if b.Plan.Path == "" { - return diag.Errorf("no plan found") - } - - cmdio.LogString(ctx, "Starting to destroy resources") - - // Apply terraform according to the computed destroy plan - err = tf.Apply(ctx, tfexec.DirOrPlan(b.Plan.Path)) - if err != nil { - return diag.Errorf("terraform destroy: %v", err) - } - - cmdio.LogString(ctx, "Successfully destroyed resources!") - return nil -} - -// Destroy returns a [bundle.Mutator] that runs the conceptual equivalent of -// `terraform destroy ./plan` from the bundle's ephemeral working directory for Terraform. -func Destroy() bundle.Mutator { - return &destroy{} -} diff --git a/bundle/deploy/terraform/interpolate.go b/bundle/deploy/terraform/interpolate.go index 608f1c7957..faa098e1cc 100644 --- a/bundle/deploy/terraform/interpolate.go +++ b/bundle/deploy/terraform/interpolate.go @@ -56,6 +56,8 @@ func (m *interpolateMutator) Apply(ctx context.Context, b *bundle.Bundle) diag.D path = dyn.NewPath(dyn.Key("databricks_registered_model")).Append(path[2:]...) case dyn.Key("quality_monitors"): path = dyn.NewPath(dyn.Key("databricks_quality_monitor")).Append(path[2:]...) + case dyn.Key("schemas"): + path = dyn.NewPath(dyn.Key("databricks_schema")).Append(path[2:]...) default: // Trigger "key not found" for unknown resource types. return dyn.GetByPath(root, path) diff --git a/bundle/deploy/terraform/interpolate_test.go b/bundle/deploy/terraform/interpolate_test.go index 9af4a1443c..5ceb243bcd 100644 --- a/bundle/deploy/terraform/interpolate_test.go +++ b/bundle/deploy/terraform/interpolate_test.go @@ -30,6 +30,7 @@ func TestInterpolate(t *testing.T) { "other_experiment": "${resources.experiments.other_experiment.id}", "other_model_serving": "${resources.model_serving_endpoints.other_model_serving.id}", "other_registered_model": "${resources.registered_models.other_registered_model.id}", + "other_schema": "${resources.schemas.other_schema.id}", }, Tasks: []jobs.Task{ { @@ -65,6 +66,7 @@ func TestInterpolate(t *testing.T) { assert.Equal(t, "${databricks_mlflow_experiment.other_experiment.id}", j.Tags["other_experiment"]) assert.Equal(t, "${databricks_model_serving.other_model_serving.id}", j.Tags["other_model_serving"]) assert.Equal(t, "${databricks_registered_model.other_registered_model.id}", j.Tags["other_registered_model"]) + assert.Equal(t, "${databricks_schema.other_schema.id}", j.Tags["other_schema"]) m := b.Config.Resources.Models["my_model"] assert.Equal(t, "my_model", m.Model.Name) diff --git a/bundle/deploy/terraform/plan.go b/bundle/deploy/terraform/plan.go index 50e0f78ca2..72f0b49a89 100644 --- a/bundle/deploy/terraform/plan.go +++ b/bundle/deploy/terraform/plan.go @@ -6,8 +6,8 @@ import ( "path/filepath" "github.com/databricks/cli/bundle" - "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/terraform" "github.com/hashicorp/terraform-exec/tfexec" ) @@ -33,8 +33,6 @@ func (p *plan) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { return diag.Errorf("terraform not initialized") } - cmdio.LogString(ctx, "Starting plan computation") - err := tf.Init(ctx, tfexec.Upgrade(true)) if err != nil { return diag.Errorf("terraform init: %v", err) @@ -55,12 +53,11 @@ func (p *plan) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { // Set plan in main bundle struct for downstream mutators b.Plan = &terraform.Plan{ - Path: planPath, - ConfirmApply: b.AutoApprove, - IsEmpty: !notEmpty, + Path: planPath, + IsEmpty: !notEmpty, } - cmdio.LogString(ctx, fmt.Sprintf("Planning complete and persisted at %s\n", planPath)) + log.Debugf(ctx, fmt.Sprintf("Planning complete and persisted at %s\n", planPath)) return nil } diff --git a/bundle/deploy/terraform/state_push.go b/bundle/deploy/terraform/state_push.go index b50983bd4b..6cdde13716 100644 --- a/bundle/deploy/terraform/state_push.go +++ b/bundle/deploy/terraform/state_push.go @@ -2,6 +2,8 @@ package terraform import ( "context" + "errors" + "io/fs" "os" "path/filepath" @@ -34,6 +36,12 @@ func (l *statePush) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostic // Expect the state file to live under dir. local, err := os.Open(filepath.Join(dir, TerraformStateFileName)) + if errors.Is(err, fs.ErrNotExist) { + // The state file can be absent if terraform apply is skipped because + // there are no changes to apply in the plan. + log.Debugf(ctx, "Local terraform state file does not exist.") + return nil + } if err != nil { return diag.FromErr(err) } diff --git a/bundle/deploy/terraform/tfdyn/convert_schema.go b/bundle/deploy/terraform/tfdyn/convert_schema.go new file mode 100644 index 0000000000..b5e6a88c0d --- /dev/null +++ b/bundle/deploy/terraform/tfdyn/convert_schema.go @@ -0,0 +1,53 @@ +package tfdyn + +import ( + "context" + "fmt" + + "github.com/databricks/cli/bundle/internal/tf/schema" + "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/dyn/convert" + "github.com/databricks/cli/libs/log" +) + +func convertSchemaResource(ctx context.Context, vin dyn.Value) (dyn.Value, error) { + // Normalize the output value to the target schema. + v, diags := convert.Normalize(schema.ResourceSchema{}, vin) + for _, diag := range diags { + log.Debugf(ctx, "schema normalization diagnostic: %s", diag.Summary) + } + + // We always set force destroy as it allows DABs to manage the lifecycle + // of the schema. It's the responsibility of the CLI to ensure the user + // is adequately warned when they try to delete a UC schema. + vout, err := dyn.SetByPath(v, dyn.MustPathFromString("force_destroy"), dyn.V(true)) + if err != nil { + return dyn.InvalidValue, err + } + + return vout, nil +} + +type schemaConverter struct{} + +func (schemaConverter) Convert(ctx context.Context, key string, vin dyn.Value, out *schema.Resources) error { + vout, err := convertSchemaResource(ctx, vin) + if err != nil { + return err + } + + // Add the converted resource to the output. + out.Schema[key] = vout.AsAny() + + // Configure grants for this resource. + if grants := convertGrantsResource(ctx, vin); grants != nil { + grants.Schema = fmt.Sprintf("${databricks_schema.%s.id}", key) + out.Grants["schema_"+key] = grants + } + + return nil +} + +func init() { + registerConverter("schemas", schemaConverter{}) +} diff --git a/bundle/deploy/terraform/tfdyn/convert_schema_test.go b/bundle/deploy/terraform/tfdyn/convert_schema_test.go new file mode 100644 index 0000000000..2efbf3e430 --- /dev/null +++ b/bundle/deploy/terraform/tfdyn/convert_schema_test.go @@ -0,0 +1,75 @@ +package tfdyn + +import ( + "context" + "testing" + + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/bundle/internal/tf/schema" + "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/dyn/convert" + "github.com/databricks/databricks-sdk-go/service/catalog" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestConvertSchema(t *testing.T) { + var src = resources.Schema{ + CreateSchema: &catalog.CreateSchema{ + Name: "name", + CatalogName: "catalog", + Comment: "comment", + Properties: map[string]string{ + "k1": "v1", + "k2": "v2", + }, + StorageRoot: "root", + }, + Grants: []resources.Grant{ + { + Privileges: []string{"EXECUTE"}, + Principal: "jack@gmail.com", + }, + { + Privileges: []string{"RUN"}, + Principal: "jane@gmail.com", + }, + }, + } + + vin, err := convert.FromTyped(src, dyn.NilValue) + require.NoError(t, err) + + ctx := context.Background() + out := schema.NewResources() + err = schemaConverter{}.Convert(ctx, "my_schema", vin, out) + require.NoError(t, err) + + // Assert equality on the schema + assert.Equal(t, map[string]any{ + "name": "name", + "catalog_name": "catalog", + "comment": "comment", + "properties": map[string]any{ + "k1": "v1", + "k2": "v2", + }, + "force_destroy": true, + "storage_root": "root", + }, out.Schema["my_schema"]) + + // Assert equality on the grants + assert.Equal(t, &schema.ResourceGrants{ + Schema: "${databricks_schema.my_schema.id}", + Grant: []schema.ResourceGrantsGrant{ + { + Privileges: []string{"EXECUTE"}, + Principal: "jack@gmail.com", + }, + { + Privileges: []string{"RUN"}, + Principal: "jane@gmail.com", + }, + }, + }, out.Grants["schema_my_schema"]) +} diff --git a/bundle/internal/bundletest/location.go b/bundle/internal/bundletest/location.go index ebec43d30a..380d6e17d2 100644 --- a/bundle/internal/bundletest/location.go +++ b/bundle/internal/bundletest/location.go @@ -29,6 +29,4 @@ func SetLocation(b *bundle.Bundle, prefix string, filePath string) { return v, dyn.ErrSkip }) }) - - b.Config.ConfigureConfigFilePath() } diff --git a/bundle/internal/tf/codegen/schema/version.go b/bundle/internal/tf/codegen/schema/version.go index 63a4b1b786..39d4f66c16 100644 --- a/bundle/internal/tf/codegen/schema/version.go +++ b/bundle/internal/tf/codegen/schema/version.go @@ -1,3 +1,3 @@ package schema -const ProviderVersion = "1.48.3" +const ProviderVersion = "1.49.1" diff --git a/bundle/internal/tf/schema/data_source_cluster.go b/bundle/internal/tf/schema/data_source_cluster.go index fff66dc932..94d67bbfab 100644 --- a/bundle/internal/tf/schema/data_source_cluster.go +++ b/bundle/internal/tf/schema/data_source_cluster.go @@ -10,7 +10,9 @@ type DataSourceClusterClusterInfoAutoscale struct { type DataSourceClusterClusterInfoAwsAttributes struct { Availability string `json:"availability,omitempty"` EbsVolumeCount int `json:"ebs_volume_count,omitempty"` + EbsVolumeIops int `json:"ebs_volume_iops,omitempty"` EbsVolumeSize int `json:"ebs_volume_size,omitempty"` + EbsVolumeThroughput int `json:"ebs_volume_throughput,omitempty"` EbsVolumeType string `json:"ebs_volume_type,omitempty"` FirstOnDemand int `json:"first_on_demand,omitempty"` InstanceProfileArn string `json:"instance_profile_arn,omitempty"` @@ -18,10 +20,16 @@ type DataSourceClusterClusterInfoAwsAttributes struct { ZoneId string `json:"zone_id,omitempty"` } +type DataSourceClusterClusterInfoAzureAttributesLogAnalyticsInfo struct { + LogAnalyticsPrimaryKey string `json:"log_analytics_primary_key,omitempty"` + LogAnalyticsWorkspaceId string `json:"log_analytics_workspace_id,omitempty"` +} + type DataSourceClusterClusterInfoAzureAttributes struct { - Availability string `json:"availability,omitempty"` - FirstOnDemand int `json:"first_on_demand,omitempty"` - SpotBidMaxPrice int `json:"spot_bid_max_price,omitempty"` + Availability string `json:"availability,omitempty"` + FirstOnDemand int `json:"first_on_demand,omitempty"` + SpotBidMaxPrice int `json:"spot_bid_max_price,omitempty"` + LogAnalyticsInfo *DataSourceClusterClusterInfoAzureAttributesLogAnalyticsInfo `json:"log_analytics_info,omitempty"` } type DataSourceClusterClusterInfoClusterLogConfDbfs struct { @@ -49,12 +57,12 @@ type DataSourceClusterClusterInfoClusterLogStatus struct { } type DataSourceClusterClusterInfoDockerImageBasicAuth struct { - Password string `json:"password"` - Username string `json:"username"` + Password string `json:"password,omitempty"` + Username string `json:"username,omitempty"` } type DataSourceClusterClusterInfoDockerImage struct { - Url string `json:"url"` + Url string `json:"url,omitempty"` BasicAuth *DataSourceClusterClusterInfoDockerImageBasicAuth `json:"basic_auth,omitempty"` } @@ -139,12 +147,212 @@ type DataSourceClusterClusterInfoInitScripts struct { Workspace *DataSourceClusterClusterInfoInitScriptsWorkspace `json:"workspace,omitempty"` } +type DataSourceClusterClusterInfoSpecAutoscale struct { + MaxWorkers int `json:"max_workers,omitempty"` + MinWorkers int `json:"min_workers,omitempty"` +} + +type DataSourceClusterClusterInfoSpecAwsAttributes struct { + Availability string `json:"availability,omitempty"` + EbsVolumeCount int `json:"ebs_volume_count,omitempty"` + EbsVolumeIops int `json:"ebs_volume_iops,omitempty"` + EbsVolumeSize int `json:"ebs_volume_size,omitempty"` + EbsVolumeThroughput int `json:"ebs_volume_throughput,omitempty"` + EbsVolumeType string `json:"ebs_volume_type,omitempty"` + FirstOnDemand int `json:"first_on_demand,omitempty"` + InstanceProfileArn string `json:"instance_profile_arn,omitempty"` + SpotBidPricePercent int `json:"spot_bid_price_percent,omitempty"` + ZoneId string `json:"zone_id,omitempty"` +} + +type DataSourceClusterClusterInfoSpecAzureAttributesLogAnalyticsInfo struct { + LogAnalyticsPrimaryKey string `json:"log_analytics_primary_key,omitempty"` + LogAnalyticsWorkspaceId string `json:"log_analytics_workspace_id,omitempty"` +} + +type DataSourceClusterClusterInfoSpecAzureAttributes struct { + Availability string `json:"availability,omitempty"` + FirstOnDemand int `json:"first_on_demand,omitempty"` + SpotBidMaxPrice int `json:"spot_bid_max_price,omitempty"` + LogAnalyticsInfo *DataSourceClusterClusterInfoSpecAzureAttributesLogAnalyticsInfo `json:"log_analytics_info,omitempty"` +} + +type DataSourceClusterClusterInfoSpecClusterLogConfDbfs struct { + Destination string `json:"destination"` +} + +type DataSourceClusterClusterInfoSpecClusterLogConfS3 struct { + CannedAcl string `json:"canned_acl,omitempty"` + Destination string `json:"destination"` + EnableEncryption bool `json:"enable_encryption,omitempty"` + EncryptionType string `json:"encryption_type,omitempty"` + Endpoint string `json:"endpoint,omitempty"` + KmsKey string `json:"kms_key,omitempty"` + Region string `json:"region,omitempty"` +} + +type DataSourceClusterClusterInfoSpecClusterLogConf struct { + Dbfs *DataSourceClusterClusterInfoSpecClusterLogConfDbfs `json:"dbfs,omitempty"` + S3 *DataSourceClusterClusterInfoSpecClusterLogConfS3 `json:"s3,omitempty"` +} + +type DataSourceClusterClusterInfoSpecClusterMountInfoNetworkFilesystemInfo struct { + MountOptions string `json:"mount_options,omitempty"` + ServerAddress string `json:"server_address"` +} + +type DataSourceClusterClusterInfoSpecClusterMountInfo struct { + LocalMountDirPath string `json:"local_mount_dir_path"` + RemoteMountDirPath string `json:"remote_mount_dir_path,omitempty"` + NetworkFilesystemInfo *DataSourceClusterClusterInfoSpecClusterMountInfoNetworkFilesystemInfo `json:"network_filesystem_info,omitempty"` +} + +type DataSourceClusterClusterInfoSpecDockerImageBasicAuth struct { + Password string `json:"password"` + Username string `json:"username"` +} + +type DataSourceClusterClusterInfoSpecDockerImage struct { + Url string `json:"url"` + BasicAuth *DataSourceClusterClusterInfoSpecDockerImageBasicAuth `json:"basic_auth,omitempty"` +} + +type DataSourceClusterClusterInfoSpecGcpAttributes struct { + Availability string `json:"availability,omitempty"` + BootDiskSize int `json:"boot_disk_size,omitempty"` + GoogleServiceAccount string `json:"google_service_account,omitempty"` + LocalSsdCount int `json:"local_ssd_count,omitempty"` + UsePreemptibleExecutors bool `json:"use_preemptible_executors,omitempty"` + ZoneId string `json:"zone_id,omitempty"` +} + +type DataSourceClusterClusterInfoSpecInitScriptsAbfss struct { + Destination string `json:"destination"` +} + +type DataSourceClusterClusterInfoSpecInitScriptsDbfs struct { + Destination string `json:"destination"` +} + +type DataSourceClusterClusterInfoSpecInitScriptsFile struct { + Destination string `json:"destination"` +} + +type DataSourceClusterClusterInfoSpecInitScriptsGcs struct { + Destination string `json:"destination"` +} + +type DataSourceClusterClusterInfoSpecInitScriptsS3 struct { + CannedAcl string `json:"canned_acl,omitempty"` + Destination string `json:"destination"` + EnableEncryption bool `json:"enable_encryption,omitempty"` + EncryptionType string `json:"encryption_type,omitempty"` + Endpoint string `json:"endpoint,omitempty"` + KmsKey string `json:"kms_key,omitempty"` + Region string `json:"region,omitempty"` +} + +type DataSourceClusterClusterInfoSpecInitScriptsVolumes struct { + Destination string `json:"destination"` +} + +type DataSourceClusterClusterInfoSpecInitScriptsWorkspace struct { + Destination string `json:"destination"` +} + +type DataSourceClusterClusterInfoSpecInitScripts struct { + Abfss *DataSourceClusterClusterInfoSpecInitScriptsAbfss `json:"abfss,omitempty"` + Dbfs *DataSourceClusterClusterInfoSpecInitScriptsDbfs `json:"dbfs,omitempty"` + File *DataSourceClusterClusterInfoSpecInitScriptsFile `json:"file,omitempty"` + Gcs *DataSourceClusterClusterInfoSpecInitScriptsGcs `json:"gcs,omitempty"` + S3 *DataSourceClusterClusterInfoSpecInitScriptsS3 `json:"s3,omitempty"` + Volumes *DataSourceClusterClusterInfoSpecInitScriptsVolumes `json:"volumes,omitempty"` + Workspace *DataSourceClusterClusterInfoSpecInitScriptsWorkspace `json:"workspace,omitempty"` +} + +type DataSourceClusterClusterInfoSpecLibraryCran struct { + Package string `json:"package"` + Repo string `json:"repo,omitempty"` +} + +type DataSourceClusterClusterInfoSpecLibraryMaven struct { + Coordinates string `json:"coordinates"` + Exclusions []string `json:"exclusions,omitempty"` + Repo string `json:"repo,omitempty"` +} + +type DataSourceClusterClusterInfoSpecLibraryPypi struct { + Package string `json:"package"` + Repo string `json:"repo,omitempty"` +} + +type DataSourceClusterClusterInfoSpecLibrary struct { + Egg string `json:"egg,omitempty"` + Jar string `json:"jar,omitempty"` + Requirements string `json:"requirements,omitempty"` + Whl string `json:"whl,omitempty"` + Cran *DataSourceClusterClusterInfoSpecLibraryCran `json:"cran,omitempty"` + Maven *DataSourceClusterClusterInfoSpecLibraryMaven `json:"maven,omitempty"` + Pypi *DataSourceClusterClusterInfoSpecLibraryPypi `json:"pypi,omitempty"` +} + +type DataSourceClusterClusterInfoSpecWorkloadTypeClients struct { + Jobs bool `json:"jobs,omitempty"` + Notebooks bool `json:"notebooks,omitempty"` +} + +type DataSourceClusterClusterInfoSpecWorkloadType struct { + Clients *DataSourceClusterClusterInfoSpecWorkloadTypeClients `json:"clients,omitempty"` +} + +type DataSourceClusterClusterInfoSpec struct { + ApplyPolicyDefaultValues bool `json:"apply_policy_default_values,omitempty"` + ClusterId string `json:"cluster_id,omitempty"` + ClusterName string `json:"cluster_name,omitempty"` + CustomTags map[string]string `json:"custom_tags,omitempty"` + DataSecurityMode string `json:"data_security_mode,omitempty"` + DriverInstancePoolId string `json:"driver_instance_pool_id,omitempty"` + DriverNodeTypeId string `json:"driver_node_type_id,omitempty"` + EnableElasticDisk bool `json:"enable_elastic_disk,omitempty"` + EnableLocalDiskEncryption bool `json:"enable_local_disk_encryption,omitempty"` + IdempotencyToken string `json:"idempotency_token,omitempty"` + InstancePoolId string `json:"instance_pool_id,omitempty"` + NodeTypeId string `json:"node_type_id,omitempty"` + NumWorkers int `json:"num_workers,omitempty"` + PolicyId string `json:"policy_id,omitempty"` + RuntimeEngine string `json:"runtime_engine,omitempty"` + SingleUserName string `json:"single_user_name,omitempty"` + SparkConf map[string]string `json:"spark_conf,omitempty"` + SparkEnvVars map[string]string `json:"spark_env_vars,omitempty"` + SparkVersion string `json:"spark_version"` + SshPublicKeys []string `json:"ssh_public_keys,omitempty"` + Autoscale *DataSourceClusterClusterInfoSpecAutoscale `json:"autoscale,omitempty"` + AwsAttributes *DataSourceClusterClusterInfoSpecAwsAttributes `json:"aws_attributes,omitempty"` + AzureAttributes *DataSourceClusterClusterInfoSpecAzureAttributes `json:"azure_attributes,omitempty"` + ClusterLogConf *DataSourceClusterClusterInfoSpecClusterLogConf `json:"cluster_log_conf,omitempty"` + ClusterMountInfo []DataSourceClusterClusterInfoSpecClusterMountInfo `json:"cluster_mount_info,omitempty"` + DockerImage *DataSourceClusterClusterInfoSpecDockerImage `json:"docker_image,omitempty"` + GcpAttributes *DataSourceClusterClusterInfoSpecGcpAttributes `json:"gcp_attributes,omitempty"` + InitScripts []DataSourceClusterClusterInfoSpecInitScripts `json:"init_scripts,omitempty"` + Library []DataSourceClusterClusterInfoSpecLibrary `json:"library,omitempty"` + WorkloadType *DataSourceClusterClusterInfoSpecWorkloadType `json:"workload_type,omitempty"` +} + type DataSourceClusterClusterInfoTerminationReason struct { Code string `json:"code,omitempty"` Parameters map[string]string `json:"parameters,omitempty"` Type string `json:"type,omitempty"` } +type DataSourceClusterClusterInfoWorkloadTypeClients struct { + Jobs bool `json:"jobs,omitempty"` + Notebooks bool `json:"notebooks,omitempty"` +} + +type DataSourceClusterClusterInfoWorkloadType struct { + Clients *DataSourceClusterClusterInfoWorkloadTypeClients `json:"clients,omitempty"` +} + type DataSourceClusterClusterInfo struct { AutoterminationMinutes int `json:"autotermination_minutes,omitempty"` ClusterCores int `json:"cluster_cores,omitempty"` @@ -155,14 +363,14 @@ type DataSourceClusterClusterInfo struct { CreatorUserName string `json:"creator_user_name,omitempty"` CustomTags map[string]string `json:"custom_tags,omitempty"` DataSecurityMode string `json:"data_security_mode,omitempty"` - DefaultTags map[string]string `json:"default_tags"` + DefaultTags map[string]string `json:"default_tags,omitempty"` DriverInstancePoolId string `json:"driver_instance_pool_id,omitempty"` DriverNodeTypeId string `json:"driver_node_type_id,omitempty"` EnableElasticDisk bool `json:"enable_elastic_disk,omitempty"` EnableLocalDiskEncryption bool `json:"enable_local_disk_encryption,omitempty"` InstancePoolId string `json:"instance_pool_id,omitempty"` JdbcPort int `json:"jdbc_port,omitempty"` - LastActivityTime int `json:"last_activity_time,omitempty"` + LastRestartedTime int `json:"last_restarted_time,omitempty"` LastStateLossTime int `json:"last_state_loss_time,omitempty"` NodeTypeId string `json:"node_type_id,omitempty"` NumWorkers int `json:"num_workers,omitempty"` @@ -172,12 +380,12 @@ type DataSourceClusterClusterInfo struct { SparkConf map[string]string `json:"spark_conf,omitempty"` SparkContextId int `json:"spark_context_id,omitempty"` SparkEnvVars map[string]string `json:"spark_env_vars,omitempty"` - SparkVersion string `json:"spark_version"` + SparkVersion string `json:"spark_version,omitempty"` SshPublicKeys []string `json:"ssh_public_keys,omitempty"` StartTime int `json:"start_time,omitempty"` - State string `json:"state"` + State string `json:"state,omitempty"` StateMessage string `json:"state_message,omitempty"` - TerminateTime int `json:"terminate_time,omitempty"` + TerminatedTime int `json:"terminated_time,omitempty"` Autoscale *DataSourceClusterClusterInfoAutoscale `json:"autoscale,omitempty"` AwsAttributes *DataSourceClusterClusterInfoAwsAttributes `json:"aws_attributes,omitempty"` AzureAttributes *DataSourceClusterClusterInfoAzureAttributes `json:"azure_attributes,omitempty"` @@ -188,7 +396,9 @@ type DataSourceClusterClusterInfo struct { Executors []DataSourceClusterClusterInfoExecutors `json:"executors,omitempty"` GcpAttributes *DataSourceClusterClusterInfoGcpAttributes `json:"gcp_attributes,omitempty"` InitScripts []DataSourceClusterClusterInfoInitScripts `json:"init_scripts,omitempty"` + Spec *DataSourceClusterClusterInfoSpec `json:"spec,omitempty"` TerminationReason *DataSourceClusterClusterInfoTerminationReason `json:"termination_reason,omitempty"` + WorkloadType *DataSourceClusterClusterInfoWorkloadType `json:"workload_type,omitempty"` } type DataSourceCluster struct { diff --git a/bundle/internal/tf/schema/data_source_schema.go b/bundle/internal/tf/schema/data_source_schema.go new file mode 100644 index 0000000000..9d778cc88f --- /dev/null +++ b/bundle/internal/tf/schema/data_source_schema.go @@ -0,0 +1,36 @@ +// Generated from Databricks Terraform provider schema. DO NOT EDIT. + +package schema + +type DataSourceSchemaSchemaInfoEffectivePredictiveOptimizationFlag struct { + InheritedFromName string `json:"inherited_from_name,omitempty"` + InheritedFromType string `json:"inherited_from_type,omitempty"` + Value string `json:"value"` +} + +type DataSourceSchemaSchemaInfo struct { + BrowseOnly bool `json:"browse_only,omitempty"` + CatalogName string `json:"catalog_name,omitempty"` + CatalogType string `json:"catalog_type,omitempty"` + Comment string `json:"comment,omitempty"` + CreatedAt int `json:"created_at,omitempty"` + CreatedBy string `json:"created_by,omitempty"` + EnablePredictiveOptimization string `json:"enable_predictive_optimization,omitempty"` + FullName string `json:"full_name,omitempty"` + MetastoreId string `json:"metastore_id,omitempty"` + Name string `json:"name,omitempty"` + Owner string `json:"owner,omitempty"` + Properties map[string]string `json:"properties,omitempty"` + SchemaId string `json:"schema_id,omitempty"` + StorageLocation string `json:"storage_location,omitempty"` + StorageRoot string `json:"storage_root,omitempty"` + UpdatedAt int `json:"updated_at,omitempty"` + UpdatedBy string `json:"updated_by,omitempty"` + EffectivePredictiveOptimizationFlag *DataSourceSchemaSchemaInfoEffectivePredictiveOptimizationFlag `json:"effective_predictive_optimization_flag,omitempty"` +} + +type DataSourceSchema struct { + Id string `json:"id,omitempty"` + Name string `json:"name"` + SchemaInfo *DataSourceSchemaSchemaInfo `json:"schema_info,omitempty"` +} diff --git a/bundle/internal/tf/schema/data_source_volume.go b/bundle/internal/tf/schema/data_source_volume.go new file mode 100644 index 0000000000..67e6100f62 --- /dev/null +++ b/bundle/internal/tf/schema/data_source_volume.go @@ -0,0 +1,38 @@ +// Generated from Databricks Terraform provider schema. DO NOT EDIT. + +package schema + +type DataSourceVolumeVolumeInfoEncryptionDetailsSseEncryptionDetails struct { + Algorithm string `json:"algorithm,omitempty"` + AwsKmsKeyArn string `json:"aws_kms_key_arn,omitempty"` +} + +type DataSourceVolumeVolumeInfoEncryptionDetails struct { + SseEncryptionDetails *DataSourceVolumeVolumeInfoEncryptionDetailsSseEncryptionDetails `json:"sse_encryption_details,omitempty"` +} + +type DataSourceVolumeVolumeInfo struct { + AccessPoint string `json:"access_point,omitempty"` + BrowseOnly bool `json:"browse_only,omitempty"` + CatalogName string `json:"catalog_name,omitempty"` + Comment string `json:"comment,omitempty"` + CreatedAt int `json:"created_at,omitempty"` + CreatedBy string `json:"created_by,omitempty"` + FullName string `json:"full_name,omitempty"` + MetastoreId string `json:"metastore_id,omitempty"` + Name string `json:"name,omitempty"` + Owner string `json:"owner,omitempty"` + SchemaName string `json:"schema_name,omitempty"` + StorageLocation string `json:"storage_location,omitempty"` + UpdatedAt int `json:"updated_at,omitempty"` + UpdatedBy string `json:"updated_by,omitempty"` + VolumeId string `json:"volume_id,omitempty"` + VolumeType string `json:"volume_type,omitempty"` + EncryptionDetails *DataSourceVolumeVolumeInfoEncryptionDetails `json:"encryption_details,omitempty"` +} + +type DataSourceVolume struct { + Id string `json:"id,omitempty"` + Name string `json:"name"` + VolumeInfo *DataSourceVolumeVolumeInfo `json:"volume_info,omitempty"` +} diff --git a/bundle/internal/tf/schema/data_sources.go b/bundle/internal/tf/schema/data_sources.go index b68df2b401..4ac78613f6 100644 --- a/bundle/internal/tf/schema/data_sources.go +++ b/bundle/internal/tf/schema/data_sources.go @@ -36,6 +36,7 @@ type DataSources struct { Notebook map[string]any `json:"databricks_notebook,omitempty"` NotebookPaths map[string]any `json:"databricks_notebook_paths,omitempty"` Pipelines map[string]any `json:"databricks_pipelines,omitempty"` + Schema map[string]any `json:"databricks_schema,omitempty"` Schemas map[string]any `json:"databricks_schemas,omitempty"` ServicePrincipal map[string]any `json:"databricks_service_principal,omitempty"` ServicePrincipals map[string]any `json:"databricks_service_principals,omitempty"` @@ -50,6 +51,7 @@ type DataSources struct { Tables map[string]any `json:"databricks_tables,omitempty"` User map[string]any `json:"databricks_user,omitempty"` Views map[string]any `json:"databricks_views,omitempty"` + Volume map[string]any `json:"databricks_volume,omitempty"` Volumes map[string]any `json:"databricks_volumes,omitempty"` Zones map[string]any `json:"databricks_zones,omitempty"` } @@ -89,6 +91,7 @@ func NewDataSources() *DataSources { Notebook: make(map[string]any), NotebookPaths: make(map[string]any), Pipelines: make(map[string]any), + Schema: make(map[string]any), Schemas: make(map[string]any), ServicePrincipal: make(map[string]any), ServicePrincipals: make(map[string]any), @@ -103,6 +106,7 @@ func NewDataSources() *DataSources { Tables: make(map[string]any), User: make(map[string]any), Views: make(map[string]any), + Volume: make(map[string]any), Volumes: make(map[string]any), Zones: make(map[string]any), } diff --git a/bundle/internal/tf/schema/resource_dashboard.go b/bundle/internal/tf/schema/resource_dashboard.go new file mode 100644 index 0000000000..0c2fa4a0fe --- /dev/null +++ b/bundle/internal/tf/schema/resource_dashboard.go @@ -0,0 +1,21 @@ +// Generated from Databricks Terraform provider schema. DO NOT EDIT. + +package schema + +type ResourceDashboard struct { + CreateTime string `json:"create_time,omitempty"` + DashboardChangeDetected bool `json:"dashboard_change_detected,omitempty"` + DashboardId string `json:"dashboard_id,omitempty"` + DisplayName string `json:"display_name"` + EmbedCredentials bool `json:"embed_credentials,omitempty"` + Etag string `json:"etag,omitempty"` + FilePath string `json:"file_path,omitempty"` + Id string `json:"id,omitempty"` + LifecycleState string `json:"lifecycle_state,omitempty"` + Md5 string `json:"md5,omitempty"` + ParentPath string `json:"parent_path"` + Path string `json:"path,omitempty"` + SerializedDashboard string `json:"serialized_dashboard,omitempty"` + UpdateTime string `json:"update_time,omitempty"` + WarehouseId string `json:"warehouse_id"` +} diff --git a/bundle/internal/tf/schema/resource_permissions.go b/bundle/internal/tf/schema/resource_permissions.go index 5d8df11e70..ee94a1a8fe 100644 --- a/bundle/internal/tf/schema/resource_permissions.go +++ b/bundle/internal/tf/schema/resource_permissions.go @@ -13,6 +13,7 @@ type ResourcePermissions struct { Authorization string `json:"authorization,omitempty"` ClusterId string `json:"cluster_id,omitempty"` ClusterPolicyId string `json:"cluster_policy_id,omitempty"` + DashboardId string `json:"dashboard_id,omitempty"` DirectoryId string `json:"directory_id,omitempty"` DirectoryPath string `json:"directory_path,omitempty"` ExperimentId string `json:"experiment_id,omitempty"` diff --git a/bundle/internal/tf/schema/resource_workspace_binding.go b/bundle/internal/tf/schema/resource_workspace_binding.go new file mode 100644 index 0000000000..f0be7a41f0 --- /dev/null +++ b/bundle/internal/tf/schema/resource_workspace_binding.go @@ -0,0 +1,12 @@ +// Generated from Databricks Terraform provider schema. DO NOT EDIT. + +package schema + +type ResourceWorkspaceBinding struct { + BindingType string `json:"binding_type,omitempty"` + CatalogName string `json:"catalog_name,omitempty"` + Id string `json:"id,omitempty"` + SecurableName string `json:"securable_name,omitempty"` + SecurableType string `json:"securable_type,omitempty"` + WorkspaceId int `json:"workspace_id,omitempty"` +} diff --git a/bundle/internal/tf/schema/resources.go b/bundle/internal/tf/schema/resources.go index 79d71a65f9..79c1b32b50 100644 --- a/bundle/internal/tf/schema/resources.go +++ b/bundle/internal/tf/schema/resources.go @@ -16,6 +16,7 @@ type Resources struct { ClusterPolicy map[string]any `json:"databricks_cluster_policy,omitempty"` ComplianceSecurityProfileWorkspaceSetting map[string]any `json:"databricks_compliance_security_profile_workspace_setting,omitempty"` Connection map[string]any `json:"databricks_connection,omitempty"` + Dashboard map[string]any `json:"databricks_dashboard,omitempty"` DbfsFile map[string]any `json:"databricks_dbfs_file,omitempty"` DefaultNamespaceSetting map[string]any `json:"databricks_default_namespace_setting,omitempty"` Directory map[string]any `json:"databricks_directory,omitempty"` @@ -96,6 +97,7 @@ type Resources struct { VectorSearchEndpoint map[string]any `json:"databricks_vector_search_endpoint,omitempty"` VectorSearchIndex map[string]any `json:"databricks_vector_search_index,omitempty"` Volume map[string]any `json:"databricks_volume,omitempty"` + WorkspaceBinding map[string]any `json:"databricks_workspace_binding,omitempty"` WorkspaceConf map[string]any `json:"databricks_workspace_conf,omitempty"` WorkspaceFile map[string]any `json:"databricks_workspace_file,omitempty"` } @@ -115,6 +117,7 @@ func NewResources() *Resources { ClusterPolicy: make(map[string]any), ComplianceSecurityProfileWorkspaceSetting: make(map[string]any), Connection: make(map[string]any), + Dashboard: make(map[string]any), DbfsFile: make(map[string]any), DefaultNamespaceSetting: make(map[string]any), Directory: make(map[string]any), @@ -195,6 +198,7 @@ func NewResources() *Resources { VectorSearchEndpoint: make(map[string]any), VectorSearchIndex: make(map[string]any), Volume: make(map[string]any), + WorkspaceBinding: make(map[string]any), WorkspaceConf: make(map[string]any), WorkspaceFile: make(map[string]any), } diff --git a/bundle/internal/tf/schema/root.go b/bundle/internal/tf/schema/root.go index a79e998cff..1711283506 100644 --- a/bundle/internal/tf/schema/root.go +++ b/bundle/internal/tf/schema/root.go @@ -21,7 +21,7 @@ type Root struct { const ProviderHost = "registry.terraform.io" const ProviderSource = "databricks/databricks" -const ProviderVersion = "1.48.3" +const ProviderVersion = "1.49.1" func NewRoot() *Root { return &Root{ diff --git a/bundle/libraries/expand_glob_references.go b/bundle/libraries/expand_glob_references.go new file mode 100644 index 0000000000..9e90a2a17f --- /dev/null +++ b/bundle/libraries/expand_glob_references.go @@ -0,0 +1,221 @@ +package libraries + +import ( + "context" + "fmt" + "path/filepath" + "strings" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dyn" +) + +type expand struct { +} + +func matchError(p dyn.Path, l []dyn.Location, message string) diag.Diagnostic { + return diag.Diagnostic{ + Severity: diag.Error, + Summary: message, + Paths: []dyn.Path{ + p.Append(), + }, + Locations: l, + } +} + +func getLibDetails(v dyn.Value) (string, string, bool) { + m := v.MustMap() + whl, ok := m.GetByString("whl") + if ok { + return whl.MustString(), "whl", true + } + + jar, ok := m.GetByString("jar") + if ok { + return jar.MustString(), "jar", true + } + + return "", "", false +} + +func findMatches(b *bundle.Bundle, path string) ([]string, error) { + matches, err := filepath.Glob(filepath.Join(b.RootPath, path)) + if err != nil { + return nil, err + } + + if len(matches) == 0 { + if isGlobPattern(path) { + return nil, fmt.Errorf("no files match pattern: %s", path) + } else { + return nil, fmt.Errorf("file doesn't exist %s", path) + } + } + + // We make the matched path relative to the root path before storing it + // to allow upload mutator to distinguish between local and remote paths + for i, match := range matches { + matches[i], err = filepath.Rel(b.RootPath, match) + if err != nil { + return nil, err + } + } + + return matches, nil +} + +// Checks if the path is a glob pattern +// It can contain *, [] or ? characters +func isGlobPattern(path string) bool { + return strings.ContainsAny(path, "*?[") +} + +func expandLibraries(b *bundle.Bundle, p dyn.Path, v dyn.Value) (diag.Diagnostics, []dyn.Value) { + var output []dyn.Value + var diags diag.Diagnostics + + libs := v.MustSequence() + for i, lib := range libs { + lp := p.Append(dyn.Index(i)) + path, libType, supported := getLibDetails(lib) + if !supported || !IsLibraryLocal(path) { + output = append(output, lib) + continue + } + + lp = lp.Append(dyn.Key(libType)) + + matches, err := findMatches(b, path) + if err != nil { + diags = diags.Append(matchError(lp, lib.Locations(), err.Error())) + continue + } + + for _, match := range matches { + output = append(output, dyn.NewValue(map[string]dyn.Value{ + libType: dyn.V(match), + }, lib.Locations())) + } + } + + return diags, output +} + +func expandEnvironmentDeps(b *bundle.Bundle, p dyn.Path, v dyn.Value) (diag.Diagnostics, []dyn.Value) { + var output []dyn.Value + var diags diag.Diagnostics + + deps := v.MustSequence() + for i, dep := range deps { + lp := p.Append(dyn.Index(i)) + path := dep.MustString() + if !IsLibraryLocal(path) { + output = append(output, dep) + continue + } + + matches, err := findMatches(b, path) + if err != nil { + diags = diags.Append(matchError(lp, dep.Locations(), err.Error())) + continue + } + + for _, match := range matches { + output = append(output, dyn.NewValue(match, dep.Locations())) + } + } + + return diags, output +} + +type expandPattern struct { + pattern dyn.Pattern + fn func(b *bundle.Bundle, p dyn.Path, v dyn.Value) (diag.Diagnostics, []dyn.Value) +} + +var taskLibrariesPattern = dyn.NewPattern( + dyn.Key("resources"), + dyn.Key("jobs"), + dyn.AnyKey(), + dyn.Key("tasks"), + dyn.AnyIndex(), + dyn.Key("libraries"), +) + +var forEachTaskLibrariesPattern = dyn.NewPattern( + dyn.Key("resources"), + dyn.Key("jobs"), + dyn.AnyKey(), + dyn.Key("tasks"), + dyn.AnyIndex(), + dyn.Key("for_each_task"), + dyn.Key("task"), + dyn.Key("libraries"), +) + +var envDepsPattern = dyn.NewPattern( + dyn.Key("resources"), + dyn.Key("jobs"), + dyn.AnyKey(), + dyn.Key("environments"), + dyn.AnyIndex(), + dyn.Key("spec"), + dyn.Key("dependencies"), +) + +func (e *expand) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { + expanders := []expandPattern{ + { + pattern: taskLibrariesPattern, + fn: expandLibraries, + }, + { + pattern: forEachTaskLibrariesPattern, + fn: expandLibraries, + }, + { + pattern: envDepsPattern, + fn: expandEnvironmentDeps, + }, + } + + var diags diag.Diagnostics + + err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { + var err error + for _, expander := range expanders { + v, err = dyn.MapByPattern(v, expander.pattern, func(p dyn.Path, lv dyn.Value) (dyn.Value, error) { + d, output := expander.fn(b, p, lv) + diags = diags.Extend(d) + return dyn.V(output), nil + }) + + if err != nil { + return dyn.InvalidValue, err + } + } + + return v, nil + }) + + if err != nil { + diags = diags.Extend(diag.FromErr(err)) + } + + return diags +} + +func (e *expand) Name() string { + return "libraries.ExpandGlobReferences" +} + +// ExpandGlobReferences expands any glob references in the libraries or environments section +// to corresponding local paths. +// We only expand local paths (i.e. paths that are relative to the root path). +// After expanding we make the paths relative to the root path to allow upload mutator later in the chain to +// distinguish between local and remote paths. +func ExpandGlobReferences() bundle.Mutator { + return &expand{} +} diff --git a/bundle/libraries/expand_glob_references_test.go b/bundle/libraries/expand_glob_references_test.go new file mode 100644 index 0000000000..34855b539d --- /dev/null +++ b/bundle/libraries/expand_glob_references_test.go @@ -0,0 +1,239 @@ +package libraries + +import ( + "context" + "path/filepath" + "testing" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/bundle/internal/bundletest" + "github.com/databricks/cli/internal/testutil" + "github.com/databricks/databricks-sdk-go/service/compute" + "github.com/databricks/databricks-sdk-go/service/jobs" + "github.com/stretchr/testify/require" +) + +func TestGlobReferencesExpandedForTaskLibraries(t *testing.T) { + dir := t.TempDir() + testutil.Touch(t, dir, "whl", "my1.whl") + testutil.Touch(t, dir, "whl", "my2.whl") + testutil.Touch(t, dir, "jar", "my1.jar") + testutil.Touch(t, dir, "jar", "my2.jar") + + b := &bundle.Bundle{ + RootPath: dir, + Config: config.Root{ + Resources: config.Resources{ + Jobs: map[string]*resources.Job{ + "job": { + JobSettings: &jobs.JobSettings{ + Tasks: []jobs.Task{ + { + TaskKey: "task", + Libraries: []compute.Library{ + { + Whl: "whl/*.whl", + }, + { + Whl: "/Workspace/path/to/whl/my.whl", + }, + { + Jar: "./jar/*.jar", + }, + { + Egg: "egg/*.egg", + }, + { + Jar: "/Workspace/path/to/jar/*.jar", + }, + { + Whl: "/some/full/path/to/whl/*.whl", + }, + }, + }, + }, + }, + }, + }, + }, + }, + } + + bundletest.SetLocation(b, ".", filepath.Join(dir, "resource.yml")) + + diags := bundle.Apply(context.Background(), b, ExpandGlobReferences()) + require.Empty(t, diags) + + job := b.Config.Resources.Jobs["job"] + task := job.JobSettings.Tasks[0] + require.Equal(t, []compute.Library{ + { + Whl: filepath.Join("whl", "my1.whl"), + }, + { + Whl: filepath.Join("whl", "my2.whl"), + }, + { + Whl: "/Workspace/path/to/whl/my.whl", + }, + { + Jar: filepath.Join("jar", "my1.jar"), + }, + { + Jar: filepath.Join("jar", "my2.jar"), + }, + { + Egg: "egg/*.egg", + }, + { + Jar: "/Workspace/path/to/jar/*.jar", + }, + { + Whl: "/some/full/path/to/whl/*.whl", + }, + }, task.Libraries) +} + +func TestGlobReferencesExpandedForForeachTaskLibraries(t *testing.T) { + dir := t.TempDir() + testutil.Touch(t, dir, "whl", "my1.whl") + testutil.Touch(t, dir, "whl", "my2.whl") + testutil.Touch(t, dir, "jar", "my1.jar") + testutil.Touch(t, dir, "jar", "my2.jar") + + b := &bundle.Bundle{ + RootPath: dir, + Config: config.Root{ + Resources: config.Resources{ + Jobs: map[string]*resources.Job{ + "job": { + JobSettings: &jobs.JobSettings{ + Tasks: []jobs.Task{ + { + TaskKey: "task", + ForEachTask: &jobs.ForEachTask{ + Task: jobs.Task{ + Libraries: []compute.Library{ + { + Whl: "whl/*.whl", + }, + { + Whl: "/Workspace/path/to/whl/my.whl", + }, + { + Jar: "./jar/*.jar", + }, + { + Egg: "egg/*.egg", + }, + { + Jar: "/Workspace/path/to/jar/*.jar", + }, + { + Whl: "/some/full/path/to/whl/*.whl", + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + } + + bundletest.SetLocation(b, ".", filepath.Join(dir, "resource.yml")) + + diags := bundle.Apply(context.Background(), b, ExpandGlobReferences()) + require.Empty(t, diags) + + job := b.Config.Resources.Jobs["job"] + task := job.JobSettings.Tasks[0].ForEachTask.Task + require.Equal(t, []compute.Library{ + { + Whl: filepath.Join("whl", "my1.whl"), + }, + { + Whl: filepath.Join("whl", "my2.whl"), + }, + { + Whl: "/Workspace/path/to/whl/my.whl", + }, + { + Jar: filepath.Join("jar", "my1.jar"), + }, + { + Jar: filepath.Join("jar", "my2.jar"), + }, + { + Egg: "egg/*.egg", + }, + { + Jar: "/Workspace/path/to/jar/*.jar", + }, + { + Whl: "/some/full/path/to/whl/*.whl", + }, + }, task.Libraries) +} + +func TestGlobReferencesExpandedForEnvironmentsDeps(t *testing.T) { + dir := t.TempDir() + testutil.Touch(t, dir, "whl", "my1.whl") + testutil.Touch(t, dir, "whl", "my2.whl") + testutil.Touch(t, dir, "jar", "my1.jar") + testutil.Touch(t, dir, "jar", "my2.jar") + + b := &bundle.Bundle{ + RootPath: dir, + Config: config.Root{ + Resources: config.Resources{ + Jobs: map[string]*resources.Job{ + "job": { + JobSettings: &jobs.JobSettings{ + Tasks: []jobs.Task{ + { + TaskKey: "task", + EnvironmentKey: "env", + }, + }, + Environments: []jobs.JobEnvironment{ + { + EnvironmentKey: "env", + Spec: &compute.Environment{ + Dependencies: []string{ + "./whl/*.whl", + "/Workspace/path/to/whl/my.whl", + "./jar/*.jar", + "/some/local/path/to/whl/*.whl", + }, + }, + }, + }, + }, + }, + }, + }, + }, + } + + bundletest.SetLocation(b, ".", filepath.Join(dir, "resource.yml")) + + diags := bundle.Apply(context.Background(), b, ExpandGlobReferences()) + require.Empty(t, diags) + + job := b.Config.Resources.Jobs["job"] + env := job.JobSettings.Environments[0] + require.Equal(t, []string{ + filepath.Join("whl", "my1.whl"), + filepath.Join("whl", "my2.whl"), + "/Workspace/path/to/whl/my.whl", + filepath.Join("jar", "my1.jar"), + filepath.Join("jar", "my2.jar"), + "/some/local/path/to/whl/*.whl", + }, env.Spec.Dependencies) +} diff --git a/bundle/libraries/libraries.go b/bundle/libraries/libraries.go index 84ead052b3..33b848dd93 100644 --- a/bundle/libraries/libraries.go +++ b/bundle/libraries/libraries.go @@ -35,7 +35,7 @@ func isEnvsWithLocalLibraries(envs []jobs.JobEnvironment) bool { } for _, l := range e.Spec.Dependencies { - if IsEnvironmentDependencyLocal(l) { + if IsLibraryLocal(l) { return true } } @@ -44,34 +44,30 @@ func isEnvsWithLocalLibraries(envs []jobs.JobEnvironment) bool { return false } -func FindAllWheelTasksWithLocalLibraries(b *bundle.Bundle) []*jobs.Task { +func FindTasksWithLocalLibraries(b *bundle.Bundle) []jobs.Task { tasks := findAllTasks(b) envs := FindAllEnvironments(b) - wheelTasks := make([]*jobs.Task, 0) + allTasks := make([]jobs.Task, 0) for k, jobTasks := range tasks { for i := range jobTasks { - task := &jobTasks[i] - if task.PythonWheelTask == nil { - continue - } - - if isTaskWithLocalLibraries(*task) { - wheelTasks = append(wheelTasks, task) + task := jobTasks[i] + if isTaskWithLocalLibraries(task) { + allTasks = append(allTasks, task) } + } - if envs[k] != nil && isEnvsWithLocalLibraries(envs[k]) { - wheelTasks = append(wheelTasks, task) - } + if envs[k] != nil && isEnvsWithLocalLibraries(envs[k]) { + allTasks = append(allTasks, jobTasks...) } } - return wheelTasks + return allTasks } func isTaskWithLocalLibraries(task jobs.Task) bool { for _, l := range task.Libraries { - if IsLocalLibrary(&l) { + if IsLibraryLocal(libraryPath(&l)) { return true } } diff --git a/bundle/libraries/local_path.go b/bundle/libraries/local_path.go index f1e3788f24..5b5ec6c076 100644 --- a/bundle/libraries/local_path.go +++ b/bundle/libraries/local_path.go @@ -4,8 +4,6 @@ import ( "net/url" "path" "strings" - - "github.com/databricks/databricks-sdk-go/service/compute" ) // IsLocalPath returns true if the specified path indicates that @@ -38,12 +36,12 @@ func IsLocalPath(p string) bool { return !path.IsAbs(p) } -// IsEnvironmentDependencyLocal returns true if the specified dependency +// IsLibraryLocal returns true if the specified library or environment dependency // should be interpreted as a local path. -// We use this to check if the dependency in environment spec is local. +// We use this to check if the dependency in environment spec is local or that library is local. // We can't use IsLocalPath beacuse environment dependencies can be // a pypi package name which can be misinterpreted as a local path by IsLocalPath. -func IsEnvironmentDependencyLocal(dep string) bool { +func IsLibraryLocal(dep string) bool { possiblePrefixes := []string{ ".", } @@ -54,7 +52,22 @@ func IsEnvironmentDependencyLocal(dep string) bool { } } - return false + // If the dependency is a requirements file, it's not a valid local path + if strings.HasPrefix(dep, "-r") { + return false + } + + // If the dependency has no extension, it's a PyPi package name + if isPackage(dep) { + return false + } + + return IsLocalPath(dep) +} + +func isPackage(name string) bool { + // If the dependency has no extension, it's a PyPi package name + return path.Ext(name) == "" } func isRemoteStorageScheme(path string) bool { @@ -67,16 +80,6 @@ func isRemoteStorageScheme(path string) bool { return false } - // If the path starts with scheme:/ format, it's a correct remote storage scheme - return strings.HasPrefix(path, url.Scheme+":/") -} - -// IsLocalLibrary returns true if the specified library refers to a local path. -func IsLocalLibrary(library *compute.Library) bool { - path := libraryPath(library) - if path == "" { - return false - } - - return IsLocalPath(path) + // If the path starts with scheme:/ format (not file), it's a correct remote storage scheme + return strings.HasPrefix(path, url.Scheme+":/") && url.Scheme != "file" } diff --git a/bundle/libraries/local_path_test.go b/bundle/libraries/local_path_test.go index d2492d6b12..be4028d522 100644 --- a/bundle/libraries/local_path_test.go +++ b/bundle/libraries/local_path_test.go @@ -3,13 +3,13 @@ package libraries import ( "testing" - "github.com/databricks/databricks-sdk-go/service/compute" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func TestIsLocalPath(t *testing.T) { // Relative paths, paths with the file scheme, and Windows paths. + assert.True(t, IsLocalPath("some/local/path")) assert.True(t, IsLocalPath("./some/local/path")) assert.True(t, IsLocalPath("file://path/to/package")) assert.True(t, IsLocalPath("C:\\path\\to\\package")) @@ -30,24 +30,13 @@ func TestIsLocalPath(t *testing.T) { assert.False(t, IsLocalPath("abfss://path/to/package")) } -func TestIsLocalLibrary(t *testing.T) { - // Local paths. - assert.True(t, IsLocalLibrary(&compute.Library{Whl: "./file.whl"})) - assert.True(t, IsLocalLibrary(&compute.Library{Jar: "../target/some.jar"})) - - // Non-local paths. - assert.False(t, IsLocalLibrary(&compute.Library{Whl: "/Workspace/path/to/file.whl"})) - assert.False(t, IsLocalLibrary(&compute.Library{Jar: "s3:/bucket/path/some.jar"})) - - // Empty. - assert.False(t, IsLocalLibrary(&compute.Library{})) -} - -func TestIsEnvironmentDependencyLocal(t *testing.T) { +func TestIsLibraryLocal(t *testing.T) { testCases := [](struct { path string expected bool }){ + {path: "local/*.whl", expected: true}, + {path: "local/test.whl", expected: true}, {path: "./local/*.whl", expected: true}, {path: ".\\local\\*.whl", expected: true}, {path: "./local/mypath.whl", expected: true}, @@ -58,15 +47,16 @@ func TestIsEnvironmentDependencyLocal(t *testing.T) { {path: ".\\..\\local\\*.whl", expected: true}, {path: "../../local/*.whl", expected: true}, {path: "..\\..\\local\\*.whl", expected: true}, + {path: "file://path/to/package/whl.whl", expected: true}, {path: "pypipackage", expected: false}, - {path: "pypipackage/test.whl", expected: false}, - {path: "pypipackage/*.whl", expected: false}, {path: "/Volumes/catalog/schema/volume/path.whl", expected: false}, {path: "/Workspace/my_project/dist.whl", expected: false}, {path: "-r /Workspace/my_project/requirements.txt", expected: false}, + {path: "s3://mybucket/path/to/package", expected: false}, + {path: "dbfs:/mnt/path/to/package", expected: false}, } - for _, tc := range testCases { - require.Equal(t, IsEnvironmentDependencyLocal(tc.path), tc.expected) + for i, tc := range testCases { + require.Equalf(t, tc.expected, IsLibraryLocal(tc.path), "failed case: %d, path: %s", i, tc.path) } } diff --git a/bundle/libraries/match.go b/bundle/libraries/match.go deleted file mode 100644 index 4feb4225d6..0000000000 --- a/bundle/libraries/match.go +++ /dev/null @@ -1,82 +0,0 @@ -package libraries - -import ( - "context" - "fmt" - "path/filepath" - - "github.com/databricks/cli/bundle" - "github.com/databricks/cli/libs/diag" - "github.com/databricks/databricks-sdk-go/service/compute" - "github.com/databricks/databricks-sdk-go/service/jobs" -) - -type match struct { -} - -func ValidateLocalLibrariesExist() bundle.Mutator { - return &match{} -} - -func (a *match) Name() string { - return "libraries.ValidateLocalLibrariesExist" -} - -func (a *match) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { - for _, job := range b.Config.Resources.Jobs { - err := validateEnvironments(job.Environments, b) - if err != nil { - return diag.FromErr(err) - } - - for _, task := range job.JobSettings.Tasks { - err := validateTaskLibraries(task.Libraries, b) - if err != nil { - return diag.FromErr(err) - } - } - } - - return nil -} - -func validateTaskLibraries(libs []compute.Library, b *bundle.Bundle) error { - for _, lib := range libs { - path := libraryPath(&lib) - if path == "" || !IsLocalPath(path) { - continue - } - - matches, err := filepath.Glob(filepath.Join(b.RootPath, path)) - if err != nil { - return err - } - - if len(matches) == 0 { - return fmt.Errorf("file %s is referenced in libraries section but doesn't exist on the local file system", libraryPath(&lib)) - } - } - - return nil -} - -func validateEnvironments(envs []jobs.JobEnvironment, b *bundle.Bundle) error { - for _, env := range envs { - if env.Spec == nil { - continue - } - - for _, dep := range env.Spec.Dependencies { - matches, err := filepath.Glob(filepath.Join(b.RootPath, dep)) - if err != nil { - return err - } - - if len(matches) == 0 && IsEnvironmentDependencyLocal(dep) { - return fmt.Errorf("file %s is referenced in environments section but doesn't exist on the local file system", dep) - } - } - } - - return nil -} diff --git a/bundle/libraries/match_test.go b/bundle/libraries/match_test.go index bb4b15107f..e60504c844 100644 --- a/bundle/libraries/match_test.go +++ b/bundle/libraries/match_test.go @@ -42,7 +42,7 @@ func TestValidateEnvironments(t *testing.T) { }, } - diags := bundle.Apply(context.Background(), b, ValidateLocalLibrariesExist()) + diags := bundle.Apply(context.Background(), b, ExpandGlobReferences()) require.Nil(t, diags) } @@ -74,9 +74,9 @@ func TestValidateEnvironmentsNoFile(t *testing.T) { }, } - diags := bundle.Apply(context.Background(), b, ValidateLocalLibrariesExist()) + diags := bundle.Apply(context.Background(), b, ExpandGlobReferences()) require.Len(t, diags, 1) - require.Equal(t, "file ./wheel.whl is referenced in environments section but doesn't exist on the local file system", diags[0].Summary) + require.Equal(t, "file doesn't exist ./wheel.whl", diags[0].Summary) } func TestValidateTaskLibraries(t *testing.T) { @@ -109,7 +109,7 @@ func TestValidateTaskLibraries(t *testing.T) { }, } - diags := bundle.Apply(context.Background(), b, ValidateLocalLibrariesExist()) + diags := bundle.Apply(context.Background(), b, ExpandGlobReferences()) require.Nil(t, diags) } @@ -142,7 +142,7 @@ func TestValidateTaskLibrariesNoFile(t *testing.T) { }, } - diags := bundle.Apply(context.Background(), b, ValidateLocalLibrariesExist()) + diags := bundle.Apply(context.Background(), b, ExpandGlobReferences()) require.Len(t, diags, 1) - require.Equal(t, "file ./wheel.whl is referenced in libraries section but doesn't exist on the local file system", diags[0].Summary) + require.Equal(t, "file doesn't exist ./wheel.whl", diags[0].Summary) } diff --git a/bundle/libraries/upload.go b/bundle/libraries/upload.go new file mode 100644 index 0000000000..be7cc41db5 --- /dev/null +++ b/bundle/libraries/upload.go @@ -0,0 +1,238 @@ +package libraries + +import ( + "context" + "errors" + "fmt" + "os" + "path" + "path/filepath" + "strings" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/filer" + "github.com/databricks/cli/libs/log" + + "github.com/databricks/databricks-sdk-go" + + "golang.org/x/sync/errgroup" +) + +// The Files API backend has a rate limit of 10 concurrent +// requests and 100 QPS. We limit the number of concurrent requests to 5 to +// avoid hitting the rate limit. +var maxFilesRequestsInFlight = 5 + +func Upload() bundle.Mutator { + return &upload{} +} + +func UploadWithClient(client filer.Filer) bundle.Mutator { + return &upload{ + client: client, + } +} + +type upload struct { + client filer.Filer +} + +type configLocation struct { + configPath dyn.Path + location dyn.Location +} + +// Collect all libraries from the bundle configuration and their config paths. +// By this stage all glob references are expanded and we have a list of all libraries that need to be uploaded. +// We collect them from task libraries, foreach task libraries, environment dependencies, and artifacts. +// We return a map of library source to a list of config paths and locations where the library is used. +// We use map so we don't upload the same library multiple times. +// Instead we upload it once and update all the config paths to point to the uploaded location. +func collectLocalLibraries(b *bundle.Bundle) (map[string][]configLocation, error) { + libs := make(map[string]([]configLocation)) + + patterns := []dyn.Pattern{ + taskLibrariesPattern.Append(dyn.AnyIndex(), dyn.Key("whl")), + taskLibrariesPattern.Append(dyn.AnyIndex(), dyn.Key("jar")), + forEachTaskLibrariesPattern.Append(dyn.AnyIndex(), dyn.Key("whl")), + forEachTaskLibrariesPattern.Append(dyn.AnyIndex(), dyn.Key("jar")), + envDepsPattern.Append(dyn.AnyIndex()), + } + + for _, pattern := range patterns { + err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { + return dyn.MapByPattern(v, pattern, func(p dyn.Path, v dyn.Value) (dyn.Value, error) { + source, ok := v.AsString() + if !ok { + return v, fmt.Errorf("expected string, got %s", v.Kind()) + } + + if !IsLibraryLocal(source) { + return v, nil + } + + source = filepath.Join(b.RootPath, source) + libs[source] = append(libs[source], configLocation{ + configPath: p.Append(), // Hack to get the copy of path + location: v.Location(), + }) + + return v, nil + }) + }) + + if err != nil { + return nil, err + } + } + + artifactPattern := dyn.NewPattern( + dyn.Key("artifacts"), + dyn.AnyKey(), + dyn.Key("files"), + dyn.AnyIndex(), + ) + + err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { + return dyn.MapByPattern(v, artifactPattern, func(p dyn.Path, v dyn.Value) (dyn.Value, error) { + file, ok := v.AsMap() + if !ok { + return v, fmt.Errorf("expected map, got %s", v.Kind()) + } + + sv, ok := file.GetByString("source") + if !ok { + return v, nil + } + + source, ok := sv.AsString() + if !ok { + return v, fmt.Errorf("expected string, got %s", v.Kind()) + } + + libs[source] = append(libs[source], configLocation{ + configPath: p.Append(dyn.Key("remote_path")), + location: v.Location(), + }) + + return v, nil + }) + }) + + if err != nil { + return nil, err + } + + return libs, nil +} + +func (u *upload) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { + uploadPath, err := GetUploadBasePath(b) + if err != nil { + return diag.FromErr(err) + } + + // If the client is not initialized, initialize it + // We use client field in mutator to allow for mocking client in testing + if u.client == nil { + filer, err := GetFilerForLibraries(b.WorkspaceClient(), uploadPath) + if err != nil { + return diag.FromErr(err) + } + + u.client = filer + } + + var diags diag.Diagnostics + + libs, err := collectLocalLibraries(b) + if err != nil { + return diag.FromErr(err) + } + + errs, errCtx := errgroup.WithContext(ctx) + errs.SetLimit(maxFilesRequestsInFlight) + + for source := range libs { + errs.Go(func() error { + return UploadFile(errCtx, source, u.client) + }) + } + + if err := errs.Wait(); err != nil { + return diag.FromErr(err) + } + + // Update all the config paths to point to the uploaded location + for source, locations := range libs { + err = b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { + remotePath := path.Join(uploadPath, filepath.Base(source)) + + // If the remote path does not start with /Workspace or /Volumes, prepend /Workspace + if !strings.HasPrefix(remotePath, "/Workspace") && !strings.HasPrefix(remotePath, "/Volumes") { + remotePath = "/Workspace" + remotePath + } + for _, location := range locations { + v, err = dyn.SetByPath(v, location.configPath, dyn.NewValue(remotePath, []dyn.Location{location.location})) + if err != nil { + return v, err + } + } + + return v, nil + }) + + if err != nil { + diags = diags.Extend(diag.FromErr(err)) + } + } + + return diags +} + +func (u *upload) Name() string { + return "libraries.Upload" +} + +func GetFilerForLibraries(w *databricks.WorkspaceClient, uploadPath string) (filer.Filer, error) { + if isVolumesPath(uploadPath) { + return filer.NewFilesClient(w, uploadPath) + } + return filer.NewWorkspaceFilesClient(w, uploadPath) +} + +func isVolumesPath(path string) bool { + return strings.HasPrefix(path, "/Volumes/") +} + +// Function to upload file (a library, artifact and etc) to Workspace or UC volume +func UploadFile(ctx context.Context, file string, client filer.Filer) error { + filename := filepath.Base(file) + cmdio.LogString(ctx, fmt.Sprintf("Uploading %s...", filename)) + + f, err := os.Open(file) + if err != nil { + return fmt.Errorf("unable to open %s: %w", file, errors.Unwrap(err)) + } + defer f.Close() + + err = client.Write(ctx, filename, f, filer.OverwriteIfExists, filer.CreateParentDirectories) + if err != nil { + return fmt.Errorf("unable to import %s: %w", filename, err) + } + + log.Infof(ctx, "Upload succeeded") + return nil +} + +func GetUploadBasePath(b *bundle.Bundle) (string, error) { + artifactPath := b.Config.Workspace.ArtifactPath + if artifactPath == "" { + return "", fmt.Errorf("remote artifact path not configured") + } + + return path.Join(artifactPath, ".internal"), nil +} diff --git a/bundle/libraries/upload_test.go b/bundle/libraries/upload_test.go new file mode 100644 index 0000000000..82fe6e7c7c --- /dev/null +++ b/bundle/libraries/upload_test.go @@ -0,0 +1,331 @@ +package libraries + +import ( + "context" + "path/filepath" + "testing" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/config/resources" + mockfiler "github.com/databricks/cli/internal/mocks/libs/filer" + "github.com/databricks/cli/internal/testutil" + "github.com/databricks/cli/libs/filer" + "github.com/databricks/databricks-sdk-go/service/compute" + "github.com/databricks/databricks-sdk-go/service/jobs" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" +) + +func TestArtifactUploadForWorkspace(t *testing.T) { + tmpDir := t.TempDir() + whlFolder := filepath.Join(tmpDir, "whl") + testutil.Touch(t, whlFolder, "source.whl") + whlLocalPath := filepath.Join(whlFolder, "source.whl") + + b := &bundle.Bundle{ + RootPath: tmpDir, + Config: config.Root{ + Workspace: config.Workspace{ + ArtifactPath: "/foo/bar/artifacts", + }, + Artifacts: config.Artifacts{ + "whl": { + Type: config.ArtifactPythonWheel, + Files: []config.ArtifactFile{ + {Source: whlLocalPath}, + }, + }, + }, + Resources: config.Resources{ + Jobs: map[string]*resources.Job{ + "job": { + JobSettings: &jobs.JobSettings{ + Tasks: []jobs.Task{ + { + Libraries: []compute.Library{ + { + Whl: filepath.Join("whl", "*.whl"), + }, + { + Whl: "/Workspace/Users/foo@bar.com/mywheel.whl", + }, + }, + }, + { + ForEachTask: &jobs.ForEachTask{ + Task: jobs.Task{ + Libraries: []compute.Library{ + { + Whl: filepath.Join("whl", "*.whl"), + }, + { + Whl: "/Workspace/Users/foo@bar.com/mywheel.whl", + }, + }, + }, + }, + }, + }, + Environments: []jobs.JobEnvironment{ + { + Spec: &compute.Environment{ + Dependencies: []string{ + filepath.Join("whl", "source.whl"), + "/Workspace/Users/foo@bar.com/mywheel.whl", + }, + }, + }, + }, + }, + }, + }, + }, + }, + } + + mockFiler := mockfiler.NewMockFiler(t) + mockFiler.EXPECT().Write( + mock.Anything, + filepath.Join("source.whl"), + mock.AnythingOfType("*os.File"), + filer.OverwriteIfExists, + filer.CreateParentDirectories, + ).Return(nil) + + diags := bundle.Apply(context.Background(), b, bundle.Seq(ExpandGlobReferences(), UploadWithClient(mockFiler))) + require.NoError(t, diags.Error()) + + // Test that libraries path is updated + require.Equal(t, "/Workspace/foo/bar/artifacts/.internal/source.whl", b.Config.Resources.Jobs["job"].JobSettings.Tasks[0].Libraries[0].Whl) + require.Equal(t, "/Workspace/Users/foo@bar.com/mywheel.whl", b.Config.Resources.Jobs["job"].JobSettings.Tasks[0].Libraries[1].Whl) + require.Equal(t, "/Workspace/foo/bar/artifacts/.internal/source.whl", b.Config.Resources.Jobs["job"].JobSettings.Environments[0].Spec.Dependencies[0]) + require.Equal(t, "/Workspace/Users/foo@bar.com/mywheel.whl", b.Config.Resources.Jobs["job"].JobSettings.Environments[0].Spec.Dependencies[1]) + require.Equal(t, "/Workspace/foo/bar/artifacts/.internal/source.whl", b.Config.Resources.Jobs["job"].JobSettings.Tasks[1].ForEachTask.Task.Libraries[0].Whl) + require.Equal(t, "/Workspace/Users/foo@bar.com/mywheel.whl", b.Config.Resources.Jobs["job"].JobSettings.Tasks[1].ForEachTask.Task.Libraries[1].Whl) +} + +func TestArtifactUploadForVolumes(t *testing.T) { + tmpDir := t.TempDir() + whlFolder := filepath.Join(tmpDir, "whl") + testutil.Touch(t, whlFolder, "source.whl") + whlLocalPath := filepath.Join(whlFolder, "source.whl") + + b := &bundle.Bundle{ + RootPath: tmpDir, + Config: config.Root{ + Workspace: config.Workspace{ + ArtifactPath: "/Volumes/foo/bar/artifacts", + }, + Artifacts: config.Artifacts{ + "whl": { + Type: config.ArtifactPythonWheel, + Files: []config.ArtifactFile{ + {Source: whlLocalPath}, + }, + }, + }, + Resources: config.Resources{ + Jobs: map[string]*resources.Job{ + "job": { + JobSettings: &jobs.JobSettings{ + Tasks: []jobs.Task{ + { + Libraries: []compute.Library{ + { + Whl: filepath.Join("whl", "*.whl"), + }, + { + Whl: "/Volumes/some/path/mywheel.whl", + }, + }, + }, + { + ForEachTask: &jobs.ForEachTask{ + Task: jobs.Task{ + Libraries: []compute.Library{ + { + Whl: filepath.Join("whl", "*.whl"), + }, + { + Whl: "/Volumes/some/path/mywheel.whl", + }, + }, + }, + }, + }, + }, + Environments: []jobs.JobEnvironment{ + { + Spec: &compute.Environment{ + Dependencies: []string{ + filepath.Join("whl", "source.whl"), + "/Volumes/some/path/mywheel.whl", + }, + }, + }, + }, + }, + }, + }, + }, + }, + } + + mockFiler := mockfiler.NewMockFiler(t) + mockFiler.EXPECT().Write( + mock.Anything, + filepath.Join("source.whl"), + mock.AnythingOfType("*os.File"), + filer.OverwriteIfExists, + filer.CreateParentDirectories, + ).Return(nil) + + diags := bundle.Apply(context.Background(), b, bundle.Seq(ExpandGlobReferences(), UploadWithClient(mockFiler))) + require.NoError(t, diags.Error()) + + // Test that libraries path is updated + require.Equal(t, "/Volumes/foo/bar/artifacts/.internal/source.whl", b.Config.Resources.Jobs["job"].JobSettings.Tasks[0].Libraries[0].Whl) + require.Equal(t, "/Volumes/some/path/mywheel.whl", b.Config.Resources.Jobs["job"].JobSettings.Tasks[0].Libraries[1].Whl) + require.Equal(t, "/Volumes/foo/bar/artifacts/.internal/source.whl", b.Config.Resources.Jobs["job"].JobSettings.Environments[0].Spec.Dependencies[0]) + require.Equal(t, "/Volumes/some/path/mywheel.whl", b.Config.Resources.Jobs["job"].JobSettings.Environments[0].Spec.Dependencies[1]) + require.Equal(t, "/Volumes/foo/bar/artifacts/.internal/source.whl", b.Config.Resources.Jobs["job"].JobSettings.Tasks[1].ForEachTask.Task.Libraries[0].Whl) + require.Equal(t, "/Volumes/some/path/mywheel.whl", b.Config.Resources.Jobs["job"].JobSettings.Tasks[1].ForEachTask.Task.Libraries[1].Whl) +} + +func TestArtifactUploadWithNoLibraryReference(t *testing.T) { + tmpDir := t.TempDir() + whlFolder := filepath.Join(tmpDir, "whl") + testutil.Touch(t, whlFolder, "source.whl") + whlLocalPath := filepath.Join(whlFolder, "source.whl") + + b := &bundle.Bundle{ + RootPath: tmpDir, + Config: config.Root{ + Workspace: config.Workspace{ + ArtifactPath: "/Workspace/foo/bar/artifacts", + }, + Artifacts: config.Artifacts{ + "whl": { + Type: config.ArtifactPythonWheel, + Files: []config.ArtifactFile{ + {Source: whlLocalPath}, + }, + }, + }, + }, + } + + mockFiler := mockfiler.NewMockFiler(t) + mockFiler.EXPECT().Write( + mock.Anything, + filepath.Join("source.whl"), + mock.AnythingOfType("*os.File"), + filer.OverwriteIfExists, + filer.CreateParentDirectories, + ).Return(nil) + + diags := bundle.Apply(context.Background(), b, bundle.Seq(ExpandGlobReferences(), UploadWithClient(mockFiler))) + require.NoError(t, diags.Error()) + + require.Equal(t, "/Workspace/foo/bar/artifacts/.internal/source.whl", b.Config.Artifacts["whl"].Files[0].RemotePath) +} + +func TestUploadMultipleLibraries(t *testing.T) { + tmpDir := t.TempDir() + whlFolder := filepath.Join(tmpDir, "whl") + testutil.Touch(t, whlFolder, "source1.whl") + testutil.Touch(t, whlFolder, "source2.whl") + testutil.Touch(t, whlFolder, "source3.whl") + testutil.Touch(t, whlFolder, "source4.whl") + + b := &bundle.Bundle{ + RootPath: tmpDir, + Config: config.Root{ + Workspace: config.Workspace{ + ArtifactPath: "/foo/bar/artifacts", + }, + Resources: config.Resources{ + Jobs: map[string]*resources.Job{ + "job": { + JobSettings: &jobs.JobSettings{ + Tasks: []jobs.Task{ + { + Libraries: []compute.Library{ + { + Whl: filepath.Join("whl", "*.whl"), + }, + { + Whl: "/Workspace/Users/foo@bar.com/mywheel.whl", + }, + }, + }, + }, + Environments: []jobs.JobEnvironment{ + { + Spec: &compute.Environment{ + Dependencies: []string{ + filepath.Join("whl", "*.whl"), + "/Workspace/Users/foo@bar.com/mywheel.whl", + }, + }, + }, + }, + }, + }, + }, + }, + }, + } + + mockFiler := mockfiler.NewMockFiler(t) + mockFiler.EXPECT().Write( + mock.Anything, + filepath.Join("source1.whl"), + mock.AnythingOfType("*os.File"), + filer.OverwriteIfExists, + filer.CreateParentDirectories, + ).Return(nil).Once() + + mockFiler.EXPECT().Write( + mock.Anything, + filepath.Join("source2.whl"), + mock.AnythingOfType("*os.File"), + filer.OverwriteIfExists, + filer.CreateParentDirectories, + ).Return(nil).Once() + + mockFiler.EXPECT().Write( + mock.Anything, + filepath.Join("source3.whl"), + mock.AnythingOfType("*os.File"), + filer.OverwriteIfExists, + filer.CreateParentDirectories, + ).Return(nil).Once() + + mockFiler.EXPECT().Write( + mock.Anything, + filepath.Join("source4.whl"), + mock.AnythingOfType("*os.File"), + filer.OverwriteIfExists, + filer.CreateParentDirectories, + ).Return(nil).Once() + + diags := bundle.Apply(context.Background(), b, bundle.Seq(ExpandGlobReferences(), UploadWithClient(mockFiler))) + require.NoError(t, diags.Error()) + + // Test that libraries path is updated + require.Len(t, b.Config.Resources.Jobs["job"].JobSettings.Tasks[0].Libraries, 5) + require.Contains(t, b.Config.Resources.Jobs["job"].JobSettings.Tasks[0].Libraries, compute.Library{Whl: "/Workspace/foo/bar/artifacts/.internal/source1.whl"}) + require.Contains(t, b.Config.Resources.Jobs["job"].JobSettings.Tasks[0].Libraries, compute.Library{Whl: "/Workspace/foo/bar/artifacts/.internal/source2.whl"}) + require.Contains(t, b.Config.Resources.Jobs["job"].JobSettings.Tasks[0].Libraries, compute.Library{Whl: "/Workspace/foo/bar/artifacts/.internal/source3.whl"}) + require.Contains(t, b.Config.Resources.Jobs["job"].JobSettings.Tasks[0].Libraries, compute.Library{Whl: "/Workspace/foo/bar/artifacts/.internal/source4.whl"}) + require.Contains(t, b.Config.Resources.Jobs["job"].JobSettings.Tasks[0].Libraries, compute.Library{Whl: "/Workspace/Users/foo@bar.com/mywheel.whl"}) + + require.Len(t, b.Config.Resources.Jobs["job"].JobSettings.Environments[0].Spec.Dependencies, 5) + require.Contains(t, b.Config.Resources.Jobs["job"].JobSettings.Environments[0].Spec.Dependencies, "/Workspace/foo/bar/artifacts/.internal/source1.whl") + require.Contains(t, b.Config.Resources.Jobs["job"].JobSettings.Environments[0].Spec.Dependencies, "/Workspace/foo/bar/artifacts/.internal/source2.whl") + require.Contains(t, b.Config.Resources.Jobs["job"].JobSettings.Environments[0].Spec.Dependencies, "/Workspace/foo/bar/artifacts/.internal/source3.whl") + require.Contains(t, b.Config.Resources.Jobs["job"].JobSettings.Environments[0].Spec.Dependencies, "/Workspace/foo/bar/artifacts/.internal/source4.whl") + require.Contains(t, b.Config.Resources.Jobs["job"].JobSettings.Environments[0].Spec.Dependencies, "/Workspace/Users/foo@bar.com/mywheel.whl") +} diff --git a/bundle/phases/build.go b/bundle/phases/build.go index 362d23be14..3ddc6b1819 100644 --- a/bundle/phases/build.go +++ b/bundle/phases/build.go @@ -16,6 +16,7 @@ func Build() bundle.Mutator { scripts.Execute(config.ScriptPreBuild), artifacts.DetectPackages(), artifacts.InferMissingProperties(), + artifacts.PrepareAll(), artifacts.BuildAll(), scripts.Execute(config.ScriptPostBuild), mutator.ResolveVariableReferences( diff --git a/bundle/phases/deploy.go b/bundle/phases/deploy.go index 46c3891895..ca967c321a 100644 --- a/bundle/phases/deploy.go +++ b/bundle/phases/deploy.go @@ -1,6 +1,9 @@ package phases import ( + "context" + "fmt" + "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/artifacts" "github.com/databricks/cli/bundle/config" @@ -14,10 +17,94 @@ import ( "github.com/databricks/cli/bundle/permissions" "github.com/databricks/cli/bundle/python" "github.com/databricks/cli/bundle/scripts" + "github.com/databricks/cli/libs/cmdio" + terraformlib "github.com/databricks/cli/libs/terraform" ) +func approvalForUcSchemaDelete(ctx context.Context, b *bundle.Bundle) (bool, error) { + tf := b.Terraform + if tf == nil { + return false, fmt.Errorf("terraform not initialized") + } + + // read plan file + plan, err := tf.ShowPlanFile(ctx, b.Plan.Path) + if err != nil { + return false, err + } + + actions := make([]terraformlib.Action, 0) + for _, rc := range plan.ResourceChanges { + // We only care about destructive actions on UC schema resources. + if rc.Type != "databricks_schema" { + continue + } + + var actionType terraformlib.ActionType + + switch { + case rc.Change.Actions.Delete(): + actionType = terraformlib.ActionTypeDelete + case rc.Change.Actions.Replace(): + actionType = terraformlib.ActionTypeRecreate + default: + // We don't need a prompt for non-destructive actions like creating + // or updating a schema. + continue + } + + actions = append(actions, terraformlib.Action{ + Action: actionType, + ResourceType: rc.Type, + ResourceName: rc.Name, + }) + } + + // No restricted actions planned. No need for approval. + if len(actions) == 0 { + return true, nil + } + + cmdio.LogString(ctx, "The following UC schemas will be deleted or recreated. Any underlying data may be lost:") + for _, action := range actions { + cmdio.Log(ctx, action) + } + + if b.AutoApprove { + return true, nil + } + + if !cmdio.IsPromptSupported(ctx) { + return false, fmt.Errorf("the deployment requires destructive actions, but current console does not support prompting. Please specify --auto-approve if you would like to skip prompts and proceed") + } + + cmdio.LogString(ctx, "") + approved, err := cmdio.AskYesOrNo(ctx, "Would you like to proceed?") + if err != nil { + return false, err + } + + return approved, nil +} + // The deploy phase deploys artifacts and resources. func Deploy() bundle.Mutator { + // Core mutators that CRUD resources and modify deployment state. These + // mutators need informed consent if they are potentially destructive. + deployCore := bundle.Defer( + bundle.Seq( + bundle.LogString("Deploying resources..."), + terraform.Apply(), + ), + bundle.Seq( + terraform.StatePush(), + terraform.Load(), + metadata.Compute(), + metadata.Upload(), + bundle.LogString("Deployment complete!"), + ), + ) + deployMutator := bundle.Seq( scripts.Execute(config.ScriptPreDeploy), lock.Acquire(), @@ -26,9 +113,9 @@ func Deploy() bundle.Mutator { terraform.StatePull(), deploy.StatePull(), mutator.ValidateGitDetails(), - libraries.ValidateLocalLibrariesExist(), artifacts.CleanUp(), - artifacts.UploadAll(), + libraries.ExpandGlobReferences(), + libraries.Upload(), python.TransformWheelTask(), files.Upload(), deploy.StateUpdate(), @@ -37,20 +124,16 @@ func Deploy() bundle.Mutator { terraform.Interpolate(), terraform.Write(), terraform.CheckRunningResource(), - bundle.Defer( - terraform.Apply(), - bundle.Seq( - terraform.StatePush(), - terraform.Load(), - metadata.Compute(), - metadata.Upload(), - ), + terraform.Plan(terraform.PlanGoal("deploy")), + bundle.If( + approvalForUcSchemaDelete, + deployCore, + bundle.LogString("Deployment cancelled!"), ), ), lock.Release(lock.GoalDeploy), ), scripts.Execute(config.ScriptPostDeploy), - bundle.LogString("Deployment complete!"), ) return newPhase( diff --git a/bundle/phases/destroy.go b/bundle/phases/destroy.go index f1beace848..6eb8b6a01f 100644 --- a/bundle/phases/destroy.go +++ b/bundle/phases/destroy.go @@ -3,13 +3,18 @@ package phases import ( "context" "errors" + "fmt" "net/http" "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/deploy/files" "github.com/databricks/cli/bundle/deploy/lock" "github.com/databricks/cli/bundle/deploy/terraform" + + "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/log" + terraformlib "github.com/databricks/cli/libs/terraform" "github.com/databricks/databricks-sdk-go/apierr" ) @@ -26,8 +31,62 @@ func assertRootPathExists(ctx context.Context, b *bundle.Bundle) (bool, error) { return true, err } +func approvalForDestroy(ctx context.Context, b *bundle.Bundle) (bool, error) { + tf := b.Terraform + if tf == nil { + return false, fmt.Errorf("terraform not initialized") + } + + // read plan file + plan, err := tf.ShowPlanFile(ctx, b.Plan.Path) + if err != nil { + return false, err + } + + deleteActions := make([]terraformlib.Action, 0) + for _, rc := range plan.ResourceChanges { + if rc.Change.Actions.Delete() { + deleteActions = append(deleteActions, terraformlib.Action{ + Action: terraformlib.ActionTypeDelete, + ResourceType: rc.Type, + ResourceName: rc.Name, + }) + } + } + + if len(deleteActions) > 0 { + cmdio.LogString(ctx, "The following resources will be deleted:") + for _, a := range deleteActions { + cmdio.Log(ctx, a) + } + cmdio.LogString(ctx, "") + + } + + cmdio.LogString(ctx, fmt.Sprintf("All files and directories at the following location will be deleted: %s", b.Config.Workspace.RootPath)) + cmdio.LogString(ctx, "") + + if b.AutoApprove { + return true, nil + } + + approved, err := cmdio.AskYesOrNo(ctx, "Would you like to proceed?") + if err != nil { + return false, err + } + + return approved, nil +} + // The destroy phase deletes artifacts and resources. func Destroy() bundle.Mutator { + // Core destructive mutators for destroy. These require informed user consent. + destroyCore := bundle.Seq( + terraform.Apply(), + files.Delete(), + bundle.LogString("Destroy complete!"), + ) + destroyMutator := bundle.Seq( lock.Acquire(), bundle.Defer( @@ -36,13 +95,14 @@ func Destroy() bundle.Mutator { terraform.Interpolate(), terraform.Write(), terraform.Plan(terraform.PlanGoal("destroy")), - terraform.Destroy(), - terraform.StatePush(), - files.Delete(), + bundle.If( + approvalForDestroy, + destroyCore, + bundle.LogString("Destroy cancelled!"), + ), ), lock.Release(lock.GoalDestroy), ), - bundle.LogString("Destroy complete!"), ) return newPhase( diff --git a/bundle/phases/initialize.go b/bundle/phases/initialize.go index 1f4af2a260..7a1081ded6 100644 --- a/bundle/phases/initialize.go +++ b/bundle/phases/initialize.go @@ -5,6 +5,7 @@ import ( "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/mutator" pythonmutator "github.com/databricks/cli/bundle/config/mutator/python" + "github.com/databricks/cli/bundle/config/validate" "github.com/databricks/cli/bundle/deploy/metadata" "github.com/databricks/cli/bundle/deploy/terraform" "github.com/databricks/cli/bundle/permissions" @@ -19,8 +20,10 @@ func Initialize() bundle.Mutator { return newPhase( "initialize", []bundle.Mutator{ + validate.AllResourcesHaveValues(), mutator.RewriteSyncPaths(), mutator.MergeJobClusters(), + mutator.MergeJobParameters(), mutator.MergeJobTasks(), mutator.MergePipelineClusters(), mutator.InitializeWorkspaceClient(), diff --git a/bundle/python/transform_test.go b/bundle/python/transform_test.go index c15feb4241..c7bddca149 100644 --- a/bundle/python/transform_test.go +++ b/bundle/python/transform_test.go @@ -7,7 +7,6 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" - "github.com/databricks/cli/bundle/config/paths" "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/databricks-sdk-go/service/compute" "github.com/databricks/databricks-sdk-go/service/jobs" @@ -124,9 +123,6 @@ func TestNoPanicWithNoPythonWheelTasks(t *testing.T) { Resources: config.Resources{ Jobs: map[string]*resources.Job{ "test": { - Paths: paths.Paths{ - ConfigFilePath: tmpDir, - }, JobSettings: &jobs.JobSettings{ Tasks: []jobs.Task{ { diff --git a/bundle/python/warning.go b/bundle/python/warning.go index 3da88b0d79..d53796d734 100644 --- a/bundle/python/warning.go +++ b/bundle/python/warning.go @@ -35,7 +35,7 @@ func isPythonWheelWrapperOn(b *bundle.Bundle) bool { } func hasIncompatibleWheelTasks(ctx context.Context, b *bundle.Bundle) bool { - tasks := libraries.FindAllWheelTasksWithLocalLibraries(b) + tasks := libraries.FindTasksWithLocalLibraries(b) for _, task := range tasks { if task.NewCluster != nil { if lowerThanExpectedVersion(ctx, task.NewCluster.SparkVersion) { diff --git a/bundle/render/render_text_output.go b/bundle/render/render_text_output.go index 439ae61323..ea0b9a944f 100644 --- a/bundle/render/render_text_output.go +++ b/bundle/render/render_text_output.go @@ -29,11 +29,11 @@ var renderFuncMap = template.FuncMap{ } const errorTemplate = `{{ "Error" | red }}: {{ .Summary }} -{{- if .Path.String }} - {{ "at " }}{{ .Path.String | green }} +{{- range $index, $element := .Paths }} + {{ if eq $index 0 }}at {{else}} {{ end}}{{ $element.String | green }} {{- end }} -{{- if .Location.File }} - {{ "in " }}{{ .Location.String | cyan }} +{{- range $index, $element := .Locations }} + {{ if eq $index 0 }}in {{else}} {{ end}}{{ $element.String | cyan }} {{- end }} {{- if .Detail }} @@ -43,11 +43,11 @@ const errorTemplate = `{{ "Error" | red }}: {{ .Summary }} ` const warningTemplate = `{{ "Warning" | yellow }}: {{ .Summary }} -{{- if .Path.String }} - {{ "at " }}{{ .Path.String | green }} +{{- range $index, $element := .Paths }} + {{ if eq $index 0 }}at {{else}} {{ end}}{{ $element.String | green }} {{- end }} -{{- if .Location.File }} - {{ "in " }}{{ .Location.String | cyan }} +{{- range $index, $element := .Locations }} + {{ if eq $index 0 }}in {{else}} {{ end}}{{ $element.String | cyan }} {{- end }} {{- if .Detail }} @@ -141,12 +141,18 @@ func renderDiagnostics(out io.Writer, b *bundle.Bundle, diags diag.Diagnostics) t = warningT } - // Make file relative to bundle root - if d.Location.File != "" && b != nil { - out, err := filepath.Rel(b.RootPath, d.Location.File) - // if we can't relativize the path, just use path as-is - if err == nil { - d.Location.File = out + for i := range d.Locations { + if b == nil { + break + } + + // Make location relative to bundle root + if d.Locations[i].File != "" { + out, err := filepath.Rel(b.RootPath, d.Locations[i].File) + // if we can't relativize the path, just use path as-is + if err == nil { + d.Locations[i].File = out + } } } diff --git a/bundle/render/render_text_output_test.go b/bundle/render/render_text_output_test.go index b7aec88648..976f86e79c 100644 --- a/bundle/render/render_text_output_test.go +++ b/bundle/render/render_text_output_test.go @@ -88,34 +88,22 @@ func TestRenderTextOutput(t *testing.T) { bundle: loadingBundle, diags: diag.Diagnostics{ diag.Diagnostic{ - Severity: diag.Error, - Summary: "error (1)", - Detail: "detail (1)", - Location: dyn.Location{ - File: "foo.py", - Line: 1, - Column: 1, - }, + Severity: diag.Error, + Summary: "error (1)", + Detail: "detail (1)", + Locations: []dyn.Location{{File: "foo.py", Line: 1, Column: 1}}, }, diag.Diagnostic{ - Severity: diag.Error, - Summary: "error (2)", - Detail: "detail (2)", - Location: dyn.Location{ - File: "foo.py", - Line: 2, - Column: 1, - }, + Severity: diag.Error, + Summary: "error (2)", + Detail: "detail (2)", + Locations: []dyn.Location{{File: "foo.py", Line: 2, Column: 1}}, }, diag.Diagnostic{ - Severity: diag.Warning, - Summary: "warning (3)", - Detail: "detail (3)", - Location: dyn.Location{ - File: "foo.py", - Line: 3, - Column: 1, - }, + Severity: diag.Warning, + Summary: "warning (3)", + Detail: "detail (3)", + Locations: []dyn.Location{{File: "foo.py", Line: 3, Column: 1}}, }, }, opts: RenderOptions{RenderSummaryTable: true}, @@ -174,24 +162,16 @@ func TestRenderTextOutput(t *testing.T) { bundle: nil, diags: diag.Diagnostics{ diag.Diagnostic{ - Severity: diag.Error, - Summary: "error (1)", - Detail: "detail (1)", - Location: dyn.Location{ - File: "foo.py", - Line: 1, - Column: 1, - }, + Severity: diag.Error, + Summary: "error (1)", + Detail: "detail (1)", + Locations: []dyn.Location{{File: "foo.py", Line: 1, Column: 1}}, }, diag.Diagnostic{ - Severity: diag.Warning, - Summary: "warning (2)", - Detail: "detail (2)", - Location: dyn.Location{ - File: "foo.py", - Line: 3, - Column: 1, - }, + Severity: diag.Warning, + Summary: "warning (2)", + Detail: "detail (2)", + Locations: []dyn.Location{{File: "foo.py", Line: 3, Column: 1}}, }, }, opts: RenderOptions{RenderSummaryTable: false}, @@ -252,17 +232,42 @@ func TestRenderDiagnostics(t *testing.T) { Severity: diag.Error, Summary: "failed to load xxx", Detail: "'name' is required", - Location: dyn.Location{ + Locations: []dyn.Location{{ File: "foo.yaml", Line: 1, - Column: 2, - }, + Column: 2}}, }, }, expected: "Error: failed to load xxx\n" + " in foo.yaml:1:2\n\n" + "'name' is required\n\n", }, + { + name: "error with multiple source locations", + diags: diag.Diagnostics{ + { + Severity: diag.Error, + Summary: "failed to load xxx", + Detail: "'name' is required", + Locations: []dyn.Location{ + { + File: "foo.yaml", + Line: 1, + Column: 2, + }, + { + File: "bar.yaml", + Line: 3, + Column: 4, + }, + }, + }, + }, + expected: "Error: failed to load xxx\n" + + " in foo.yaml:1:2\n" + + " bar.yaml:3:4\n\n" + + "'name' is required\n\n", + }, { name: "error with path", diags: diag.Diagnostics{ @@ -270,11 +275,32 @@ func TestRenderDiagnostics(t *testing.T) { Severity: diag.Error, Detail: "'name' is required", Summary: "failed to load xxx", - Path: dyn.MustPathFromString("resources.jobs.xxx"), + Paths: []dyn.Path{dyn.MustPathFromString("resources.jobs.xxx")}, + }, + }, + expected: "Error: failed to load xxx\n" + + " at resources.jobs.xxx\n" + + "\n" + + "'name' is required\n\n", + }, + { + name: "error with multiple paths", + diags: diag.Diagnostics{ + { + Severity: diag.Error, + Detail: "'name' is required", + Summary: "failed to load xxx", + Paths: []dyn.Path{ + dyn.MustPathFromString("resources.jobs.xxx"), + dyn.MustPathFromString("resources.jobs.yyy"), + dyn.MustPathFromString("resources.jobs.zzz"), + }, }, }, expected: "Error: failed to load xxx\n" + " at resources.jobs.xxx\n" + + " resources.jobs.yyy\n" + + " resources.jobs.zzz\n" + "\n" + "'name' is required\n\n", }, diff --git a/bundle/tests/conflicting_resource_ids_test.go b/bundle/tests/conflicting_resource_ids_test.go deleted file mode 100644 index e7f0aa28f2..0000000000 --- a/bundle/tests/conflicting_resource_ids_test.go +++ /dev/null @@ -1,42 +0,0 @@ -package config_tests - -import ( - "context" - "fmt" - "path/filepath" - "testing" - - "github.com/databricks/cli/bundle" - "github.com/databricks/cli/bundle/phases" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestConflictingResourceIdsNoSubconfig(t *testing.T) { - ctx := context.Background() - b, err := bundle.Load(ctx, "./conflicting_resource_ids/no_subconfigurations") - require.NoError(t, err) - diags := bundle.Apply(ctx, b, phases.Load()) - bundleConfigPath := filepath.FromSlash("conflicting_resource_ids/no_subconfigurations/databricks.yml") - assert.ErrorContains(t, diags.Error(), fmt.Sprintf("multiple resources named foo (job at %s, pipeline at %s)", bundleConfigPath, bundleConfigPath)) -} - -func TestConflictingResourceIdsOneSubconfig(t *testing.T) { - ctx := context.Background() - b, err := bundle.Load(ctx, "./conflicting_resource_ids/one_subconfiguration") - require.NoError(t, err) - diags := bundle.Apply(ctx, b, phases.Load()) - bundleConfigPath := filepath.FromSlash("conflicting_resource_ids/one_subconfiguration/databricks.yml") - resourcesConfigPath := filepath.FromSlash("conflicting_resource_ids/one_subconfiguration/resources.yml") - assert.ErrorContains(t, diags.Error(), fmt.Sprintf("multiple resources named foo (job at %s, pipeline at %s)", bundleConfigPath, resourcesConfigPath)) -} - -func TestConflictingResourceIdsTwoSubconfigs(t *testing.T) { - ctx := context.Background() - b, err := bundle.Load(ctx, "./conflicting_resource_ids/two_subconfigurations") - require.NoError(t, err) - diags := bundle.Apply(ctx, b, phases.Load()) - resources1ConfigPath := filepath.FromSlash("conflicting_resource_ids/two_subconfigurations/resources1.yml") - resources2ConfigPath := filepath.FromSlash("conflicting_resource_ids/two_subconfigurations/resources2.yml") - assert.ErrorContains(t, diags.Error(), fmt.Sprintf("multiple resources named foo (job at %s, pipeline at %s)", resources1ConfigPath, resources2ConfigPath)) -} diff --git a/bundle/tests/enviroment_key_test.go b/bundle/tests/enviroment_key_test.go index aed3964db5..135ef19177 100644 --- a/bundle/tests/enviroment_key_test.go +++ b/bundle/tests/enviroment_key_test.go @@ -18,6 +18,6 @@ func TestEnvironmentKeyProvidedAndNoPanic(t *testing.T) { b, diags := loadTargetWithDiags("./environment_key_only", "default") require.Empty(t, diags) - diags = bundle.Apply(context.Background(), b, libraries.ValidateLocalLibrariesExist()) + diags = bundle.Apply(context.Background(), b, libraries.ExpandGlobReferences()) require.Empty(t, diags) } diff --git a/bundle/tests/environments_job_and_pipeline_test.go b/bundle/tests/environments_job_and_pipeline_test.go index a18daf90c8..0abeb487c6 100644 --- a/bundle/tests/environments_job_and_pipeline_test.go +++ b/bundle/tests/environments_job_and_pipeline_test.go @@ -15,7 +15,8 @@ func TestJobAndPipelineDevelopmentWithEnvironment(t *testing.T) { assert.Len(t, b.Config.Resources.Pipelines, 1) p := b.Config.Resources.Pipelines["nyc_taxi_pipeline"] - assert.Equal(t, "environments_job_and_pipeline/databricks.yml", filepath.ToSlash(p.ConfigFilePath)) + l := b.Config.GetLocation("resources.pipelines.nyc_taxi_pipeline") + assert.Equal(t, "environments_job_and_pipeline/databricks.yml", filepath.ToSlash(l.File)) assert.Equal(t, b.Config.Bundle.Mode, config.Development) assert.True(t, p.Development) require.Len(t, p.Libraries, 1) @@ -29,7 +30,8 @@ func TestJobAndPipelineStagingWithEnvironment(t *testing.T) { assert.Len(t, b.Config.Resources.Pipelines, 1) p := b.Config.Resources.Pipelines["nyc_taxi_pipeline"] - assert.Equal(t, "environments_job_and_pipeline/databricks.yml", filepath.ToSlash(p.ConfigFilePath)) + l := b.Config.GetLocation("resources.pipelines.nyc_taxi_pipeline") + assert.Equal(t, "environments_job_and_pipeline/databricks.yml", filepath.ToSlash(l.File)) assert.False(t, p.Development) require.Len(t, p.Libraries, 1) assert.Equal(t, "./dlt/nyc_taxi_loader", p.Libraries[0].Notebook.Path) @@ -42,14 +44,16 @@ func TestJobAndPipelineProductionWithEnvironment(t *testing.T) { assert.Len(t, b.Config.Resources.Pipelines, 1) p := b.Config.Resources.Pipelines["nyc_taxi_pipeline"] - assert.Equal(t, "environments_job_and_pipeline/databricks.yml", filepath.ToSlash(p.ConfigFilePath)) + pl := b.Config.GetLocation("resources.pipelines.nyc_taxi_pipeline") + assert.Equal(t, "environments_job_and_pipeline/databricks.yml", filepath.ToSlash(pl.File)) assert.False(t, p.Development) require.Len(t, p.Libraries, 1) assert.Equal(t, "./dlt/nyc_taxi_loader", p.Libraries[0].Notebook.Path) assert.Equal(t, "nyc_taxi_production", p.Target) j := b.Config.Resources.Jobs["pipeline_schedule"] - assert.Equal(t, "environments_job_and_pipeline/databricks.yml", filepath.ToSlash(j.ConfigFilePath)) + jl := b.Config.GetLocation("resources.jobs.pipeline_schedule") + assert.Equal(t, "environments_job_and_pipeline/databricks.yml", filepath.ToSlash(jl.File)) assert.Equal(t, "Daily refresh of production pipeline", j.Name) require.Len(t, j.Tasks, 1) assert.NotEmpty(t, j.Tasks[0].PipelineTask.PipelineId) diff --git a/bundle/tests/include_test.go b/bundle/tests/include_test.go index 5b0235f605..15f8fcec18 100644 --- a/bundle/tests/include_test.go +++ b/bundle/tests/include_test.go @@ -31,7 +31,8 @@ func TestIncludeWithGlob(t *testing.T) { job := b.Config.Resources.Jobs["my_job"] assert.Equal(t, "1", job.ID) - assert.Equal(t, "include_with_glob/job.yml", filepath.ToSlash(job.ConfigFilePath)) + l := b.Config.GetLocation("resources.jobs.my_job") + assert.Equal(t, "include_with_glob/job.yml", filepath.ToSlash(l.File)) } func TestIncludeDefault(t *testing.T) { @@ -51,9 +52,11 @@ func TestIncludeForMultipleMatches(t *testing.T) { first := b.Config.Resources.Jobs["my_first_job"] assert.Equal(t, "1", first.ID) - assert.Equal(t, "include_multiple/my_first_job/resource.yml", filepath.ToSlash(first.ConfigFilePath)) + fl := b.Config.GetLocation("resources.jobs.my_first_job") + assert.Equal(t, "include_multiple/my_first_job/resource.yml", filepath.ToSlash(fl.File)) second := b.Config.Resources.Jobs["my_second_job"] assert.Equal(t, "2", second.ID) - assert.Equal(t, "include_multiple/my_second_job/resource.yml", filepath.ToSlash(second.ConfigFilePath)) + sl := b.Config.GetLocation("resources.jobs.my_second_job") + assert.Equal(t, "include_multiple/my_second_job/resource.yml", filepath.ToSlash(sl.File)) } diff --git a/bundle/tests/job_and_pipeline_test.go b/bundle/tests/job_and_pipeline_test.go index 5e8febc333..65aa5bdc49 100644 --- a/bundle/tests/job_and_pipeline_test.go +++ b/bundle/tests/job_and_pipeline_test.go @@ -1,7 +1,6 @@ package config_tests import ( - "path/filepath" "testing" "github.com/databricks/cli/bundle/config" @@ -15,7 +14,6 @@ func TestJobAndPipelineDevelopment(t *testing.T) { assert.Len(t, b.Config.Resources.Pipelines, 1) p := b.Config.Resources.Pipelines["nyc_taxi_pipeline"] - assert.Equal(t, "job_and_pipeline/databricks.yml", filepath.ToSlash(p.ConfigFilePath)) assert.Equal(t, b.Config.Bundle.Mode, config.Development) assert.True(t, p.Development) require.Len(t, p.Libraries, 1) @@ -29,7 +27,6 @@ func TestJobAndPipelineStaging(t *testing.T) { assert.Len(t, b.Config.Resources.Pipelines, 1) p := b.Config.Resources.Pipelines["nyc_taxi_pipeline"] - assert.Equal(t, "job_and_pipeline/databricks.yml", filepath.ToSlash(p.ConfigFilePath)) assert.False(t, p.Development) require.Len(t, p.Libraries, 1) assert.Equal(t, "./dlt/nyc_taxi_loader", p.Libraries[0].Notebook.Path) @@ -42,14 +39,12 @@ func TestJobAndPipelineProduction(t *testing.T) { assert.Len(t, b.Config.Resources.Pipelines, 1) p := b.Config.Resources.Pipelines["nyc_taxi_pipeline"] - assert.Equal(t, "job_and_pipeline/databricks.yml", filepath.ToSlash(p.ConfigFilePath)) assert.False(t, p.Development) require.Len(t, p.Libraries, 1) assert.Equal(t, "./dlt/nyc_taxi_loader", p.Libraries[0].Notebook.Path) assert.Equal(t, "nyc_taxi_production", p.Target) j := b.Config.Resources.Jobs["pipeline_schedule"] - assert.Equal(t, "job_and_pipeline/databricks.yml", filepath.ToSlash(j.ConfigFilePath)) assert.Equal(t, "Daily refresh of production pipeline", j.Name) require.Len(t, j.Tasks, 1) assert.NotEmpty(t, j.Tasks[0].PipelineTask.PipelineId) diff --git a/bundle/tests/loader.go b/bundle/tests/loader.go index 8eddcf9a11..069f09358c 100644 --- a/bundle/tests/loader.go +++ b/bundle/tests/loader.go @@ -37,6 +37,7 @@ func loadTargetWithDiags(path, env string) (*bundle.Bundle, diag.Diagnostics) { phases.LoadNamedTarget(env), mutator.RewriteSyncPaths(), mutator.MergeJobClusters(), + mutator.MergeJobParameters(), mutator.MergeJobTasks(), mutator.MergePipelineClusters(), )) diff --git a/bundle/tests/model_serving_endpoint_test.go b/bundle/tests/model_serving_endpoint_test.go index bfa1a31b41..b8b8008639 100644 --- a/bundle/tests/model_serving_endpoint_test.go +++ b/bundle/tests/model_serving_endpoint_test.go @@ -1,7 +1,6 @@ package config_tests import ( - "path/filepath" "testing" "github.com/databricks/cli/bundle/config" @@ -10,7 +9,6 @@ import ( ) func assertExpected(t *testing.T, p *resources.ModelServingEndpoint) { - assert.Equal(t, "model_serving_endpoint/databricks.yml", filepath.ToSlash(p.ConfigFilePath)) assert.Equal(t, "model-name", p.Config.ServedModels[0].ModelName) assert.Equal(t, "1", p.Config.ServedModels[0].ModelVersion) assert.Equal(t, "model-name-1", p.Config.TrafficConfig.Routes[0].ServedModelName) diff --git a/bundle/tests/override_job_parameters/databricks.yml b/bundle/tests/override_job_parameters/databricks.yml new file mode 100644 index 0000000000..9c333c3234 --- /dev/null +++ b/bundle/tests/override_job_parameters/databricks.yml @@ -0,0 +1,32 @@ +bundle: + name: override_job_parameters + +workspace: + host: https://acme.cloud.databricks.com/ + +resources: + jobs: + foo: + name: job + parameters: + - name: foo + default: v1 + - name: bar + default: v1 + +targets: + development: + resources: + jobs: + foo: + parameters: + - name: foo + default: v2 + + staging: + resources: + jobs: + foo: + parameters: + - name: bar + default: v2 diff --git a/bundle/tests/override_job_parameters_test.go b/bundle/tests/override_job_parameters_test.go new file mode 100644 index 0000000000..21e0e35a67 --- /dev/null +++ b/bundle/tests/override_job_parameters_test.go @@ -0,0 +1,31 @@ +package config_tests + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestOverrideJobParametersDev(t *testing.T) { + b := loadTarget(t, "./override_job_parameters", "development") + assert.Equal(t, "job", b.Config.Resources.Jobs["foo"].Name) + + p := b.Config.Resources.Jobs["foo"].Parameters + assert.Len(t, p, 2) + assert.Equal(t, "foo", p[0].Name) + assert.Equal(t, "v2", p[0].Default) + assert.Equal(t, "bar", p[1].Name) + assert.Equal(t, "v1", p[1].Default) +} + +func TestOverrideJobParametersStaging(t *testing.T) { + b := loadTarget(t, "./override_job_parameters", "staging") + assert.Equal(t, "job", b.Config.Resources.Jobs["foo"].Name) + + p := b.Config.Resources.Jobs["foo"].Parameters + assert.Len(t, p, 2) + assert.Equal(t, "foo", p[0].Name) + assert.Equal(t, "v1", p[0].Default) + assert.Equal(t, "bar", p[1].Name) + assert.Equal(t, "v2", p[1].Default) +} diff --git a/bundle/tests/python_wheel/python_wheel_multiple/.gitignore b/bundle/tests/python_wheel/python_wheel_multiple/.gitignore new file mode 100644 index 0000000000..f03e23bc26 --- /dev/null +++ b/bundle/tests/python_wheel/python_wheel_multiple/.gitignore @@ -0,0 +1,3 @@ +build/ +*.egg-info +.databricks diff --git a/bundle/tests/python_wheel/python_wheel_multiple/bundle.yml b/bundle/tests/python_wheel/python_wheel_multiple/bundle.yml new file mode 100644 index 0000000000..6964c58a4f --- /dev/null +++ b/bundle/tests/python_wheel/python_wheel_multiple/bundle.yml @@ -0,0 +1,25 @@ +bundle: + name: python-wheel + +artifacts: + my_test_code: + type: whl + path: "./my_test_code" + build: "python3 setup.py bdist_wheel" + my_test_code_2: + type: whl + path: "./my_test_code" + build: "python3 setup2.py bdist_wheel" + +resources: + jobs: + test_job: + name: "[${bundle.environment}] My Wheel Job" + tasks: + - task_key: TestTask + existing_cluster_id: "0717-132531-5opeqon1" + python_wheel_task: + package_name: "my_test_code" + entry_point: "run" + libraries: + - whl: ./my_test_code/dist/*.whl diff --git a/bundle/tests/python_wheel/python_wheel_multiple/my_test_code/setup.py b/bundle/tests/python_wheel/python_wheel_multiple/my_test_code/setup.py new file mode 100644 index 0000000000..0bd871dd34 --- /dev/null +++ b/bundle/tests/python_wheel/python_wheel_multiple/my_test_code/setup.py @@ -0,0 +1,15 @@ +from setuptools import setup, find_packages + +import src + +setup( + name="my_test_code", + version=src.__version__, + author=src.__author__, + url="https://databricks.com", + author_email="john.doe@databricks.com", + description="my test wheel", + packages=find_packages(include=["src"]), + entry_points={"group_1": "run=src.__main__:main"}, + install_requires=["setuptools"], +) diff --git a/bundle/tests/python_wheel/python_wheel_multiple/my_test_code/setup2.py b/bundle/tests/python_wheel/python_wheel_multiple/my_test_code/setup2.py new file mode 100644 index 0000000000..424bec9f18 --- /dev/null +++ b/bundle/tests/python_wheel/python_wheel_multiple/my_test_code/setup2.py @@ -0,0 +1,15 @@ +from setuptools import setup, find_packages + +import src + +setup( + name="my_test_code_2", + version=src.__version__, + author=src.__author__, + url="https://databricks.com", + author_email="john.doe@databricks.com", + description="my test wheel", + packages=find_packages(include=["src"]), + entry_points={"group_1": "run=src.__main__:main"}, + install_requires=["setuptools"], +) diff --git a/bundle/tests/python_wheel/python_wheel_multiple/my_test_code/src/__init__.py b/bundle/tests/python_wheel/python_wheel_multiple/my_test_code/src/__init__.py new file mode 100644 index 0000000000..909f1f3220 --- /dev/null +++ b/bundle/tests/python_wheel/python_wheel_multiple/my_test_code/src/__init__.py @@ -0,0 +1,2 @@ +__version__ = "0.0.1" +__author__ = "Databricks" diff --git a/bundle/tests/python_wheel/python_wheel_multiple/my_test_code/src/__main__.py b/bundle/tests/python_wheel/python_wheel_multiple/my_test_code/src/__main__.py new file mode 100644 index 0000000000..73d045afb4 --- /dev/null +++ b/bundle/tests/python_wheel/python_wheel_multiple/my_test_code/src/__main__.py @@ -0,0 +1,16 @@ +""" +The entry point of the Python Wheel +""" + +import sys + + +def main(): + # This method will print the provided arguments + print('Hello from my func') + print('Got arguments:') + print(sys.argv) + + +if __name__ == '__main__': + main() diff --git a/bundle/tests/python_wheel/python_wheel_no_artifact_no_setup/bundle.yml b/bundle/tests/python_wheel/python_wheel_no_artifact_no_setup/bundle.yml index 1bac4ebadf..492861969d 100644 --- a/bundle/tests/python_wheel/python_wheel_no_artifact_no_setup/bundle.yml +++ b/bundle/tests/python_wheel/python_wheel_no_artifact_no_setup/bundle.yml @@ -13,10 +13,3 @@ resources: entry_point: "run" libraries: - whl: ./package/*.whl - - task_key: TestTask2 - existing_cluster_id: "0717-aaaaa-bbbbbb" - python_wheel_task: - package_name: "my_test_code" - entry_point: "run" - libraries: - - whl: ./non-existing/*.whl diff --git a/bundle/tests/python_wheel/python_wheel_no_artifact_notebook/.gitignore b/bundle/tests/python_wheel/python_wheel_no_artifact_notebook/.gitignore new file mode 100644 index 0000000000..f03e23bc26 --- /dev/null +++ b/bundle/tests/python_wheel/python_wheel_no_artifact_notebook/.gitignore @@ -0,0 +1,3 @@ +build/ +*.egg-info +.databricks diff --git a/bundle/tests/python_wheel/python_wheel_no_artifact_notebook/bundle.yml b/bundle/tests/python_wheel/python_wheel_no_artifact_notebook/bundle.yml new file mode 100644 index 0000000000..93e4e6918b --- /dev/null +++ b/bundle/tests/python_wheel/python_wheel_no_artifact_notebook/bundle.yml @@ -0,0 +1,14 @@ +bundle: + name: python-wheel-notebook + +resources: + jobs: + test_job: + name: "[${bundle.environment}] My Wheel Job" + tasks: + - task_key: TestTask + existing_cluster_id: "0717-aaaaa-bbbbbb" + notebook_task: + notebook_path: "/notebook.py" + libraries: + - whl: ./dist/*.whl diff --git a/bundle/tests/python_wheel/python_wheel_no_artifact_notebook/my_test_code/__init__.py b/bundle/tests/python_wheel/python_wheel_no_artifact_notebook/my_test_code/__init__.py new file mode 100644 index 0000000000..909f1f3220 --- /dev/null +++ b/bundle/tests/python_wheel/python_wheel_no_artifact_notebook/my_test_code/__init__.py @@ -0,0 +1,2 @@ +__version__ = "0.0.1" +__author__ = "Databricks" diff --git a/bundle/tests/python_wheel/python_wheel_no_artifact_notebook/my_test_code/__main__.py b/bundle/tests/python_wheel/python_wheel_no_artifact_notebook/my_test_code/__main__.py new file mode 100644 index 0000000000..73d045afb4 --- /dev/null +++ b/bundle/tests/python_wheel/python_wheel_no_artifact_notebook/my_test_code/__main__.py @@ -0,0 +1,16 @@ +""" +The entry point of the Python Wheel +""" + +import sys + + +def main(): + # This method will print the provided arguments + print('Hello from my func') + print('Got arguments:') + print(sys.argv) + + +if __name__ == '__main__': + main() diff --git a/bundle/tests/python_wheel/python_wheel_no_artifact_notebook/notebook.py b/bundle/tests/python_wheel/python_wheel_no_artifact_notebook/notebook.py new file mode 100644 index 0000000000..24dc150ffb --- /dev/null +++ b/bundle/tests/python_wheel/python_wheel_no_artifact_notebook/notebook.py @@ -0,0 +1,3 @@ +# Databricks notebook source + +print("Hello, World!") diff --git a/bundle/tests/python_wheel/python_wheel_no_artifact_notebook/setup.py b/bundle/tests/python_wheel/python_wheel_no_artifact_notebook/setup.py new file mode 100644 index 0000000000..7a1317b2f5 --- /dev/null +++ b/bundle/tests/python_wheel/python_wheel_no_artifact_notebook/setup.py @@ -0,0 +1,15 @@ +from setuptools import setup, find_packages + +import my_test_code + +setup( + name="my_test_code", + version=my_test_code.__version__, + author=my_test_code.__author__, + url="https://databricks.com", + author_email="john.doe@databricks.com", + description="my test wheel", + packages=find_packages(include=["my_test_code"]), + entry_points={"group_1": "run=my_test_code.__main__:main"}, + install_requires=["setuptools"], +) diff --git a/bundle/tests/python_wheel/python_wheel_no_build/.gitignore b/bundle/tests/python_wheel/python_wheel_no_build/.gitignore new file mode 100644 index 0000000000..f03e23bc26 --- /dev/null +++ b/bundle/tests/python_wheel/python_wheel_no_build/.gitignore @@ -0,0 +1,3 @@ +build/ +*.egg-info +.databricks diff --git a/bundle/tests/python_wheel/python_wheel_no_build/bundle.yml b/bundle/tests/python_wheel/python_wheel_no_build/bundle.yml new file mode 100644 index 0000000000..91b8b1556d --- /dev/null +++ b/bundle/tests/python_wheel/python_wheel_no_build/bundle.yml @@ -0,0 +1,16 @@ +bundle: + name: python-wheel + +resources: + jobs: + test_job: + name: "[${bundle.environment}] My Wheel Job" + tasks: + - task_key: TestTask + existing_cluster_id: "0717-132531-5opeqon1" + python_wheel_task: + package_name: "my_test_code" + entry_point: "run" + libraries: + - whl: ./dist/*.whl + - whl: ./dist/lib/my_test_code-0.0.1-py3-none-any.whl diff --git a/bundle/tests/python_wheel/python_wheel_no_build/dist/lib/my_test_code-0.0.1-py3-none-any.whl b/bundle/tests/python_wheel/python_wheel_no_build/dist/lib/my_test_code-0.0.1-py3-none-any.whl new file mode 100644 index 0000000000..4bb80477ca Binary files /dev/null and b/bundle/tests/python_wheel/python_wheel_no_build/dist/lib/my_test_code-0.0.1-py3-none-any.whl differ diff --git a/bundle/tests/python_wheel/python_wheel_no_build/dist/my_test_code-0.0.1-py3-none-any.whl b/bundle/tests/python_wheel/python_wheel_no_build/dist/my_test_code-0.0.1-py3-none-any.whl new file mode 100644 index 0000000000..4bb80477ca Binary files /dev/null and b/bundle/tests/python_wheel/python_wheel_no_build/dist/my_test_code-0.0.1-py3-none-any.whl differ diff --git a/bundle/tests/python_wheel_test.go b/bundle/tests/python_wheel_test.go index 8d0036a7bb..c4d85703cc 100644 --- a/bundle/tests/python_wheel_test.go +++ b/bundle/tests/python_wheel_test.go @@ -8,6 +8,9 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/libraries" "github.com/databricks/cli/bundle/phases" + mockfiler "github.com/databricks/cli/internal/mocks/libs/filer" + "github.com/databricks/cli/libs/filer" + "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" ) @@ -23,7 +26,7 @@ func TestPythonWheelBuild(t *testing.T) { require.NoError(t, err) require.Equal(t, 1, len(matches)) - match := libraries.ValidateLocalLibrariesExist() + match := libraries.ExpandGlobReferences() diags = bundle.Apply(ctx, b, match) require.NoError(t, diags.Error()) } @@ -40,7 +43,24 @@ func TestPythonWheelBuildAutoDetect(t *testing.T) { require.NoError(t, err) require.Equal(t, 1, len(matches)) - match := libraries.ValidateLocalLibrariesExist() + match := libraries.ExpandGlobReferences() + diags = bundle.Apply(ctx, b, match) + require.NoError(t, diags.Error()) +} + +func TestPythonWheelBuildAutoDetectWithNotebookTask(t *testing.T) { + ctx := context.Background() + b, err := bundle.Load(ctx, "./python_wheel/python_wheel_no_artifact_notebook") + require.NoError(t, err) + + diags := bundle.Apply(ctx, b, bundle.Seq(phases.Load(), phases.Build())) + require.NoError(t, diags.Error()) + + matches, err := filepath.Glob("./python_wheel/python_wheel_no_artifact_notebook/dist/my_test_code-*.whl") + require.NoError(t, err) + require.Equal(t, 1, len(matches)) + + match := libraries.ExpandGlobReferences() diags = bundle.Apply(ctx, b, match) require.NoError(t, diags.Error()) } @@ -53,7 +73,7 @@ func TestPythonWheelWithDBFSLib(t *testing.T) { diags := bundle.Apply(ctx, b, bundle.Seq(phases.Load(), phases.Build())) require.NoError(t, diags.Error()) - match := libraries.ValidateLocalLibrariesExist() + match := libraries.ExpandGlobReferences() diags = bundle.Apply(ctx, b, match) require.NoError(t, diags.Error()) } @@ -63,21 +83,23 @@ func TestPythonWheelBuildNoBuildJustUpload(t *testing.T) { b, err := bundle.Load(ctx, "./python_wheel/python_wheel_no_artifact_no_setup") require.NoError(t, err) - diags := bundle.Apply(ctx, b, bundle.Seq(phases.Load(), phases.Build())) - require.NoError(t, diags.Error()) + b.Config.Workspace.ArtifactPath = "/foo/bar" - match := libraries.ValidateLocalLibrariesExist() - diags = bundle.Apply(ctx, b, match) - require.ErrorContains(t, diags.Error(), "./non-existing/*.whl") + mockFiler := mockfiler.NewMockFiler(t) + mockFiler.EXPECT().Write( + mock.Anything, + filepath.Join("my_test_code-0.0.1-py3-none-any.whl"), + mock.AnythingOfType("*os.File"), + filer.OverwriteIfExists, + filer.CreateParentDirectories, + ).Return(nil) - require.NotZero(t, len(b.Config.Artifacts)) + u := libraries.UploadWithClient(mockFiler) + diags := bundle.Apply(ctx, b, bundle.Seq(phases.Load(), phases.Build(), libraries.ExpandGlobReferences(), u)) + require.NoError(t, diags.Error()) + require.Empty(t, diags) - artifact := b.Config.Artifacts["my_test_code-0.0.1-py3-none-any.whl"] - require.NotNil(t, artifact) - require.Empty(t, artifact.BuildCommand) - require.Contains(t, artifact.Files[0].Source, filepath.Join(b.RootPath, "package", - "my_test_code-0.0.1-py3-none-any.whl", - )) + require.Equal(t, "/Workspace/foo/bar/.internal/my_test_code-0.0.1-py3-none-any.whl", b.Config.Resources.Jobs["test_job"].JobSettings.Tasks[0].Libraries[0].Whl) } func TestPythonWheelBuildWithEnvironmentKey(t *testing.T) { @@ -92,7 +114,37 @@ func TestPythonWheelBuildWithEnvironmentKey(t *testing.T) { require.NoError(t, err) require.Equal(t, 1, len(matches)) - match := libraries.ValidateLocalLibrariesExist() + match := libraries.ExpandGlobReferences() + diags = bundle.Apply(ctx, b, match) + require.NoError(t, diags.Error()) +} + +func TestPythonWheelBuildMultiple(t *testing.T) { + ctx := context.Background() + b, err := bundle.Load(ctx, "./python_wheel/python_wheel_multiple") + require.NoError(t, err) + + diags := bundle.Apply(ctx, b, bundle.Seq(phases.Load(), phases.Build())) + require.NoError(t, diags.Error()) + + matches, err := filepath.Glob("./python_wheel/python_wheel_multiple/my_test_code/dist/my_test_code*.whl") + require.NoError(t, err) + require.Equal(t, 2, len(matches)) + + match := libraries.ExpandGlobReferences() + diags = bundle.Apply(ctx, b, match) + require.NoError(t, diags.Error()) +} + +func TestPythonWheelNoBuild(t *testing.T) { + ctx := context.Background() + b, err := bundle.Load(ctx, "./python_wheel/python_wheel_no_build") + require.NoError(t, err) + + diags := bundle.Apply(ctx, b, bundle.Seq(phases.Load(), phases.Build())) + require.NoError(t, diags.Error()) + + match := libraries.ExpandGlobReferences() diags = bundle.Apply(ctx, b, match) require.NoError(t, diags.Error()) } diff --git a/bundle/tests/registered_model_test.go b/bundle/tests/registered_model_test.go index 920a2ac78e..008db8bdd7 100644 --- a/bundle/tests/registered_model_test.go +++ b/bundle/tests/registered_model_test.go @@ -1,7 +1,6 @@ package config_tests import ( - "path/filepath" "testing" "github.com/databricks/cli/bundle/config" @@ -10,7 +9,6 @@ import ( ) func assertExpectedModel(t *testing.T, p *resources.RegisteredModel) { - assert.Equal(t, "registered_model/databricks.yml", filepath.ToSlash(p.ConfigFilePath)) assert.Equal(t, "main", p.CatalogName) assert.Equal(t, "default", p.SchemaName) assert.Equal(t, "comment", p.Comment) diff --git a/bundle/tests/sync/negate/databricks.yml b/bundle/tests/sync/negate/databricks.yml new file mode 100644 index 0000000000..3d591d19b3 --- /dev/null +++ b/bundle/tests/sync/negate/databricks.yml @@ -0,0 +1,22 @@ +bundle: + name: sync_negate + +workspace: + host: https://acme.cloud.databricks.com/ + +sync: + exclude: + - ./* + - '!*.txt' + include: + - '*.txt' + +targets: + default: + dev: + sync: + exclude: + - ./* + - '!*.txt2' + include: + - '*.txt' diff --git a/libs/template/testdata/template-in-path/template/{{template `dir_name`}}/{{template `file_name`}} b/bundle/tests/sync/negate/test.txt similarity index 100% rename from libs/template/testdata/template-in-path/template/{{template `dir_name`}}/{{template `file_name`}} rename to bundle/tests/sync/negate/test.txt diff --git a/libs/template/testdata/templated-defaults/template/{{template `dir_name`}}/{{template `file_name`}} b/bundle/tests/sync/negate/test.yml similarity index 100% rename from libs/template/testdata/templated-defaults/template/{{template `dir_name`}}/{{template `file_name`}} rename to bundle/tests/sync/negate/test.yml diff --git a/bundle/tests/sync_include_exclude_no_matches_test.go b/bundle/tests/sync_include_exclude_no_matches_test.go index 94cedbaa62..0192b61e65 100644 --- a/bundle/tests/sync_include_exclude_no_matches_test.go +++ b/bundle/tests/sync_include_exclude_no_matches_test.go @@ -9,6 +9,7 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config/validate" "github.com/databricks/cli/libs/diag" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -21,10 +22,14 @@ func TestSyncIncludeExcludeNoMatchesTest(t *testing.T) { require.Equal(t, diags[0].Severity, diag.Warning) require.Equal(t, diags[0].Summary, "Pattern dist does not match any files") - require.Equal(t, diags[0].Location.File, filepath.Join("sync", "override", "databricks.yml")) - require.Equal(t, diags[0].Location.Line, 17) - require.Equal(t, diags[0].Location.Column, 11) - require.Equal(t, diags[0].Path.String(), "sync.exclude[0]") + + require.Len(t, diags[0].Paths, 1) + require.Equal(t, diags[0].Paths[0].String(), "sync.exclude[0]") + + assert.Len(t, diags[0].Locations, 1) + require.Equal(t, diags[0].Locations[0].File, filepath.Join("sync", "override", "databricks.yml")) + require.Equal(t, diags[0].Locations[0].Line, 17) + require.Equal(t, diags[0].Locations[0].Column, 11) summaries := []string{ fmt.Sprintf("Pattern %s does not match any files", filepath.Join("src", "*")), @@ -37,3 +42,22 @@ func TestSyncIncludeExcludeNoMatchesTest(t *testing.T) { require.Equal(t, diags[2].Severity, diag.Warning) require.Contains(t, summaries, diags[2].Summary) } + +func TestSyncIncludeWithNegate(t *testing.T) { + b := loadTarget(t, "./sync/negate", "default") + + diags := bundle.ApplyReadOnly(context.Background(), bundle.ReadOnly(b), validate.ValidateSyncPatterns()) + require.Len(t, diags, 0) + require.NoError(t, diags.Error()) +} + +func TestSyncIncludeWithNegateNoMatches(t *testing.T) { + b := loadTarget(t, "./sync/negate", "dev") + + diags := bundle.ApplyReadOnly(context.Background(), bundle.ReadOnly(b), validate.ValidateSyncPatterns()) + require.Len(t, diags, 1) + require.NoError(t, diags.Error()) + + require.Equal(t, diags[0].Severity, diag.Warning) + require.Equal(t, diags[0].Summary, "Pattern !*.txt2 does not match any files") +} diff --git a/bundle/tests/undefined_job_test.go b/bundle/tests/undefined_job_test.go index ed502c471c..4596f20695 100644 --- a/bundle/tests/undefined_job_test.go +++ b/bundle/tests/undefined_job_test.go @@ -1,12 +1,22 @@ package config_tests import ( + "context" "testing" + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config/validate" "github.com/stretchr/testify/assert" ) func TestUndefinedJobLoadsWithError(t *testing.T) { - _, diags := loadTargetWithDiags("./undefined_job", "default") + b := load(t, "./undefined_job") + diags := bundle.Apply(context.Background(), b, validate.AllResourcesHaveValues()) assert.ErrorContains(t, diags.Error(), "job undefined is not defined") } + +func TestUndefinedPipelineLoadsWithError(t *testing.T) { + b := load(t, "./undefined_pipeline") + diags := bundle.Apply(context.Background(), b, validate.AllResourcesHaveValues()) + assert.ErrorContains(t, diags.Error(), "pipeline undefined is not defined") +} diff --git a/bundle/tests/undefined_pipeline/databricks.yml b/bundle/tests/undefined_pipeline/databricks.yml new file mode 100644 index 0000000000..a52fda38c4 --- /dev/null +++ b/bundle/tests/undefined_pipeline/databricks.yml @@ -0,0 +1,8 @@ +bundle: + name: undefined-pipeline + +resources: + pipelines: + undefined: + test: + name: "Test Pipeline" diff --git a/bundle/tests/conflicting_resource_ids/no_subconfigurations/databricks.yml b/bundle/tests/validate/duplicate_resource_name_in_multiple_locations/databricks.yml similarity index 53% rename from bundle/tests/conflicting_resource_ids/no_subconfigurations/databricks.yml rename to bundle/tests/validate/duplicate_resource_name_in_multiple_locations/databricks.yml index 1e9aa10b1f..ebb1f90053 100644 --- a/bundle/tests/conflicting_resource_ids/no_subconfigurations/databricks.yml +++ b/bundle/tests/validate/duplicate_resource_name_in_multiple_locations/databricks.yml @@ -4,10 +4,10 @@ bundle: workspace: profile: test +include: + - ./*.yml + resources: jobs: foo: - name: job foo - pipelines: - foo: - name: pipeline foo + name: job foo 1 diff --git a/bundle/tests/conflicting_resource_ids/one_subconfiguration/resources.yml b/bundle/tests/validate/duplicate_resource_name_in_multiple_locations/resources1.yml similarity index 59% rename from bundle/tests/conflicting_resource_ids/one_subconfiguration/resources.yml rename to bundle/tests/validate/duplicate_resource_name_in_multiple_locations/resources1.yml index c3dcb6e2fe..deb81caa1c 100644 --- a/bundle/tests/conflicting_resource_ids/one_subconfiguration/resources.yml +++ b/bundle/tests/validate/duplicate_resource_name_in_multiple_locations/resources1.yml @@ -1,4 +1,8 @@ resources: + jobs: + foo: + name: job foo 2 + pipelines: foo: name: pipeline foo diff --git a/bundle/tests/validate/duplicate_resource_name_in_multiple_locations/resources2.yml b/bundle/tests/validate/duplicate_resource_name_in_multiple_locations/resources2.yml new file mode 100644 index 0000000000..4e0a342b30 --- /dev/null +++ b/bundle/tests/validate/duplicate_resource_name_in_multiple_locations/resources2.yml @@ -0,0 +1,8 @@ +resources: + jobs: + foo: + name: job foo 3 + + experiments: + foo: + name: experiment foo diff --git a/bundle/tests/conflicting_resource_ids/one_subconfiguration/databricks.yml b/bundle/tests/validate/duplicate_resource_name_in_subconfiguration/databricks.yml similarity index 84% rename from bundle/tests/conflicting_resource_ids/one_subconfiguration/databricks.yml rename to bundle/tests/validate/duplicate_resource_name_in_subconfiguration/databricks.yml index ea4dec2e1e..5bec674839 100644 --- a/bundle/tests/conflicting_resource_ids/one_subconfiguration/databricks.yml +++ b/bundle/tests/validate/duplicate_resource_name_in_subconfiguration/databricks.yml @@ -5,7 +5,7 @@ workspace: profile: test include: - - "*.yml" + - ./resources.yml resources: jobs: diff --git a/bundle/config/testdata/duplicate_resource_name_in_subconfiguration/resources.yml b/bundle/tests/validate/duplicate_resource_name_in_subconfiguration/resources.yml similarity index 100% rename from bundle/config/testdata/duplicate_resource_name_in_subconfiguration/resources.yml rename to bundle/tests/validate/duplicate_resource_name_in_subconfiguration/resources.yml diff --git a/bundle/config/testdata/duplicate_resource_name_in_subconfiguration/databricks.yml b/bundle/tests/validate/duplicate_resource_name_in_subconfiguration_job_and_job/databricks.yml similarity index 76% rename from bundle/config/testdata/duplicate_resource_name_in_subconfiguration/databricks.yml rename to bundle/tests/validate/duplicate_resource_name_in_subconfiguration_job_and_job/databricks.yml index a816029204..5bec674839 100644 --- a/bundle/config/testdata/duplicate_resource_name_in_subconfiguration/databricks.yml +++ b/bundle/tests/validate/duplicate_resource_name_in_subconfiguration_job_and_job/databricks.yml @@ -4,6 +4,9 @@ bundle: workspace: profile: test +include: + - ./resources.yml + resources: jobs: foo: diff --git a/bundle/tests/validate/duplicate_resource_name_in_subconfiguration_job_and_job/resources.yml b/bundle/tests/validate/duplicate_resource_name_in_subconfiguration_job_and_job/resources.yml new file mode 100644 index 0000000000..83fb75735c --- /dev/null +++ b/bundle/tests/validate/duplicate_resource_name_in_subconfiguration_job_and_job/resources.yml @@ -0,0 +1,4 @@ +resources: + jobs: + foo: + name: job foo 2 diff --git a/bundle/tests/conflicting_resource_ids/two_subconfigurations/databricks.yml b/bundle/tests/validate/duplicate_resource_names_in_different_subconfiguations/databricks.yml similarity index 100% rename from bundle/tests/conflicting_resource_ids/two_subconfigurations/databricks.yml rename to bundle/tests/validate/duplicate_resource_names_in_different_subconfiguations/databricks.yml diff --git a/bundle/tests/conflicting_resource_ids/two_subconfigurations/resources1.yml b/bundle/tests/validate/duplicate_resource_names_in_different_subconfiguations/resources1.yml similarity index 100% rename from bundle/tests/conflicting_resource_ids/two_subconfigurations/resources1.yml rename to bundle/tests/validate/duplicate_resource_names_in_different_subconfiguations/resources1.yml diff --git a/bundle/tests/conflicting_resource_ids/two_subconfigurations/resources2.yml b/bundle/tests/validate/duplicate_resource_names_in_different_subconfiguations/resources2.yml similarity index 100% rename from bundle/tests/conflicting_resource_ids/two_subconfigurations/resources2.yml rename to bundle/tests/validate/duplicate_resource_names_in_different_subconfiguations/resources2.yml diff --git a/bundle/tests/validate/duplicate_resource_names_in_root_job_and_experiment/databricks.yml b/bundle/tests/validate/duplicate_resource_names_in_root_job_and_experiment/databricks.yml new file mode 100644 index 0000000000..d286f10496 --- /dev/null +++ b/bundle/tests/validate/duplicate_resource_names_in_root_job_and_experiment/databricks.yml @@ -0,0 +1,18 @@ +bundle: + name: test + +workspace: + profile: test + +resources: + jobs: + foo: + name: job foo + bar: + name: job bar + pipelines: + baz: + name: pipeline baz + experiments: + foo: + name: experiment foo diff --git a/bundle/config/testdata/duplicate_resource_names_in_root/databricks.yml b/bundle/tests/validate/duplicate_resource_names_in_root_job_and_pipeline/databricks.yml similarity index 100% rename from bundle/config/testdata/duplicate_resource_names_in_root/databricks.yml rename to bundle/tests/validate/duplicate_resource_names_in_root_job_and_pipeline/databricks.yml diff --git a/bundle/tests/validate_test.go b/bundle/tests/validate_test.go new file mode 100644 index 0000000000..9cd7c201b2 --- /dev/null +++ b/bundle/tests/validate_test.go @@ -0,0 +1,139 @@ +package config_tests + +import ( + "context" + "path/filepath" + "testing" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/phases" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dyn" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestValidateUniqueResourceIdentifiers(t *testing.T) { + tcases := []struct { + name string + diagnostics diag.Diagnostics + }{ + { + name: "duplicate_resource_names_in_root_job_and_pipeline", + diagnostics: diag.Diagnostics{ + { + Severity: diag.Error, + Summary: "multiple resources have been defined with the same key: foo", + Locations: []dyn.Location{ + {File: filepath.FromSlash("validate/duplicate_resource_names_in_root_job_and_pipeline/databricks.yml"), Line: 10, Column: 7}, + {File: filepath.FromSlash("validate/duplicate_resource_names_in_root_job_and_pipeline/databricks.yml"), Line: 13, Column: 7}, + }, + Paths: []dyn.Path{ + dyn.MustPathFromString("jobs.foo"), + dyn.MustPathFromString("pipelines.foo"), + }, + }, + }, + }, + { + name: "duplicate_resource_names_in_root_job_and_experiment", + diagnostics: diag.Diagnostics{ + { + Severity: diag.Error, + Summary: "multiple resources have been defined with the same key: foo", + Locations: []dyn.Location{ + {File: filepath.FromSlash("validate/duplicate_resource_names_in_root_job_and_experiment/databricks.yml"), Line: 10, Column: 7}, + {File: filepath.FromSlash("validate/duplicate_resource_names_in_root_job_and_experiment/databricks.yml"), Line: 18, Column: 7}, + }, + Paths: []dyn.Path{ + dyn.MustPathFromString("experiments.foo"), + dyn.MustPathFromString("jobs.foo"), + }, + }, + }, + }, + { + name: "duplicate_resource_name_in_subconfiguration", + diagnostics: diag.Diagnostics{ + { + Severity: diag.Error, + Summary: "multiple resources have been defined with the same key: foo", + Locations: []dyn.Location{ + {File: filepath.FromSlash("validate/duplicate_resource_name_in_subconfiguration/databricks.yml"), Line: 13, Column: 7}, + {File: filepath.FromSlash("validate/duplicate_resource_name_in_subconfiguration/resources.yml"), Line: 4, Column: 7}, + }, + Paths: []dyn.Path{ + dyn.MustPathFromString("jobs.foo"), + dyn.MustPathFromString("pipelines.foo"), + }, + }, + }, + }, + { + name: "duplicate_resource_name_in_subconfiguration_job_and_job", + diagnostics: diag.Diagnostics{ + { + Severity: diag.Error, + Summary: "multiple resources have been defined with the same key: foo", + Locations: []dyn.Location{ + {File: filepath.FromSlash("validate/duplicate_resource_name_in_subconfiguration_job_and_job/databricks.yml"), Line: 13, Column: 7}, + {File: filepath.FromSlash("validate/duplicate_resource_name_in_subconfiguration_job_and_job/resources.yml"), Line: 4, Column: 7}, + }, + Paths: []dyn.Path{ + dyn.MustPathFromString("jobs.foo"), + }, + }, + }, + }, + { + name: "duplicate_resource_names_in_different_subconfiguations", + diagnostics: diag.Diagnostics{ + { + Severity: diag.Error, + Summary: "multiple resources have been defined with the same key: foo", + Locations: []dyn.Location{ + {File: filepath.FromSlash("validate/duplicate_resource_names_in_different_subconfiguations/resources1.yml"), Line: 4, Column: 7}, + {File: filepath.FromSlash("validate/duplicate_resource_names_in_different_subconfiguations/resources2.yml"), Line: 4, Column: 7}, + }, + Paths: []dyn.Path{ + dyn.MustPathFromString("jobs.foo"), + dyn.MustPathFromString("pipelines.foo"), + }, + }, + }, + }, + { + name: "duplicate_resource_name_in_multiple_locations", + diagnostics: diag.Diagnostics{ + { + Severity: diag.Error, + Summary: "multiple resources have been defined with the same key: foo", + Locations: []dyn.Location{ + {File: filepath.FromSlash("validate/duplicate_resource_name_in_multiple_locations/databricks.yml"), Line: 13, Column: 7}, + {File: filepath.FromSlash("validate/duplicate_resource_name_in_multiple_locations/resources1.yml"), Line: 4, Column: 7}, + {File: filepath.FromSlash("validate/duplicate_resource_name_in_multiple_locations/resources1.yml"), Line: 8, Column: 7}, + {File: filepath.FromSlash("validate/duplicate_resource_name_in_multiple_locations/resources2.yml"), Line: 4, Column: 7}, + {File: filepath.FromSlash("validate/duplicate_resource_name_in_multiple_locations/resources2.yml"), Line: 8, Column: 7}, + }, + Paths: []dyn.Path{ + dyn.MustPathFromString("experiments.foo"), + dyn.MustPathFromString("jobs.foo"), + dyn.MustPathFromString("pipelines.foo"), + }, + }, + }, + }, + } + + for _, tc := range tcases { + t.Run(tc.name, func(t *testing.T) { + ctx := context.Background() + b, err := bundle.Load(ctx, "./validate/"+tc.name) + require.NoError(t, err) + + // The UniqueResourceKeys mutator is run as part of the Load phase. + diags := bundle.Apply(ctx, b, phases.Load()) + assert.Equal(t, tc.diagnostics, diags) + }) + } +} diff --git a/cmd/auth/auth.go b/cmd/auth/auth.go index 79e1063b18..ceceae25c5 100644 --- a/cmd/auth/auth.go +++ b/cmd/auth/auth.go @@ -2,6 +2,7 @@ package auth import ( "context" + "fmt" "github.com/databricks/cli/libs/auth" "github.com/databricks/cli/libs/cmdio" @@ -34,25 +35,23 @@ GCP: https://docs.gcp.databricks.com/dev-tools/auth/index.html`, } func promptForHost(ctx context.Context) (string, error) { - prompt := cmdio.Prompt(ctx) - prompt.Label = "Databricks Host (e.g. https://.cloud.databricks.com)" - // Validate? - host, err := prompt.Run() - if err != nil { - return "", err + if !cmdio.IsInTTY(ctx) { + return "", fmt.Errorf("the command is being run in a non-interactive environment, please specify a host using --host") } - return host, nil + + prompt := cmdio.Prompt(ctx) + prompt.Label = "Databricks host (e.g. https://.cloud.databricks.com)" + return prompt.Run() } func promptForAccountID(ctx context.Context) (string, error) { + if !cmdio.IsInTTY(ctx) { + return "", fmt.Errorf("the command is being run in a non-interactive environment, please specify an account ID using --account-id") + } + prompt := cmdio.Prompt(ctx) - prompt.Label = "Databricks Account ID" + prompt.Label = "Databricks account ID" prompt.Default = "" prompt.AllowEdit = true - // Validate? - accountId, err := prompt.Run() - if err != nil { - return "", err - } - return accountId, nil + return prompt.Run() } diff --git a/cmd/auth/login.go b/cmd/auth/login.go index 11cba8e5f1..f87a2a0277 100644 --- a/cmd/auth/login.go +++ b/cmd/auth/login.go @@ -17,18 +17,16 @@ import ( "github.com/spf13/cobra" ) -func configureHost(ctx context.Context, persistentAuth *auth.PersistentAuth, args []string, argIndex int) error { - if len(args) > argIndex { - persistentAuth.Host = args[argIndex] - return nil +func promptForProfile(ctx context.Context, defaultValue string) (string, error) { + if !cmdio.IsInTTY(ctx) { + return "", fmt.Errorf("the command is being run in a non-interactive environment, please specify a profile using --profile") } - host, err := promptForHost(ctx) - if err != nil { - return err - } - persistentAuth.Host = host - return nil + prompt := cmdio.Prompt(ctx) + prompt.Label = "Databricks profile name" + prompt.Default = defaultValue + prompt.AllowEdit = true + return prompt.Run() } const minimalDbConnectVersion = "13.1" @@ -93,23 +91,18 @@ depends on the existing profiles you have set in your configuration file cmd.RunE = func(cmd *cobra.Command, args []string) error { ctx := cmd.Context() + profileName := cmd.Flag("profile").Value.String() - var profileName string - profileFlag := cmd.Flag("profile") - if profileFlag != nil && profileFlag.Value.String() != "" { - profileName = profileFlag.Value.String() - } else if cmdio.IsInTTY(ctx) { - prompt := cmdio.Prompt(ctx) - prompt.Label = "Databricks Profile Name" - prompt.Default = persistentAuth.ProfileName() - prompt.AllowEdit = true - profile, err := prompt.Run() + // If the user has not specified a profile name, prompt for one. + if profileName == "" { + var err error + profileName, err = promptForProfile(ctx, persistentAuth.ProfileName()) if err != nil { return err } - profileName = profile } + // Set the host and account-id based on the provided arguments and flags. err := setHostAndAccountId(ctx, profileName, persistentAuth, args) if err != nil { return err @@ -167,7 +160,23 @@ depends on the existing profiles you have set in your configuration file return cmd } +// Sets the host in the persistentAuth object based on the provided arguments and flags. +// Follows the following precedence: +// 1. [HOST] (first positional argument) or --host flag. Error if both are specified. +// 2. Profile host, if available. +// 3. Prompt the user for the host. +// +// Set the account in the persistentAuth object based on the flags. +// Follows the following precedence: +// 1. --account-id flag. +// 2. account-id from the specified profile, if available. +// 3. Prompt the user for the account-id. func setHostAndAccountId(ctx context.Context, profileName string, persistentAuth *auth.PersistentAuth, args []string) error { + // If both [HOST] and --host are provided, return an error. + if len(args) > 0 && persistentAuth.Host != "" { + return fmt.Errorf("please only provide a host as an argument or a flag, not both") + } + profiler := profile.GetProfiler(ctx) // If the chosen profile has a hostname and the user hasn't specified a host, infer the host from the profile. profiles, err := profiler.LoadProfiles(ctx, profile.WithName(profileName)) @@ -177,17 +186,32 @@ func setHostAndAccountId(ctx context.Context, profileName string, persistentAuth } if persistentAuth.Host == "" { - if len(profiles) > 0 && profiles[0].Host != "" { + if len(args) > 0 { + // If [HOST] is provided, set the host to the provided positional argument. + persistentAuth.Host = args[0] + } else if len(profiles) > 0 && profiles[0].Host != "" { + // If neither [HOST] nor --host are provided, and the profile has a host, use it. persistentAuth.Host = profiles[0].Host } else { - configureHost(ctx, persistentAuth, args, 0) + // If neither [HOST] nor --host are provided, and the profile does not have a host, + // then prompt the user for a host. + hostName, err := promptForHost(ctx) + if err != nil { + return err + } + persistentAuth.Host = hostName } } + + // If the account-id was not provided as a cmd line flag, try to read it from + // the specified profile. isAccountClient := (&config.Config{Host: persistentAuth.Host}).IsAccountClient() if isAccountClient && persistentAuth.AccountID == "" { if len(profiles) > 0 && profiles[0].AccountID != "" { persistentAuth.AccountID = profiles[0].AccountID } else { + // Prompt user for the account-id if it we could not get it from a + // profile. accountId, err := promptForAccountID(ctx) if err != nil { return err diff --git a/cmd/auth/login_test.go b/cmd/auth/login_test.go index ce3ca5ae57..d0fa5a16b8 100644 --- a/cmd/auth/login_test.go +++ b/cmd/auth/login_test.go @@ -5,8 +5,10 @@ import ( "testing" "github.com/databricks/cli/libs/auth" + "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/env" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestSetHostDoesNotFailWithNoDatabrickscfg(t *testing.T) { @@ -15,3 +17,69 @@ func TestSetHostDoesNotFailWithNoDatabrickscfg(t *testing.T) { err := setHostAndAccountId(ctx, "foo", &auth.PersistentAuth{Host: "test"}, []string{}) assert.NoError(t, err) } + +func TestSetHost(t *testing.T) { + var persistentAuth auth.PersistentAuth + t.Setenv("DATABRICKS_CONFIG_FILE", "./testdata/.databrickscfg") + ctx, _ := cmdio.SetupTest(context.Background()) + + // Test error when both flag and argument are provided + persistentAuth.Host = "val from --host" + err := setHostAndAccountId(ctx, "profile-1", &persistentAuth, []string{"val from [HOST]"}) + assert.EqualError(t, err, "please only provide a host as an argument or a flag, not both") + + // Test setting host from flag + persistentAuth.Host = "val from --host" + err = setHostAndAccountId(ctx, "profile-1", &persistentAuth, []string{}) + assert.NoError(t, err) + assert.Equal(t, "val from --host", persistentAuth.Host) + + // Test setting host from argument + persistentAuth.Host = "" + err = setHostAndAccountId(ctx, "profile-1", &persistentAuth, []string{"val from [HOST]"}) + assert.NoError(t, err) + assert.Equal(t, "val from [HOST]", persistentAuth.Host) + + // Test setting host from profile + persistentAuth.Host = "" + err = setHostAndAccountId(ctx, "profile-1", &persistentAuth, []string{}) + assert.NoError(t, err) + assert.Equal(t, "https://www.host1.com", persistentAuth.Host) + + // Test setting host from profile + persistentAuth.Host = "" + err = setHostAndAccountId(ctx, "profile-2", &persistentAuth, []string{}) + assert.NoError(t, err) + assert.Equal(t, "https://www.host2.com", persistentAuth.Host) + + // Test host is not set. Should prompt. + persistentAuth.Host = "" + err = setHostAndAccountId(ctx, "", &persistentAuth, []string{}) + assert.EqualError(t, err, "the command is being run in a non-interactive environment, please specify a host using --host") +} + +func TestSetAccountId(t *testing.T) { + var persistentAuth auth.PersistentAuth + t.Setenv("DATABRICKS_CONFIG_FILE", "./testdata/.databrickscfg") + ctx, _ := cmdio.SetupTest(context.Background()) + + // Test setting account-id from flag + persistentAuth.AccountID = "val from --account-id" + err := setHostAndAccountId(ctx, "account-profile", &persistentAuth, []string{}) + assert.NoError(t, err) + assert.Equal(t, "https://accounts.cloud.databricks.com", persistentAuth.Host) + assert.Equal(t, "val from --account-id", persistentAuth.AccountID) + + // Test setting account_id from profile + persistentAuth.AccountID = "" + err = setHostAndAccountId(ctx, "account-profile", &persistentAuth, []string{}) + require.NoError(t, err) + assert.Equal(t, "https://accounts.cloud.databricks.com", persistentAuth.Host) + assert.Equal(t, "id-from-profile", persistentAuth.AccountID) + + // Neither flag nor profile account-id is set, should prompt + persistentAuth.AccountID = "" + persistentAuth.Host = "https://accounts.cloud.databricks.com" + err = setHostAndAccountId(ctx, "", &persistentAuth, []string{}) + assert.EqualError(t, err, "the command is being run in a non-interactive environment, please specify an account ID using --account-id") +} diff --git a/cmd/auth/testdata/.databrickscfg b/cmd/auth/testdata/.databrickscfg new file mode 100644 index 0000000000..06e55224a1 --- /dev/null +++ b/cmd/auth/testdata/.databrickscfg @@ -0,0 +1,9 @@ +[profile-1] +host = https://www.host1.com + +[profile-2] +host = https://www.host2.com + +[account-profile] +host = https://accounts.cloud.databricks.com +account_id = id-from-profile diff --git a/cmd/bundle/deploy.go b/cmd/bundle/deploy.go index 1232c8de51..1166875ab3 100644 --- a/cmd/bundle/deploy.go +++ b/cmd/bundle/deploy.go @@ -24,10 +24,12 @@ func newDeployCommand() *cobra.Command { var forceLock bool var failOnActiveRuns bool var computeID string + var autoApprove bool cmd.Flags().BoolVar(&force, "force", false, "Force-override Git branch validation.") cmd.Flags().BoolVar(&forceLock, "force-lock", false, "Force acquisition of deployment lock.") cmd.Flags().BoolVar(&failOnActiveRuns, "fail-on-active-runs", false, "Fail if there are running jobs or pipelines in the deployment.") cmd.Flags().StringVarP(&computeID, "compute-id", "c", "", "Override compute in the deployment with the given compute ID.") + cmd.Flags().BoolVar(&autoApprove, "auto-approve", false, "Skip interactive approvals that might be required for deployment.") cmd.RunE = func(cmd *cobra.Command, args []string) error { ctx := cmd.Context() @@ -37,10 +39,11 @@ func newDeployCommand() *cobra.Command { bundle.ApplyFunc(ctx, b, func(context.Context, *bundle.Bundle) diag.Diagnostics { b.Config.Bundle.Force = force b.Config.Bundle.Deployment.Lock.Force = forceLock + b.AutoApprove = autoApprove + if cmd.Flag("compute-id").Changed { b.Config.Bundle.ComputeID = computeID } - if cmd.Flag("fail-on-active-runs").Changed { b.Config.Bundle.Deployment.FailOnActiveRuns = failOnActiveRuns } diff --git a/cmd/bundle/init.go b/cmd/bundle/init.go index c25391577c..7f2c0efc58 100644 --- a/cmd/bundle/init.go +++ b/cmd/bundle/init.go @@ -148,7 +148,7 @@ See https://docs.databricks.com/en/dev-tools/bundles/templates.html for more inf var templateDir string var tag string var branch string - cmd.Flags().StringVar(&configFile, "config-file", "", "File containing input parameters for template initialization.") + cmd.Flags().StringVar(&configFile, "config-file", "", "JSON file containing key value pairs of input parameters required for template initialization.") cmd.Flags().StringVar(&templateDir, "template-dir", "", "Directory path within a Git repository containing the template.") cmd.Flags().StringVar(&outputDir, "output-dir", "", "Directory to write the initialized template to.") cmd.Flags().StringVar(&branch, "tag", "", "Git tag to use for template initialization") diff --git a/cmd/fs/cat.go b/cmd/fs/cat.go index 7a6f42cba2..28df80d70d 100644 --- a/cmd/fs/cat.go +++ b/cmd/fs/cat.go @@ -30,5 +30,8 @@ func newCatCommand() *cobra.Command { return cmdio.Render(ctx, r) } + v := newValidArgs() + cmd.ValidArgsFunction = v.Validate + return cmd } diff --git a/cmd/fs/cp.go b/cmd/fs/cp.go index 52feb89051..6fb3e5e6f1 100644 --- a/cmd/fs/cp.go +++ b/cmd/fs/cp.go @@ -200,5 +200,10 @@ func newCpCommand() *cobra.Command { return c.cpFileToFile(sourcePath, targetPath) } + v := newValidArgs() + // The copy command has two paths that can be completed (SOURCE_PATH & TARGET_PATH) + v.pathArgCount = 2 + cmd.ValidArgsFunction = v.Validate + return cmd } diff --git a/cmd/fs/helpers.go b/cmd/fs/helpers.go index 43d65b5dd7..bda3239cf2 100644 --- a/cmd/fs/helpers.go +++ b/cmd/fs/helpers.go @@ -8,6 +8,8 @@ import ( "github.com/databricks/cli/cmd/root" "github.com/databricks/cli/libs/filer" + "github.com/databricks/cli/libs/filer/completer" + "github.com/spf13/cobra" ) func filerForPath(ctx context.Context, fullPath string) (filer.Filer, string, error) { @@ -46,6 +48,58 @@ func filerForPath(ctx context.Context, fullPath string) (filer.Filer, string, er return f, path, err } +const dbfsPrefix string = "dbfs:" + func isDbfsPath(path string) bool { - return strings.HasPrefix(path, "dbfs:/") + return strings.HasPrefix(path, dbfsPrefix) +} + +type validArgs struct { + mustWorkspaceClientFunc func(cmd *cobra.Command, args []string) error + filerForPathFunc func(ctx context.Context, fullPath string) (filer.Filer, string, error) + pathArgCount int + onlyDirs bool +} + +func newValidArgs() *validArgs { + return &validArgs{ + mustWorkspaceClientFunc: root.MustWorkspaceClient, + filerForPathFunc: filerForPath, + pathArgCount: 1, + onlyDirs: false, + } +} + +func (v *validArgs) Validate(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { + cmd.SetContext(root.SkipPrompt(cmd.Context())) + + if len(args) >= v.pathArgCount { + return nil, cobra.ShellCompDirectiveNoFileComp + } + + err := v.mustWorkspaceClientFunc(cmd, args) + if err != nil { + return nil, cobra.ShellCompDirectiveError + } + + filer, toCompletePath, err := v.filerForPathFunc(cmd.Context(), toComplete) + if err != nil { + return nil, cobra.ShellCompDirectiveError + } + + completer := completer.New(cmd.Context(), filer, v.onlyDirs) + + // Dbfs should have a prefix and always use the "/" separator + isDbfsPath := isDbfsPath(toComplete) + if isDbfsPath { + completer.SetPrefix(dbfsPrefix) + completer.SetIsLocalPath(false) + } + + completions, directive, err := completer.CompletePath(toCompletePath) + if err != nil { + return nil, cobra.ShellCompDirectiveError + } + + return completions, directive } diff --git a/cmd/fs/helpers_test.go b/cmd/fs/helpers_test.go index d86bd46e1e..10b4aa1604 100644 --- a/cmd/fs/helpers_test.go +++ b/cmd/fs/helpers_test.go @@ -3,9 +3,13 @@ package fs import ( "context" "runtime" + "strings" "testing" + "github.com/databricks/cli/cmd/root" "github.com/databricks/cli/libs/filer" + "github.com/databricks/databricks-sdk-go/experimental/mocks" + "github.com/spf13/cobra" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -60,3 +64,88 @@ func TestFilerForWindowsLocalPaths(t *testing.T) { testWindowsFilerForPath(t, ctx, `d:\abc`) testWindowsFilerForPath(t, ctx, `f:\abc\ef`) } + +func mockMustWorkspaceClientFunc(cmd *cobra.Command, args []string) error { + return nil +} + +func setupCommand(t *testing.T) (*cobra.Command, *mocks.MockWorkspaceClient) { + m := mocks.NewMockWorkspaceClient(t) + ctx := context.Background() + ctx = root.SetWorkspaceClient(ctx, m.WorkspaceClient) + + cmd := &cobra.Command{} + cmd.SetContext(ctx) + + return cmd, m +} + +func setupTest(t *testing.T) (*validArgs, *cobra.Command, *mocks.MockWorkspaceClient) { + cmd, m := setupCommand(t) + + fakeFilerForPath := func(ctx context.Context, fullPath string) (filer.Filer, string, error) { + fakeFiler := filer.NewFakeFiler(map[string]filer.FakeFileInfo{ + "dir": {FakeName: "root", FakeDir: true}, + "dir/dirA": {FakeDir: true}, + "dir/dirB": {FakeDir: true}, + "dir/fileA": {}, + }) + return fakeFiler, strings.TrimPrefix(fullPath, "dbfs:/"), nil + } + + v := newValidArgs() + v.filerForPathFunc = fakeFilerForPath + v.mustWorkspaceClientFunc = mockMustWorkspaceClientFunc + + return v, cmd, m +} + +func TestGetValidArgsFunctionDbfsCompletion(t *testing.T) { + v, cmd, _ := setupTest(t) + completions, directive := v.Validate(cmd, []string{}, "dbfs:/dir/") + assert.Equal(t, []string{"dbfs:/dir/dirA/", "dbfs:/dir/dirB/", "dbfs:/dir/fileA"}, completions) + assert.Equal(t, cobra.ShellCompDirectiveNoSpace, directive) +} + +func TestGetValidArgsFunctionLocalCompletion(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip() + } + + v, cmd, _ := setupTest(t) + completions, directive := v.Validate(cmd, []string{}, "dir/") + assert.Equal(t, []string{"dir/dirA/", "dir/dirB/", "dir/fileA", "dbfs:/"}, completions) + assert.Equal(t, cobra.ShellCompDirectiveNoSpace, directive) +} + +func TestGetValidArgsFunctionLocalCompletionWindows(t *testing.T) { + if runtime.GOOS != "windows" { + t.Skip() + } + + v, cmd, _ := setupTest(t) + completions, directive := v.Validate(cmd, []string{}, "dir/") + assert.Equal(t, []string{"dir\\dirA\\", "dir\\dirB\\", "dir\\fileA", "dbfs:/"}, completions) + assert.Equal(t, cobra.ShellCompDirectiveNoSpace, directive) +} + +func TestGetValidArgsFunctionCompletionOnlyDirs(t *testing.T) { + v, cmd, _ := setupTest(t) + v.onlyDirs = true + completions, directive := v.Validate(cmd, []string{}, "dbfs:/dir/") + assert.Equal(t, []string{"dbfs:/dir/dirA/", "dbfs:/dir/dirB/"}, completions) + assert.Equal(t, cobra.ShellCompDirectiveNoSpace, directive) +} + +func TestGetValidArgsFunctionNotCompletedArgument(t *testing.T) { + cmd, _ := setupCommand(t) + + v := newValidArgs() + v.pathArgCount = 0 + v.mustWorkspaceClientFunc = mockMustWorkspaceClientFunc + + completions, directive := v.Validate(cmd, []string{}, "dbfs:/") + + assert.Nil(t, completions) + assert.Equal(t, cobra.ShellCompDirectiveNoFileComp, directive) +} diff --git a/cmd/fs/ls.go b/cmd/fs/ls.go index cec9b98ba0..d7eac513a5 100644 --- a/cmd/fs/ls.go +++ b/cmd/fs/ls.go @@ -89,5 +89,9 @@ func newLsCommand() *cobra.Command { `)) } + v := newValidArgs() + v.onlyDirs = true + cmd.ValidArgsFunction = v.Validate + return cmd } diff --git a/cmd/fs/mkdir.go b/cmd/fs/mkdir.go index 074a7543da..5e9ac78429 100644 --- a/cmd/fs/mkdir.go +++ b/cmd/fs/mkdir.go @@ -28,5 +28,9 @@ func newMkdirCommand() *cobra.Command { return f.Mkdir(ctx, path) } + v := newValidArgs() + v.onlyDirs = true + cmd.ValidArgsFunction = v.Validate + return cmd } diff --git a/cmd/fs/rm.go b/cmd/fs/rm.go index 5f2904e715..a133a83097 100644 --- a/cmd/fs/rm.go +++ b/cmd/fs/rm.go @@ -32,5 +32,8 @@ func newRmCommand() *cobra.Command { return f.Delete(ctx, path) } + v := newValidArgs() + cmd.ValidArgsFunction = v.Validate + return cmd } diff --git a/cmd/labs/project/installer.go b/cmd/labs/project/installer.go index 92dfe9e7c7..041415964f 100644 --- a/cmd/labs/project/installer.go +++ b/cmd/labs/project/installer.go @@ -132,14 +132,14 @@ func (i *installer) Upgrade(ctx context.Context) error { if err != nil { return fmt.Errorf("record version: %w", err) } - err = i.runInstallHook(ctx) - if err != nil { - return fmt.Errorf("installer: %w", err) - } err = i.installPythonDependencies(ctx, ".") if err != nil { return fmt.Errorf("python dependencies: %w", err) } + err = i.runInstallHook(ctx) + if err != nil { + return fmt.Errorf("installer: %w", err) + } return nil } @@ -272,8 +272,10 @@ func (i *installer) installPythonDependencies(ctx context.Context, spec string) // - python3 -m ensurepip --default-pip // - curl -o https://bootstrap.pypa.io/get-pip.py | python3 var buf bytes.Buffer + // Ensure latest version(s) is installed with the `--upgrade` and `--upgrade-strategy eager` flags + // https://pip.pypa.io/en/stable/cli/pip_install/#cmdoption-U _, err := process.Background(ctx, - []string{i.virtualEnvPython(ctx), "-m", "pip", "install", spec}, + []string{i.virtualEnvPython(ctx), "-m", "pip", "install", "--upgrade", "--upgrade-strategy", "eager", spec}, process.WithCombinedOutput(&buf), process.WithDir(libDir)) if err != nil { diff --git a/cmd/labs/project/installer_test.go b/cmd/labs/project/installer_test.go index 0e049b4c06..8754a560bf 100644 --- a/cmd/labs/project/installer_test.go +++ b/cmd/labs/project/installer_test.go @@ -199,7 +199,7 @@ func TestInstallerWorksForReleases(t *testing.T) { stub.WithStdoutFor(`python[\S]+ --version`, "Python 3.10.5") // on Unix, we call `python3`, but on Windows it is `python.exe` stub.WithStderrFor(`python[\S]+ -m venv .*/.databricks/labs/blueprint/state/venv`, "[mock venv create]") - stub.WithStderrFor(`python[\S]+ -m pip install .`, "[mock pip install]") + stub.WithStderrFor(`python[\S]+ -m pip install --upgrade --upgrade-strategy eager .`, "[mock pip install]") stub.WithStdoutFor(`python[\S]+ install.py`, "setting up important infrastructure") // simulate the case of GitHub Actions @@ -406,7 +406,7 @@ func TestUpgraderWorksForReleases(t *testing.T) { // Install stubs for the python calls we need to ensure were run in the // upgrade process. ctx, stub := process.WithStub(ctx) - stub.WithStderrFor(`python[\S]+ -m pip install .`, "[mock pip install]") + stub.WithStderrFor(`python[\S]+ -m pip install --upgrade --upgrade-strategy eager .`, "[mock pip install]") stub.WithStdoutFor(`python[\S]+ install.py`, "setting up important infrastructure") py, _ := python.DetectExecutable(ctx) @@ -430,13 +430,13 @@ func TestUpgraderWorksForReleases(t *testing.T) { // Check if the stub was called with the 'python -m pip install' command pi := false for _, call := range stub.Commands() { - if strings.HasSuffix(call, "-m pip install .") { + if strings.HasSuffix(call, "-m pip install --upgrade --upgrade-strategy eager .") { pi = true break } } if !pi { - t.Logf(`Expected stub command 'python[\S]+ -m pip install .' not found`) + t.Logf(`Expected stub command 'python[\S]+ -m pip install --upgrade --upgrade-strategy eager .' not found`) t.FailNow() } } diff --git a/go.mod b/go.mod index 5e29d295e7..3f5af08153 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ require ( github.com/ghodss/yaml v1.0.0 // MIT + NOTICE github.com/google/uuid v1.6.0 // BSD-3-Clause github.com/hashicorp/go-version v1.7.0 // MPL 2.0 - github.com/hashicorp/hc-install v0.7.0 // MPL 2.0 + github.com/hashicorp/hc-install v0.8.0 // MPL 2.0 github.com/hashicorp/terraform-exec v0.21.0 // MPL 2.0 github.com/hashicorp/terraform-json v0.22.1 // MPL 2.0 github.com/manifoldco/promptui v0.9.0 // BSD-3-Clause @@ -22,11 +22,11 @@ require ( github.com/spf13/pflag v1.0.5 // BSD-3-Clause github.com/stretchr/testify v1.9.0 // MIT golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 - golang.org/x/mod v0.19.0 - golang.org/x/oauth2 v0.21.0 - golang.org/x/sync v0.7.0 - golang.org/x/term v0.22.0 - golang.org/x/text v0.16.0 + golang.org/x/mod v0.20.0 + golang.org/x/oauth2 v0.22.0 + golang.org/x/sync v0.8.0 + golang.org/x/term v0.23.0 + golang.org/x/text v0.17.0 gopkg.in/ini.v1 v1.67.0 // Apache 2.0 gopkg.in/yaml.v3 v3.0.1 ) @@ -49,6 +49,7 @@ require ( github.com/google/s2a-go v0.1.7 // indirect github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect + github.com/hashicorp/go-retryablehttp v0.7.7 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect @@ -61,7 +62,7 @@ require ( go.opentelemetry.io/otel/trace v1.24.0 // indirect golang.org/x/crypto v0.23.0 // indirect golang.org/x/net v0.25.0 // indirect - golang.org/x/sys v0.22.0 // indirect + golang.org/x/sys v0.23.0 // indirect golang.org/x/time v0.5.0 // indirect google.golang.org/api v0.182.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240521202816-d264139d666e // indirect diff --git a/go.sum b/go.sum index 8f774a47ab..f33a9562ab 100644 --- a/go.sum +++ b/go.sum @@ -99,10 +99,14 @@ github.com/googleapis/gax-go/v2 v2.12.4 h1:9gWcmF85Wvq4ryPFvGFaOgPIs1AQX0d0bcbGw github.com/googleapis/gax-go/v2 v2.12.4/go.mod h1:KYEYLorsnIGDi/rPC8b5TdlB9kbKoFubselGIoBMCwI= github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= +github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB11/k= +github.com/hashicorp/go-hclog v1.6.3/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M= +github.com/hashicorp/go-retryablehttp v0.7.7 h1:C8hUCYzor8PIfXHa4UrZkU4VvK8o9ISHxT2Q8+VepXU= +github.com/hashicorp/go-retryablehttp v0.7.7/go.mod h1:pkQpWZeYWskR+D1tR2O5OcBFOxfA7DoAO6xtkuQnHTk= github.com/hashicorp/go-version v1.7.0 h1:5tqGy27NaOTB8yJKUZELlFAS/LTKJkrmONwQKeRZfjY= github.com/hashicorp/go-version v1.7.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= -github.com/hashicorp/hc-install v0.7.0 h1:Uu9edVqjKQxxuD28mR5TikkKDd/p55S8vzPC1659aBk= -github.com/hashicorp/hc-install v0.7.0/go.mod h1:ELmmzZlGnEcqoUMKUuykHaPCIR1sYLYX+KSggWSKZuA= +github.com/hashicorp/hc-install v0.8.0 h1:LdpZeXkZYMQhoKPCecJHlKvUkQFixN/nvyR1CdfOLjI= +github.com/hashicorp/hc-install v0.8.0/go.mod h1:+MwJYjDfCruSD/udvBmRB22Nlkwwkwf5sAB6uTIhSaU= github.com/hashicorp/terraform-exec v0.21.0 h1:uNkLAe95ey5Uux6KJdua6+cv8asgILFVWkd/RG0D2XQ= github.com/hashicorp/terraform-exec v0.21.0/go.mod h1:1PPeMYou+KDUSSeRE9szMZ/oHf4fYUmB923Wzbq1ICg= github.com/hashicorp/terraform-json v0.22.1 h1:xft84GZR0QzjPVWs4lRUwvTcPnegqlyS7orfb5Ltvec= @@ -180,8 +184,8 @@ golang.org/x/exp v0.0.0-20240222234643-814bf88cf225/go.mod h1:CxmFvTBINI24O/j8iY golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/mod v0.19.0 h1:fEdghXQSo20giMthA7cd28ZC+jts4amQ3YMXiP5oMQ8= -golang.org/x/mod v0.19.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.20.0 h1:utOm6MM3R3dnawAiJgn0y+xvuYRsm1RKM/4giyfDgV0= +golang.org/x/mod v0.20.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -191,13 +195,13 @@ golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.21.0 h1:tsimM75w1tF/uws5rbeHzIWxEqElMehnc+iW793zsZs= -golang.org/x/oauth2 v0.21.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= +golang.org/x/oauth2 v0.22.0 h1:BzDx2FehcG7jJwgWLELCdmLuxk2i+x9UDpSiss2u0ZA= +golang.org/x/oauth2 v0.22.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= -golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= +golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -208,14 +212,14 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210616045830-e2b7044e8c71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= -golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/term v0.22.0 h1:BbsgPEJULsl2fV/AT3v15Mjva5yXKQDyKf+TbDz7QJk= -golang.org/x/term v0.22.0/go.mod h1:F3qCibpT5AMpCRfhfT53vVJwhLtIVHhB9XDjfFvnMI4= +golang.org/x/sys v0.23.0 h1:YfKFowiIMvtgl1UERQoTPPToxltDeZfbj4H7dVUCwmM= +golang.org/x/sys v0.23.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.23.0 h1:F6D4vR+EHoL9/sWAWgAR1H2DcHr4PareCbAaCo1RpuU= +golang.org/x/term v0.23.0/go.mod h1:DgV24QBUrK6jhZXl+20l6UWznPlwAHm1Q1mGHtydmSk= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= -golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= +golang.org/x/text v0.17.0 h1:XtiM5bkSOt+ewxlOE/aE/AKEHibwj/6gvWMl9Rsh0Qc= +golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= diff --git a/internal/acc/workspace.go b/internal/acc/workspace.go index 8944e199f9..39374f229e 100644 --- a/internal/acc/workspace.go +++ b/internal/acc/workspace.go @@ -2,6 +2,7 @@ package acc import ( "context" + "os" "testing" "github.com/databricks/databricks-sdk-go" @@ -38,6 +39,33 @@ func WorkspaceTest(t *testing.T) (context.Context, *WorkspaceT) { return wt.ctx, wt } +// Run the workspace test only on UC workspaces. +func UcWorkspaceTest(t *testing.T) (context.Context, *WorkspaceT) { + loadDebugEnvIfRunFromIDE(t, "workspace") + + t.Log(GetEnvOrSkipTest(t, "CLOUD_ENV")) + + if os.Getenv("TEST_METASTORE_ID") == "" { + t.Skipf("Skipping on non-UC workspaces") + } + if os.Getenv("DATABRICKS_ACCOUNT_ID") != "" { + t.Skipf("Skipping on accounts") + } + + w, err := databricks.NewWorkspaceClient() + require.NoError(t, err) + + wt := &WorkspaceT{ + T: t, + + W: w, + + ctx: context.Background(), + } + + return wt.ctx, wt +} + func (t *WorkspaceT) TestClusterID() string { clusterID := GetEnvOrSkipTest(t.T, "TEST_BRICKS_CLUSTER_ID") err := t.W.Clusters.EnsureClusterIsRunning(t.ctx, clusterID) diff --git a/internal/bundle/artifacts_test.go b/internal/bundle/artifacts_test.go index 46c236a4e9..bae8073fcb 100644 --- a/internal/bundle/artifacts_test.go +++ b/internal/bundle/artifacts_test.go @@ -8,9 +8,9 @@ import ( "testing" "github.com/databricks/cli/bundle" - "github.com/databricks/cli/bundle/artifacts" "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/bundle/libraries" "github.com/databricks/cli/internal" "github.com/databricks/cli/internal/acc" "github.com/databricks/databricks-sdk-go/service/compute" @@ -74,7 +74,7 @@ func TestAccUploadArtifactFileToCorrectRemotePath(t *testing.T) { }, } - diags := bundle.Apply(ctx, b, artifacts.BasicUpload("test")) + diags := bundle.Apply(ctx, b, bundle.Seq(libraries.ExpandGlobReferences(), libraries.Upload())) require.NoError(t, diags.Error()) // The remote path attribute on the artifact file should have been set. @@ -138,7 +138,7 @@ func TestAccUploadArtifactFileToCorrectRemotePathWithEnvironments(t *testing.T) }, } - diags := bundle.Apply(ctx, b, artifacts.BasicUpload("test")) + diags := bundle.Apply(ctx, b, bundle.Seq(libraries.ExpandGlobReferences(), libraries.Upload())) require.NoError(t, diags.Error()) // The remote path attribute on the artifact file should have been set. @@ -207,7 +207,7 @@ func TestAccUploadArtifactFileToCorrectRemotePathForVolumes(t *testing.T) { }, } - diags := bundle.Apply(ctx, b, artifacts.BasicUpload("test")) + diags := bundle.Apply(ctx, b, bundle.Seq(libraries.ExpandGlobReferences(), libraries.Upload())) require.NoError(t, diags.Error()) // The remote path attribute on the artifact file should have been set. diff --git a/internal/bundle/bundles/python_wheel_task/databricks_template_schema.json b/internal/bundle/bundles/python_wheel_task/databricks_template_schema.json index 0695eb2ba7..c4a74df070 100644 --- a/internal/bundle/bundles/python_wheel_task/databricks_template_schema.json +++ b/internal/bundle/bundles/python_wheel_task/databricks_template_schema.json @@ -20,6 +20,10 @@ "python_wheel_wrapper": { "type": "boolean", "description": "Whether or not to enable python wheel wrapper" + }, + "instance_pool_id": { + "type": "string", + "description": "Instance pool id for job cluster" } } } diff --git a/internal/bundle/bundles/python_wheel_task/template/databricks.yml.tmpl b/internal/bundle/bundles/python_wheel_task/template/databricks.yml.tmpl index 8729dcba50..30b0a5eaea 100644 --- a/internal/bundle/bundles/python_wheel_task/template/databricks.yml.tmpl +++ b/internal/bundle/bundles/python_wheel_task/template/databricks.yml.tmpl @@ -20,6 +20,7 @@ resources: spark_version: "{{.spark_version}}" node_type_id: "{{.node_type_id}}" data_security_mode: USER_ISOLATION + instance_pool_id: "{{.instance_pool_id}}" python_wheel_task: package_name: my_test_code entry_point: run diff --git a/internal/bundle/bundles/spark_jar_task/databricks_template_schema.json b/internal/bundle/bundles/spark_jar_task/databricks_template_schema.json index 078dff976c..1381da1ddc 100644 --- a/internal/bundle/bundles/spark_jar_task/databricks_template_schema.json +++ b/internal/bundle/bundles/spark_jar_task/databricks_template_schema.json @@ -24,6 +24,10 @@ "artifact_path": { "type": "string", "description": "Path to the remote base path for artifacts" + }, + "instance_pool_id": { + "type": "string", + "description": "Instance pool id for job cluster" } } } diff --git a/internal/bundle/bundles/spark_jar_task/template/databricks.yml.tmpl b/internal/bundle/bundles/spark_jar_task/template/databricks.yml.tmpl index 24a6d7d8a9..db451cd93b 100644 --- a/internal/bundle/bundles/spark_jar_task/template/databricks.yml.tmpl +++ b/internal/bundle/bundles/spark_jar_task/template/databricks.yml.tmpl @@ -3,7 +3,6 @@ bundle: workspace: root_path: "~/.bundle/{{.unique_id}}" - artifact_path: {{.artifact_path}} artifacts: my_java_code: @@ -22,7 +21,35 @@ resources: num_workers: 1 spark_version: "{{.spark_version}}" node_type_id: "{{.node_type_id}}" + instance_pool_id: "{{.instance_pool_id}}" spark_jar_task: main_class_name: PrintArgs libraries: - jar: ./{{.project_name}}/PrintArgs.jar + +targets: + volume: + # Override the artifact path to upload artifacts to a volume path + workspace: + artifact_path: {{.artifact_path}} + + resources: + jobs: + jar_job: + tasks: + - task_key: TestSparkJarTask + new_cluster: + + # Force cluster to run in single user mode (force it to be a UC cluster) + data_security_mode: SINGLE_USER + + workspace: + resources: + jobs: + jar_job: + tasks: + - task_key: TestSparkJarTask + new_cluster: + + # Force cluster to run in no isolation mode (force it to be a non-UC cluster) + data_security_mode: NONE diff --git a/internal/bundle/bundles/uc_schema/databricks_template_schema.json b/internal/bundle/bundles/uc_schema/databricks_template_schema.json new file mode 100644 index 0000000000..762f4470c2 --- /dev/null +++ b/internal/bundle/bundles/uc_schema/databricks_template_schema.json @@ -0,0 +1,8 @@ +{ + "properties": { + "unique_id": { + "type": "string", + "description": "Unique ID for the schema and pipeline names" + } + } +} diff --git a/internal/bundle/bundles/uc_schema/template/databricks.yml.tmpl b/internal/bundle/bundles/uc_schema/template/databricks.yml.tmpl new file mode 100644 index 0000000000..961af25e86 --- /dev/null +++ b/internal/bundle/bundles/uc_schema/template/databricks.yml.tmpl @@ -0,0 +1,19 @@ +bundle: + name: "bundle-playground" + +resources: + pipelines: + foo: + name: test-pipeline-{{.unique_id}} + libraries: + - notebook: + path: ./nb.sql + development: true + catalog: main + +include: + - "*.yml" + +targets: + development: + default: true diff --git a/internal/bundle/bundles/uc_schema/template/nb.sql b/internal/bundle/bundles/uc_schema/template/nb.sql new file mode 100644 index 0000000000..199ff50788 --- /dev/null +++ b/internal/bundle/bundles/uc_schema/template/nb.sql @@ -0,0 +1,2 @@ +-- Databricks notebook source +select 1 diff --git a/internal/bundle/bundles/uc_schema/template/schema.yml.tmpl b/internal/bundle/bundles/uc_schema/template/schema.yml.tmpl new file mode 100644 index 0000000000..50067036e6 --- /dev/null +++ b/internal/bundle/bundles/uc_schema/template/schema.yml.tmpl @@ -0,0 +1,13 @@ +resources: + schemas: + bar: + name: test-schema-{{.unique_id}} + catalog_name: main + comment: This schema was created from DABs + +targets: + development: + resources: + pipelines: + foo: + target: ${resources.schemas.bar.id} diff --git a/internal/bundle/deploy_test.go b/internal/bundle/deploy_test.go new file mode 100644 index 0000000000..3da885705d --- /dev/null +++ b/internal/bundle/deploy_test.go @@ -0,0 +1,125 @@ +package bundle + +import ( + "context" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/databricks/cli/cmd/root" + "github.com/databricks/cli/internal" + "github.com/databricks/cli/internal/acc" + "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/apierr" + "github.com/databricks/databricks-sdk-go/service/catalog" + "github.com/databricks/databricks-sdk-go/service/files" + "github.com/google/uuid" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func setupUcSchemaBundle(t *testing.T, ctx context.Context, w *databricks.WorkspaceClient, uniqueId string) string { + bundleRoot, err := initTestTemplate(t, ctx, "uc_schema", map[string]any{ + "unique_id": uniqueId, + }) + require.NoError(t, err) + + err = deployBundle(t, ctx, bundleRoot) + require.NoError(t, err) + + t.Cleanup(func() { + destroyBundle(t, ctx, bundleRoot) + }) + + // Assert the schema is created + catalogName := "main" + schemaName := "test-schema-" + uniqueId + schema, err := w.Schemas.GetByFullName(ctx, strings.Join([]string{catalogName, schemaName}, ".")) + require.NoError(t, err) + require.Equal(t, strings.Join([]string{catalogName, schemaName}, "."), schema.FullName) + require.Equal(t, "This schema was created from DABs", schema.Comment) + + // Assert the pipeline is created + pipelineName := "test-pipeline-" + uniqueId + pipeline, err := w.Pipelines.GetByName(ctx, pipelineName) + require.NoError(t, err) + require.Equal(t, pipelineName, pipeline.Name) + id := pipeline.PipelineId + + // Assert the pipeline uses the schema + i, err := w.Pipelines.GetByPipelineId(ctx, id) + require.NoError(t, err) + require.Equal(t, catalogName, i.Spec.Catalog) + require.Equal(t, strings.Join([]string{catalogName, schemaName}, "."), i.Spec.Target) + + // Create a volume in the schema, and add a file to it. This ensures that the + // schema has some data in it and deletion will fail unless the generated + // terraform configuration has force_destroy set to true. + volumeName := "test-volume-" + uniqueId + volume, err := w.Volumes.Create(ctx, catalog.CreateVolumeRequestContent{ + CatalogName: catalogName, + SchemaName: schemaName, + Name: volumeName, + VolumeType: catalog.VolumeTypeManaged, + }) + require.NoError(t, err) + require.Equal(t, volume.Name, volumeName) + + fileName := "test-file-" + uniqueId + err = w.Files.Upload(ctx, files.UploadRequest{ + Contents: io.NopCloser(strings.NewReader("Hello, world!")), + FilePath: fmt.Sprintf("/Volumes/%s/%s/%s/%s", catalogName, schemaName, volumeName, fileName), + }) + require.NoError(t, err) + + return bundleRoot +} + +func TestAccBundleDeployUcSchema(t *testing.T) { + ctx, wt := acc.UcWorkspaceTest(t) + w := wt.W + + uniqueId := uuid.New().String() + schemaName := "test-schema-" + uniqueId + catalogName := "main" + + bundleRoot := setupUcSchemaBundle(t, ctx, w, uniqueId) + + // Remove the UC schema from the resource configuration. + err := os.Remove(filepath.Join(bundleRoot, "schema.yml")) + require.NoError(t, err) + + // Redeploy the bundle + err = deployBundle(t, ctx, bundleRoot) + require.NoError(t, err) + + // Assert the schema is deleted + _, err = w.Schemas.GetByFullName(ctx, strings.Join([]string{catalogName, schemaName}, ".")) + apiErr := &apierr.APIError{} + assert.True(t, errors.As(err, &apiErr)) + assert.Equal(t, "SCHEMA_DOES_NOT_EXIST", apiErr.ErrorCode) +} + +func TestAccBundleDeployUcSchemaFailsWithoutAutoApprove(t *testing.T) { + ctx, wt := acc.UcWorkspaceTest(t) + w := wt.W + + uniqueId := uuid.New().String() + bundleRoot := setupUcSchemaBundle(t, ctx, w, uniqueId) + + // Remove the UC schema from the resource configuration. + err := os.Remove(filepath.Join(bundleRoot, "schema.yml")) + require.NoError(t, err) + + // Redeploy the bundle + t.Setenv("BUNDLE_ROOT", bundleRoot) + t.Setenv("TERM", "dumb") + c := internal.NewCobraTestRunnerWithContext(t, ctx, "bundle", "deploy", "--force-lock") + stdout, _, err := c.Run() + assert.EqualError(t, err, root.ErrAlreadyPrinted.Error()) + assert.Contains(t, stdout.String(), "the deployment requires destructive actions, but current console does not support prompting. Please specify --auto-approve if you would like to skip prompts and proceed") +} diff --git a/internal/bundle/helpers.go b/internal/bundle/helpers.go index c33c153313..03d9cff70c 100644 --- a/internal/bundle/helpers.go +++ b/internal/bundle/helpers.go @@ -12,6 +12,7 @@ import ( "github.com/databricks/cli/cmd/root" "github.com/databricks/cli/internal" "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/env" "github.com/databricks/cli/libs/flags" "github.com/databricks/cli/libs/template" "github.com/databricks/databricks-sdk-go" @@ -56,21 +57,21 @@ func writeConfigFile(t *testing.T, config map[string]any) (string, error) { } func validateBundle(t *testing.T, ctx context.Context, path string) ([]byte, error) { - t.Setenv("BUNDLE_ROOT", path) + ctx = env.Set(ctx, "BUNDLE_ROOT", path) c := internal.NewCobraTestRunnerWithContext(t, ctx, "bundle", "validate", "--output", "json") stdout, _, err := c.Run() return stdout.Bytes(), err } func deployBundle(t *testing.T, ctx context.Context, path string) error { - t.Setenv("BUNDLE_ROOT", path) - c := internal.NewCobraTestRunnerWithContext(t, ctx, "bundle", "deploy", "--force-lock") + ctx = env.Set(ctx, "BUNDLE_ROOT", path) + c := internal.NewCobraTestRunnerWithContext(t, ctx, "bundle", "deploy", "--force-lock", "--auto-approve") _, _, err := c.Run() return err } func deployBundleWithFlags(t *testing.T, ctx context.Context, path string, flags []string) error { - t.Setenv("BUNDLE_ROOT", path) + ctx = env.Set(ctx, "BUNDLE_ROOT", path) args := []string{"bundle", "deploy", "--force-lock"} args = append(args, flags...) c := internal.NewCobraTestRunnerWithContext(t, ctx, args...) @@ -79,6 +80,7 @@ func deployBundleWithFlags(t *testing.T, ctx context.Context, path string, flags } func runResource(t *testing.T, ctx context.Context, path string, key string) (string, error) { + ctx = env.Set(ctx, "BUNDLE_ROOT", path) ctx = cmdio.NewContext(ctx, cmdio.Default()) c := internal.NewCobraTestRunnerWithContext(t, ctx, "bundle", "run", key) @@ -87,6 +89,7 @@ func runResource(t *testing.T, ctx context.Context, path string, key string) (st } func runResourceWithParams(t *testing.T, ctx context.Context, path string, key string, params ...string) (string, error) { + ctx = env.Set(ctx, "BUNDLE_ROOT", path) ctx = cmdio.NewContext(ctx, cmdio.Default()) args := make([]string, 0) @@ -98,7 +101,7 @@ func runResourceWithParams(t *testing.T, ctx context.Context, path string, key s } func destroyBundle(t *testing.T, ctx context.Context, path string) error { - t.Setenv("BUNDLE_ROOT", path) + ctx = env.Set(ctx, "BUNDLE_ROOT", path) c := internal.NewCobraTestRunnerWithContext(t, ctx, "bundle", "destroy", "--auto-approve") _, _, err := c.Run() return err diff --git a/internal/bundle/python_wheel_test.go b/internal/bundle/python_wheel_test.go index bf2462920d..ed98efecd5 100644 --- a/internal/bundle/python_wheel_test.go +++ b/internal/bundle/python_wheel_test.go @@ -14,11 +14,13 @@ func runPythonWheelTest(t *testing.T, sparkVersion string, pythonWheelWrapper bo ctx, _ := acc.WorkspaceTest(t) nodeTypeId := internal.GetNodeTypeId(env.Get(ctx, "CLOUD_ENV")) + instancePoolId := env.Get(ctx, "TEST_INSTANCE_POOL_ID") bundleRoot, err := initTestTemplate(t, ctx, "python_wheel_task", map[string]any{ "node_type_id": nodeTypeId, "unique_id": uuid.New().String(), "spark_version": sparkVersion, "python_wheel_wrapper": pythonWheelWrapper, + "instance_pool_id": instancePoolId, }) require.NoError(t, err) diff --git a/internal/bundle/spark_jar_test.go b/internal/bundle/spark_jar_test.go index c981e77504..4b469617c5 100644 --- a/internal/bundle/spark_jar_test.go +++ b/internal/bundle/spark_jar_test.go @@ -1,37 +1,29 @@ package bundle import ( - "os" + "context" "testing" "github.com/databricks/cli/internal" "github.com/databricks/cli/internal/acc" + "github.com/databricks/cli/internal/testutil" + "github.com/databricks/cli/libs/env" "github.com/google/uuid" "github.com/stretchr/testify/require" ) -func runSparkJarTest(t *testing.T, sparkVersion string) { - t.Skip("Temporarily skipping the test until auth / permission issues for UC volumes are resolved.") - - env := internal.GetEnvOrSkipTest(t, "CLOUD_ENV") - t.Log(env) - - if os.Getenv("TEST_METASTORE_ID") == "" { - t.Skip("Skipping tests that require a UC Volume when metastore id is not set.") - } - - ctx, wt := acc.WorkspaceTest(t) - w := wt.W - volumePath := internal.TemporaryUcVolume(t, w) - - nodeTypeId := internal.GetNodeTypeId(env) +func runSparkJarTestCommon(t *testing.T, ctx context.Context, sparkVersion string, artifactPath string) { + cloudEnv := internal.GetEnvOrSkipTest(t, "CLOUD_ENV") + nodeTypeId := internal.GetNodeTypeId(cloudEnv) tmpDir := t.TempDir() + instancePoolId := env.Get(ctx, "TEST_INSTANCE_POOL_ID") bundleRoot, err := initTestTemplateWithBundleRoot(t, ctx, "spark_jar_task", map[string]any{ - "node_type_id": nodeTypeId, - "unique_id": uuid.New().String(), - "spark_version": sparkVersion, - "root": tmpDir, - "artifact_path": volumePath, + "node_type_id": nodeTypeId, + "unique_id": uuid.New().String(), + "spark_version": sparkVersion, + "root": tmpDir, + "artifact_path": artifactPath, + "instance_pool_id": instancePoolId, }, tmpDir) require.NoError(t, err) @@ -47,6 +39,62 @@ func runSparkJarTest(t *testing.T, sparkVersion string) { require.Contains(t, out, "Hello from Jar!") } +func runSparkJarTestFromVolume(t *testing.T, sparkVersion string) { + ctx, wt := acc.UcWorkspaceTest(t) + volumePath := internal.TemporaryUcVolume(t, wt.W) + ctx = env.Set(ctx, "DATABRICKS_BUNDLE_TARGET", "volume") + runSparkJarTestCommon(t, ctx, sparkVersion, volumePath) +} + +func runSparkJarTestFromWorkspace(t *testing.T, sparkVersion string) { + ctx, _ := acc.WorkspaceTest(t) + ctx = env.Set(ctx, "DATABRICKS_BUNDLE_TARGET", "workspace") + runSparkJarTestCommon(t, ctx, sparkVersion, "n/a") +} + func TestAccSparkJarTaskDeployAndRunOnVolumes(t *testing.T) { - runSparkJarTest(t, "14.3.x-scala2.12") + internal.GetEnvOrSkipTest(t, "CLOUD_ENV") + testutil.RequireJDK(t, context.Background(), "1.8.0") + + // Failure on earlier DBR versions: + // + // JAR installation from Volumes is supported on UC Clusters with DBR >= 13.3. + // Denied library is Jar(/Volumes/main/test-schema-ldgaklhcahlg/my-volume/.internal/PrintArgs.jar) + // + + versions := []string{ + "13.3.x-scala2.12", // 13.3 LTS (includes Apache Spark 3.4.1, Scala 2.12) + "14.3.x-scala2.12", // 14.3 LTS (includes Apache Spark 3.5.0, Scala 2.12) + "15.4.x-scala2.12", // 15.4 LTS Beta (includes Apache Spark 3.5.0, Scala 2.12) + } + + for _, version := range versions { + t.Run(version, func(t *testing.T) { + t.Parallel() + runSparkJarTestFromVolume(t, version) + }) + } +} + +func TestAccSparkJarTaskDeployAndRunOnWorkspace(t *testing.T) { + internal.GetEnvOrSkipTest(t, "CLOUD_ENV") + testutil.RequireJDK(t, context.Background(), "1.8.0") + + // Failure on earlier DBR versions: + // + // Library from /Workspace is not allowed on this cluster. + // Please switch to using DBR 14.1+ No Isolation Shared or DBR 13.1+ Shared cluster or 13.2+ Assigned cluster to use /Workspace libraries. + // + + versions := []string{ + "14.3.x-scala2.12", // 14.3 LTS (includes Apache Spark 3.5.0, Scala 2.12) + "15.4.x-scala2.12", // 15.4 LTS Beta (includes Apache Spark 3.5.0, Scala 2.12) + } + + for _, version := range versions { + t.Run(version, func(t *testing.T) { + t.Parallel() + runSparkJarTestFromWorkspace(t, version) + }) + } } diff --git a/internal/completer_test.go b/internal/completer_test.go new file mode 100644 index 0000000000..b2c9368862 --- /dev/null +++ b/internal/completer_test.go @@ -0,0 +1,27 @@ +package internal + +import ( + "context" + "fmt" + "strings" + "testing" + + _ "github.com/databricks/cli/cmd/fs" + "github.com/databricks/cli/libs/filer" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func setupCompletionFile(t *testing.T, f filer.Filer) { + err := f.Write(context.Background(), "dir1/file1.txt", strings.NewReader("abc"), filer.CreateParentDirectories) + require.NoError(t, err) +} + +func TestAccFsCompletion(t *testing.T) { + f, tmpDir := setupDbfsFiler(t) + setupCompletionFile(t, f) + + stdout, _ := RequireSuccessfulRun(t, "__complete", "fs", "ls", tmpDir+"/") + expectedOutput := fmt.Sprintf("%s/dir1/\n:2\n", tmpDir) + assert.Equal(t, expectedOutput, stdout.String()) +} diff --git a/internal/helpers.go b/internal/helpers.go index 972a2322b5..5d9aead1f3 100644 --- a/internal/helpers.go +++ b/internal/helpers.go @@ -94,10 +94,16 @@ func consumeLines(ctx context.Context, wg *sync.WaitGroup, r io.Reader) <-chan s defer wg.Done() scanner := bufio.NewScanner(r) for scanner.Scan() { + // We expect to be able to always send these lines into the channel. + // If we can't, it means the channel is full and likely there is a problem + // in either the test or the code under test. select { case <-ctx.Done(): return case ch <- scanner.Text(): + continue + default: + panic("line buffer is full") } } }() diff --git a/internal/testutil/copy.go b/internal/testutil/copy.go new file mode 100644 index 0000000000..21faece003 --- /dev/null +++ b/internal/testutil/copy.go @@ -0,0 +1,48 @@ +package testutil + +import ( + "io" + "io/fs" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +// CopyDirectory copies the contents of a directory to another directory. +// The destination directory is created if it does not exist. +func CopyDirectory(t *testing.T, src, dst string) { + err := filepath.WalkDir(src, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + + rel, err := filepath.Rel(src, path) + require.NoError(t, err) + + if d.IsDir() { + return os.MkdirAll(filepath.Join(dst, rel), 0755) + } + + // Copy the file to the temporary directory + in, err := os.Open(path) + if err != nil { + return err + } + + defer in.Close() + + out, err := os.Create(filepath.Join(dst, rel)) + if err != nil { + return err + } + + defer out.Close() + + _, err = io.Copy(out, in) + return err + }) + + require.NoError(t, err) +} diff --git a/internal/testutil/jdk.go b/internal/testutil/jdk.go new file mode 100644 index 0000000000..05bd7d6d68 --- /dev/null +++ b/internal/testutil/jdk.go @@ -0,0 +1,24 @@ +package testutil + +import ( + "bytes" + "context" + "os/exec" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +func RequireJDK(t *testing.T, ctx context.Context, version string) { + var stderr bytes.Buffer + + cmd := exec.Command("javac", "-version") + cmd.Stderr = &stderr + err := cmd.Run() + require.NoError(t, err, "Unable to run javac -version") + + // Get the first line of the output + line := strings.Split(stderr.String(), "\n")[0] + require.Contains(t, line, version, "Expected JDK version %s, got %s", version, line) +} diff --git a/libs/auth/oauth.go b/libs/auth/oauth.go index 1f3e032de9..7c1cb95768 100644 --- a/libs/auth/oauth.go +++ b/libs/auth/oauth.go @@ -105,7 +105,6 @@ func (a *PersistentAuth) Load(ctx context.Context) (*oauth2.Token, error) { } func (a *PersistentAuth) ProfileName() string { - // TODO: get profile name from interactive input if a.AccountID != "" { return fmt.Sprintf("ACCOUNT-%s", a.AccountID) } diff --git a/libs/diag/diagnostic.go b/libs/diag/diagnostic.go index 6215275512..93334c067a 100644 --- a/libs/diag/diagnostic.go +++ b/libs/diag/diagnostic.go @@ -17,13 +17,13 @@ type Diagnostic struct { // This may be multiple lines and may be nil. Detail string - // Location is a source code location associated with the diagnostic message. - // It may be zero if there is no associated location. - Location dyn.Location + // Locations are the source code locations associated with the diagnostic message. + // It may be empty if there are no associated locations. + Locations []dyn.Location - // Path is a path to the value in a configuration tree that the diagnostic is associated with. - // It may be nil if there is no associated path. - Path dyn.Path + // Paths are paths to the values in the configuration tree that the diagnostic is associated with. + // It may be nil if there are no associated paths. + Paths []dyn.Path } // Errorf creates a new error diagnostic. diff --git a/libs/dyn/convert/normalize.go b/libs/dyn/convert/normalize.go index 246c97eaf9..c80a914f14 100644 --- a/libs/dyn/convert/normalize.go +++ b/libs/dyn/convert/normalize.go @@ -65,19 +65,19 @@ func (n normalizeOptions) normalizeType(typ reflect.Type, src dyn.Value, seen [] func nullWarning(expected dyn.Kind, src dyn.Value, path dyn.Path) diag.Diagnostic { return diag.Diagnostic{ - Severity: diag.Warning, - Summary: fmt.Sprintf("expected a %s value, found null", expected), - Location: src.Location(), - Path: path, + Severity: diag.Warning, + Summary: fmt.Sprintf("expected a %s value, found null", expected), + Locations: []dyn.Location{src.Location()}, + Paths: []dyn.Path{path}, } } func typeMismatch(expected dyn.Kind, src dyn.Value, path dyn.Path) diag.Diagnostic { return diag.Diagnostic{ - Severity: diag.Warning, - Summary: fmt.Sprintf("expected %s, found %s", expected, src.Kind()), - Location: src.Location(), - Path: path, + Severity: diag.Warning, + Summary: fmt.Sprintf("expected %s, found %s", expected, src.Kind()), + Locations: []dyn.Location{src.Location()}, + Paths: []dyn.Path{path}, } } @@ -98,8 +98,9 @@ func (n normalizeOptions) normalizeStruct(typ reflect.Type, src dyn.Value, seen diags = diags.Append(diag.Diagnostic{ Severity: diag.Warning, Summary: fmt.Sprintf("unknown field: %s", pk.MustString()), - Location: pk.Location(), - Path: path, + // Show all locations the unknown field is defined at. + Locations: pk.Locations(), + Paths: []dyn.Path{path}, }) } continue @@ -320,10 +321,10 @@ func (n normalizeOptions) normalizeInt(typ reflect.Type, src dyn.Value, path dyn out = int64(src.MustFloat()) if src.MustFloat() != float64(out) { return dyn.InvalidValue, diags.Append(diag.Diagnostic{ - Severity: diag.Warning, - Summary: fmt.Sprintf(`cannot accurately represent "%g" as integer due to precision loss`, src.MustFloat()), - Location: src.Location(), - Path: path, + Severity: diag.Warning, + Summary: fmt.Sprintf(`cannot accurately represent "%g" as integer due to precision loss`, src.MustFloat()), + Locations: []dyn.Location{src.Location()}, + Paths: []dyn.Path{path}, }) } case dyn.KindString: @@ -336,10 +337,10 @@ func (n normalizeOptions) normalizeInt(typ reflect.Type, src dyn.Value, path dyn } return dyn.InvalidValue, diags.Append(diag.Diagnostic{ - Severity: diag.Warning, - Summary: fmt.Sprintf("cannot parse %q as an integer", src.MustString()), - Location: src.Location(), - Path: path, + Severity: diag.Warning, + Summary: fmt.Sprintf("cannot parse %q as an integer", src.MustString()), + Locations: []dyn.Location{src.Location()}, + Paths: []dyn.Path{path}, }) } case dyn.KindNil: @@ -363,10 +364,10 @@ func (n normalizeOptions) normalizeFloat(typ reflect.Type, src dyn.Value, path d out = float64(src.MustInt()) if src.MustInt() != int64(out) { return dyn.InvalidValue, diags.Append(diag.Diagnostic{ - Severity: diag.Warning, - Summary: fmt.Sprintf(`cannot accurately represent "%d" as floating point number due to precision loss`, src.MustInt()), - Location: src.Location(), - Path: path, + Severity: diag.Warning, + Summary: fmt.Sprintf(`cannot accurately represent "%d" as floating point number due to precision loss`, src.MustInt()), + Locations: []dyn.Location{src.Location()}, + Paths: []dyn.Path{path}, }) } case dyn.KindString: @@ -379,10 +380,10 @@ func (n normalizeOptions) normalizeFloat(typ reflect.Type, src dyn.Value, path d } return dyn.InvalidValue, diags.Append(diag.Diagnostic{ - Severity: diag.Warning, - Summary: fmt.Sprintf("cannot parse %q as a floating point number", src.MustString()), - Location: src.Location(), - Path: path, + Severity: diag.Warning, + Summary: fmt.Sprintf("cannot parse %q as a floating point number", src.MustString()), + Locations: []dyn.Location{src.Location()}, + Paths: []dyn.Path{path}, }) } case dyn.KindNil: diff --git a/libs/dyn/convert/normalize_test.go b/libs/dyn/convert/normalize_test.go index 452ed4eb1d..c2256615e9 100644 --- a/libs/dyn/convert/normalize_test.go +++ b/libs/dyn/convert/normalize_test.go @@ -40,10 +40,10 @@ func TestNormalizeStructElementDiagnostic(t *testing.T) { vout, err := Normalize(typ, vin) assert.Len(t, err, 1) assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `expected string, found map`, - Location: dyn.Location{}, - Path: dyn.NewPath(dyn.Key("bar")), + Severity: diag.Warning, + Summary: `expected string, found map`, + Locations: []dyn.Location{{}}, + Paths: []dyn.Path{dyn.NewPath(dyn.Key("bar"))}, }, err[0]) // Elements that encounter an error during normalization are dropped. @@ -58,23 +58,33 @@ func TestNormalizeStructUnknownField(t *testing.T) { } var typ Tmp - vin := dyn.V(map[string]dyn.Value{ - "foo": dyn.V("bar"), - "bar": dyn.V("baz"), - }) + + m := dyn.NewMapping() + m.Set(dyn.V("foo"), dyn.V("val-foo")) + // Set the unknown field, with location information. + m.Set(dyn.NewValue("bar", []dyn.Location{ + {File: "hello.yaml", Line: 1, Column: 1}, + {File: "world.yaml", Line: 2, Column: 2}, + }), dyn.V("var-bar")) + + vin := dyn.V(m) vout, err := Normalize(typ, vin) assert.Len(t, err, 1) assert.Equal(t, diag.Diagnostic{ Severity: diag.Warning, Summary: `unknown field: bar`, - Location: vin.Get("foo").Location(), - Path: dyn.EmptyPath, + // Assert location of the unknown field is included in the diagnostic. + Locations: []dyn.Location{ + {File: "hello.yaml", Line: 1, Column: 1}, + {File: "world.yaml", Line: 2, Column: 2}, + }, + Paths: []dyn.Path{dyn.EmptyPath}, }, err[0]) // The field that can be mapped to the struct field is retained. assert.Equal(t, map[string]any{ - "foo": "bar", + "foo": "val-foo", }, vout.AsAny()) } @@ -100,10 +110,10 @@ func TestNormalizeStructError(t *testing.T) { _, err := Normalize(typ, vin) assert.Len(t, err, 1) assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `expected map, found string`, - Location: vin.Get("foo").Location(), - Path: dyn.EmptyPath, + Severity: diag.Warning, + Summary: `expected map, found string`, + Locations: []dyn.Location{vin.Get("foo").Location()}, + Paths: []dyn.Path{dyn.EmptyPath}, }, err[0]) } @@ -245,10 +255,10 @@ func TestNormalizeStructRandomStringError(t *testing.T) { _, err := Normalize(typ, vin) assert.Len(t, err, 1) assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `expected map, found string`, - Location: vin.Location(), - Path: dyn.EmptyPath, + Severity: diag.Warning, + Summary: `expected map, found string`, + Locations: []dyn.Location{vin.Location()}, + Paths: []dyn.Path{dyn.EmptyPath}, }, err[0]) } @@ -262,10 +272,10 @@ func TestNormalizeStructIntError(t *testing.T) { _, err := Normalize(typ, vin) assert.Len(t, err, 1) assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `expected map, found int`, - Location: vin.Location(), - Path: dyn.EmptyPath, + Severity: diag.Warning, + Summary: `expected map, found int`, + Locations: []dyn.Location{vin.Location()}, + Paths: []dyn.Path{dyn.EmptyPath}, }, err[0]) } @@ -291,10 +301,10 @@ func TestNormalizeMapElementDiagnostic(t *testing.T) { vout, err := Normalize(typ, vin) assert.Len(t, err, 1) assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `expected string, found map`, - Location: dyn.Location{}, - Path: dyn.NewPath(dyn.Key("bar")), + Severity: diag.Warning, + Summary: `expected string, found map`, + Locations: []dyn.Location{{}}, + Paths: []dyn.Path{dyn.NewPath(dyn.Key("bar"))}, }, err[0]) // Elements that encounter an error during normalization are dropped. @@ -317,10 +327,10 @@ func TestNormalizeMapError(t *testing.T) { _, err := Normalize(typ, vin) assert.Len(t, err, 1) assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `expected map, found string`, - Location: vin.Location(), - Path: dyn.EmptyPath, + Severity: diag.Warning, + Summary: `expected map, found string`, + Locations: []dyn.Location{vin.Location()}, + Paths: []dyn.Path{dyn.EmptyPath}, }, err[0]) } @@ -372,10 +382,10 @@ func TestNormalizeMapRandomStringError(t *testing.T) { _, err := Normalize(typ, vin) assert.Len(t, err, 1) assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `expected map, found string`, - Location: vin.Location(), - Path: dyn.EmptyPath, + Severity: diag.Warning, + Summary: `expected map, found string`, + Locations: []dyn.Location{vin.Location()}, + Paths: []dyn.Path{dyn.EmptyPath}, }, err[0]) } @@ -385,10 +395,10 @@ func TestNormalizeMapIntError(t *testing.T) { _, err := Normalize(typ, vin) assert.Len(t, err, 1) assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `expected map, found int`, - Location: vin.Location(), - Path: dyn.EmptyPath, + Severity: diag.Warning, + Summary: `expected map, found int`, + Locations: []dyn.Location{vin.Location()}, + Paths: []dyn.Path{dyn.EmptyPath}, }, err[0]) } @@ -415,10 +425,10 @@ func TestNormalizeSliceElementDiagnostic(t *testing.T) { vout, err := Normalize(typ, vin) assert.Len(t, err, 1) assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `expected string, found map`, - Location: dyn.Location{}, - Path: dyn.NewPath(dyn.Index(2)), + Severity: diag.Warning, + Summary: `expected string, found map`, + Locations: []dyn.Location{{}}, + Paths: []dyn.Path{dyn.NewPath(dyn.Index(2))}, }, err[0]) // Elements that encounter an error during normalization are dropped. @@ -439,10 +449,10 @@ func TestNormalizeSliceError(t *testing.T) { _, err := Normalize(typ, vin) assert.Len(t, err, 1) assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `expected sequence, found string`, - Location: vin.Location(), - Path: dyn.EmptyPath, + Severity: diag.Warning, + Summary: `expected sequence, found string`, + Locations: []dyn.Location{vin.Location()}, + Paths: []dyn.Path{dyn.EmptyPath}, }, err[0]) } @@ -494,10 +504,10 @@ func TestNormalizeSliceRandomStringError(t *testing.T) { _, err := Normalize(typ, vin) assert.Len(t, err, 1) assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `expected sequence, found string`, - Location: vin.Location(), - Path: dyn.EmptyPath, + Severity: diag.Warning, + Summary: `expected sequence, found string`, + Locations: []dyn.Location{vin.Location()}, + Paths: []dyn.Path{dyn.EmptyPath}, }, err[0]) } @@ -507,10 +517,10 @@ func TestNormalizeSliceIntError(t *testing.T) { _, err := Normalize(typ, vin) assert.Len(t, err, 1) assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `expected sequence, found int`, - Location: vin.Location(), - Path: dyn.EmptyPath, + Severity: diag.Warning, + Summary: `expected sequence, found int`, + Locations: []dyn.Location{vin.Location()}, + Paths: []dyn.Path{dyn.EmptyPath}, }, err[0]) } @@ -528,10 +538,10 @@ func TestNormalizeStringNil(t *testing.T) { _, err := Normalize(&typ, vin) assert.Len(t, err, 1) assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `expected a string value, found null`, - Location: vin.Location(), - Path: dyn.EmptyPath, + Severity: diag.Warning, + Summary: `expected a string value, found null`, + Locations: []dyn.Location{vin.Location()}, + Paths: []dyn.Path{dyn.EmptyPath}, }, err[0]) } @@ -565,10 +575,10 @@ func TestNormalizeStringError(t *testing.T) { _, err := Normalize(&typ, vin) assert.Len(t, err, 1) assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `expected string, found map`, - Location: dyn.Location{}, - Path: dyn.EmptyPath, + Severity: diag.Warning, + Summary: `expected string, found map`, + Locations: []dyn.Location{{}}, + Paths: []dyn.Path{dyn.EmptyPath}, }, err[0]) } @@ -586,10 +596,10 @@ func TestNormalizeBoolNil(t *testing.T) { _, err := Normalize(&typ, vin) assert.Len(t, err, 1) assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `expected a bool value, found null`, - Location: vin.Location(), - Path: dyn.EmptyPath, + Severity: diag.Warning, + Summary: `expected a bool value, found null`, + Locations: []dyn.Location{vin.Location()}, + Paths: []dyn.Path{dyn.EmptyPath}, }, err[0]) } @@ -628,10 +638,10 @@ func TestNormalizeBoolFromStringError(t *testing.T) { _, err := Normalize(&typ, vin) assert.Len(t, err, 1) assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `expected bool, found string`, - Location: vin.Location(), - Path: dyn.EmptyPath, + Severity: diag.Warning, + Summary: `expected bool, found string`, + Locations: []dyn.Location{vin.Location()}, + Paths: []dyn.Path{dyn.EmptyPath}, }, err[0]) } @@ -641,10 +651,10 @@ func TestNormalizeBoolError(t *testing.T) { _, err := Normalize(&typ, vin) assert.Len(t, err, 1) assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `expected bool, found map`, - Location: dyn.Location{}, - Path: dyn.EmptyPath, + Severity: diag.Warning, + Summary: `expected bool, found map`, + Locations: []dyn.Location{{}}, + Paths: []dyn.Path{dyn.EmptyPath}, }, err[0]) } @@ -662,10 +672,10 @@ func TestNormalizeIntNil(t *testing.T) { _, err := Normalize(&typ, vin) assert.Len(t, err, 1) assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `expected a int value, found null`, - Location: vin.Location(), - Path: dyn.EmptyPath, + Severity: diag.Warning, + Summary: `expected a int value, found null`, + Locations: []dyn.Location{vin.Location()}, + Paths: []dyn.Path{dyn.EmptyPath}, }, err[0]) } @@ -683,10 +693,10 @@ func TestNormalizeIntFromFloatError(t *testing.T) { _, err := Normalize(&typ, vin) assert.Len(t, err, 1) assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `cannot accurately represent "1.5" as integer due to precision loss`, - Location: vin.Location(), - Path: dyn.EmptyPath, + Severity: diag.Warning, + Summary: `cannot accurately represent "1.5" as integer due to precision loss`, + Locations: []dyn.Location{vin.Location()}, + Paths: []dyn.Path{dyn.EmptyPath}, }, err[0]) } @@ -712,10 +722,10 @@ func TestNormalizeIntFromStringError(t *testing.T) { _, err := Normalize(&typ, vin) assert.Len(t, err, 1) assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `cannot parse "abc" as an integer`, - Location: vin.Location(), - Path: dyn.EmptyPath, + Severity: diag.Warning, + Summary: `cannot parse "abc" as an integer`, + Locations: []dyn.Location{vin.Location()}, + Paths: []dyn.Path{dyn.EmptyPath}, }, err[0]) } @@ -725,10 +735,10 @@ func TestNormalizeIntError(t *testing.T) { _, err := Normalize(&typ, vin) assert.Len(t, err, 1) assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `expected int, found map`, - Location: dyn.Location{}, - Path: dyn.EmptyPath, + Severity: diag.Warning, + Summary: `expected int, found map`, + Locations: []dyn.Location{{}}, + Paths: []dyn.Path{dyn.EmptyPath}, }, err[0]) } @@ -746,10 +756,10 @@ func TestNormalizeFloatNil(t *testing.T) { _, err := Normalize(&typ, vin) assert.Len(t, err, 1) assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `expected a float value, found null`, - Location: vin.Location(), - Path: dyn.EmptyPath, + Severity: diag.Warning, + Summary: `expected a float value, found null`, + Locations: []dyn.Location{vin.Location()}, + Paths: []dyn.Path{dyn.EmptyPath}, }, err[0]) } @@ -771,10 +781,10 @@ func TestNormalizeFloatFromIntError(t *testing.T) { _, err := Normalize(&typ, vin) assert.Len(t, err, 1) assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `cannot accurately represent "9007199254740993" as floating point number due to precision loss`, - Location: vin.Location(), - Path: dyn.EmptyPath, + Severity: diag.Warning, + Summary: `cannot accurately represent "9007199254740993" as floating point number due to precision loss`, + Locations: []dyn.Location{vin.Location()}, + Paths: []dyn.Path{dyn.EmptyPath}, }, err[0]) } @@ -800,10 +810,10 @@ func TestNormalizeFloatFromStringError(t *testing.T) { _, err := Normalize(&typ, vin) assert.Len(t, err, 1) assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `cannot parse "abc" as a floating point number`, - Location: vin.Location(), - Path: dyn.EmptyPath, + Severity: diag.Warning, + Summary: `cannot parse "abc" as a floating point number`, + Locations: []dyn.Location{vin.Location()}, + Paths: []dyn.Path{dyn.EmptyPath}, }, err[0]) } @@ -813,10 +823,10 @@ func TestNormalizeFloatError(t *testing.T) { _, err := Normalize(&typ, vin) assert.Len(t, err, 1) assert.Equal(t, diag.Diagnostic{ - Severity: diag.Warning, - Summary: `expected float, found map`, - Location: dyn.Location{}, - Path: dyn.EmptyPath, + Severity: diag.Warning, + Summary: `expected float, found map`, + Locations: []dyn.Location{{}}, + Paths: []dyn.Path{dyn.EmptyPath}, }, err[0]) } diff --git a/libs/dyn/convert/to_typed.go b/libs/dyn/convert/to_typed.go index 181c88cc97..839d0111ab 100644 --- a/libs/dyn/convert/to_typed.go +++ b/libs/dyn/convert/to_typed.go @@ -9,6 +9,12 @@ import ( "github.com/databricks/cli/libs/dyn/dynvar" ) +// Populate a destination typed value from a source dynamic value. +// +// At any point while walking the destination type tree using +// reflection, if this function sees an exported field with type dyn.Value it +// will populate that field with the appropriate source dynamic value. +// see PR: https://github.com/databricks/cli/pull/1010 func ToTyped(dst any, src dyn.Value) error { dstv := reflect.ValueOf(dst) diff --git a/libs/dyn/mapping.go b/libs/dyn/mapping.go index 668f57ecc4..f9f2d2e97e 100644 --- a/libs/dyn/mapping.go +++ b/libs/dyn/mapping.go @@ -46,7 +46,8 @@ func newMappingFromGoMap(vin map[string]Value) Mapping { return m } -// Pairs returns all the key-value pairs in the Mapping. +// Pairs returns all the key-value pairs in the Mapping. The pairs are sorted by +// their key in lexicographic order. func (m Mapping) Pairs() []Pair { return m.pairs } diff --git a/libs/filer/completer/completer.go b/libs/filer/completer/completer.go new file mode 100644 index 0000000000..569286ca38 --- /dev/null +++ b/libs/filer/completer/completer.go @@ -0,0 +1,95 @@ +package completer + +import ( + "context" + "path" + "path/filepath" + "strings" + + "github.com/databricks/cli/libs/filer" + "github.com/spf13/cobra" +) + +type completer struct { + ctx context.Context + + // The filer to use for completing remote or local paths. + filer filer.Filer + + // CompletePath will only return directories when onlyDirs is true. + onlyDirs bool + + // Prefix to prepend to completions. + prefix string + + // Whether the path is local or remote. If the path is local we use the `filepath` + // package for path manipulation. Otherwise we use the `path` package. + isLocalPath bool +} + +// General completer that takes a filer to complete remote paths when TAB-ing through a path. +func New(ctx context.Context, filer filer.Filer, onlyDirs bool) *completer { + return &completer{ctx: ctx, filer: filer, onlyDirs: onlyDirs, prefix: "", isLocalPath: true} +} + +func (c *completer) SetPrefix(p string) { + c.prefix = p +} + +func (c *completer) SetIsLocalPath(i bool) { + c.isLocalPath = i +} + +func (c *completer) CompletePath(p string) ([]string, cobra.ShellCompDirective, error) { + trailingSeparator := "/" + joinFunc := path.Join + + // Use filepath functions if we are in a local path. + if c.isLocalPath { + joinFunc = filepath.Join + trailingSeparator = string(filepath.Separator) + } + + // If the user is TAB-ing their way through a path and the + // path ends in a trailing slash, we should list nested directories. + // If the path is incomplete, however, then we should list adjacent + // directories. + dirPath := p + if !strings.HasSuffix(p, trailingSeparator) { + dirPath = path.Dir(p) + } + + entries, err := c.filer.ReadDir(c.ctx, dirPath) + if err != nil { + return nil, cobra.ShellCompDirectiveError, err + } + + completions := []string{} + for _, entry := range entries { + if c.onlyDirs && !entry.IsDir() { + continue + } + + // Join directory path and entry name + completion := joinFunc(dirPath, entry.Name()) + + // Prepend prefix if it has been set + if c.prefix != "" { + completion = joinFunc(c.prefix, completion) + } + + // Add trailing separator for directories. + if entry.IsDir() { + completion += trailingSeparator + } + + completions = append(completions, completion) + } + + // If the path is local, we add the dbfs:/ prefix suggestion as an option + if c.isLocalPath { + completions = append(completions, "dbfs:/") + } + + return completions, cobra.ShellCompDirectiveNoSpace, err +} diff --git a/libs/filer/completer/completer_test.go b/libs/filer/completer/completer_test.go new file mode 100644 index 0000000000..c533f0b6cf --- /dev/null +++ b/libs/filer/completer/completer_test.go @@ -0,0 +1,104 @@ +package completer + +import ( + "context" + "runtime" + "testing" + + "github.com/databricks/cli/cmd/root" + "github.com/databricks/cli/libs/filer" + "github.com/databricks/databricks-sdk-go/experimental/mocks" + "github.com/spf13/cobra" + "github.com/stretchr/testify/assert" +) + +func setupCompleter(t *testing.T, onlyDirs bool) *completer { + ctx := context.Background() + // Needed to make type context.valueCtx for mockFilerForPath + ctx = root.SetWorkspaceClient(ctx, mocks.NewMockWorkspaceClient(t).WorkspaceClient) + + fakeFiler := filer.NewFakeFiler(map[string]filer.FakeFileInfo{ + "dir": {FakeName: "root", FakeDir: true}, + "dir/dirA": {FakeDir: true}, + "dir/dirB": {FakeDir: true}, + "dir/fileA": {}, + }) + + completer := New(ctx, fakeFiler, onlyDirs) + completer.SetIsLocalPath(false) + return completer +} + +func TestFilerCompleterSetsPrefix(t *testing.T) { + completer := setupCompleter(t, true) + completer.SetPrefix("dbfs:") + completions, directive, err := completer.CompletePath("dir/") + + assert.Equal(t, []string{"dbfs:/dir/dirA/", "dbfs:/dir/dirB/"}, completions) + assert.Equal(t, cobra.ShellCompDirectiveNoSpace, directive) + assert.Nil(t, err) +} + +func TestFilerCompleterReturnsNestedDirs(t *testing.T) { + completer := setupCompleter(t, true) + completions, directive, err := completer.CompletePath("dir/") + + assert.Equal(t, []string{"dir/dirA/", "dir/dirB/"}, completions) + assert.Equal(t, cobra.ShellCompDirectiveNoSpace, directive) + assert.Nil(t, err) +} + +func TestFilerCompleterReturnsAdjacentDirs(t *testing.T) { + completer := setupCompleter(t, true) + completions, directive, err := completer.CompletePath("dir/wrong_path") + + assert.Equal(t, []string{"dir/dirA/", "dir/dirB/"}, completions) + assert.Equal(t, cobra.ShellCompDirectiveNoSpace, directive) + assert.Nil(t, err) +} + +func TestFilerCompleterReturnsNestedDirsAndFiles(t *testing.T) { + completer := setupCompleter(t, false) + completions, directive, err := completer.CompletePath("dir/") + + assert.Equal(t, []string{"dir/dirA/", "dir/dirB/", "dir/fileA"}, completions) + assert.Equal(t, cobra.ShellCompDirectiveNoSpace, directive) + assert.Nil(t, err) +} + +func TestFilerCompleterAddsDbfsPath(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip() + } + + completer := setupCompleter(t, true) + completer.SetIsLocalPath(true) + completions, directive, err := completer.CompletePath("dir/") + + assert.Equal(t, []string{"dir/dirA/", "dir/dirB/", "dbfs:/"}, completions) + assert.Equal(t, cobra.ShellCompDirectiveNoSpace, directive) + assert.Nil(t, err) +} + +func TestFilerCompleterWindowsSeparator(t *testing.T) { + if runtime.GOOS != "windows" { + t.Skip() + } + + completer := setupCompleter(t, true) + completer.SetIsLocalPath(true) + completions, directive, err := completer.CompletePath("dir/") + + assert.Equal(t, []string{"dir\\dirA\\", "dir\\dirB\\", "dbfs:/"}, completions) + assert.Equal(t, cobra.ShellCompDirectiveNoSpace, directive) + assert.Nil(t, err) +} + +func TestFilerCompleterNoCompletions(t *testing.T) { + completer := setupCompleter(t, true) + completions, directive, err := completer.CompletePath("wrong_dir/wrong_dir") + + assert.Nil(t, completions) + assert.Equal(t, cobra.ShellCompDirectiveError, directive) + assert.Error(t, err) +} diff --git a/libs/filer/fake_filer.go b/libs/filer/fake_filer.go new file mode 100644 index 0000000000..0e650ff605 --- /dev/null +++ b/libs/filer/fake_filer.go @@ -0,0 +1,134 @@ +package filer + +import ( + "context" + "fmt" + "io" + "io/fs" + "path" + "sort" + "strings" + "time" +) + +type FakeDirEntry struct { + FakeFileInfo +} + +func (entry FakeDirEntry) Type() fs.FileMode { + typ := fs.ModePerm + if entry.FakeDir { + typ |= fs.ModeDir + } + return typ +} + +func (entry FakeDirEntry) Info() (fs.FileInfo, error) { + return entry.FakeFileInfo, nil +} + +type FakeFileInfo struct { + FakeName string + FakeSize int64 + FakeDir bool + FakeMode fs.FileMode +} + +func (info FakeFileInfo) Name() string { + return info.FakeName +} + +func (info FakeFileInfo) Size() int64 { + return info.FakeSize +} + +func (info FakeFileInfo) Mode() fs.FileMode { + return info.FakeMode +} + +func (info FakeFileInfo) ModTime() time.Time { + return time.Now() +} + +func (info FakeFileInfo) IsDir() bool { + return info.FakeDir +} + +func (info FakeFileInfo) Sys() any { + return nil +} + +type FakeFiler struct { + entries map[string]FakeFileInfo +} + +func (f *FakeFiler) Write(ctx context.Context, p string, reader io.Reader, mode ...WriteMode) error { + return fmt.Errorf("not implemented") +} + +func (f *FakeFiler) Read(ctx context.Context, p string) (io.ReadCloser, error) { + _, ok := f.entries[p] + if !ok { + return nil, fs.ErrNotExist + } + + return io.NopCloser(strings.NewReader("foo")), nil +} + +func (f *FakeFiler) Delete(ctx context.Context, p string, mode ...DeleteMode) error { + return fmt.Errorf("not implemented") +} + +func (f *FakeFiler) ReadDir(ctx context.Context, p string) ([]fs.DirEntry, error) { + p = strings.TrimSuffix(p, "/") + entry, ok := f.entries[p] + if !ok { + return nil, NoSuchDirectoryError{p} + } + + if !entry.FakeDir { + return nil, fs.ErrInvalid + } + + // Find all entries contained in the specified directory `p`. + var out []fs.DirEntry + for k, v := range f.entries { + if k == p || path.Dir(k) != p { + continue + } + + out = append(out, FakeDirEntry{v}) + } + + sort.Slice(out, func(i, j int) bool { return out[i].Name() < out[j].Name() }) + return out, nil +} + +func (f *FakeFiler) Mkdir(ctx context.Context, path string) error { + return fmt.Errorf("not implemented") +} + +func (f *FakeFiler) Stat(ctx context.Context, path string) (fs.FileInfo, error) { + entry, ok := f.entries[path] + if !ok { + return nil, fs.ErrNotExist + } + + return entry, nil +} + +func NewFakeFiler(entries map[string]FakeFileInfo) *FakeFiler { + fakeFiler := &FakeFiler{ + entries: entries, + } + + for k, v := range fakeFiler.entries { + if v.FakeName != "" { + continue + } + v.FakeName = path.Base(k) + fakeFiler.entries[k] = v + } + + return fakeFiler +} diff --git a/libs/filer/fs_test.go b/libs/filer/fs_test.go index 03ed312b46..a74c10f0bb 100644 --- a/libs/filer/fs_test.go +++ b/libs/filer/fs_test.go @@ -2,124 +2,14 @@ package filer import ( "context" - "fmt" "io" "io/fs" - "path" - "sort" - "strings" "testing" - "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) -type fakeDirEntry struct { - fakeFileInfo -} - -func (entry fakeDirEntry) Type() fs.FileMode { - typ := fs.ModePerm - if entry.dir { - typ |= fs.ModeDir - } - return typ -} - -func (entry fakeDirEntry) Info() (fs.FileInfo, error) { - return entry.fakeFileInfo, nil -} - -type fakeFileInfo struct { - name string - size int64 - dir bool - mode fs.FileMode -} - -func (info fakeFileInfo) Name() string { - return info.name -} - -func (info fakeFileInfo) Size() int64 { - return info.size -} - -func (info fakeFileInfo) Mode() fs.FileMode { - return info.mode -} - -func (info fakeFileInfo) ModTime() time.Time { - return time.Now() -} - -func (info fakeFileInfo) IsDir() bool { - return info.dir -} - -func (info fakeFileInfo) Sys() any { - return nil -} - -type fakeFiler struct { - entries map[string]fakeFileInfo -} - -func (f *fakeFiler) Write(ctx context.Context, p string, reader io.Reader, mode ...WriteMode) error { - return fmt.Errorf("not implemented") -} - -func (f *fakeFiler) Read(ctx context.Context, p string) (io.ReadCloser, error) { - _, ok := f.entries[p] - if !ok { - return nil, fs.ErrNotExist - } - - return io.NopCloser(strings.NewReader("foo")), nil -} - -func (f *fakeFiler) Delete(ctx context.Context, p string, mode ...DeleteMode) error { - return fmt.Errorf("not implemented") -} - -func (f *fakeFiler) ReadDir(ctx context.Context, p string) ([]fs.DirEntry, error) { - entry, ok := f.entries[p] - if !ok { - return nil, fs.ErrNotExist - } - - if !entry.dir { - return nil, fs.ErrInvalid - } - - // Find all entries contained in the specified directory `p`. - var out []fs.DirEntry - for k, v := range f.entries { - if k == p || path.Dir(k) != p { - continue - } - - out = append(out, fakeDirEntry{v}) - } - - sort.Slice(out, func(i, j int) bool { return out[i].Name() < out[j].Name() }) - return out, nil -} - -func (f *fakeFiler) Mkdir(ctx context.Context, path string) error { - return fmt.Errorf("not implemented") -} - -func (f *fakeFiler) Stat(ctx context.Context, path string) (fs.FileInfo, error) { - entry, ok := f.entries[path] - if !ok { - return nil, fs.ErrNotExist - } - - return entry, nil -} - func TestFsImplementsFS(t *testing.T) { var _ fs.FS = &filerFS{} } @@ -145,22 +35,12 @@ func TestFsDirImplementsFsReadDirFile(t *testing.T) { } func fakeFS() fs.FS { - fakeFiler := &fakeFiler{ - entries: map[string]fakeFileInfo{ - ".": {name: "root", dir: true}, - "dirA": {dir: true}, - "dirB": {dir: true}, - "fileA": {size: 3}, - }, - } - - for k, v := range fakeFiler.entries { - if v.name != "" { - continue - } - v.name = path.Base(k) - fakeFiler.entries[k] = v - } + fakeFiler := NewFakeFiler(map[string]FakeFileInfo{ + ".": {FakeName: "root", FakeDir: true}, + "dirA": {FakeDir: true}, + "dirB": {FakeDir: true}, + "fileA": {FakeSize: 3}, + }) return NewFS(context.Background(), fakeFiler) } diff --git a/libs/fileset/glob_test.go b/libs/fileset/glob_test.go index 70b9c444b2..8418df73a2 100644 --- a/libs/fileset/glob_test.go +++ b/libs/fileset/glob_test.go @@ -20,7 +20,7 @@ func collectRelativePaths(files []File) []string { } func TestGlobFileset(t *testing.T) { - root := vfs.MustNew("../filer") + root := vfs.MustNew("./") entries, err := root.ReadDir(".") require.NoError(t, err) @@ -32,6 +32,7 @@ func TestGlobFileset(t *testing.T) { files, err := g.All() require.NoError(t, err) + // +1 as there's one folder in ../filer require.Equal(t, len(files), len(entries)) for _, f := range files { exists := slices.ContainsFunc(entries, func(de fs.DirEntry) bool { @@ -51,7 +52,7 @@ func TestGlobFileset(t *testing.T) { } func TestGlobFilesetWithRelativeRoot(t *testing.T) { - root := vfs.MustNew("../filer") + root := vfs.MustNew("../set") entries, err := root.ReadDir(".") require.NoError(t, err) diff --git a/libs/sync/sync_test.go b/libs/sync/sync_test.go index 292586e8d1..2d800f4668 100644 --- a/libs/sync/sync_test.go +++ b/libs/sync/sync_test.go @@ -2,70 +2,32 @@ package sync import ( "context" - "os" - "path/filepath" "testing" + "github.com/databricks/cli/internal/testutil" "github.com/databricks/cli/libs/fileset" "github.com/databricks/cli/libs/git" "github.com/databricks/cli/libs/vfs" "github.com/stretchr/testify/require" ) -func createFile(dir string, name string) error { - f, err := os.Create(filepath.Join(dir, name)) - if err != nil { - return err - } - - return f.Close() -} - func setupFiles(t *testing.T) string { dir := t.TempDir() - err := createFile(dir, "a.go") - require.NoError(t, err) - - err = createFile(dir, "b.go") - require.NoError(t, err) - - err = createFile(dir, "ab.go") - require.NoError(t, err) - - err = createFile(dir, "abc.go") - require.NoError(t, err) - - err = createFile(dir, "c.go") - require.NoError(t, err) - - err = createFile(dir, "d.go") - require.NoError(t, err) - - dbDir := filepath.Join(dir, ".databricks") - err = os.Mkdir(dbDir, 0755) - require.NoError(t, err) - - err = createFile(dbDir, "e.go") - require.NoError(t, err) - - testDir := filepath.Join(dir, "test") - err = os.Mkdir(testDir, 0755) - require.NoError(t, err) - - sub1 := filepath.Join(testDir, "sub1") - err = os.Mkdir(sub1, 0755) - require.NoError(t, err) - - err = createFile(sub1, "f.go") - require.NoError(t, err) - - sub2 := filepath.Join(sub1, "sub2") - err = os.Mkdir(sub2, 0755) - require.NoError(t, err) - - err = createFile(sub2, "g.go") - require.NoError(t, err) + for _, f := range []([]string){ + []string{dir, "a.go"}, + []string{dir, "b.go"}, + []string{dir, "ab.go"}, + []string{dir, "abc.go"}, + []string{dir, "c.go"}, + []string{dir, "d.go"}, + []string{dir, ".databricks", "e.go"}, + []string{dir, "test", "sub1", "f.go"}, + []string{dir, "test", "sub1", "sub2", "g.go"}, + []string{dir, "test", "sub1", "sub2", "h.txt"}, + } { + testutil.Touch(t, f...) + } return dir } @@ -97,7 +59,7 @@ func TestGetFileSet(t *testing.T) { fileList, err := s.GetFileList(ctx) require.NoError(t, err) - require.Equal(t, len(fileList), 9) + require.Equal(t, len(fileList), 10) inc, err = fileset.NewGlobSet(root, []string{}) require.NoError(t, err) @@ -115,9 +77,9 @@ func TestGetFileSet(t *testing.T) { fileList, err = s.GetFileList(ctx) require.NoError(t, err) - require.Equal(t, len(fileList), 1) + require.Equal(t, len(fileList), 2) - inc, err = fileset.NewGlobSet(root, []string{".databricks/*"}) + inc, err = fileset.NewGlobSet(root, []string{"./.databricks/*.go"}) require.NoError(t, err) excl, err = fileset.NewGlobSet(root, []string{}) @@ -133,7 +95,7 @@ func TestGetFileSet(t *testing.T) { fileList, err = s.GetFileList(ctx) require.NoError(t, err) - require.Equal(t, len(fileList), 10) + require.Equal(t, len(fileList), 11) } func TestRecursiveExclude(t *testing.T) { @@ -165,3 +127,34 @@ func TestRecursiveExclude(t *testing.T) { require.NoError(t, err) require.Equal(t, len(fileList), 7) } + +func TestNegateExclude(t *testing.T) { + ctx := context.Background() + + dir := setupFiles(t) + root := vfs.MustNew(dir) + fileSet, err := git.NewFileSet(root) + require.NoError(t, err) + + err = fileSet.EnsureValidGitIgnoreExists() + require.NoError(t, err) + + inc, err := fileset.NewGlobSet(root, []string{}) + require.NoError(t, err) + + excl, err := fileset.NewGlobSet(root, []string{"./*", "!*.txt"}) + require.NoError(t, err) + + s := &Sync{ + SyncOptions: &SyncOptions{}, + + fileSet: fileSet, + includeFileSet: inc, + excludeFileSet: excl, + } + + fileList, err := s.GetFileList(ctx) + require.NoError(t, err) + require.Equal(t, len(fileList), 1) + require.Equal(t, fileList[0].Relative, "test/sub1/sub2/h.txt") +} diff --git a/libs/template/config_test.go b/libs/template/config_test.go index 1af2e5f5ae..73b47f2891 100644 --- a/libs/template/config_test.go +++ b/libs/template/config_test.go @@ -3,59 +3,70 @@ package template import ( "context" "fmt" + "path/filepath" "testing" "text/template" - "github.com/databricks/cli/cmd/root" "github.com/databricks/cli/libs/jsonschema" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) -func testConfig(t *testing.T) *config { - c, err := newConfig(context.Background(), "./testdata/config-test-schema/test-schema.json") - require.NoError(t, err) - return c -} - func TestTemplateConfigAssignValuesFromFile(t *testing.T) { - c := testConfig(t) + testDir := "./testdata/config-assign-from-file" - err := c.assignValuesFromFile("./testdata/config-assign-from-file/config.json") - assert.NoError(t, err) + ctx := context.Background() + c, err := newConfig(ctx, filepath.Join(testDir, "schema.json")) + require.NoError(t, err) - assert.Equal(t, int64(1), c.values["int_val"]) - assert.Equal(t, float64(2), c.values["float_val"]) - assert.Equal(t, true, c.values["bool_val"]) - assert.Equal(t, "hello", c.values["string_val"]) + err = c.assignValuesFromFile(filepath.Join(testDir, "config.json")) + if assert.NoError(t, err) { + assert.Equal(t, int64(1), c.values["int_val"]) + assert.Equal(t, float64(2), c.values["float_val"]) + assert.Equal(t, true, c.values["bool_val"]) + assert.Equal(t, "hello", c.values["string_val"]) + } } -func TestTemplateConfigAssignValuesFromFileForInvalidIntegerValue(t *testing.T) { - c := testConfig(t) +func TestTemplateConfigAssignValuesFromFileDoesNotOverwriteExistingConfigs(t *testing.T) { + testDir := "./testdata/config-assign-from-file" - err := c.assignValuesFromFile("./testdata/config-assign-from-file-invalid-int/config.json") - assert.EqualError(t, err, "failed to load config from file ./testdata/config-assign-from-file-invalid-int/config.json: failed to parse property int_val: cannot convert \"abc\" to an integer") -} + ctx := context.Background() + c, err := newConfig(ctx, filepath.Join(testDir, "schema.json")) + require.NoError(t, err) -func TestTemplateConfigAssignValuesFromFileDoesNotOverwriteExistingConfigs(t *testing.T) { - c := testConfig(t) c.values = map[string]any{ "string_val": "this-is-not-overwritten", } - err := c.assignValuesFromFile("./testdata/config-assign-from-file/config.json") - assert.NoError(t, err) + err = c.assignValuesFromFile(filepath.Join(testDir, "config.json")) + if assert.NoError(t, err) { + assert.Equal(t, int64(1), c.values["int_val"]) + assert.Equal(t, float64(2), c.values["float_val"]) + assert.Equal(t, true, c.values["bool_val"]) + assert.Equal(t, "this-is-not-overwritten", c.values["string_val"]) + } +} + +func TestTemplateConfigAssignValuesFromFileForInvalidIntegerValue(t *testing.T) { + testDir := "./testdata/config-assign-from-file-invalid-int" + + ctx := context.Background() + c, err := newConfig(ctx, filepath.Join(testDir, "schema.json")) + require.NoError(t, err) - assert.Equal(t, int64(1), c.values["int_val"]) - assert.Equal(t, float64(2), c.values["float_val"]) - assert.Equal(t, true, c.values["bool_val"]) - assert.Equal(t, "this-is-not-overwritten", c.values["string_val"]) + err = c.assignValuesFromFile(filepath.Join(testDir, "config.json")) + assert.EqualError(t, err, fmt.Sprintf("failed to load config from file %s: failed to parse property int_val: cannot convert \"abc\" to an integer", filepath.Join(testDir, "config.json"))) } func TestTemplateConfigAssignValuesFromFileFiltersPropertiesNotInTheSchema(t *testing.T) { - c := testConfig(t) + testDir := "./testdata/config-assign-from-file-unknown-property" - err := c.assignValuesFromFile("./testdata/config-assign-from-file-unknown-property/config.json") + ctx := context.Background() + c, err := newConfig(ctx, filepath.Join(testDir, "schema.json")) + require.NoError(t, err) + + err = c.assignValuesFromFile(filepath.Join(testDir, "config.json")) assert.NoError(t, err) // assert only the known property is loaded @@ -63,37 +74,66 @@ func TestTemplateConfigAssignValuesFromFileFiltersPropertiesNotInTheSchema(t *te assert.Equal(t, "i am a known property", c.values["string_val"]) } -func TestTemplateConfigAssignDefaultValues(t *testing.T) { - c := testConfig(t) +func TestTemplateConfigAssignValuesFromDefaultValues(t *testing.T) { + testDir := "./testdata/config-assign-from-default-value" ctx := context.Background() - ctx = root.SetWorkspaceClient(ctx, nil) - helpers := loadHelpers(ctx) - r, err := newRenderer(ctx, nil, helpers, "./testdata/template-in-path/template", "./testdata/template-in-path/library", t.TempDir()) + c, err := newConfig(ctx, filepath.Join(testDir, "schema.json")) + require.NoError(t, err) + + r, err := newRenderer(ctx, nil, nil, "./testdata/empty/template", "./testdata/empty/library", t.TempDir()) require.NoError(t, err) err = c.assignDefaultValues(r) - assert.NoError(t, err) + if assert.NoError(t, err) { + assert.Equal(t, int64(123), c.values["int_val"]) + assert.Equal(t, float64(123), c.values["float_val"]) + assert.Equal(t, true, c.values["bool_val"]) + assert.Equal(t, "hello", c.values["string_val"]) + } +} + +func TestTemplateConfigAssignValuesFromTemplatedDefaultValues(t *testing.T) { + testDir := "./testdata/config-assign-from-templated-default-value" + + ctx := context.Background() + c, err := newConfig(ctx, filepath.Join(testDir, "schema.json")) + require.NoError(t, err) + + r, err := newRenderer(ctx, nil, nil, filepath.Join(testDir, "template/template"), filepath.Join(testDir, "template/library"), t.TempDir()) + require.NoError(t, err) - assert.Len(t, c.values, 2) - assert.Equal(t, "my_file", c.values["string_val"]) - assert.Equal(t, int64(123), c.values["int_val"]) + // Note: only the string value is templated. + // The JSON schema package doesn't allow using a string default for integer types. + err = c.assignDefaultValues(r) + if assert.NoError(t, err) { + assert.Equal(t, int64(123), c.values["int_val"]) + assert.Equal(t, float64(123), c.values["float_val"]) + assert.Equal(t, true, c.values["bool_val"]) + assert.Equal(t, "world", c.values["string_val"]) + } } func TestTemplateConfigValidateValuesDefined(t *testing.T) { - c := testConfig(t) + ctx := context.Background() + c, err := newConfig(ctx, "testdata/config-test-schema/test-schema.json") + require.NoError(t, err) + c.values = map[string]any{ "int_val": 1, "float_val": 1.0, "bool_val": false, } - err := c.validate() + err = c.validate() assert.EqualError(t, err, "validation for template input parameters failed. no value provided for required property string_val") } func TestTemplateConfigValidateTypeForValidConfig(t *testing.T) { - c := testConfig(t) + ctx := context.Background() + c, err := newConfig(ctx, "testdata/config-test-schema/test-schema.json") + require.NoError(t, err) + c.values = map[string]any{ "int_val": 1, "float_val": 1.1, @@ -101,12 +141,15 @@ func TestTemplateConfigValidateTypeForValidConfig(t *testing.T) { "string_val": "abcd", } - err := c.validate() + err = c.validate() assert.NoError(t, err) } func TestTemplateConfigValidateTypeForUnknownField(t *testing.T) { - c := testConfig(t) + ctx := context.Background() + c, err := newConfig(ctx, "testdata/config-test-schema/test-schema.json") + require.NoError(t, err) + c.values = map[string]any{ "unknown_prop": 1, "int_val": 1, @@ -115,12 +158,15 @@ func TestTemplateConfigValidateTypeForUnknownField(t *testing.T) { "string_val": "abcd", } - err := c.validate() + err = c.validate() assert.EqualError(t, err, "validation for template input parameters failed. property unknown_prop is not defined in the schema") } func TestTemplateConfigValidateTypeForInvalidType(t *testing.T) { - c := testConfig(t) + ctx := context.Background() + c, err := newConfig(ctx, "testdata/config-test-schema/test-schema.json") + require.NoError(t, err) + c.values = map[string]any{ "int_val": "this-should-be-an-int", "float_val": 1.1, @@ -128,7 +174,7 @@ func TestTemplateConfigValidateTypeForInvalidType(t *testing.T) { "string_val": "abcd", } - err := c.validate() + err = c.validate() assert.EqualError(t, err, "validation for template input parameters failed. incorrect type for property int_val: expected type integer, but value is \"this-should-be-an-int\"") } @@ -224,19 +270,6 @@ func TestTemplateEnumValidation(t *testing.T) { assert.NoError(t, c.validate()) } -func TestAssignDefaultValuesWithTemplatedDefaults(t *testing.T) { - c := testConfig(t) - ctx := context.Background() - ctx = root.SetWorkspaceClient(ctx, nil) - helpers := loadHelpers(ctx) - r, err := newRenderer(ctx, nil, helpers, "./testdata/templated-defaults/template", "./testdata/templated-defaults/library", t.TempDir()) - require.NoError(t, err) - - err = c.assignDefaultValues(r) - assert.NoError(t, err) - assert.Equal(t, "my_file", c.values["string_val"]) -} - func TestTemplateSchemaErrorsWithEmptyDescription(t *testing.T) { _, err := newConfig(context.Background(), "./testdata/config-test-schema/invalid-test-schema.json") assert.EqualError(t, err, "template property property-without-description is missing a description") diff --git a/libs/template/renderer_test.go b/libs/template/renderer_test.go index a8678a5251..92133c5fea 100644 --- a/libs/template/renderer_test.go +++ b/libs/template/renderer_test.go @@ -16,6 +16,7 @@ import ( bundleConfig "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/phases" "github.com/databricks/cli/cmd/root" + "github.com/databricks/cli/internal/testutil" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/tags" "github.com/databricks/databricks-sdk-go" @@ -655,15 +656,27 @@ func TestRendererFileTreeRendering(t *testing.T) { func TestRendererSubTemplateInPath(t *testing.T) { ctx := context.Background() ctx = root.SetWorkspaceClient(ctx, nil) - tmpDir := t.TempDir() - helpers := loadHelpers(ctx) - r, err := newRenderer(ctx, nil, helpers, "./testdata/template-in-path/template", "./testdata/template-in-path/library", tmpDir) + // Copy the template directory to a temporary directory where we can safely include a templated file path. + // These paths include characters that are forbidden in Go modules, so we can't use the testdata directory. + // Also see https://github.com/databricks/cli/pull/1671. + templateDir := t.TempDir() + testutil.CopyDirectory(t, "./testdata/template-in-path", templateDir) + + // Use a backtick-quoted string; double quotes are a reserved character for Windows paths: + // https://learn.microsoft.com/en-us/windows/win32/fileio/naming-a-file. + testutil.Touch(t, filepath.Join(templateDir, "template/{{template `dir_name`}}/{{template `file_name`}}")) + + tmpDir := t.TempDir() + r, err := newRenderer(ctx, nil, nil, filepath.Join(templateDir, "template"), filepath.Join(templateDir, "library"), tmpDir) require.NoError(t, err) err = r.walk() require.NoError(t, err) - assert.Equal(t, filepath.Join(tmpDir, "my_directory", "my_file"), r.files[0].DstPath().absPath()) - assert.Equal(t, "my_directory/my_file", r.files[0].DstPath().relPath) + if assert.Len(t, r.files, 2) { + f := r.files[1] + assert.Equal(t, filepath.Join(tmpDir, "my_directory", "my_file"), f.DstPath().absPath()) + assert.Equal(t, "my_directory/my_file", f.DstPath().relPath) + } } diff --git a/libs/template/testdata/config-assign-from-default-value/schema.json b/libs/template/testdata/config-assign-from-default-value/schema.json new file mode 100644 index 0000000000..259bb9a7f8 --- /dev/null +++ b/libs/template/testdata/config-assign-from-default-value/schema.json @@ -0,0 +1,24 @@ +{ + "properties": { + "int_val": { + "type": "integer", + "description": "This is an integer value", + "default": 123 + }, + "float_val": { + "type": "number", + "description": "This is a float value", + "default": 123 + }, + "bool_val": { + "type": "boolean", + "description": "This is a boolean value", + "default": true + }, + "string_val": { + "type": "string", + "description": "This is a string value", + "default": "hello" + } + } +} diff --git a/libs/template/testdata/config-assign-from-file-invalid-int/schema.json b/libs/template/testdata/config-assign-from-file-invalid-int/schema.json new file mode 100644 index 0000000000..80c44d6d9f --- /dev/null +++ b/libs/template/testdata/config-assign-from-file-invalid-int/schema.json @@ -0,0 +1,20 @@ +{ + "properties": { + "int_val": { + "type": "integer", + "description": "This is an integer value" + }, + "float_val": { + "type": "number", + "description": "This is a float value" + }, + "bool_val": { + "type": "boolean", + "description": "This is a boolean value" + }, + "string_val": { + "type": "string", + "description": "This is a string value" + } + } +} diff --git a/libs/template/testdata/config-assign-from-file-unknown-property/schema.json b/libs/template/testdata/config-assign-from-file-unknown-property/schema.json new file mode 100644 index 0000000000..80c44d6d9f --- /dev/null +++ b/libs/template/testdata/config-assign-from-file-unknown-property/schema.json @@ -0,0 +1,20 @@ +{ + "properties": { + "int_val": { + "type": "integer", + "description": "This is an integer value" + }, + "float_val": { + "type": "number", + "description": "This is a float value" + }, + "bool_val": { + "type": "boolean", + "description": "This is a boolean value" + }, + "string_val": { + "type": "string", + "description": "This is a string value" + } + } +} diff --git a/libs/template/testdata/config-assign-from-file/schema.json b/libs/template/testdata/config-assign-from-file/schema.json new file mode 100644 index 0000000000..80c44d6d9f --- /dev/null +++ b/libs/template/testdata/config-assign-from-file/schema.json @@ -0,0 +1,20 @@ +{ + "properties": { + "int_val": { + "type": "integer", + "description": "This is an integer value" + }, + "float_val": { + "type": "number", + "description": "This is a float value" + }, + "bool_val": { + "type": "boolean", + "description": "This is a boolean value" + }, + "string_val": { + "type": "string", + "description": "This is a string value" + } + } +} diff --git a/libs/template/testdata/config-assign-from-templated-default-value/schema.json b/libs/template/testdata/config-assign-from-templated-default-value/schema.json new file mode 100644 index 0000000000..fe664430b9 --- /dev/null +++ b/libs/template/testdata/config-assign-from-templated-default-value/schema.json @@ -0,0 +1,24 @@ +{ + "properties": { + "int_val": { + "type": "integer", + "description": "This is an integer value", + "default": 123 + }, + "float_val": { + "type": "number", + "description": "This is a float value", + "default": 123 + }, + "bool_val": { + "type": "boolean", + "description": "This is a boolean value", + "default": true + }, + "string_val": { + "type": "string", + "description": "This is a string value", + "default": "{{ template \"string_val\" }}" + } + } +} diff --git a/libs/template/testdata/config-assign-from-templated-default-value/template/library/my_funcs.tmpl b/libs/template/testdata/config-assign-from-templated-default-value/template/library/my_funcs.tmpl new file mode 100644 index 0000000000..41c50d7e58 --- /dev/null +++ b/libs/template/testdata/config-assign-from-templated-default-value/template/library/my_funcs.tmpl @@ -0,0 +1,3 @@ +{{define "string_val" -}} +world +{{- end}} diff --git a/libs/template/testdata/config-assign-from-templated-default-value/template/template/.gitkeep b/libs/template/testdata/config-assign-from-templated-default-value/template/template/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/libs/template/testdata/config-test-schema/test-schema.json b/libs/template/testdata/config-test-schema/test-schema.json index 10f8652f4c..80c44d6d9f 100644 --- a/libs/template/testdata/config-test-schema/test-schema.json +++ b/libs/template/testdata/config-test-schema/test-schema.json @@ -2,8 +2,7 @@ "properties": { "int_val": { "type": "integer", - "description": "This is an integer value", - "default": 123 + "description": "This is an integer value" }, "float_val": { "type": "number", @@ -15,8 +14,7 @@ }, "string_val": { "type": "string", - "description": "This is a string value", - "default": "{{template \"file_name\"}}" + "description": "This is a string value" } } } diff --git a/libs/template/testdata/empty/library/.gitkeep b/libs/template/testdata/empty/library/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/libs/template/testdata/empty/template/.gitkeep b/libs/template/testdata/empty/template/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/libs/template/testdata/template-in-path/template/.gitkeep b/libs/template/testdata/template-in-path/template/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/libs/template/testdata/templated-defaults/library/my_funcs.tmpl b/libs/template/testdata/templated-defaults/library/my_funcs.tmpl deleted file mode 100644 index 3415ad774c..0000000000 --- a/libs/template/testdata/templated-defaults/library/my_funcs.tmpl +++ /dev/null @@ -1,7 +0,0 @@ -{{define "dir_name" -}} -my_directory -{{- end}} - -{{define "file_name" -}} -my_file -{{- end}} diff --git a/libs/terraform/plan.go b/libs/terraform/plan.go index 22fea62063..36383cc248 100644 --- a/libs/terraform/plan.go +++ b/libs/terraform/plan.go @@ -1,13 +1,44 @@ package terraform +import "strings" + type Plan struct { // Path to the plan Path string - // Holds whether the user can consented to destruction. Either by interactive - // confirmation or by passing a command line flag - ConfirmApply bool - // If true, the plan is empty and applying it will not do anything IsEmpty bool } + +type Action struct { + // Type and name of the resource + ResourceType string `json:"resource_type"` + ResourceName string `json:"resource_name"` + + Action ActionType `json:"action"` +} + +func (a Action) String() string { + // terraform resources have the databricks_ prefix, which is not needed. + rtype := strings.TrimPrefix(a.ResourceType, "databricks_") + return strings.Join([]string{" ", string(a.Action), rtype, a.ResourceName}, " ") +} + +func (c Action) IsInplaceSupported() bool { + return false +} + +// These enum values correspond to action types defined in the tfjson library. +// "recreate" maps to the tfjson.Actions.Replace() function. +// "update" maps to tfjson.Actions.Update() and so on. source: +// https://github.com/hashicorp/terraform-json/blob/0104004301ca8e7046d089cdc2e2db2179d225be/action.go#L14 +type ActionType string + +const ( + ActionTypeCreate ActionType = "create" + ActionTypeDelete ActionType = "delete" + ActionTypeUpdate ActionType = "update" + ActionTypeNoOp ActionType = "no-op" + ActionTypeRead ActionType = "read" + ActionTypeRecreate ActionType = "recreate" +) diff --git a/main_test.go b/main_test.go index 34ecdca0f7..dea82e9b93 100644 --- a/main_test.go +++ b/main_test.go @@ -2,11 +2,14 @@ package main import ( "context" + "io/fs" + "path/filepath" "testing" "github.com/databricks/cli/cmd" "github.com/spf13/cobra" "github.com/stretchr/testify/assert" + "golang.org/x/mod/module" ) func TestCommandsDontUseUnderscoreInName(t *testing.T) { @@ -23,3 +26,25 @@ func TestCommandsDontUseUnderscoreInName(t *testing.T) { queue = append(queue[1:], cmd.Commands()...) } } + +func TestFilePath(t *testing.T) { + // To import this repository as a library, all files must match the + // file path constraints made by Go. This test ensures that all files + // in the repository have a valid file path. + // + // See https://github.com/databricks/cli/issues/1629 + // + err := filepath.WalkDir(".", func(path string, _ fs.DirEntry, err error) error { + switch path { + case ".": + return nil + case ".git": + return filepath.SkipDir + } + if assert.NoError(t, err) { + assert.NoError(t, module.CheckFilePath(filepath.ToSlash(path))) + } + return nil + }) + assert.NoError(t, err) +}