Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add bundle summary to display URLs for deployed resources #1731

Merged
merged 23 commits into from
Oct 18, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
23 commits
Select commit Hold shift + click to select a range
869637d
Add a textual bundle summary command
lennartkats-db Aug 24, 2024
462ee2e
Merge remote-tracking branch 'databricks/main' into cp-summary-with-urls
lennartkats-db Aug 29, 2024
1325fd8
Fix typo
lennartkats-db Aug 29, 2024
44110d1
Merge branch 'main' into cp-summary-with-urls
pietern Sep 13, 2024
189d40b
Address reviewer comments
lennartkats-db Oct 5, 2024
8cd03d1
Merge branch 'cp-summary-with-urls' of github.com:lennartkats-db/cli …
lennartkats-db Oct 5, 2024
7911c67
Merge remote-tracking branch 'databricks/main' into cp-summary-with-urls
lennartkats-db Oct 5, 2024
65bad56
Fix test failure based on code from main
lennartkats-db Oct 5, 2024
ef2400f
Merge remote-tracking branch 'databricks/main' into cp-summary-with-urls
lennartkats-db Oct 11, 2024
d54f641
Fix test name
lennartkats-db Oct 11, 2024
6218539
Merge remote-tracking branch 'databricks/main' into cp-summary-with-urls
lennartkats-db Oct 11, 2024
aea4a6e
Styling fix
lennartkats-db Oct 17, 2024
2765a41
Merge remote-tracking branch 'databricks/main' into cp-summary-with-urls
lennartkats-db Oct 17, 2024
13049a5
Remove unused name field
pietern Oct 17, 2024
2c8bb75
Add singular/plural titles to config.SupportedResources()
pietern Oct 17, 2024
85bc79f
Use plural title resource header
pietern Oct 17, 2024
dccd70b
require -> assert
pietern Oct 17, 2024
b1dd60c
Include test case for clusters
pietern Oct 17, 2024
7bf2ec3
Always use ID to synthesize URLs
pietern Oct 17, 2024
7658dbd
Use net/url to build resource URLs such that proper escaping is done
pietern Oct 17, 2024
530a819
Merge branch 'main' into cp-summary-with-urls
lennartkats-db Oct 17, 2024
0cd3fcd
Update bundle/config/resources.go
pietern Oct 18, 2024
c45b058
Update bundle/config/mutator/initialize_urls_test.go
pietern Oct 18, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
65 changes: 65 additions & 0 deletions bundle/config/mutator/initialize_urls.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,65 @@
package mutator

import (
"context"
"net/url"
"strconv"
"strings"

"github.com/databricks/cli/bundle"
"github.com/databricks/cli/libs/diag"
)

type initializeURLs struct {
}

// InitializeURLs makes sure the URL field of each resource is configured.
// NOTE: since this depends on an extra API call, this mutator adds some extra
// latency. As such, it should only be used when needed.
// This URL field is used for the output of the 'bundle summary' CLI command.
lennartkats-db marked this conversation as resolved.
Show resolved Hide resolved
func InitializeURLs() bundle.Mutator {
return &initializeURLs{}
}

func (m *initializeURLs) Name() string {
return "InitializeURLs"
}

func (m *initializeURLs) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
workspaceId, err := b.WorkspaceClient().CurrentWorkspaceID(ctx)
if err != nil {
return diag.FromErr(err)
}
orgId := strconv.FormatInt(workspaceId, 10)
host := b.WorkspaceClient().Config.CanonicalHostName()
initializeForWorkspace(b, orgId, host)
return nil
}

func initializeForWorkspace(b *bundle.Bundle, orgId string, host string) error {
baseURL, err := url.Parse(host)
if err != nil {
return err
}

// Add ?o=<workspace id> only if <workspace id> wasn't in the subdomain already.
// The ?o= is needed when vanity URLs / legacy workspace URLs are used.
// If it's not needed we prefer to leave it out since these URLs are rather
// long for most terminals.
pietern marked this conversation as resolved.
Show resolved Hide resolved
//
// See https://docs.databricks.com/en/workspace/workspace-details.html for
// further reading about the '?o=' suffix.
if !strings.Contains(baseURL.Hostname(), orgId) {
values := baseURL.Query()
values.Add("o", orgId)
baseURL.RawQuery = values.Encode()
}

for _, group := range b.Config.Resources.AllResources() {
for _, r := range group.Resources {
r.InitializeURL(*baseURL)
}
}

return nil
}
130 changes: 130 additions & 0 deletions bundle/config/mutator/initialize_urls_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,130 @@
package mutator

import (
"testing"

"github.com/databricks/cli/bundle"
"github.com/databricks/cli/bundle/config"
"github.com/databricks/cli/bundle/config/resources"
"github.com/databricks/databricks-sdk-go/service/catalog"
"github.com/databricks/databricks-sdk-go/service/compute"
"github.com/databricks/databricks-sdk-go/service/jobs"
"github.com/databricks/databricks-sdk-go/service/ml"
"github.com/databricks/databricks-sdk-go/service/pipelines"
"github.com/databricks/databricks-sdk-go/service/serving"
"github.com/stretchr/testify/require"
)

func TestInitializeURLs(t *testing.T) {
b := &bundle.Bundle{
Config: config.Root{
Workspace: config.Workspace{
Host: "https://mycompany.databricks.com/",
},
Resources: config.Resources{
Jobs: map[string]*resources.Job{
"job1": {
ID: "1",
JobSettings: &jobs.JobSettings{Name: "job1"},
},
},
Pipelines: map[string]*resources.Pipeline{
"pipeline1": {
ID: "3",
PipelineSpec: &pipelines.PipelineSpec{Name: "pipeline1"},
},
},
Experiments: map[string]*resources.MlflowExperiment{
"experiment1": {
ID: "4",
Experiment: &ml.Experiment{Name: "experiment1"},
},
},
Models: map[string]*resources.MlflowModel{
"model1": {
ID: "a model uses its name for identifier",
Model: &ml.Model{Name: "a model uses its name for identifier"},
},
},
ModelServingEndpoints: map[string]*resources.ModelServingEndpoint{
"servingendpoint1": {
ID: "my_serving_endpoint",
CreateServingEndpoint: &serving.CreateServingEndpoint{
Name: "my_serving_endpoint",
},
},
},
RegisteredModels: map[string]*resources.RegisteredModel{
"registeredmodel1": {
ID: "8",
CreateRegisteredModelRequest: &catalog.CreateRegisteredModelRequest{
Name: "my_registered_model",
},
},
},
QualityMonitors: map[string]*resources.QualityMonitor{
"qualityMonitor1": {
CreateMonitor: &catalog.CreateMonitor{
TableName: "catalog.schema.qualityMonitor1",
},
},
},
Schemas: map[string]*resources.Schema{
"schema1": {
ID: "catalog.schema",
CreateSchema: &catalog.CreateSchema{
Name: "schema",
},
},
},
Clusters: map[string]*resources.Cluster{
"cluster1": {
ID: "1017-103929-vlr7jzcf",
ClusterSpec: &compute.ClusterSpec{
ClusterName: "cluster1",
},
},
},
},
},
}

expectedURLs := map[string]string{
"job1": "https://mycompany.databricks.com/jobs/1?o=123456",
"pipeline1": "https://mycompany.databricks.com/pipelines/3?o=123456",
"experiment1": "https://mycompany.databricks.com/ml/experiments/4?o=123456",
"model1": "https://mycompany.databricks.com/ml/models/a%20model%20uses%20its%20name%20for%20identifier?o=123456",
pietern marked this conversation as resolved.
Show resolved Hide resolved
"servingendpoint1": "https://mycompany.databricks.com/ml/endpoints/my_serving_endpoint?o=123456",
"registeredmodel1": "https://mycompany.databricks.com/explore/data/models/8?o=123456",
"qualityMonitor1": "https://mycompany.databricks.com/explore/data/catalog/schema/qualityMonitor1?o=123456",
"schema1": "https://mycompany.databricks.com/explore/data/catalog/schema?o=123456",
lennartkats-db marked this conversation as resolved.
Show resolved Hide resolved
"cluster1": "https://mycompany.databricks.com/compute/clusters/1017-103929-vlr7jzcf?o=123456",
}

initializeForWorkspace(b, "123456", "https://mycompany.databricks.com/")

for _, group := range b.Config.Resources.AllResources() {
for key, r := range group.Resources {
require.Equal(t, expectedURLs[key], r.GetURL(), "Unexpected URL for "+key)
}
}
}

func TestInitializeURLsWithoutOrgId(t *testing.T) {
b := &bundle.Bundle{
Config: config.Root{
Resources: config.Resources{
Jobs: map[string]*resources.Job{
"job1": {
ID: "1",
JobSettings: &jobs.JobSettings{Name: "job1"},
},
},
},
},
}

initializeForWorkspace(b, "123456", "https://adb-123456.azuredatabricks.net/")

require.Equal(t, "https://adb-123456.azuredatabricks.net/jobs/1", b.Config.Resources.Jobs["job1"].URL)
}
117 changes: 108 additions & 9 deletions bundle/config/resources.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package config
import (
"context"
"fmt"
"net/url"

"github.com/databricks/cli/bundle/config/resources"
"github.com/databricks/databricks-sdk-go"
Expand Down Expand Up @@ -30,6 +31,53 @@ type ConfigResource interface {
// Terraform equivalent name of the resource. For example "databricks_job"
// for jobs and "databricks_pipeline" for pipelines.
TerraformResourceName() string

// GetName returns the in-product name of the resource.
GetName() string

// GetURL returns the URL of the resource.
GetURL() string

// InitializeURL initializes the URL field of the resource.
InitializeURL(baseURL url.URL)
}

// ResourceGroup represents a group of resources of the same type.
// It includes a description of the resource type and a map of resources.
type ResourceGroup struct {
Description ResourceDescription
Resources map[string]ConfigResource
}

// collectResourceMap collects resources of a specific type into a ResourceGroup.
func collectResourceMap[T ConfigResource](
description ResourceDescription,
input map[string]T,
) ResourceGroup {
resources := make(map[string]ConfigResource)
for key, resource := range input {
resources[key] = resource
}
return ResourceGroup{
Description: description,
Resources: resources,
}
}

// AllResources returns all resources in the bundle grouped by their resource type.
func (r *Resources) AllResources() []ResourceGroup {
descriptions := SupportedResources()
return []ResourceGroup{
collectResourceMap(descriptions["jobs"], r.Jobs),
collectResourceMap(descriptions["pipelines"], r.Pipelines),
collectResourceMap(descriptions["models"], r.Models),
collectResourceMap(descriptions["experiments"], r.Experiments),
collectResourceMap(descriptions["model_serving_endpoints"], r.ModelServingEndpoints),
collectResourceMap(descriptions["registered_models"], r.RegisteredModels),
collectResourceMap(descriptions["quality_monitors"], r.QualityMonitors),
collectResourceMap(descriptions["schemas"], r.Schemas),
collectResourceMap(descriptions["clusters"], r.Clusters),
}
}

func (r *Resources) FindResourceByConfigKey(key string) (ConfigResource, error) {
Expand Down Expand Up @@ -61,20 +109,71 @@ func (r *Resources) FindResourceByConfigKey(key string) (ConfigResource, error)
}

type ResourceDescription struct {
// Singular and plural name when used to refer to the configuration.
SingularName string
PluralName string

// Singular and plural title when used in summaries / terminal UI.
SingularTitle string
PluralTitle string
}

// The keys of the map corresponds to the resource key in the bundle configuration.
func SupportedResources() map[string]ResourceDescription {
return map[string]ResourceDescription{
"jobs": {SingularName: "job"},
"pipelines": {SingularName: "pipeline"},
"models": {SingularName: "model"},
"experiments": {SingularName: "experiment"},
"model_serving_endpoints": {SingularName: "model_serving_endpoint"},
"registered_models": {SingularName: "registered_model"},
"quality_monitors": {SingularName: "quality_monitor"},
"schemas": {SingularName: "schema"},
"clusters": {SingularName: "cluster"},
"jobs": {
SingularName: "job",
PluralName: "jobs",
SingularTitle: "Job",
PluralTitle: "Jobs",
},
"pipelines": {
SingularName: "pipeline",
PluralName: "pipelines",
SingularTitle: "Pipeline",
PluralTitle: "Pipelines",
},
"models": {
SingularName: "model",
PluralName: "models",
SingularTitle: "Model",
PluralTitle: "Models",
},
"experiments": {
SingularName: "experiment",
PluralName: "experiments",
SingularTitle: "Experiment",
PluralTitle: "Experiments",
},
"model_serving_endpoints": {
SingularName: "model_serving_endpoint",
PluralName: "model_serving_endpoints",
SingularTitle: "Model Serving Endpoint",
PluralTitle: "Model Serving Endpoints",
},
"registered_models": {
SingularName: "registered_model",
PluralName: "registered_models",
SingularTitle: "Registered Model",
PluralTitle: "Registered Models",
},
"quality_monitors": {
SingularName: "quality_monitor",
PluralName: "quality_monitors",
SingularTitle: "Quality Monitor",
PluralTitle: "Quality Monitors",
},
"schemas": {
SingularName: "schema",
PluralName: "schemas",
SingularTitle: "Schema",
PluralTitle: "Schemas",
},
"clusters": {
SingularName: "cluster",
PluralName: "clusters",
SingularTitle: "Cluster",
PluralTitle: "Clusters",
},
}
}
19 changes: 19 additions & 0 deletions bundle/config/resources/clusters.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@ package resources

import (
"context"
"fmt"
"net/url"

"github.com/databricks/cli/libs/log"
"github.com/databricks/databricks-sdk-go"
Expand All @@ -13,6 +15,7 @@ type Cluster struct {
ID string `json:"id,omitempty" bundle:"readonly"`
Permissions []Permission `json:"permissions,omitempty"`
ModifiedStatus ModifiedStatus `json:"modified_status,omitempty" bundle:"internal"`
URL string `json:"url,omitempty" bundle:"internal"`

*compute.ClusterSpec
}
Expand All @@ -37,3 +40,19 @@ func (s *Cluster) Exists(ctx context.Context, w *databricks.WorkspaceClient, id
func (s *Cluster) TerraformResourceName() string {
return "databricks_cluster"
}

func (s *Cluster) InitializeURL(baseURL url.URL) {
if s.ID == "" {
return
}
baseURL.Path = fmt.Sprintf("compute/clusters/%s", s.ID)
s.URL = baseURL.String()
}

func (s *Cluster) GetName() string {
return s.ClusterName
}

func (s *Cluster) GetURL() string {
return s.URL
}
Loading