Move bundle schema update to an internal module (#1012)

## Changes

This PR:
1. Move code to load bundle JSON Schema descriptions from the OpenAPI
spec to an internal Go module
2. Remove command line flags from the `bundle schema` command. These
flags were meant for internal processes and at no point were meant for
customer use.
3. Regenerate `bundle_descriptions.json`
4. Add support for `bundle: "deprecated"`. The `environments` field is
tagged as deprecated in this PR and consequently will no longer be a
part of the bundle schema.

## Tests
Tested by regenerating the CLI against its current OpenAPI spec (as
defined in `__openapi_sha`). The `bundle_descriptions.json` in this PR
was generated from the code generator.

Manually checked that the autocompletion / descriptions from the new
bundle schema are correct.

.codegen.json

@@ -8,6 +8,12 @@
         ".codegen/cmds-account.go.tmpl": "cmd/account/cmd.go"
     },
     "toolchain": {
-        "required": ["go"]
+        "required": ["go"],
+        "post_generate": [
+            "go run ./bundle/internal/bundle/schema/main.go ./bundle/schema/docs/bundle_descriptions.json",
+            "echo 'bundle/internal/tf/schema/\\*.go linguist-generated=true' >> ./.gitattributes",
+            "echo 'go.sum linguist-generated=true' >> ./.gitattributes",
+            "echo 'bundle/schema/docs/bundle_descriptions.json linguist-generated=true' >> ./.gitattributes"
+        ]
     }
 }

.gitattributes

@@ -83,3 +83,6 @@ cmd/workspace/warehouses/warehouses.go linguist-generated=true
 cmd/workspace/workspace-bindings/workspace-bindings.go linguist-generated=true
 cmd/workspace/workspace-conf/workspace-conf.go linguist-generated=true
 cmd/workspace/workspace/workspace.go linguist-generated=true
+bundle/internal/tf/schema/\*.go linguist-generated=true
+go.sum linguist-generated=true
+bundle/schema/docs/bundle_descriptions.json linguist-generated=true

bundle/config/root.go

@@ -48,7 +48,7 @@ type Root struct {
 	Targets map[string]*Target `json:"targets,omitempty"`
 
 	// DEPRECATED. Left for backward compatibility with Targets
-	Environments map[string]*Target `json:"environments,omitempty"`
+	Environments map[string]*Target `json:"environments,omitempty" bundle:"deprecated"`
 
 	// Sync section specifies options for files synchronization
 	Sync Sync `json:"sync,omitempty"`

bundle/internal/bundle/schema/main.go

@@ -0,0 +1,42 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"log"
+	"os"
+
+	"github.com/databricks/cli/bundle/schema"
+)
+
+func main() {
+	if len(os.Args) != 2 {
+		fmt.Println("Usage: go run main.go <output-file>")
+		os.Exit(1)
+	}
+
+	// Output file, to write the generated schema descriptions to.
+	outputFile := os.Args[1]
+
+	// Input file, the databricks openapi spec.
+	inputFile := os.Getenv("DATABRICKS_OPENAPI_SPEC")
+	if inputFile == "" {
+		log.Fatal("DATABRICKS_OPENAPI_SPEC environment variable not set")
+	}
+
+	// Generate the schema descriptions.
+	docs, err := schema.UpdateBundleDescriptions(inputFile)
+	if err != nil {
+		log.Fatal(err)
+	}
+	result, err := json.MarshalIndent(docs, "", "  ")
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	// Write the schema descriptions to the output file.
+	err = os.WriteFile(outputFile, result, 0644)
+	if err != nil {
+		log.Fatal(err)
+	}
+}

bundle/schema/README.md

@@ -13,15 +13,6 @@ These descriptions are rendered in the inline documentation in an IDE
 
 ### SOP: Add schema descriptions for new fields in bundle config
 
-1. You can autogenerate empty descriptions for the new fields by running
-`databricks bundle schema --only-docs > ~/databricks/bundle/schema/docs/bundle_descriptions.json`
-2. Manually edit bundle_descriptions.json to add your descriptions
-3. Build again to embed the new `bundle_descriptions.json` into the binary (`go build`)
-4. Again run `databricks bundle schema --only-docs > ~/databricks/bundle/schema/docs/bundle_descriptions.json` to copy over any applicable descriptions to `targets`
-5. push to repo
-
-
-### SOP: Update descriptions in resources from a newer openapi spec
-
-1. Run `databricks bundle schema --only-docs --openapi PATH_TO_SPEC > ~/databricks/bundle/schema/docs/bundle_descriptions.json`
-2. push to repo
+Manually edit bundle_descriptions.json to add your descriptions. Note that the
+descriptions in `resources` block is generated from the OpenAPI spec, and thus
+any changes there will be overwritten.

bundle/schema/docs.go

@@ -23,39 +23,6 @@ type Docs struct {
 //go:embed docs/bundle_descriptions.json
 var bundleDocs []byte
 
-func BundleDocs(openapiSpecPath string) (*Docs, error) {
-	docs, err := initializeBundleDocs()
-	if err != nil {
-		return nil, err
-	}
-	if openapiSpecPath != "" {
-		openapiSpec, err := os.ReadFile(openapiSpecPath)
-		if err != nil {
-			return nil, err
-		}
-		spec := &openapi.Specification{}
-		err = json.Unmarshal(openapiSpec, spec)
-		if err != nil {
-			return nil, err
-		}
-		openapiReader := &OpenapiReader{
-			OpenapiSpec: spec,
-			Memo:        make(map[string]*jsonschema.Schema),
-		}
-		resourcesDocs, err := openapiReader.ResourcesDocs()
-		if err != nil {
-			return nil, err
-		}
-		resourceSchema, err := New(reflect.TypeOf(config.Resources{}), resourcesDocs)
-		if err != nil {
-			return nil, err
-		}
-		docs.Properties["resources"] = schemaToDocs(resourceSchema)
-	}
-	docs.refreshTargetsDocs()
-	return docs, nil
-}
-
 func (docs *Docs) refreshTargetsDocs() error {
 	targetsDocs, ok := docs.Properties["targets"]
 	if !ok || targetsDocs.AdditionalProperties == nil ||
@@ -70,21 +37,53 @@ func (docs *Docs) refreshTargetsDocs() error {
 	return nil
 }
 
-func initializeBundleDocs() (*Docs, error) {
-	// load embedded descriptions
+func LoadBundleDescriptions() (*Docs, error) {
 	embedded := Docs{}
 	err := json.Unmarshal(bundleDocs, &embedded)
+	return &embedded, err
+}
+
+func UpdateBundleDescriptions(openapiSpecPath string) (*Docs, error) {
+	embedded, err := LoadBundleDescriptions()
 	if err != nil {
 		return nil, err
 	}
-	// generate schema with the embedded descriptions
-	schema, err := New(reflect.TypeOf(config.Root{}), &embedded)
+
+	// Generate schema from the embedded descriptions, and convert it back to docs.
+	// This creates empty descriptions for any properties that were missing in the
+	// embedded descriptions.
+	schema, err := New(reflect.TypeOf(config.Root{}), embedded)
 	if err != nil {
 		return nil, err
 	}
-	// converting the schema back to docs. This creates empty descriptions
-	// for any properties that were missing in the embedded descriptions
 	docs := schemaToDocs(schema)
+
+	// Load the Databricks OpenAPI spec
+	openapiSpec, err := os.ReadFile(openapiSpecPath)
+	if err != nil {
+		return nil, err
+	}
+	spec := &openapi.Specification{}
+	err = json.Unmarshal(openapiSpec, spec)
+	if err != nil {
+		return nil, err
+	}
+	openapiReader := &OpenapiReader{
+		OpenapiSpec: spec,
+		Memo:        make(map[string]*jsonschema.Schema),
+	}
+
+	// Generate descriptions for the "resources" field
+	resourcesDocs, err := openapiReader.ResourcesDocs()
+	if err != nil {
+		return nil, err
+	}
+	resourceSchema, err := New(reflect.TypeOf(config.Resources{}), resourcesDocs)
+	if err != nil {
+		return nil, err
+	}
+	docs.Properties["resources"] = schemaToDocs(resourceSchema)
+	docs.refreshTargetsDocs()
 	return docs, nil
 }