Make bundle JSON schema modular with $defs

.codegen.json

@@ -11,10 +11,10 @@
     "toolchain": {
         "required": ["go"],
         "post_generate": [
-            "go run ./bundle/internal/bundle/schema/main.go ./bundle/schema/docs/bundle_descriptions.json",
+            "go run ./bundle/internal/schema/*.go ./cmd/bundle/_generated/jsonschema.json",
             "echo 'bundle/internal/tf/schema/\\*.go linguist-generated=true' >> ./.gitattributes",
             "echo 'go.sum linguist-generated=true' >> ./.gitattributes",
-            "echo 'bundle/schema/docs/bundle_descriptions.json linguist-generated=true' >> ./.gitattributes"
+            "echo 'cmd/bundle/_generated linguist-generated=true' >> ./.gitattributes"
         ]
     }
 }

.gitattributes

@@ -120,4 +120,4 @@ cmd/workspace/workspace-conf/workspace-conf.go linguist-generated=true
 cmd/workspace/workspace/workspace.go linguist-generated=true
 bundle/internal/tf/schema/\*.go linguist-generated=true
 go.sum linguist-generated=true
-bundle/schema/docs/bundle_descriptions.json linguist-generated=true
+cmd/bundle/_generated linguist-generated=true

bundle/internal/schema/main.go

@@ -0,0 +1,89 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"log"
+	"os"
+	"reflect"
+
+	"github.com/databricks/cli/bundle/config"
+	"github.com/databricks/cli/bundle/config/variable"
+	"github.com/databricks/cli/libs/jsonschema"
+)
+
+func interpolationPattern(s string) string {
+	return fmt.Sprintf(`\$\{(%s(\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\[[0-9]+\])*)*(\[[0-9]+\])*)\}`, s)
+}
+
+func addInterpolationPatterns(typ reflect.Type, s jsonschema.Schema) jsonschema.Schema {
+	if typ == reflect.TypeOf(config.Root{}) || typ == reflect.TypeOf(variable.Variable{}) {
+		return s
+	}
+
+	switch s.Type {
+	case jsonschema.ArrayType, jsonschema.ObjectType:
+		// arrays and objects can have complex variable values specified.
+		return jsonschema.Schema{
+			AnyOf: []jsonschema.Schema{s, {
+				Type:    jsonschema.StringType,
+				Pattern: interpolationPattern("var"),
+			}},
+		}
+	case jsonschema.IntegerType, jsonschema.NumberType, jsonschema.BooleanType:
+		// primitives can have variable values, or references like ${bundle.xyz}
+		// or ${workspace.xyz}
+		return jsonschema.Schema{
+			AnyOf: []jsonschema.Schema{s,
+				{Type: jsonschema.StringType, Pattern: interpolationPattern("resources")},
+				{Type: jsonschema.StringType, Pattern: interpolationPattern("bundle")},
+				{Type: jsonschema.StringType, Pattern: interpolationPattern("workspace")},
+				{Type: jsonschema.StringType, Pattern: interpolationPattern("var")},
+			},
+		}
+	default:
+		return s
+	}
+}
+
+func main() {
+	if len(os.Args) != 2 {
+		fmt.Println("Usage: go run main.go <output-file>")
+		os.Exit(1)
+	}
+
+	// Output file, where the generated JSON schema will be written to.
+	outputFile := os.Args[1]
+
+	// Input file, the databricks openapi spec.
+	inputFile := os.Getenv("DATABRICKS_OPENAPI_SPEC")
+	if inputFile == "" {
+		log.Fatal("DATABRICKS_OPENAPI_SPEC environment variable not set")
+	}
+
+	p, err := newParser(inputFile)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	// Generate the JSON schema from the bundle Go struct.
+	s, err := jsonschema.FromType(reflect.TypeOf(config.Root{}), []func(reflect.Type, jsonschema.Schema) jsonschema.Schema{
+		p.addDescriptions,
+		p.addEnums,
+		addInterpolationPatterns,
+	})
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	b, err := json.MarshalIndent(s, "", "  ")
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	// Write the schema descriptions to the output file.
+	err = os.WriteFile(outputFile, b, 0644)
+	if err != nil {
+		log.Fatal(err)
+	}
+}

bundle/internal/schema/parser.go

@@ -0,0 +1,123 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"path"
+	"reflect"
+	"strings"
+
+	"github.com/databricks/cli/libs/jsonschema"
+)
+
+type Components struct {
+	Schemas map[string]jsonschema.Schema `json:"schemas,omitempty"`
+}
+
+type Specification struct {
+	Components Components `json:"components"`
+}
+
+type openapiParser struct {
+	ref map[string]jsonschema.Schema
+}
+
+func newParser(path string) (*openapiParser, error) {
+	b, err := os.ReadFile(path)
+	if err != nil {
+		return nil, err
+	}
+
+	spec := Specification{}
+	err = json.Unmarshal(b, &spec)
+	if err != nil {
+		return nil, err
+	}
+
+	p := &openapiParser{}
+	p.ref = spec.Components.Schemas
+	return p, nil
+}
+
+// This function checks if the input type:
+// 1. Is a Databricks Go SDK type.
+// 2. Has a Databricks Go SDK type embedded in it.
+//
+// If the above conditions are met, the function returns the JSON schema
+// corresponding to the Databricks Go SDK type from the OpenAPI spec.
+func (p *openapiParser) findRef(typ reflect.Type) (jsonschema.Schema, bool) {
+	typs := []reflect.Type{typ}
+
+	// Check for embedded Databricks Go SDK types.
+	if typ.Kind() == reflect.Struct {
+		for i := 0; i < typ.NumField(); i++ {
+			if !typ.Field(i).Anonymous {
+				continue
+			}
+
+			// Deference current type if it's a pointer.
+			ctyp := typ.Field(i).Type
+			for ctyp.Kind() == reflect.Ptr {
+				ctyp = ctyp.Elem()
+			}
+
+			typs = append(typs, ctyp)
+		}
+	}
+
+	for _, ctyp := range typs {
+		// Skip if it's not a Go SDK type.
+		if !strings.HasPrefix(ctyp.PkgPath(), "github.com/databricks/databricks-sdk-go") {
+			continue
+		}
+
+		pkgName := path.Base(ctyp.PkgPath())
+		k := fmt.Sprintf("%s.%s", pkgName, ctyp.Name())
+
+		// Skip if the type is not in the openapi spec.
+		_, ok := p.ref[k]
+		if !ok {
+			continue
+		}
+
+		// Return the first Go SDK type found in the openapi spec.
+		return p.ref[k], true
+	}
+
+	return jsonschema.Schema{}, false
+}
+
+// Use the OpenAPI spec to load descriptions for the given type.
+func (p *openapiParser) addDescriptions(typ reflect.Type, s jsonschema.Schema) jsonschema.Schema {
+	ref, ok := p.findRef(typ)
+	if !ok {
+		return s
+	}
+
+	s.Description = ref.Description
+	for k, v := range s.Properties {
+		if refProp, ok := ref.Properties[k]; ok {
+			v.Description = refProp.Description
+		}
+	}
+
+	return s
+}
+
+// Use the OpenAPI spec add enum values for the given type.
+func (p *openapiParser) addEnums(typ reflect.Type, s jsonschema.Schema) jsonschema.Schema {
+	ref, ok := p.findRef(typ)
+	if !ok {
+		return s
+	}
+
+	s.Enum = append(s.Enum, ref.Enum...)
+	for k, v := range s.Properties {
+		if refProp, ok := ref.Properties[k]; ok {
+			v.Enum = append(v.Enum, refProp.Enum...)
+		}
+	}
+
+	return s
+}