Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Make bundle JSON schema modular with $defs #1700

Merged
merged 62 commits into from
Sep 10, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
62 commits
Select commit Hold shift + click to select a range
75f252e
Improve JSON schema
shreyas-goenka Aug 20, 2024
43325fd
add some more todos:
shreyas-goenka Aug 20, 2024
6d2f882
more progress on the internal functionality
shreyas-goenka Aug 20, 2024
65f1c75
-
shreyas-goenka Aug 20, 2024
7db32fc
-
shreyas-goenka Aug 20, 2024
b899285
-
shreyas-goenka Aug 20, 2024
b023ba0
removed more tests
shreyas-goenka Aug 20, 2024
e24725d
added nested tests
shreyas-goenka Aug 20, 2024
460eeb9
more tests
shreyas-goenka Aug 20, 2024
00e5896
add test for recursive
shreyas-goenka Aug 20, 2024
0813312
Merge remote-tracking branch 'origin' into improve/json-schema
shreyas-goenka Aug 22, 2024
7c70179
add self referential tesT
shreyas-goenka Aug 22, 2024
39efb70
-
shreyas-goenka Aug 22, 2024
790731f
remove generic schema test
shreyas-goenka Aug 22, 2024
535f670
remove most of the debug helpers
shreyas-goenka Aug 22, 2024
11dfdc0
more iteration
shreyas-goenka Aug 26, 2024
870e419
from_type method is mostly complete
shreyas-goenka Aug 27, 2024
2dea889
delete the old schema code
shreyas-goenka Aug 27, 2024
d07192f
merge
shreyas-goenka Aug 27, 2024
fcdccb3
cleanu unused code
shreyas-goenka Aug 27, 2024
e7fd063
-
shreyas-goenka Aug 27, 2024
5b79747
-
shreyas-goenka Aug 27, 2024
ac60163
move gen file
shreyas-goenka Aug 27, 2024
2c30cfa
-
shreyas-goenka Aug 27, 2024
483480f
-
shreyas-goenka Aug 27, 2024
727036b
bundle schema command is OK now
shreyas-goenka Aug 27, 2024
acc4309
fix lint
shreyas-goenka Aug 27, 2024
f194a5b
skip adding patterns for string schemas
shreyas-goenka Aug 27, 2024
ad7503a
cleanup todos
shreyas-goenka Aug 27, 2024
40f4d35
cleanup todos
shreyas-goenka Aug 27, 2024
aac6687
remove more todos
shreyas-goenka Aug 27, 2024
4141f4e
-
shreyas-goenka Aug 27, 2024
cb8d6a9
self pass 1
shreyas-goenka Aug 27, 2024
578019b
-
shreyas-goenka Sep 3, 2024
176ced1
add tests for bfs assertion
shreyas-goenka Sep 4, 2024
3d5c076
jsonTags[0] -> fieldName
shreyas-goenka Sep 4, 2024
be0ad48
str bool
shreyas-goenka Sep 4, 2024
8575c47
json pointer comment
shreyas-goenka Sep 4, 2024
46585bb
type defs as map[string]any
shreyas-goenka Sep 4, 2024
6bd7ad0
clarify interface comments
shreyas-goenka Sep 4, 2024
66cbb54
return typPath from walk
shreyas-goenka Sep 4, 2024
ba3f004
schema local to generator
shreyas-goenka Sep 4, 2024
9d9c612
Merge remote-tracking branch 'origin' into improve/json-schema
shreyas-goenka Sep 4, 2024
446463b
update schema
shreyas-goenka Sep 4, 2024
bfe9cc4
create generated package
shreyas-goenka Sep 4, 2024
a64857e
-
shreyas-goenka Sep 4, 2024
42da1c9
Merge remote-tracking branch 'origin' into improve/json-schema
shreyas-goenka Sep 4, 2024
d0aa493
-
shreyas-goenka Sep 5, 2024
4379d7f
allow artifacts referencing
shreyas-goenka Sep 5, 2024
e55df7b
Merge remote-tracking branch 'origin' into improve/json-schema
shreyas-goenka Sep 5, 2024
1dd399f
new line for s
shreyas-goenka Sep 6, 2024
75a571a
slice for tags
shreyas-goenka Sep 6, 2024
8cd2631
fix test
shreyas-goenka Sep 6, 2024
c69e6d9
loop over keys
shreyas-goenka Sep 6, 2024
2d62c0c
Added testS
shreyas-goenka Sep 6, 2024
5f48b58
add test for bfs final schema
shreyas-goenka Sep 9, 2024
ab3dd7a
add comment
shreyas-goenka Sep 9, 2024
804d379
address comments
shreyas-goenka Sep 9, 2024
4c3facc
Merge remote-tracking branch 'origin' into improve/json-schema
shreyas-goenka Sep 9, 2024
d9432e5
Merge remote-tracking branch 'origin' into improve/json-schema
shreyas-goenka Sep 10, 2024
65ed301
make interpolation pattern better
shreyas-goenka Sep 10, 2024
e1a8e03
regenerate schema with interpolation patterns
shreyas-goenka Sep 10, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions .codegen.json
Original file line number Diff line number Diff line change
Expand Up @@ -11,10 +11,10 @@
"toolchain": {
"required": ["go"],
"post_generate": [
"go run ./bundle/internal/bundle/schema/main.go ./bundle/schema/docs/bundle_descriptions.json",
"go run ./bundle/internal/schema/*.go ./bundle/schema/jsonschema.json",
"echo 'bundle/internal/tf/schema/\\*.go linguist-generated=true' >> ./.gitattributes",
"echo 'go.sum linguist-generated=true' >> ./.gitattributes",
"echo 'bundle/schema/docs/bundle_descriptions.json linguist-generated=true' >> ./.gitattributes"
"echo 'bundle/schema/jsonschema.json linguist-generated=true' >> ./.gitattributes"
]
}
}
2 changes: 1 addition & 1 deletion .gitattributes
Original file line number Diff line number Diff line change
Expand Up @@ -120,4 +120,4 @@ cmd/workspace/workspace-conf/workspace-conf.go linguist-generated=true
cmd/workspace/workspace/workspace.go linguist-generated=true
bundle/internal/tf/schema/\*.go linguist-generated=true
go.sum linguist-generated=true
bundle/schema/docs/bundle_descriptions.json linguist-generated=true
bundle/schema/jsonschema.json linguist-generated=true
42 changes: 0 additions & 42 deletions bundle/internal/bundle/schema/main.go

This file was deleted.

93 changes: 93 additions & 0 deletions bundle/internal/schema/main.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,93 @@
package main

import (
"encoding/json"
"fmt"
"log"
"os"
"reflect"

"github.com/databricks/cli/bundle/config"
"github.com/databricks/cli/bundle/config/variable"
"github.com/databricks/cli/libs/jsonschema"
)

func interpolationPattern(s string) string {
return fmt.Sprintf(`\$\{(%s(\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\[[0-9]+\])*)+)\}`, s)
}

func addInterpolationPatterns(typ reflect.Type, s jsonschema.Schema) jsonschema.Schema {
if typ == reflect.TypeOf(config.Root{}) || typ == reflect.TypeOf(variable.Variable{}) {
return s
}

switch s.Type {
case jsonschema.ArrayType, jsonschema.ObjectType:
// arrays and objects can have complex variable values specified.
return jsonschema.Schema{
AnyOf: []jsonschema.Schema{
s,
{
Type: jsonschema.StringType,
Pattern: interpolationPattern("var"),
}},
}
case jsonschema.IntegerType, jsonschema.NumberType, jsonschema.BooleanType:
// primitives can have variable values, or references like ${bundle.xyz}
// or ${workspace.xyz}
return jsonschema.Schema{
AnyOf: []jsonschema.Schema{
s,
{Type: jsonschema.StringType, Pattern: interpolationPattern("resources")},
{Type: jsonschema.StringType, Pattern: interpolationPattern("bundle")},
{Type: jsonschema.StringType, Pattern: interpolationPattern("workspace")},
{Type: jsonschema.StringType, Pattern: interpolationPattern("artifacts")},
{Type: jsonschema.StringType, Pattern: interpolationPattern("var")},
},
}
default:
return s
}
}

func main() {
if len(os.Args) != 2 {
fmt.Println("Usage: go run main.go <output-file>")
os.Exit(1)
}

// Output file, where the generated JSON schema will be written to.
outputFile := os.Args[1]

// Input file, the databricks openapi spec.
inputFile := os.Getenv("DATABRICKS_OPENAPI_SPEC")
if inputFile == "" {
log.Fatal("DATABRICKS_OPENAPI_SPEC environment variable not set")
}

p, err := newParser(inputFile)
if err != nil {
log.Fatal(err)
}

// Generate the JSON schema from the bundle Go struct.
s, err := jsonschema.FromType(reflect.TypeOf(config.Root{}), []func(reflect.Type, jsonschema.Schema) jsonschema.Schema{
p.addDescriptions,
p.addEnums,
addInterpolationPatterns,
})
if err != nil {
log.Fatal(err)
}

b, err := json.MarshalIndent(s, "", " ")
if err != nil {
log.Fatal(err)
}

// Write the schema descriptions to the output file.
err = os.WriteFile(outputFile, b, 0644)
if err != nil {
log.Fatal(err)
}
}
123 changes: 123 additions & 0 deletions bundle/internal/schema/parser.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,123 @@
package main

import (
"encoding/json"
"fmt"
"os"
"path"
"reflect"
"strings"

"github.com/databricks/cli/libs/jsonschema"
)

type Components struct {
Schemas map[string]jsonschema.Schema `json:"schemas,omitempty"`
}

type Specification struct {
Components Components `json:"components"`
}

type openapiParser struct {
ref map[string]jsonschema.Schema
}

func newParser(path string) (*openapiParser, error) {
b, err := os.ReadFile(path)
if err != nil {
return nil, err
}

spec := Specification{}
err = json.Unmarshal(b, &spec)
if err != nil {
return nil, err
}

p := &openapiParser{}
p.ref = spec.Components.Schemas
return p, nil
}

// This function checks if the input type:
// 1. Is a Databricks Go SDK type.
// 2. Has a Databricks Go SDK type embedded in it.
//
// If the above conditions are met, the function returns the JSON schema
// corresponding to the Databricks Go SDK type from the OpenAPI spec.
func (p *openapiParser) findRef(typ reflect.Type) (jsonschema.Schema, bool) {
typs := []reflect.Type{typ}

// Check for embedded Databricks Go SDK types.
if typ.Kind() == reflect.Struct {
for i := 0; i < typ.NumField(); i++ {
if !typ.Field(i).Anonymous {
continue
}

// Deference current type if it's a pointer.
ctyp := typ.Field(i).Type
for ctyp.Kind() == reflect.Ptr {
ctyp = ctyp.Elem()
}

typs = append(typs, ctyp)
shreyas-goenka marked this conversation as resolved.
Show resolved Hide resolved
}
}

for _, ctyp := range typs {
// Skip if it's not a Go SDK type.
if !strings.HasPrefix(ctyp.PkgPath(), "github.com/databricks/databricks-sdk-go") {
continue
}

pkgName := path.Base(ctyp.PkgPath())
k := fmt.Sprintf("%s.%s", pkgName, ctyp.Name())

// Skip if the type is not in the openapi spec.
_, ok := p.ref[k]
if !ok {
continue
}

// Return the first Go SDK type found in the openapi spec.
return p.ref[k], true
}

return jsonschema.Schema{}, false
}

// Use the OpenAPI spec to load descriptions for the given type.
func (p *openapiParser) addDescriptions(typ reflect.Type, s jsonschema.Schema) jsonschema.Schema {
ref, ok := p.findRef(typ)
if !ok {
return s
}

s.Description = ref.Description
for k, v := range s.Properties {
if refProp, ok := ref.Properties[k]; ok {
v.Description = refProp.Description
}
}

return s
}

// Use the OpenAPI spec add enum values for the given type.
func (p *openapiParser) addEnums(typ reflect.Type, s jsonschema.Schema) jsonschema.Schema {
ref, ok := p.findRef(typ)
if !ok {
return s
}

s.Enum = append(s.Enum, ref.Enum...)
for k, v := range s.Properties {
if refProp, ok := ref.Properties[k]; ok {
v.Enum = append(v.Enum, refProp.Enum...)
}
}

return s
}
18 changes: 0 additions & 18 deletions bundle/schema/README.md

This file was deleted.

Loading
Loading