From 75f252e51c7ece6f7a9d65c674ec6dbc5a8ead47 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 20 Aug 2024 15:34:49 +0200 Subject: [PATCH 01/62] Improve JSON schema --- bundle/schema/schema.go | 446 ++++++++++++++++++----------------- libs/dyn/path.go | 4 +- libs/dyn/visit.go | 8 +- libs/dyn/visit_set.go | 4 +- libs/jsonschema/from_type.go | 204 ++++++++++++++++ libs/jsonschema/schema.go | 16 +- 6 files changed, 459 insertions(+), 223 deletions(-) create mode 100644 libs/jsonschema/from_type.go diff --git a/bundle/schema/schema.go b/bundle/schema/schema.go index ac0b4f2ec3..d5c6c1f616 100644 --- a/bundle/schema/schema.go +++ b/bundle/schema/schema.go @@ -1,26 +1,23 @@ package schema import ( - "container/list" - "fmt" "reflect" - "strings" "github.com/databricks/cli/libs/dyn/dynvar" "github.com/databricks/cli/libs/jsonschema" ) -// Fields tagged "readonly" should not be emitted in the schema as they are -// computed at runtime, and should not be assigned a value by the bundle author. -const readonlyTag = "readonly" +// // Fields tagged "readonly" should not be emitted in the schema as they are +// // computed at runtime, and should not be assigned a value by the bundle author. +// const readonlyTag = "readonly" -// Annotation for internal bundle fields that should not be exposed to customers. -// Fields can be tagged as "internal" to remove them from the generated schema. -const internalTag = "internal" +// // Annotation for internal bundle fields that should not be exposed to customers. +// // Fields can be tagged as "internal" to remove them from the generated schema. +// const internalTag = "internal" -// Annotation for bundle fields that have been deprecated. -// Fields tagged as "deprecated" are removed/omitted from the generated schema. -const deprecatedTag = "deprecated" +// // Annotation for bundle fields that have been deprecated. +// // Fields tagged as "deprecated" are removed/omitted from the generated schema. +// const deprecatedTag = "deprecated" // This function translates golang types into json schema. Here is the mapping // between json schema types and golang types @@ -44,39 +41,62 @@ const deprecatedTag = "deprecated" // - []MyStruct -> {type: object, properties: {}, additionalProperties: false} // for details visit: https://json-schema.org/understanding-json-schema/reference/object.html#properties func New(golangType reflect.Type, docs *Docs) (*jsonschema.Schema, error) { - tracker := newTracker() - schema, err := safeToSchema(golangType, docs, "", tracker) + + s, err := jsonschema.FromType(golangType, jsonschema.FromTypeOptions{ + Transform: func(s jsonschema.Schema) jsonschema.Schema { + if s.Type == jsonschema.NumberType || s.Type == jsonschema.BooleanType { + s = jsonschema.Schema{ + AnyOf: []jsonschema.Schema{ + s, + { + Type: jsonschema.StringType, + // TODO: + Pattern: dynvar.VariableRegex, + }, + }, + } + } + return s + }, + }) if err != nil { - return nil, tracker.errWithTrace(err.Error(), "root") + return nil, err } - return schema, nil -} + return &s, nil -func jsonSchemaType(golangType reflect.Type) (jsonschema.Type, error) { - switch golangType.Kind() { - case reflect.Bool: - return jsonschema.BooleanType, nil - case reflect.String: - return jsonschema.StringType, nil - case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, - reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, - reflect.Float32, reflect.Float64: - - return jsonschema.NumberType, nil - case reflect.Struct: - return jsonschema.ObjectType, nil - case reflect.Map: - if golangType.Key().Kind() != reflect.String { - return jsonschema.InvalidType, fmt.Errorf("only strings map keys are valid. key type: %v", golangType.Key().Kind()) - } - return jsonschema.ObjectType, nil - case reflect.Array, reflect.Slice: - return jsonschema.ArrayType, nil - default: - return jsonschema.InvalidType, fmt.Errorf("unhandled golang type: %s", golangType) - } + // tracker := newTracker() + // schema, err := safeToSchema(golangType, docs, "", tracker) + // if err != nil { + // return nil, tracker.errWithTrace(err.Error(), "root") + // } + // return schema, nil } +// func jsonSchemaType(golangType reflect.Type) (jsonschema.Type, error) { +// switch golangType.Kind() { +// case reflect.Bool: +// return jsonschema.BooleanType, nil +// case reflect.String: +// return jsonschema.StringType, nil +// case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, +// reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, +// reflect.Float32, reflect.Float64: + +// return jsonschema.NumberType, nil +// case reflect.Struct: +// return jsonschema.ObjectType, nil +// case reflect.Map: +// if golangType.Key().Kind() != reflect.String { +// return jsonschema.InvalidType, fmt.Errorf("only strings map keys are valid. key type: %v", golangType.Key().Kind()) +// } +// return jsonschema.ObjectType, nil +// case reflect.Array, reflect.Slice: +// return jsonschema.ArrayType, nil +// default: +// return jsonschema.InvalidType, fmt.Errorf("unhandled golang type: %s", golangType) +// } +// } + // A wrapper over toSchema function to: // 1. Detect cycles in the bundle config struct. // 2. Update tracker @@ -92,196 +112,196 @@ func jsonSchemaType(golangType reflect.Type) (jsonschema.Type, error) { // like array, map or no json tags // // - tracker: Keeps track of types / traceIds seen during recursive traversal -func safeToSchema(golangType reflect.Type, docs *Docs, traceId string, tracker *tracker) (*jsonschema.Schema, error) { - // HACK to unblock CLI release (13th Feb 2024). This is temporary until proper - // support for recursive types is added to the schema generator. PR: https://github.com/databricks/cli/pull/1204 - if traceId == "for_each_task" { - return &jsonschema.Schema{ - Type: jsonschema.ObjectType, - }, nil - } +// func safeToSchema(golangType reflect.Type, docs *Docs, traceId string, tracker *tracker) (*jsonschema.Schema, error) { +// // HACK to unblock CLI release (13th Feb 2024). This is temporary until proper +// // support for recursive types is added to the schema generator. PR: https://github.com/databricks/cli/pull/1204 +// if traceId == "for_each_task" { +// return &jsonschema.Schema{ +// Type: jsonschema.ObjectType, +// }, nil +// } - // WE ERROR OUT IF THERE ARE CYCLES IN THE JSON SCHEMA - // There are mechanisms to deal with cycles though recursive identifiers in json - // schema. However if we use them, we would need to make sure we are able to detect - // cycles where two properties (directly or indirectly) pointing to each other - // - // see: https://json-schema.org/understanding-json-schema/structuring.html#recursion - // for details - if tracker.hasCycle(golangType) { - return nil, fmt.Errorf("cycle detected") - } +// // WE ERROR OUT IF THERE ARE CYCLES IN THE JSON SCHEMA +// // There are mechanisms to deal with cycles though recursive identifiers in json +// // schema. However if we use them, we would need to make sure we are able to detect +// // cycles where two properties (directly or indirectly) pointing to each other +// // +// // see: https://json-schema.org/understanding-json-schema/structuring.html#recursion +// // for details +// if tracker.hasCycle(golangType) { +// return nil, fmt.Errorf("cycle detected") +// } - tracker.push(golangType, traceId) - props, err := toSchema(golangType, docs, tracker) - if err != nil { - return nil, err - } - tracker.pop(golangType) - return props, nil -} +// tracker.push(golangType, traceId) +// props, err := toSchema(golangType, docs, tracker) +// if err != nil { +// return nil, err +// } +// tracker.pop(golangType) +// return props, nil +// } // This function returns all member fields of the provided type. // If the type has embedded (aka anonymous) fields, this function traverses // those in a breadth first manner -func getStructFields(golangType reflect.Type) []reflect.StructField { - fields := []reflect.StructField{} - bfsQueue := list.New() +// func getStructFields(golangType reflect.Type) []reflect.StructField { +// fields := []reflect.StructField{} +// bfsQueue := list.New() - for i := 0; i < golangType.NumField(); i++ { - bfsQueue.PushBack(golangType.Field(i)) - } - for bfsQueue.Len() > 0 { - front := bfsQueue.Front() - field := front.Value.(reflect.StructField) - bfsQueue.Remove(front) - - if !field.Anonymous { - fields = append(fields, field) - continue - } - - fieldType := field.Type - if fieldType.Kind() == reflect.Pointer { - fieldType = fieldType.Elem() - } - - for i := 0; i < fieldType.NumField(); i++ { - bfsQueue.PushBack(fieldType.Field(i)) - } - } - return fields -} +// for i := 0; i < golangType.NumField(); i++ { +// bfsQueue.PushBack(golangType.Field(i)) +// } +// for bfsQueue.Len() > 0 { +// front := bfsQueue.Front() +// field := front.Value.(reflect.StructField) +// bfsQueue.Remove(front) -func toSchema(golangType reflect.Type, docs *Docs, tracker *tracker) (*jsonschema.Schema, error) { - // *Struct and Struct generate identical json schemas - if golangType.Kind() == reflect.Pointer { - return safeToSchema(golangType.Elem(), docs, "", tracker) - } - if golangType.Kind() == reflect.Interface { - return &jsonschema.Schema{}, nil - } +// if !field.Anonymous { +// fields = append(fields, field) +// continue +// } - rootJavascriptType, err := jsonSchemaType(golangType) - if err != nil { - return nil, err - } - jsonSchema := &jsonschema.Schema{Type: rootJavascriptType} - - // If the type is a non-string primitive, then we allow it to be a string - // provided it's a pure variable reference (ie only a single variable reference). - if rootJavascriptType == jsonschema.BooleanType || rootJavascriptType == jsonschema.NumberType { - jsonSchema = &jsonschema.Schema{ - AnyOf: []*jsonschema.Schema{ - { - Type: rootJavascriptType, - }, - { - Type: jsonschema.StringType, - Pattern: dynvar.VariableRegex, - }, - }, - } - } +// fieldType := field.Type +// if fieldType.Kind() == reflect.Pointer { +// fieldType = fieldType.Elem() +// } - if docs != nil { - jsonSchema.Description = docs.Description - } +// for i := 0; i < fieldType.NumField(); i++ { +// bfsQueue.PushBack(fieldType.Field(i)) +// } +// } +// return fields +// } - // case array/slice - if golangType.Kind() == reflect.Array || golangType.Kind() == reflect.Slice { - elemGolangType := golangType.Elem() - elemJavascriptType, err := jsonSchemaType(elemGolangType) - if err != nil { - return nil, err - } - var childDocs *Docs - if docs != nil { - childDocs = docs.Items - } - elemProps, err := safeToSchema(elemGolangType, childDocs, "", tracker) - if err != nil { - return nil, err - } - jsonSchema.Items = &jsonschema.Schema{ - Type: elemJavascriptType, - Properties: elemProps.Properties, - AdditionalProperties: elemProps.AdditionalProperties, - Items: elemProps.Items, - Required: elemProps.Required, - } - } +// func toSchema(golangType reflect.Type, docs *Docs, tracker *tracker) (*jsonschema.Schema, error) { +// // *Struct and Struct generate identical json schemas +// if golangType.Kind() == reflect.Pointer { +// return safeToSchema(golangType.Elem(), docs, "", tracker) +// } +// if golangType.Kind() == reflect.Interface { +// return &jsonschema.Schema{}, nil +// } - // case map - if golangType.Kind() == reflect.Map { - if golangType.Key().Kind() != reflect.String { - return nil, fmt.Errorf("only string keyed maps allowed") - } - var childDocs *Docs - if docs != nil { - childDocs = docs.AdditionalProperties - } - jsonSchema.AdditionalProperties, err = safeToSchema(golangType.Elem(), childDocs, "", tracker) - if err != nil { - return nil, err - } - } +// rootJavascriptType, err := jsonSchemaType(golangType) +// if err != nil { +// return nil, err +// } +// jsonSchema := &jsonschema.Schema{Type: rootJavascriptType} - // case struct - if golangType.Kind() == reflect.Struct { - children := getStructFields(golangType) - properties := map[string]*jsonschema.Schema{} - required := []string{} - for _, child := range children { - bundleTag := child.Tag.Get("bundle") - // Fields marked as "readonly", "internal" or "deprecated" are skipped - // while generating the schema - if bundleTag == readonlyTag || bundleTag == internalTag || bundleTag == deprecatedTag { - continue - } +// // If the type is a non-string primitive, then we allow it to be a string +// // provided it's a pure variable reference (ie only a single variable reference). +// if rootJavascriptType == jsonschema.BooleanType || rootJavascriptType == jsonschema.NumberType { +// jsonSchema = &jsonschema.Schema{ +// AnyOf: []*jsonschema.Schema{ +// { +// Type: rootJavascriptType, +// }, +// { +// Type: jsonschema.StringType, +// Pattern: dynvar.VariableRegex, +// }, +// }, +// } +// } - // get child json tags - childJsonTag := strings.Split(child.Tag.Get("json"), ",") - childName := childJsonTag[0] +// if docs != nil { +// jsonSchema.Description = docs.Description +// } - // skip children that have no json tags, the first json tag is "" - // or the first json tag is "-" - if childName == "" || childName == "-" { - continue - } +// // case array/slice +// if golangType.Kind() == reflect.Array || golangType.Kind() == reflect.Slice { +// elemGolangType := golangType.Elem() +// elemJavascriptType, err := jsonSchemaType(elemGolangType) +// if err != nil { +// return nil, err +// } +// var childDocs *Docs +// if docs != nil { +// childDocs = docs.Items +// } +// elemProps, err := safeToSchema(elemGolangType, childDocs, "", tracker) +// if err != nil { +// return nil, err +// } +// jsonSchema.Items = &jsonschema.Schema{ +// Type: elemJavascriptType, +// Properties: elemProps.Properties, +// AdditionalProperties: elemProps.AdditionalProperties, +// Items: elemProps.Items, +// Required: elemProps.Required, +// } +// } - // get docs for the child if they exist - var childDocs *Docs - if docs != nil { - if val, ok := docs.Properties[childName]; ok { - childDocs = val - } - } +// // case map +// if golangType.Kind() == reflect.Map { +// if golangType.Key().Kind() != reflect.String { +// return nil, fmt.Errorf("only string keyed maps allowed") +// } +// var childDocs *Docs +// if docs != nil { +// childDocs = docs.AdditionalProperties +// } +// jsonSchema.AdditionalProperties, err = safeToSchema(golangType.Elem(), childDocs, "", tracker) +// if err != nil { +// return nil, err +// } +// } - // compute if the child is a required field. Determined by the - // presence of "omitempty" in the json tags - hasOmitEmptyTag := false - for i := 1; i < len(childJsonTag); i++ { - if childJsonTag[i] == "omitempty" { - hasOmitEmptyTag = true - } - } - if !hasOmitEmptyTag { - required = append(required, childName) - } +// // case struct +// if golangType.Kind() == reflect.Struct { +// children := getStructFields(golangType) +// properties := map[string]*jsonschema.Schema{} +// required := []string{} +// for _, child := range children { +// bundleTag := child.Tag.Get("bundle") +// // Fields marked as "readonly", "internal" or "deprecated" are skipped +// // while generating the schema +// if bundleTag == readonlyTag || bundleTag == internalTag || bundleTag == deprecatedTag { +// continue +// } - // compute Schema.Properties for the child recursively - fieldProps, err := safeToSchema(child.Type, childDocs, childName, tracker) - if err != nil { - return nil, err - } - properties[childName] = fieldProps - } +// // get child json tags +// childJsonTag := strings.Split(child.Tag.Get("json"), ",") +// childName := childJsonTag[0] - jsonSchema.AdditionalProperties = false - jsonSchema.Properties = properties - jsonSchema.Required = required - } +// // skip children that have no json tags, the first json tag is "" +// // or the first json tag is "-" +// if childName == "" || childName == "-" { +// continue +// } - return jsonSchema, nil -} +// // get docs for the child if they exist +// var childDocs *Docs +// if docs != nil { +// if val, ok := docs.Properties[childName]; ok { +// childDocs = val +// } +// } + +// // compute if the child is a required field. Determined by the +// // presence of "omitempty" in the json tags +// hasOmitEmptyTag := false +// for i := 1; i < len(childJsonTag); i++ { +// if childJsonTag[i] == "omitempty" { +// hasOmitEmptyTag = true +// } +// } +// if !hasOmitEmptyTag { +// required = append(required, childName) +// } + +// // compute Schema.Properties for the child recursively +// fieldProps, err := safeToSchema(child.Type, childDocs, childName, tracker) +// if err != nil { +// return nil, err +// } +// properties[childName] = fieldProps +// } + +// jsonSchema.AdditionalProperties = false +// jsonSchema.Properties = properties +// jsonSchema.Required = required +// } + +// return jsonSchema, nil +// } diff --git a/libs/dyn/path.go b/libs/dyn/path.go index 76377e2dce..1d0d4afabd 100644 --- a/libs/dyn/path.go +++ b/libs/dyn/path.go @@ -18,11 +18,11 @@ func (c pathComponent) Index() int { return c.index } -func (c pathComponent) isKey() bool { +func (c pathComponent) IsKey() bool { return c.key != "" } -func (c pathComponent) isIndex() bool { +func (c pathComponent) IsIndex() bool { return c.key == "" } diff --git a/libs/dyn/visit.go b/libs/dyn/visit.go index 4d3cf50142..ee30227d7d 100644 --- a/libs/dyn/visit.go +++ b/libs/dyn/visit.go @@ -14,9 +14,9 @@ type cannotTraverseNilError struct { func (e cannotTraverseNilError) Error() string { component := e.p[len(e.p)-1] switch { - case component.isKey(): + case component.IsKey(): return fmt.Sprintf("expected a map to index %q, found nil", e.p) - case component.isIndex(): + case component.IsIndex(): return fmt.Sprintf("expected a sequence to index %q, found nil", e.p) default: panic("invalid component") @@ -90,7 +90,7 @@ func (component pathComponent) visit(v Value, prefix Path, suffix Pattern, opts path := append(prefix, component) switch { - case component.isKey(): + case component.IsKey(): // Expect a map to be set if this is a key. switch v.Kind() { case KindMap: @@ -129,7 +129,7 @@ func (component pathComponent) visit(v Value, prefix Path, suffix Pattern, opts l: v.l, }, nil - case component.isIndex(): + case component.IsIndex(): // Expect a sequence to be set if this is an index. switch v.Kind() { case KindSequence: diff --git a/libs/dyn/visit_set.go b/libs/dyn/visit_set.go index b086fb8a91..cf3b3b3127 100644 --- a/libs/dyn/visit_set.go +++ b/libs/dyn/visit_set.go @@ -32,7 +32,7 @@ func SetByPath(v Value, p Path, nv Value) (Value, error) { path := append(prefix, component) switch { - case component.isKey(): + case component.IsKey(): // Expect a map to be set if this is a key. m, ok := v.AsMap() if !ok { @@ -48,7 +48,7 @@ func SetByPath(v Value, p Path, nv Value) (Value, error) { l: v.l, }, nil - case component.isIndex(): + case component.IsIndex(): // Expect a sequence to be set if this is an index. s, ok := v.AsSequence() if !ok { diff --git a/libs/jsonschema/from_type.go b/libs/jsonschema/from_type.go new file mode 100644 index 0000000000..43ce9a85a9 --- /dev/null +++ b/libs/jsonschema/from_type.go @@ -0,0 +1,204 @@ +package jsonschema + +import ( + "container/list" + "fmt" + "reflect" + "slices" + "strings" +) + +var InvalidSchema = Schema{ + Type: InvalidType, +} + +// Fields tagged "readonly" should not be emitted in the schema as they are +// computed at runtime, and should not be assigned a value by the bundle author. +const readonlyTag = "readonly" + +// Annotation for internal bundle fields that should not be exposed to customers. +// Fields can be tagged as "internal" to remove them from the generated schema. +const internalTag = "internal" + +// Annotation for bundle fields that have been deprecated. +// Fields tagged as "deprecated" are removed/omitted from the generated schema. +const deprecatedTag = "deprecated" + +// TODO: Test what happens with invalid cycles? Do integration tests fail? +// TODO: Call out in the PR description that recursive types like "for_each_task" +// are now supported. + +type FromTypeOptions struct { + // Transformation function to apply after generating the schema. + Transform func(s Schema) Schema +} + +// TODO: Skip generating schema for interface fields. +func FromType(typ reflect.Type, opts FromTypeOptions) (Schema, error) { + // Dereference pointers if necessary. + for typ.Kind() == reflect.Ptr { + typ = typ.Elem() + } + + // An interface value can never be serialized from text, and thus is explicitly + // set to null and disallowed in the schema. + if typ.Kind() == reflect.Interface { + return Schema{Type: NullType}, nil + } + + var res Schema + var err error + + // TODO: Narrow down the number of Go types handled here. + switch typ.Kind() { + case reflect.Struct: + res, err = fromTypeStruct(typ, opts) + case reflect.Slice: + res, err = fromTypeSlice(typ, opts) + case reflect.Map: + res, err = fromTypeMap(typ, opts) + // TODO: Should the primitive functions below be inlined? + case reflect.String: + res = Schema{Type: StringType} + case reflect.Bool: + res = Schema{Type: BooleanType} + // case reflect.Int, reflect.Int32, reflect.Int64: + // res = Schema{Type: IntegerType} + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, + reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, + reflect.Float32, reflect.Float64: + res = Schema{Type: NumberType} + default: + return InvalidSchema, fmt.Errorf("unsupported type: %s", typ.Kind()) + } + if err != nil { + return InvalidSchema, err + } + + if opts.Transform != nil { + res = opts.Transform(res) + } + return res, nil +} + +// This function returns all member fields of the provided type. +// If the type has embedded (aka anonymous) fields, this function traverses +// those in a breadth first manner +func getStructFields(golangType reflect.Type) []reflect.StructField { + fields := []reflect.StructField{} + bfsQueue := list.New() + + for i := 0; i < golangType.NumField(); i++ { + bfsQueue.PushBack(golangType.Field(i)) + } + for bfsQueue.Len() > 0 { + front := bfsQueue.Front() + field := front.Value.(reflect.StructField) + bfsQueue.Remove(front) + + if !field.Anonymous { + fields = append(fields, field) + continue + } + + fieldType := field.Type + if fieldType.Kind() == reflect.Pointer { + fieldType = fieldType.Elem() + } + + for i := 0; i < fieldType.NumField(); i++ { + bfsQueue.PushBack(fieldType.Field(i)) + } + } + return fields +} + +func fromTypeStruct(typ reflect.Type, opts FromTypeOptions) (Schema, error) { + if typ.Kind() != reflect.Struct { + return InvalidSchema, fmt.Errorf("expected struct, got %s", typ.Kind()) + } + + res := Schema{ + Type: ObjectType, + + Properties: make(map[string]*Schema), + + // TODO: Confirm that empty arrays are not serialized. + Required: []string{}, + + AdditionalProperties: false, + } + + structFields := getStructFields(typ) + + for _, structField := range structFields { + bundleTags := strings.Split(structField.Tag.Get("bundle"), ",") + // Fields marked as "readonly", "internal" or "deprecated" are skipped + // while generating the schema + if slices.Contains(bundleTags, readonlyTag) || + slices.Contains(bundleTags, internalTag) || + slices.Contains(bundleTags, deprecatedTag) { + continue + } + + jsonTags := strings.Split(structField.Tag.Get("json"), ",") + // Do not include fields in the schema that will not be serialized during + // JSON marshalling. + if jsonTags[0] == "" || jsonTags[0] == "-" { + continue + } + // "omitempty" tags in the Go SDK structs represent fields that not are + // required to be present in the API payload. Thus its absence in the + // tags list indicates that the field is required. + if !slices.Contains(jsonTags, "omitempty") { + res.Required = append(res.Required, jsonTags[0]) + } + + s, err := FromType(structField.Type, opts) + if err != nil { + return InvalidSchema, err + } + res.Properties[jsonTags[0]] = &s + } + + return res, nil +} + +func fromTypeSlice(typ reflect.Type, opts FromTypeOptions) (Schema, error) { + if typ.Kind() != reflect.Slice { + return InvalidSchema, fmt.Errorf("expected slice, got %s", typ.Kind()) + } + + res := Schema{ + Type: ArrayType, + } + + items, err := FromType(typ.Elem(), opts) + if err != nil { + return InvalidSchema, err + } + + res.Items = &items + return res, nil +} + +func fromTypeMap(typ reflect.Type, opts FromTypeOptions) (Schema, error) { + if typ.Kind() != reflect.Map { + return InvalidSchema, fmt.Errorf("expected map, got %s", typ.Kind()) + } + + if typ.Key().Kind() != reflect.String { + return InvalidSchema, fmt.Errorf("found map with non-string key: %v", typ.Key()) + } + + res := Schema{ + Type: ObjectType, + } + + additionalProperties, err := FromType(typ.Elem(), opts) + if err != nil { + return InvalidSchema, err + } + res.AdditionalProperties = additionalProperties + return res, nil +} diff --git a/libs/jsonschema/schema.go b/libs/jsonschema/schema.go index f1e223ec73..6c6a89c9d8 100644 --- a/libs/jsonschema/schema.go +++ b/libs/jsonschema/schema.go @@ -13,6 +13,13 @@ import ( ) // defines schema for a json object +// TODO: Remove pointers from properties and AnyOf. +// TODO: Can / should we emulate dyn.V here in having a readonly model for the data +// structure? Makes it easier to reason about. +// +// Any performance issues can be addressed by storing the schema +// +// as an embedded file. type Schema struct { // Type of the object Type Type `json:"type,omitempty"` @@ -23,6 +30,7 @@ type Schema struct { // Expected value for the JSON object. The object value must be equal to this // field if it's specified in the schema. + // TODO: Generics here? OR maybe a type from the reflection package. Const any `json:"const,omitempty"` // Schemas for the fields of an struct. The keys are the first json tag. @@ -38,7 +46,8 @@ type Schema struct { // A boolean type with value false. Setting false here validates that all // properties in the config have been defined in the json schema as properties // - // Its type during runtime will either be *Schema or bool + // Its type during runtime will either be Schema or bool + // TODO: Generics to represent either a Schema{} or a bool. AdditionalProperties any `json:"additionalProperties,omitempty"` // Required properties for the object. Any fields missing the "omitempty" @@ -63,7 +72,7 @@ type Schema struct { Extension // Schema that must match any of the schemas in the array - AnyOf []*Schema `json:"anyOf,omitempty"` + AnyOf []Schema `json:"anyOf,omitempty"` } // Default value defined in a JSON Schema, represented as a string. @@ -120,7 +129,10 @@ func (s *Schema) SetByPath(path string, v Schema) error { type Type string const ( + // Default zero value of a schema. This does not correspond to a type in the + // JSON schema spec and is an internal type defined for convenience. InvalidType Type = "invalid" + NullType Type = "null" BooleanType Type = "boolean" StringType Type = "string" NumberType Type = "number" From 43325fdd0ab11ed5b63c8daae1d919230fd27b1e Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 20 Aug 2024 15:39:31 +0200 Subject: [PATCH 02/62] add some more todos: --- bundle/schema/schema.go | 3 ++- libs/jsonschema/from_type.go | 11 ++++++----- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/bundle/schema/schema.go b/bundle/schema/schema.go index d5c6c1f616..8290139c42 100644 --- a/bundle/schema/schema.go +++ b/bundle/schema/schema.go @@ -50,7 +50,8 @@ func New(golangType reflect.Type, docs *Docs) (*jsonschema.Schema, error) { s, { Type: jsonschema.StringType, - // TODO: + // TODO: Narrow down the scope of the regex match. + // Also likely need to rename this variable. Pattern: dynvar.VariableRegex, }, }, diff --git a/libs/jsonschema/from_type.go b/libs/jsonschema/from_type.go index 43ce9a85a9..fa9c1efa65 100644 --- a/libs/jsonschema/from_type.go +++ b/libs/jsonschema/from_type.go @@ -62,11 +62,10 @@ func FromType(typ reflect.Type, opts FromTypeOptions) (Schema, error) { res = Schema{Type: StringType} case reflect.Bool: res = Schema{Type: BooleanType} - // case reflect.Int, reflect.Int32, reflect.Int64: - // res = Schema{Type: IntegerType} - case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, - reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, - reflect.Float32, reflect.Float64: + // TODO: Add comment about reduced coverage of primitive Go types in the code paths here. + case reflect.Int: + res = Schema{Type: IntegerType} + case reflect.Float32, reflect.Float64: res = Schema{Type: NumberType} default: return InvalidSchema, fmt.Errorf("unsupported type: %s", typ.Kind()) @@ -164,6 +163,8 @@ func fromTypeStruct(typ reflect.Type, opts FromTypeOptions) (Schema, error) { return res, nil } +// TODO: Add comments explaining the translation between struct, map, slice and +// the JSON schema representation. func fromTypeSlice(typ reflect.Type, opts FromTypeOptions) (Schema, error) { if typ.Kind() != reflect.Slice { return InvalidSchema, fmt.Errorf("expected slice, got %s", typ.Kind()) From 6d2f88282f2bb81baf085f2cff67558c0b0ffdf6 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 20 Aug 2024 18:40:30 +0200 Subject: [PATCH 03/62] more progress on the internal functionality --- bundle/schema/schema.go | 28 +++-- bundle/schema/schema_test.go | 6 ++ libs/jsonschema/from_type.go | 166 ++++++++++++++++++++++++------ libs/jsonschema/from_type_test.go | 119 +++++++++++++++++++++ libs/jsonschema/schema.go | 4 + 5 files changed, 275 insertions(+), 48 deletions(-) create mode 100644 libs/jsonschema/from_type_test.go diff --git a/bundle/schema/schema.go b/bundle/schema/schema.go index 8290139c42..f8c77ab268 100644 --- a/bundle/schema/schema.go +++ b/bundle/schema/schema.go @@ -42,23 +42,21 @@ import ( // for details visit: https://json-schema.org/understanding-json-schema/reference/object.html#properties func New(golangType reflect.Type, docs *Docs) (*jsonschema.Schema, error) { - s, err := jsonschema.FromType(golangType, jsonschema.FromTypeOptions{ - Transform: func(s jsonschema.Schema) jsonschema.Schema { - if s.Type == jsonschema.NumberType || s.Type == jsonschema.BooleanType { - s = jsonschema.Schema{ - AnyOf: []jsonschema.Schema{ - s, - { - Type: jsonschema.StringType, - // TODO: Narrow down the scope of the regex match. - // Also likely need to rename this variable. - Pattern: dynvar.VariableRegex, - }, + s, err := jsonschema.FromType(golangType, func(s jsonschema.Schema) jsonschema.Schema { + if s.Type == jsonschema.NumberType || s.Type == jsonschema.BooleanType { + s = jsonschema.Schema{ + AnyOf: []jsonschema.Schema{ + s, + { + Type: jsonschema.StringType, + // TODO: Narrow down the scope of the regex match. + // Also likely need to rename this variable. + Pattern: dynvar.VariableRegex, }, - } + }, } - return s - }, + } + return s }) if err != nil { return nil, err diff --git a/bundle/schema/schema_test.go b/bundle/schema/schema_test.go index 6d9df0cc7c..a75c588bc1 100644 --- a/bundle/schema/schema_test.go +++ b/bundle/schema/schema_test.go @@ -2,6 +2,7 @@ package schema import ( "encoding/json" + "fmt" "reflect" "testing" @@ -12,6 +13,11 @@ import ( func TestIntSchema(t *testing.T) { var elemInt int + type Bae struct{} + + typ := reflect.TypeOf(Bae{}) + fmt.Println(typ.PkgPath()) + expected := `{ "anyOf": [ diff --git a/libs/jsonschema/from_type.go b/libs/jsonschema/from_type.go index fa9c1efa65..8332e70df6 100644 --- a/libs/jsonschema/from_type.go +++ b/libs/jsonschema/from_type.go @@ -3,11 +3,13 @@ package jsonschema import ( "container/list" "fmt" + "path" "reflect" "slices" "strings" ) +// TODO: Maybe can be removed? var InvalidSchema = Schema{ Type: InvalidType, } @@ -28,56 +30,130 @@ const deprecatedTag = "deprecated" // TODO: Call out in the PR description that recursive types like "for_each_task" // are now supported. -type FromTypeOptions struct { - // Transformation function to apply after generating the schema. - Transform func(s Schema) Schema +type constructor struct { + // Map of typ.PkgPath() + "." + typ.Name() to the schema for that type. + // Example key: github.com/databricks/databricks-sdk-go/service/jobs.JobSettings + definitions map[string]Schema + + // Transformation function to apply after generating a node in the schema. + fn func(s Schema) Schema +} + +// The $defs block in a JSON schema cannot contain "/", otherwise it will not be +// correctly parsed by a JSON schema validator. So we replace "/" with an additional +// level of nesting in the output map. +// +// For example: +// {"a/b/c": "value"} is converted to {"a": {"b": {"c": "value"}}} +func (c *constructor) nestedDefinitions() any { + if len(c.definitions) == 0 { + return nil + } + + res := make(map[string]any) + for k, v := range c.definitions { + parts := strings.Split(k, "/") + cur := res + for i, p := range parts { + if i == len(parts)-1 { + cur[p] = v + break + } + + if _, ok := cur[p]; !ok { + cur[p] = make(map[string]any) + } + cur = cur[p].(map[string]any) + } + } + + return res } // TODO: Skip generating schema for interface fields. -func FromType(typ reflect.Type, opts FromTypeOptions) (Schema, error) { +func FromType(typ reflect.Type, fn func(s Schema) Schema) (Schema, error) { + c := constructor{ + definitions: make(map[string]Schema), + fn: fn, + } + + err := c.walk(typ) + if err != nil { + return InvalidSchema, err + } + + res := c.definitions[typePath(typ)] + // No need to include the root type in the definitions. + delete(c.definitions, typePath(typ)) + res.Definitions = c.nestedDefinitions() + return res, nil +} + +func typePath(typ reflect.Type) string { + // For built-in types, return the type name directly. + if typ.PkgPath() == "" { + return typ.Name() + } + + return strings.Join([]string{typ.PkgPath(), typ.Name()}, ".") +} + +// TODO: would a worked based model fit better here? Is this internal API not +// the right fit? +func (c *constructor) walk(typ reflect.Type) error { // Dereference pointers if necessary. for typ.Kind() == reflect.Ptr { typ = typ.Elem() } - // An interface value can never be serialized from text, and thus is explicitly - // set to null and disallowed in the schema. - if typ.Kind() == reflect.Interface { - return Schema{Type: NullType}, nil + typPath := typePath(typ) + + // Return value directly if it's already been processed. + if _, ok := c.definitions[typPath]; ok { + return nil } - var res Schema + var s Schema var err error - // TODO: Narrow down the number of Go types handled here. + // TODO: Narrow / widen down the number of Go types handled here. switch typ.Kind() { case reflect.Struct: - res, err = fromTypeStruct(typ, opts) + s, err = c.fromTypeStruct(typ) case reflect.Slice: - res, err = fromTypeSlice(typ, opts) + s, err = c.fromTypeSlice(typ) case reflect.Map: - res, err = fromTypeMap(typ, opts) + s, err = c.fromTypeMap(typ) // TODO: Should the primitive functions below be inlined? case reflect.String: - res = Schema{Type: StringType} + s = Schema{Type: StringType} case reflect.Bool: - res = Schema{Type: BooleanType} + s = Schema{Type: BooleanType} // TODO: Add comment about reduced coverage of primitive Go types in the code paths here. case reflect.Int: - res = Schema{Type: IntegerType} + s = Schema{Type: IntegerType} case reflect.Float32, reflect.Float64: - res = Schema{Type: NumberType} + s = Schema{Type: NumberType} + case reflect.Interface: + // An interface value can never be serialized from text, and thus is explicitly + // set to null and disallowed in the schema. + s = Schema{Type: NullType} default: - return InvalidSchema, fmt.Errorf("unsupported type: %s", typ.Kind()) + return fmt.Errorf("unsupported type: %s", typ.Kind()) } if err != nil { - return InvalidSchema, err + return err } - if opts.Transform != nil { - res = opts.Transform(res) + if c.fn != nil { + s = c.fn(s) } - return res, nil + + // Store definition for the type if it's part of a Go package and not a built-in type. + // TODO: Apply transformation at the end, to all definitions instead of + // during recursive traversal? + c.definitions[typPath] = s + return nil } // This function returns all member fields of the provided type. @@ -112,7 +188,7 @@ func getStructFields(golangType reflect.Type) []reflect.StructField { return fields } -func fromTypeStruct(typ reflect.Type, opts FromTypeOptions) (Schema, error) { +func (c *constructor) fromTypeStruct(typ reflect.Type) (Schema, error) { if typ.Kind() != reflect.Struct { return InvalidSchema, fmt.Errorf("expected struct, got %s", typ.Kind()) } @@ -129,7 +205,6 @@ func fromTypeStruct(typ reflect.Type, opts FromTypeOptions) (Schema, error) { } structFields := getStructFields(typ) - for _, structField := range structFields { bundleTags := strings.Split(structField.Tag.Get("bundle"), ",") // Fields marked as "readonly", "internal" or "deprecated" are skipped @@ -143,7 +218,7 @@ func fromTypeStruct(typ reflect.Type, opts FromTypeOptions) (Schema, error) { jsonTags := strings.Split(structField.Tag.Get("json"), ",") // Do not include fields in the schema that will not be serialized during // JSON marshalling. - if jsonTags[0] == "" || jsonTags[0] == "-" { + if jsonTags[0] == "" || jsonTags[0] == "-" || !structField.IsExported() { continue } // "omitempty" tags in the Go SDK structs represent fields that not are @@ -153,11 +228,19 @@ func fromTypeStruct(typ reflect.Type, opts FromTypeOptions) (Schema, error) { res.Required = append(res.Required, jsonTags[0]) } - s, err := FromType(structField.Type, opts) + // Trigger call to fromType, to recursively generate definitions for + // the struct field. + err := c.walk(structField.Type) if err != nil { return InvalidSchema, err } - res.Properties[jsonTags[0]] = &s + + typPath := typePath(structField.Type) + refPath := path.Join("#/$defs", typPath) + // For non-built-in types, refer to the definition. + res.Properties[jsonTags[0]] = &Schema{ + Reference: &refPath, + } } return res, nil @@ -165,7 +248,7 @@ func fromTypeStruct(typ reflect.Type, opts FromTypeOptions) (Schema, error) { // TODO: Add comments explaining the translation between struct, map, slice and // the JSON schema representation. -func fromTypeSlice(typ reflect.Type, opts FromTypeOptions) (Schema, error) { +func (c *constructor) fromTypeSlice(typ reflect.Type) (Schema, error) { if typ.Kind() != reflect.Slice { return InvalidSchema, fmt.Errorf("expected slice, got %s", typ.Kind()) } @@ -174,16 +257,24 @@ func fromTypeSlice(typ reflect.Type, opts FromTypeOptions) (Schema, error) { Type: ArrayType, } - items, err := FromType(typ.Elem(), opts) + // Trigger call to fromType, to recursively generate definitions for + // the slice element. + err := c.walk(typ.Elem()) if err != nil { return InvalidSchema, err } - res.Items = &items + typPath := typePath(typ.Elem()) + refPath := path.Join("#/$defs", typPath) + + // For non-built-in types, refer to the definition + res.Items = &Schema{ + Reference: &refPath, + } return res, nil } -func fromTypeMap(typ reflect.Type, opts FromTypeOptions) (Schema, error) { +func (c *constructor) fromTypeMap(typ reflect.Type) (Schema, error) { if typ.Kind() != reflect.Map { return InvalidSchema, fmt.Errorf("expected map, got %s", typ.Kind()) } @@ -196,10 +287,19 @@ func fromTypeMap(typ reflect.Type, opts FromTypeOptions) (Schema, error) { Type: ObjectType, } - additionalProperties, err := FromType(typ.Elem(), opts) + // Trigger call to fromType, to recursively generate definitions for + // the map value. + err := c.walk(typ.Elem()) if err != nil { return InvalidSchema, err } - res.AdditionalProperties = additionalProperties + + typPath := typePath(typ.Elem()) + refPath := path.Join("#/$defs", typPath) + + // For non-built-in types, refer to the definition + res.AdditionalProperties = &Schema{ + Reference: &refPath, + } return res, nil } diff --git a/libs/jsonschema/from_type_test.go b/libs/jsonschema/from_type_test.go new file mode 100644 index 0000000000..5992100b98 --- /dev/null +++ b/libs/jsonschema/from_type_test.go @@ -0,0 +1,119 @@ +package jsonschema + +import ( + "reflect" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestFromTypeBasic(t *testing.T) { + type myStruct struct { + V string `json:"v"` + } + + strRef := "#/$defs/string" + boolRef := "#/$defs/bool" + intRef := "#/$defs/int" + + tcases := []struct { + name string + typ reflect.Type + expected Schema + }{ + { + name: "int", + typ: reflect.TypeOf(int(0)), + expected: Schema{ + Type: "integer", + }, + }, + { + name: "string", + typ: reflect.TypeOf(""), + expected: Schema{ + Type: "string", + }, + }, + { + name: "bool", + typ: reflect.TypeOf(true), + expected: Schema{ + Type: "boolean", + }, + }, + { + name: "float64", + typ: reflect.TypeOf(float64(0)), + expected: Schema{ + Type: "number", + }, + }, + { + name: "struct", + typ: reflect.TypeOf(myStruct{}), + expected: Schema{ + Type: "object", + Definitions: map[string]any{ + "string": Schema{ + Type: "string", + }, + }, + Properties: map[string]*Schema{ + "v": { + Reference: &strRef, + }, + }, + AdditionalProperties: false, + Required: []string{"v"}, + }, + }, + { + name: "slice", + typ: reflect.TypeOf([]bool{}), + expected: Schema{ + Type: "array", + Definitions: map[string]any{ + "bool": Schema{ + Type: "boolean", + }, + }, + Items: &Schema{ + Reference: &boolRef, + }, + }, + }, + { + name: "map", + typ: reflect.TypeOf(map[string]int{}), + expected: Schema{ + Type: "object", + Definitions: map[string]any{ + "int": Schema{ + Type: "integer", + }, + }, + AdditionalProperties: &Schema{ + Reference: &intRef, + }, + }, + }, + } + + for _, tc := range tcases { + t.Run(tc.name, func(t *testing.T) { + s, err := FromType(tc.typ, nil) + assert.NoError(t, err) + assert.Equal(t, tc.expected, s) + + // jsonSchema, err := json.MarshalIndent(s, " ", " ") + // assert.NoError(t, err) + + // expectedJson, err := json.MarshalIndent(tc.expected, " ", " ") + // assert.NoError(t, err) + + // t.Log("[DEBUG] actual: ", string(jsonSchema)) + // t.Log("[DEBUG] expected: ", string(expectedJson)) + }) + } +} diff --git a/libs/jsonschema/schema.go b/libs/jsonschema/schema.go index 6c6a89c9d8..44cc3b9c89 100644 --- a/libs/jsonschema/schema.go +++ b/libs/jsonschema/schema.go @@ -21,6 +21,9 @@ import ( // // as an embedded file. type Schema struct { + // TODO: Comments for this field + Definitions any `json:"$defs,omitempty"` + // Type of the object Type Type `json:"type,omitempty"` @@ -55,6 +58,7 @@ type Schema struct { Required []string `json:"required,omitempty"` // URI to a json schema + // TODO: Would be nice to make items as well as this a non-pointer. Reference *string `json:"$ref,omitempty"` // Default value for the property / object From 65f1c750f607275dc29b7bb5dc7888a68af9fbde Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 20 Aug 2024 18:46:23 +0200 Subject: [PATCH 04/62] - --- bundle/schema/schema_test.go | 283 +----------------------------- libs/jsonschema/from_type_test.go | 13 +- 2 files changed, 11 insertions(+), 285 deletions(-) diff --git a/bundle/schema/schema_test.go b/bundle/schema/schema_test.go index a75c588bc1..e754e5c6ca 100644 --- a/bundle/schema/schema_test.go +++ b/bundle/schema/schema_test.go @@ -10,288 +10,7 @@ import ( "github.com/stretchr/testify/require" ) -func TestIntSchema(t *testing.T) { - var elemInt int - - type Bae struct{} - - typ := reflect.TypeOf(Bae{}) - fmt.Println(typ.PkgPath()) - - expected := - `{ - "anyOf": [ - { - "type": "number" - }, - { - "type": "string", - "pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - } - ] - }` - - schema, err := New(reflect.TypeOf(elemInt), nil) - require.NoError(t, err) - - jsonSchema, err := json.MarshalIndent(schema, " ", " ") - assert.NoError(t, err) - - t.Log("[DEBUG] actual: ", string(jsonSchema)) - t.Log("[DEBUG] expected: ", expected) - assert.Equal(t, expected, string(jsonSchema)) -} - -func TestBooleanSchema(t *testing.T) { - var elem bool - - expected := - `{ - "anyOf": [ - { - "type": "boolean" - }, - { - "type": "string", - "pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - } - ] - }` - - schema, err := New(reflect.TypeOf(elem), nil) - require.NoError(t, err) - - jsonSchema, err := json.MarshalIndent(schema, " ", " ") - assert.NoError(t, err) - - t.Log("[DEBUG] actual: ", string(jsonSchema)) - t.Log("[DEBUG] expected: ", expected) - assert.Equal(t, expected, string(jsonSchema)) -} - -func TestStringSchema(t *testing.T) { - var elem string - - expected := - `{ - "type": "string" - }` - - schema, err := New(reflect.TypeOf(elem), nil) - require.NoError(t, err) - - jsonSchema, err := json.MarshalIndent(schema, " ", " ") - assert.NoError(t, err) - - t.Log("[DEBUG] actual: ", string(jsonSchema)) - t.Log("[DEBUG] expected: ", expected) - assert.Equal(t, expected, string(jsonSchema)) -} - -func TestStructOfPrimitivesSchema(t *testing.T) { - type Foo struct { - IntVal int `json:"int_val"` - Int8Val int8 `json:"int8_val"` - Int16Val int16 `json:"int16_val"` - Int32Val int32 `json:"int32_val"` - Int64Val int64 `json:"int64_val"` - - UIntVal uint `json:"uint_val"` - Uint8Val uint8 `json:"uint8_val"` - Uint16Val uint16 `json:"uint16_val"` - Uint32Val uint32 `json:"uint32_val"` - Uint64Val uint64 `json:"uint64_val"` - - Float32Val float32 `json:"float32_val"` - Float64Val float64 `json:"float64_val"` - - StringVal string `json:"string_val"` - - BoolVal bool `json:"bool_val"` - } - - elem := Foo{} - - schema, err := New(reflect.TypeOf(elem), nil) - assert.NoError(t, err) - - jsonSchema, err := json.MarshalIndent(schema, " ", " ") - assert.NoError(t, err) - - expected := - `{ - "type": "object", - "properties": { - "bool_val": { - "anyOf": [ - { - "type": "boolean" - }, - { - "type": "string", - "pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - } - ] - }, - "float32_val": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "string", - "pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - } - ] - }, - "float64_val": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "string", - "pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - } - ] - }, - "int16_val": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "string", - "pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - } - ] - }, - "int32_val": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "string", - "pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - } - ] - }, - "int64_val": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "string", - "pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - } - ] - }, - "int8_val": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "string", - "pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - } - ] - }, - "int_val": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "string", - "pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - } - ] - }, - "string_val": { - "type": "string" - }, - "uint16_val": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "string", - "pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - } - ] - }, - "uint32_val": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "string", - "pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - } - ] - }, - "uint64_val": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "string", - "pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - } - ] - }, - "uint8_val": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "string", - "pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - } - ] - }, - "uint_val": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "string", - "pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - } - ] - } - }, - "additionalProperties": false, - "required": [ - "int_val", - "int8_val", - "int16_val", - "int32_val", - "int64_val", - "uint_val", - "uint8_val", - "uint16_val", - "uint32_val", - "uint64_val", - "float32_val", - "float64_val", - "string_val", - "bool_val" - ] - }` - - t.Log("[DEBUG] actual: ", string(jsonSchema)) - t.Log("[DEBUG] expected: ", expected) - assert.Equal(t, expected, string(jsonSchema)) -} +// TODO: Add a test that checks the primitive overrides for reference regexs wor. func TestStructOfStructsSchema(t *testing.T) { type Bar struct { diff --git a/libs/jsonschema/from_type_test.go b/libs/jsonschema/from_type_test.go index 5992100b98..beea80e336 100644 --- a/libs/jsonschema/from_type_test.go +++ b/libs/jsonschema/from_type_test.go @@ -9,7 +9,8 @@ import ( func TestFromTypeBasic(t *testing.T) { type myStruct struct { - V string `json:"v"` + S string `json:"s"` + I int `json:"i"` } strRef := "#/$defs/string" @@ -58,14 +59,20 @@ func TestFromTypeBasic(t *testing.T) { "string": Schema{ Type: "string", }, + "int": Schema{ + Type: "integer", + }, }, Properties: map[string]*Schema{ - "v": { + "s": { Reference: &strRef, }, + "i": { + Reference: &intRef, + }, }, AdditionalProperties: false, - Required: []string{"v"}, + Required: []string{"s", "i"}, }, }, { From 7db32fc211d2c35bc1a4dff3a2ebe6e23ccc5b66 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 20 Aug 2024 19:09:21 +0200 Subject: [PATCH 05/62] - --- bundle/schema/schema_test.go | 53 ------------------------------- libs/jsonschema/from_type.go | 6 ++-- libs/jsonschema/from_type_test.go | 18 +++++++++++ 3 files changed, 21 insertions(+), 56 deletions(-) diff --git a/bundle/schema/schema_test.go b/bundle/schema/schema_test.go index e754e5c6ca..1955dce1a4 100644 --- a/bundle/schema/schema_test.go +++ b/bundle/schema/schema_test.go @@ -2,7 +2,6 @@ package schema import ( "encoding/json" - "fmt" "reflect" "testing" @@ -187,36 +186,6 @@ func TestStructOfSliceSchema(t *testing.T) { assert.Equal(t, expected, string(jsonSchema)) } -func TestMapOfPrimitivesSchema(t *testing.T) { - var elem map[string]int - - schema, err := New(reflect.TypeOf(elem), nil) - assert.NoError(t, err) - - jsonSchema, err := json.MarshalIndent(schema, " ", " ") - assert.NoError(t, err) - - expected := - `{ - "type": "object", - "additionalProperties": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "string", - "pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - } - ] - } - }` - - t.Log("[DEBUG] actual: ", string(jsonSchema)) - t.Log("[DEBUG] expected: ", expected) - assert.Equal(t, expected, string(jsonSchema)) -} - func TestMapOfStructSchema(t *testing.T) { type Foo struct { MyInt int `json:"my_int"` @@ -318,28 +287,6 @@ func TestMapOfSliceSchema(t *testing.T) { assert.Equal(t, expected, string(jsonSchema)) } -func TestSliceOfPrimitivesSchema(t *testing.T) { - var elem []float32 - - schema, err := New(reflect.TypeOf(elem), nil) - assert.NoError(t, err) - - jsonSchema, err := json.MarshalIndent(schema, " ", " ") - assert.NoError(t, err) - - expected := - `{ - "type": "array", - "items": { - "type": "number" - } - }` - - t.Log("[DEBUG] actual: ", string(jsonSchema)) - t.Log("[DEBUG] expected: ", expected) - assert.Equal(t, expected, string(jsonSchema)) -} - func TestSliceOfSliceSchema(t *testing.T) { var elem [][]string diff --git a/libs/jsonschema/from_type.go b/libs/jsonschema/from_type.go index 8332e70df6..c5f6250a9e 100644 --- a/libs/jsonschema/from_type.go +++ b/libs/jsonschema/from_type.go @@ -159,12 +159,12 @@ func (c *constructor) walk(typ reflect.Type) error { // This function returns all member fields of the provided type. // If the type has embedded (aka anonymous) fields, this function traverses // those in a breadth first manner -func getStructFields(golangType reflect.Type) []reflect.StructField { +func getStructFields(typ reflect.Type) []reflect.StructField { fields := []reflect.StructField{} bfsQueue := list.New() - for i := 0; i < golangType.NumField(); i++ { - bfsQueue.PushBack(golangType.Field(i)) + for i := 0; i < typ.NumField(); i++ { + bfsQueue.PushBack(typ.Field(i)) } for bfsQueue.Len() > 0 { front := bfsQueue.Front() diff --git a/libs/jsonschema/from_type_test.go b/libs/jsonschema/from_type_test.go index beea80e336..fd60bf44ff 100644 --- a/libs/jsonschema/from_type_test.go +++ b/libs/jsonschema/from_type_test.go @@ -124,3 +124,21 @@ func TestFromTypeBasic(t *testing.T) { }) } } + +func TestGetStructFields(t *testing.T) { + type EmbeddedStruct struct { + I int + B bool + } + + type MyStruct struct { + S string + *EmbeddedStruct + } + + fields := getStructFields(reflect.TypeOf(MyStruct{})) + assert.Len(t, fields, 3) + assert.Equal(t, "S", fields[0].Name) + assert.Equal(t, "I", fields[1].Name) + assert.Equal(t, "B", fields[2].Name) +} From b89928513e81cdee5575045e5f0918fc8fb7a9fa Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 20 Aug 2024 19:10:50 +0200 Subject: [PATCH 06/62] - --- bundle/schema/schema_test.go | 135 ----------------------------------- 1 file changed, 135 deletions(-) diff --git a/bundle/schema/schema_test.go b/bundle/schema/schema_test.go index 1955dce1a4..bf15002070 100644 --- a/bundle/schema/schema_test.go +++ b/bundle/schema/schema_test.go @@ -388,141 +388,6 @@ func TestSliceOfStructSchema(t *testing.T) { assert.Equal(t, expected, string(jsonSchema)) } -func TestEmbeddedStructSchema(t *testing.T) { - type Location struct { - Country string `json:"country"` - State string `json:"state,omitempty"` - } - - type Person struct { - Name string `json:"name"` - Age int `json:"age,omitempty"` - Home Location `json:"home"` - } - - type Plot struct { - Events map[string]Person `json:"events"` - } - - type Story struct { - Plot Plot `json:"plot"` - *Person - Location - } - - elem := Story{} - - schema, err := New(reflect.TypeOf(elem), nil) - assert.NoError(t, err) - - jsonSchema, err := json.MarshalIndent(schema, " ", " ") - assert.NoError(t, err) - - expected := - `{ - "type": "object", - "properties": { - "age": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "string", - "pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - } - ] - }, - "country": { - "type": "string" - }, - "home": { - "type": "object", - "properties": { - "country": { - "type": "string" - }, - "state": { - "type": "string" - } - }, - "additionalProperties": false, - "required": [ - "country" - ] - }, - "name": { - "type": "string" - }, - "plot": { - "type": "object", - "properties": { - "events": { - "type": "object", - "additionalProperties": { - "type": "object", - "properties": { - "age": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "string", - "pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - } - ] - }, - "home": { - "type": "object", - "properties": { - "country": { - "type": "string" - }, - "state": { - "type": "string" - } - }, - "additionalProperties": false, - "required": [ - "country" - ] - }, - "name": { - "type": "string" - } - }, - "additionalProperties": false, - "required": [ - "name", - "home" - ] - } - } - }, - "additionalProperties": false, - "required": [ - "events" - ] - }, - "state": { - "type": "string" - } - }, - "additionalProperties": false, - "required": [ - "plot", - "name", - "home", - "country" - ] - }` - - t.Log("[DEBUG] actual: ", string(jsonSchema)) - t.Log("[DEBUG] expected: ", expected) - assert.Equal(t, expected, string(jsonSchema)) -} - func TestErrorWithTrace(t *testing.T) { tracker := newTracker() dummyType := reflect.TypeOf(struct{}{}) From b023ba0dd4f53b5f7a33400bbca0ec22d268a96a Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 20 Aug 2024 19:57:57 +0200 Subject: [PATCH 07/62] removed more tests --- bundle/schema/schema_test.go | 442 ------------------------------ libs/jsonschema/from_type.go | 26 +- libs/jsonschema/from_type_test.go | 35 ++- 3 files changed, 40 insertions(+), 463 deletions(-) diff --git a/bundle/schema/schema_test.go b/bundle/schema/schema_test.go index bf15002070..ad9cb530bf 100644 --- a/bundle/schema/schema_test.go +++ b/bundle/schema/schema_test.go @@ -404,198 +404,6 @@ func TestErrorWithTrace(t *testing.T) { assert.ErrorContains(t, err, "with depth = 4. traversal trace: root -> resources -> pipelines -> datasets") } -func TestNonAnnotatedFieldsAreSkipped(t *testing.T) { - type MyStruct struct { - Foo string - Bar int `json:"bar"` - } - - elem := MyStruct{} - - schema, err := New(reflect.TypeOf(elem), nil) - require.NoError(t, err) - - jsonSchema, err := json.MarshalIndent(schema, " ", " ") - assert.NoError(t, err) - - expectedSchema := - `{ - "type": "object", - "properties": { - "bar": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "string", - "pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - } - ] - } - }, - "additionalProperties": false, - "required": [ - "bar" - ] - }` - - t.Log("[DEBUG] actual: ", string(jsonSchema)) - t.Log("[DEBUG] expected: ", expectedSchema) - - assert.Equal(t, expectedSchema, string(jsonSchema)) -} - -func TestDashFieldsAreSkipped(t *testing.T) { - type MyStruct struct { - Foo string `json:"-"` - Bar int `json:"bar"` - } - - elem := MyStruct{} - - schema, err := New(reflect.TypeOf(elem), nil) - require.NoError(t, err) - - jsonSchema, err := json.MarshalIndent(schema, " ", " ") - assert.NoError(t, err) - - expectedSchema := - `{ - "type": "object", - "properties": { - "bar": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "string", - "pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - } - ] - } - }, - "additionalProperties": false, - "required": [ - "bar" - ] - }` - - t.Log("[DEBUG] actual: ", string(jsonSchema)) - t.Log("[DEBUG] expected: ", expectedSchema) - - assert.Equal(t, expectedSchema, string(jsonSchema)) -} - -func TestPointerInStructSchema(t *testing.T) { - - type Bar struct { - PtrVal2 *int `json:"ptr_val2"` - } - - type Foo struct { - PtrInt *int `json:"ptr_int"` - PtrString *string `json:"ptr_string"` - FloatVal float32 `json:"float_val"` - PtrBar *Bar `json:"ptr_bar"` - Bar *Bar `json:"bar"` - } - - elem := Foo{} - - schema, err := New(reflect.TypeOf(elem), nil) - require.NoError(t, err) - - jsonSchema, err := json.MarshalIndent(schema, " ", " ") - assert.NoError(t, err) - - expectedSchema := - `{ - "type": "object", - "properties": { - "bar": { - "type": "object", - "properties": { - "ptr_val2": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "string", - "pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - } - ] - } - }, - "additionalProperties": false, - "required": [ - "ptr_val2" - ] - }, - "float_val": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "string", - "pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - } - ] - }, - "ptr_bar": { - "type": "object", - "properties": { - "ptr_val2": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "string", - "pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - } - ] - } - }, - "additionalProperties": false, - "required": [ - "ptr_val2" - ] - }, - "ptr_int": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "string", - "pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - } - ] - }, - "ptr_string": { - "type": "string" - } - }, - "additionalProperties": false, - "required": [ - "ptr_int", - "ptr_string", - "float_val", - "ptr_bar", - "bar" - ] - }` - - t.Log("[DEBUG] actual: ", string(jsonSchema)) - t.Log("[DEBUG] expected: ", expectedSchema) - - assert.Equal(t, expectedSchema, string(jsonSchema)) -} - func TestGenericSchema(t *testing.T) { type Person struct { Name string `json:"name"` @@ -824,93 +632,6 @@ func TestGenericSchema(t *testing.T) { assert.Equal(t, expected, string(jsonSchema)) } -func TestFieldsWithoutOmitEmptyAreRequired(t *testing.T) { - - type Papaya struct { - A int `json:"a,string,omitempty"` - B string `json:"b"` - } - - type MyStruct struct { - Foo string `json:"-,omitempty"` - Bar int `json:"bar"` - Apple int `json:"apple,omitempty"` - Mango int `json:",omitempty"` - Guava int `json:","` - Papaya *Papaya `json:"papaya,"` - } - - elem := MyStruct{} - - schema, err := New(reflect.TypeOf(elem), nil) - require.NoError(t, err) - - jsonSchema, err := json.MarshalIndent(schema, " ", " ") - assert.NoError(t, err) - - expectedSchema := - `{ - "type": "object", - "properties": { - "apple": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "string", - "pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - } - ] - }, - "bar": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "string", - "pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - } - ] - }, - "papaya": { - "type": "object", - "properties": { - "a": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "string", - "pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - } - ] - }, - "b": { - "type": "string" - } - }, - "additionalProperties": false, - "required": [ - "b" - ] - } - }, - "additionalProperties": false, - "required": [ - "bar", - "papaya" - ] - }` - - t.Log("[DEBUG] actual: ", string(jsonSchema)) - t.Log("[DEBUG] expected: ", expectedSchema) - - assert.Equal(t, expectedSchema, string(jsonSchema)) -} - func TestDocIngestionForObject(t *testing.T) { docs := &Docs{ Description: "docs for root", @@ -1272,166 +993,3 @@ func TestErrorIfStructHasLoop(t *testing.T) { _, err := New(reflect.TypeOf(elem), nil) assert.ErrorContains(t, err, "cycle detected. traversal trace: root -> my_mango -> my_guava -> my_papaya -> my_apple") } - -func TestInterfaceGeneratesEmptySchema(t *testing.T) { - type Foo struct { - Apple int `json:"apple"` - Mango interface{} `json:"mango"` - } - - elem := Foo{} - - schema, err := New(reflect.TypeOf(elem), nil) - assert.NoError(t, err) - - jsonSchema, err := json.MarshalIndent(schema, " ", " ") - assert.NoError(t, err) - - expected := - `{ - "type": "object", - "properties": { - "apple": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "string", - "pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - } - ] - }, - "mango": {} - }, - "additionalProperties": false, - "required": [ - "apple", - "mango" - ] - }` - - t.Log("[DEBUG] actual: ", string(jsonSchema)) - t.Log("[DEBUG] expected: ", expected) - assert.Equal(t, expected, string(jsonSchema)) -} - -func TestBundleReadOnlytag(t *testing.T) { - type Pokemon struct { - Pikachu string `json:"pikachu" bundle:"readonly"` - Raichu string `json:"raichu"` - } - - type Foo struct { - Pokemon *Pokemon `json:"pokemon"` - Apple int `json:"apple"` - Mango string `json:"mango" bundle:"readonly"` - } - - elem := Foo{} - - schema, err := New(reflect.TypeOf(elem), nil) - assert.NoError(t, err) - - jsonSchema, err := json.MarshalIndent(schema, " ", " ") - assert.NoError(t, err) - - expected := - `{ - "type": "object", - "properties": { - "apple": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "string", - "pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - } - ] - }, - "pokemon": { - "type": "object", - "properties": { - "raichu": { - "type": "string" - } - }, - "additionalProperties": false, - "required": [ - "raichu" - ] - } - }, - "additionalProperties": false, - "required": [ - "pokemon", - "apple" - ] - }` - - t.Log("[DEBUG] actual: ", string(jsonSchema)) - t.Log("[DEBUG] expected: ", expected) - assert.Equal(t, expected, string(jsonSchema)) -} - -func TestBundleInternalTag(t *testing.T) { - type Pokemon struct { - Pikachu string `json:"pikachu" bundle:"internal"` - Raichu string `json:"raichu"` - } - - type Foo struct { - Pokemon *Pokemon `json:"pokemon"` - Apple int `json:"apple"` - Mango string `json:"mango" bundle:"internal"` - } - - elem := Foo{} - - schema, err := New(reflect.TypeOf(elem), nil) - assert.NoError(t, err) - - jsonSchema, err := json.MarshalIndent(schema, " ", " ") - assert.NoError(t, err) - - expected := - `{ - "type": "object", - "properties": { - "apple": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "string", - "pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - } - ] - }, - "pokemon": { - "type": "object", - "properties": { - "raichu": { - "type": "string" - } - }, - "additionalProperties": false, - "required": [ - "raichu" - ] - } - }, - "additionalProperties": false, - "required": [ - "pokemon", - "apple" - ] - }` - - t.Log("[DEBUG] actual: ", string(jsonSchema)) - t.Log("[DEBUG] expected: ", expected) - assert.Equal(t, expected, string(jsonSchema)) -} diff --git a/libs/jsonschema/from_type.go b/libs/jsonschema/from_type.go index c5f6250a9e..d0e4e2acdf 100644 --- a/libs/jsonschema/from_type.go +++ b/libs/jsonschema/from_type.go @@ -77,7 +77,7 @@ func FromType(typ reflect.Type, fn func(s Schema) Schema) (Schema, error) { fn: fn, } - err := c.walk(typ) + _, err := c.walk(typ) if err != nil { return InvalidSchema, err } @@ -90,6 +90,11 @@ func FromType(typ reflect.Type, fn func(s Schema) Schema) (Schema, error) { } func typePath(typ reflect.Type) string { + // typ.Name() resolves to "" for any type. + if typ.Kind() == reflect.Interface { + return "interface" + } + // For built-in types, return the type name directly. if typ.PkgPath() == "" { return typ.Name() @@ -100,7 +105,7 @@ func typePath(typ reflect.Type) string { // TODO: would a worked based model fit better here? Is this internal API not // the right fit? -func (c *constructor) walk(typ reflect.Type) error { +func (c *constructor) walk(typ reflect.Type) (string, error) { // Dereference pointers if necessary. for typ.Kind() == reflect.Ptr { typ = typ.Elem() @@ -110,7 +115,7 @@ func (c *constructor) walk(typ reflect.Type) error { // Return value directly if it's already been processed. if _, ok := c.definitions[typPath]; ok { - return nil + return "", nil } var s Schema @@ -139,10 +144,10 @@ func (c *constructor) walk(typ reflect.Type) error { // set to null and disallowed in the schema. s = Schema{Type: NullType} default: - return fmt.Errorf("unsupported type: %s", typ.Kind()) + return "", fmt.Errorf("unsupported type: %s", typ.Kind()) } if err != nil { - return err + return "", err } if c.fn != nil { @@ -153,7 +158,7 @@ func (c *constructor) walk(typ reflect.Type) error { // TODO: Apply transformation at the end, to all definitions instead of // during recursive traversal? c.definitions[typPath] = s - return nil + return typPath, nil } // This function returns all member fields of the provided type. @@ -230,12 +235,11 @@ func (c *constructor) fromTypeStruct(typ reflect.Type) (Schema, error) { // Trigger call to fromType, to recursively generate definitions for // the struct field. - err := c.walk(structField.Type) + typPath, err := c.walk(structField.Type) if err != nil { return InvalidSchema, err } - typPath := typePath(structField.Type) refPath := path.Join("#/$defs", typPath) // For non-built-in types, refer to the definition. res.Properties[jsonTags[0]] = &Schema{ @@ -259,12 +263,11 @@ func (c *constructor) fromTypeSlice(typ reflect.Type) (Schema, error) { // Trigger call to fromType, to recursively generate definitions for // the slice element. - err := c.walk(typ.Elem()) + typPath, err := c.walk(typ.Elem()) if err != nil { return InvalidSchema, err } - typPath := typePath(typ.Elem()) refPath := path.Join("#/$defs", typPath) // For non-built-in types, refer to the definition @@ -289,12 +292,11 @@ func (c *constructor) fromTypeMap(typ reflect.Type) (Schema, error) { // Trigger call to fromType, to recursively generate definitions for // the map value. - err := c.walk(typ.Elem()) + typPath, err := c.walk(typ.Elem()) if err != nil { return InvalidSchema, err } - typPath := typePath(typ.Elem()) refPath := path.Join("#/$defs", typPath) // For non-built-in types, refer to the definition diff --git a/libs/jsonschema/from_type_test.go b/libs/jsonschema/from_type_test.go index fd60bf44ff..9ba06ba910 100644 --- a/libs/jsonschema/from_type_test.go +++ b/libs/jsonschema/from_type_test.go @@ -1,6 +1,7 @@ package jsonschema import ( + "encoding/json" "reflect" "testing" @@ -9,13 +10,23 @@ import ( func TestFromTypeBasic(t *testing.T) { type myStruct struct { - S string `json:"s"` - I int `json:"i"` + S string `json:"s"` + I *int `json:"i,omitempty"` + V interface{} `json:"v,omitempty"` + + // These fields should be ignored in the resulting schema. + NotAnnotated string + DashedTag string `json:"-"` + notExported string `json:"not_exported"` + InternalTagged string `json:"internal_tagged" bundle:"internal"` + DeprecatedTagged string `json:"deprecated_tagged" bundle:"deprecated"` + ReadOnlyTagged string `json:"readonly_tagged" bundle:"readonly"` } strRef := "#/$defs/string" boolRef := "#/$defs/bool" intRef := "#/$defs/int" + interfaceRef := "#/$defs/interface" tcases := []struct { name string @@ -56,6 +67,9 @@ func TestFromTypeBasic(t *testing.T) { expected: Schema{ Type: "object", Definitions: map[string]any{ + "interface": Schema{ + Type: "null", + }, "string": Schema{ Type: "string", }, @@ -70,9 +84,12 @@ func TestFromTypeBasic(t *testing.T) { "i": { Reference: &intRef, }, + "v": { + Reference: &interfaceRef, + }, }, AdditionalProperties: false, - Required: []string{"s", "i"}, + Required: []string{"s"}, }, }, { @@ -113,14 +130,14 @@ func TestFromTypeBasic(t *testing.T) { assert.NoError(t, err) assert.Equal(t, tc.expected, s) - // jsonSchema, err := json.MarshalIndent(s, " ", " ") - // assert.NoError(t, err) + jsonSchema, err := json.MarshalIndent(s, " ", " ") + assert.NoError(t, err) - // expectedJson, err := json.MarshalIndent(tc.expected, " ", " ") - // assert.NoError(t, err) + expectedJson, err := json.MarshalIndent(tc.expected, " ", " ") + assert.NoError(t, err) - // t.Log("[DEBUG] actual: ", string(jsonSchema)) - // t.Log("[DEBUG] expected: ", string(expectedJson)) + t.Log("[DEBUG] actual: ", string(jsonSchema)) + t.Log("[DEBUG] expected: ", string(expectedJson)) }) } } From e24725d230a21baec8a5db4cb088df0e797bdb6a Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 20 Aug 2024 21:06:42 +0200 Subject: [PATCH 08/62] added nested tests --- libs/jsonschema/from_type.go | 2 +- libs/jsonschema/from_type_test.go | 114 ++++++++++++++++++++++++++++-- 2 files changed, 108 insertions(+), 8 deletions(-) diff --git a/libs/jsonschema/from_type.go b/libs/jsonschema/from_type.go index d0e4e2acdf..ee29684b1c 100644 --- a/libs/jsonschema/from_type.go +++ b/libs/jsonschema/from_type.go @@ -115,7 +115,7 @@ func (c *constructor) walk(typ reflect.Type) (string, error) { // Return value directly if it's already been processed. if _, ok := c.definitions[typPath]; ok { - return "", nil + return typPath, nil } var s Schema diff --git a/libs/jsonschema/from_type_test.go b/libs/jsonschema/from_type_test.go index 9ba06ba910..e70e75c487 100644 --- a/libs/jsonschema/from_type_test.go +++ b/libs/jsonschema/from_type_test.go @@ -1,7 +1,6 @@ package jsonschema import ( - "encoding/json" "reflect" "testing" @@ -130,14 +129,14 @@ func TestFromTypeBasic(t *testing.T) { assert.NoError(t, err) assert.Equal(t, tc.expected, s) - jsonSchema, err := json.MarshalIndent(s, " ", " ") - assert.NoError(t, err) + // jsonSchema, err := json.MarshalIndent(s, " ", " ") + // assert.NoError(t, err) - expectedJson, err := json.MarshalIndent(tc.expected, " ", " ") - assert.NoError(t, err) + // expectedJson, err := json.MarshalIndent(tc.expected, " ", " ") + // assert.NoError(t, err) - t.Log("[DEBUG] actual: ", string(jsonSchema)) - t.Log("[DEBUG] expected: ", string(expectedJson)) + // t.Log("[DEBUG] actual: ", string(jsonSchema)) + // t.Log("[DEBUG] expected: ", string(expectedJson)) }) } } @@ -159,3 +158,104 @@ func TestGetStructFields(t *testing.T) { assert.Equal(t, "I", fields[1].Name) assert.Equal(t, "B", fields[2].Name) } + +func TestFromTypeNested(t *testing.T) { + type Inner struct { + S string `json:"s"` + } + + type Outer struct { + I string `json:"i"` + Inner Inner `json:"inner"` + } + + innerRef := "#/$defs/github.com/databricks/cli/libs/jsonschema.Inner" + strRef := "#/$defs/string" + + expectedDefinitions := map[string]any{ + "github.com": map[string]any{ + "databricks": map[string]any{ + "cli": map[string]any{ + "libs": map[string]any{ + "jsonschema.Inner": Schema{ + Type: "object", + Properties: map[string]*Schema{ + "s": { + Reference: &strRef, + }, + }, + AdditionalProperties: false, + Required: []string{"s"}, + }, + }, + }, + }, + }, + "string": Schema{ + Type: "string", + }, + } + + tcases := []struct { + name string + typ reflect.Type + expected Schema + }{ + { + name: "struct in struct", + typ: reflect.TypeOf(Outer{}), + expected: Schema{ + Type: "object", + Definitions: expectedDefinitions, + Properties: map[string]*Schema{ + "i": { + Reference: &strRef, + }, + "inner": { + Reference: &innerRef, + }, + }, + AdditionalProperties: false, + Required: []string{"i", "inner"}, + }, + }, + { + name: "struct as a map value", + typ: reflect.TypeOf(map[string]Inner{}), + expected: Schema{ + Type: "object", + Definitions: expectedDefinitions, + AdditionalProperties: &Schema{ + Reference: &innerRef, + }, + }, + }, + { + name: "struct as a slice element", + typ: reflect.TypeOf([]Inner{}), + expected: Schema{ + Type: "array", + Definitions: expectedDefinitions, + Items: &Schema{ + Reference: &innerRef, + }, + }, + }, + } + for _, tc := range tcases { + t.Run(tc.name, func(t *testing.T) { + s, err := FromType(tc.typ, nil) + assert.NoError(t, err) + assert.Equal(t, tc.expected, s) + + // jsonSchema, err := json.MarshalIndent(s, " ", " ") + // assert.NoError(t, err) + + // expectedJson, err := json.MarshalIndent(tc.expected, " ", " ") + // assert.NoError(t, err) + + // t.Log("[DEBUG] actual: ", string(jsonSchema)) + // t.Log("[DEBUG] expected: ", string(expectedJson)) + }) + } +} From 460eeb928d6dafe0c6a6fd336343290890cabac9 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 20 Aug 2024 21:11:42 +0200 Subject: [PATCH 09/62] more tests --- bundle/schema/schema_test.go | 377 ----------------------------------- 1 file changed, 377 deletions(-) diff --git a/bundle/schema/schema_test.go b/bundle/schema/schema_test.go index ad9cb530bf..9a16c89bf9 100644 --- a/bundle/schema/schema_test.go +++ b/bundle/schema/schema_test.go @@ -11,383 +11,6 @@ import ( // TODO: Add a test that checks the primitive overrides for reference regexs wor. -func TestStructOfStructsSchema(t *testing.T) { - type Bar struct { - A int `json:"a"` - B string `json:"b,string"` - } - - type Foo struct { - Bar Bar `json:"bar"` - } - - type MyStruct struct { - Foo Foo `json:"foo"` - } - - elem := MyStruct{} - - schema, err := New(reflect.TypeOf(elem), nil) - assert.NoError(t, err) - - jsonSchema, err := json.MarshalIndent(schema, " ", " ") - assert.NoError(t, err) - - expected := - `{ - "type": "object", - "properties": { - "foo": { - "type": "object", - "properties": { - "bar": { - "type": "object", - "properties": { - "a": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "string", - "pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - } - ] - }, - "b": { - "type": "string" - } - }, - "additionalProperties": false, - "required": [ - "a", - "b" - ] - } - }, - "additionalProperties": false, - "required": [ - "bar" - ] - } - }, - "additionalProperties": false, - "required": [ - "foo" - ] - }` - - t.Log("[DEBUG] actual: ", string(jsonSchema)) - t.Log("[DEBUG] expected: ", expected) - assert.Equal(t, expected, string(jsonSchema)) -} - -func TestStructOfMapsSchema(t *testing.T) { - type Bar struct { - MyMap map[string]int `json:"my_map"` - } - - type Foo struct { - Bar Bar `json:"bar"` - } - - elem := Foo{} - - schema, err := New(reflect.TypeOf(elem), nil) - assert.NoError(t, err) - - jsonSchema, err := json.MarshalIndent(schema, " ", " ") - assert.NoError(t, err) - - expected := - `{ - "type": "object", - "properties": { - "bar": { - "type": "object", - "properties": { - "my_map": { - "type": "object", - "additionalProperties": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "string", - "pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - } - ] - } - } - }, - "additionalProperties": false, - "required": [ - "my_map" - ] - } - }, - "additionalProperties": false, - "required": [ - "bar" - ] - }` - - t.Log("[DEBUG] actual: ", string(jsonSchema)) - t.Log("[DEBUG] expected: ", expected) - assert.Equal(t, expected, string(jsonSchema)) -} - -func TestStructOfSliceSchema(t *testing.T) { - type Bar struct { - MySlice []string `json:"my_slice"` - } - - type Foo struct { - Bar Bar `json:"bar"` - } - - elem := Foo{} - - schema, err := New(reflect.TypeOf(elem), nil) - assert.NoError(t, err) - - jsonSchema, err := json.MarshalIndent(schema, " ", " ") - assert.NoError(t, err) - - expected := - `{ - "type": "object", - "properties": { - "bar": { - "type": "object", - "properties": { - "my_slice": { - "type": "array", - "items": { - "type": "string" - } - } - }, - "additionalProperties": false, - "required": [ - "my_slice" - ] - } - }, - "additionalProperties": false, - "required": [ - "bar" - ] - }` - - t.Log("[DEBUG] actual: ", string(jsonSchema)) - t.Log("[DEBUG] expected: ", expected) - assert.Equal(t, expected, string(jsonSchema)) -} - -func TestMapOfStructSchema(t *testing.T) { - type Foo struct { - MyInt int `json:"my_int"` - } - - var elem map[string]Foo - - schema, err := New(reflect.TypeOf(elem), nil) - assert.NoError(t, err) - - jsonSchema, err := json.MarshalIndent(schema, " ", " ") - assert.NoError(t, err) - - expected := - `{ - "type": "object", - "additionalProperties": { - "type": "object", - "properties": { - "my_int": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "string", - "pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - } - ] - } - }, - "additionalProperties": false, - "required": [ - "my_int" - ] - } - }` - - t.Log("[DEBUG] actual: ", string(jsonSchema)) - t.Log("[DEBUG] expected: ", expected) - assert.Equal(t, expected, string(jsonSchema)) -} - -func TestMapOfMapSchema(t *testing.T) { - var elem map[string]map[string]int - - schema, err := New(reflect.TypeOf(elem), nil) - assert.NoError(t, err) - - jsonSchema, err := json.MarshalIndent(schema, " ", " ") - assert.NoError(t, err) - - expected := - `{ - "type": "object", - "additionalProperties": { - "type": "object", - "additionalProperties": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "string", - "pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - } - ] - } - } - }` - - t.Log("[DEBUG] actual: ", string(jsonSchema)) - t.Log("[DEBUG] expected: ", expected) - assert.Equal(t, expected, string(jsonSchema)) -} - -func TestMapOfSliceSchema(t *testing.T) { - var elem map[string][]string - - schema, err := New(reflect.TypeOf(elem), nil) - assert.NoError(t, err) - - jsonSchema, err := json.MarshalIndent(schema, " ", " ") - assert.NoError(t, err) - - expected := - `{ - "type": "object", - "additionalProperties": { - "type": "array", - "items": { - "type": "string" - } - } - }` - - t.Log("[DEBUG] actual: ", string(jsonSchema)) - t.Log("[DEBUG] expected: ", expected) - assert.Equal(t, expected, string(jsonSchema)) -} - -func TestSliceOfSliceSchema(t *testing.T) { - var elem [][]string - - schema, err := New(reflect.TypeOf(elem), nil) - assert.NoError(t, err) - - jsonSchema, err := json.MarshalIndent(schema, " ", " ") - assert.NoError(t, err) - - expected := - `{ - "type": "array", - "items": { - "type": "array", - "items": { - "type": "string" - } - } - }` - - t.Log("[DEBUG] actual: ", string(jsonSchema)) - t.Log("[DEBUG] expected: ", expected) - assert.Equal(t, expected, string(jsonSchema)) -} - -func TestSliceOfMapSchema(t *testing.T) { - var elem []map[string]int - - schema, err := New(reflect.TypeOf(elem), nil) - assert.NoError(t, err) - - jsonSchema, err := json.MarshalIndent(schema, " ", " ") - assert.NoError(t, err) - - expected := - `{ - "type": "array", - "items": { - "type": "object", - "additionalProperties": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "string", - "pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - } - ] - } - } - }` - - t.Log("[DEBUG] actual: ", string(jsonSchema)) - t.Log("[DEBUG] expected: ", expected) - assert.Equal(t, expected, string(jsonSchema)) -} - -func TestSliceOfStructSchema(t *testing.T) { - type Foo struct { - MyInt int `json:"my_int"` - } - - var elem []Foo - - schema, err := New(reflect.TypeOf(elem), nil) - assert.NoError(t, err) - - jsonSchema, err := json.MarshalIndent(schema, " ", " ") - assert.NoError(t, err) - - expected := - `{ - "type": "array", - "items": { - "type": "object", - "properties": { - "my_int": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "string", - "pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - } - ] - } - }, - "additionalProperties": false, - "required": [ - "my_int" - ] - } - }` - - t.Log("[DEBUG] actual: ", string(jsonSchema)) - t.Log("[DEBUG] expected: ", expected) - assert.Equal(t, expected, string(jsonSchema)) -} - func TestErrorWithTrace(t *testing.T) { tracker := newTracker() dummyType := reflect.TypeOf(struct{}{}) From 00e5896966b5b386ceff2eddceaa3689ab9a8792 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 20 Aug 2024 21:53:04 +0200 Subject: [PATCH 10/62] add test for recursive --- libs/jsonschema/from_type.go | 70 ++++++++++++++++-------- libs/jsonschema/from_type_test.go | 64 ++++++++++++++++++++++ libs/jsonschema/test_types/test_types.go | 15 +++++ 3 files changed, 127 insertions(+), 22 deletions(-) create mode 100644 libs/jsonschema/test_types/test_types.go diff --git a/libs/jsonschema/from_type.go b/libs/jsonschema/from_type.go index ee29684b1c..2c1d4ca62f 100644 --- a/libs/jsonschema/from_type.go +++ b/libs/jsonschema/from_type.go @@ -35,6 +35,8 @@ type constructor struct { // Example key: github.com/databricks/databricks-sdk-go/service/jobs.JobSettings definitions map[string]Schema + seen map[string]struct{} + // Transformation function to apply after generating a node in the schema. fn func(s Schema) Schema } @@ -74,10 +76,11 @@ func (c *constructor) nestedDefinitions() any { func FromType(typ reflect.Type, fn func(s Schema) Schema) (Schema, error) { c := constructor{ definitions: make(map[string]Schema), + seen: make(map[string]struct{}), fn: fn, } - _, err := c.walk(typ) + err := c.walk(typ) if err != nil { return InvalidSchema, err } @@ -90,6 +93,11 @@ func FromType(typ reflect.Type, fn func(s Schema) Schema) (Schema, error) { } func typePath(typ reflect.Type) string { + // Pointers have a typ.Name() of "". Dereference them to get the underlying type. + for typ.Kind() == reflect.Ptr { + typ = typ.Elem() + } + // typ.Name() resolves to "" for any type. if typ.Kind() == reflect.Interface { return "interface" @@ -105,7 +113,7 @@ func typePath(typ reflect.Type) string { // TODO: would a worked based model fit better here? Is this internal API not // the right fit? -func (c *constructor) walk(typ reflect.Type) (string, error) { +func (c *constructor) walk(typ reflect.Type) error { // Dereference pointers if necessary. for typ.Kind() == reflect.Ptr { typ = typ.Elem() @@ -113,9 +121,14 @@ func (c *constructor) walk(typ reflect.Type) (string, error) { typPath := typePath(typ) - // Return value directly if it's already been processed. + // Keep track of seen types to avoid infinite recursion. + if _, ok := c.seen[typPath]; !ok { + c.seen[typPath] = struct{}{} + } + + // Return early directly if it's already been processed. if _, ok := c.definitions[typPath]; ok { - return typPath, nil + return nil } var s Schema @@ -144,10 +157,10 @@ func (c *constructor) walk(typ reflect.Type) (string, error) { // set to null and disallowed in the schema. s = Schema{Type: NullType} default: - return "", fmt.Errorf("unsupported type: %s", typ.Kind()) + return fmt.Errorf("unsupported type: %s", typ.Kind()) } if err != nil { - return "", err + return err } if c.fn != nil { @@ -158,7 +171,7 @@ func (c *constructor) walk(typ reflect.Type) (string, error) { // TODO: Apply transformation at the end, to all definitions instead of // during recursive traversal? c.definitions[typPath] = s - return typPath, nil + return nil } // This function returns all member fields of the provided type. @@ -193,6 +206,7 @@ func getStructFields(typ reflect.Type) []reflect.StructField { return fields } +// TODO: get rid of the errors here and panic instead? func (c *constructor) fromTypeStruct(typ reflect.Type) (Schema, error) { if typ.Kind() != reflect.Struct { return InvalidSchema, fmt.Errorf("expected struct, got %s", typ.Kind()) @@ -233,11 +247,15 @@ func (c *constructor) fromTypeStruct(typ reflect.Type) (Schema, error) { res.Required = append(res.Required, jsonTags[0]) } - // Trigger call to fromType, to recursively generate definitions for - // the struct field. - typPath, err := c.walk(structField.Type) - if err != nil { - return InvalidSchema, err + typPath := typePath(structField.Type) + // Only walk if the type has not been seen yet. + if _, ok := c.seen[typPath]; !ok { + // Trigger call to fromType, to recursively generate definitions for + // the struct field. + err := c.walk(structField.Type) + if err != nil { + return InvalidSchema, err + } } refPath := path.Join("#/$defs", typPath) @@ -261,11 +279,15 @@ func (c *constructor) fromTypeSlice(typ reflect.Type) (Schema, error) { Type: ArrayType, } - // Trigger call to fromType, to recursively generate definitions for - // the slice element. - typPath, err := c.walk(typ.Elem()) - if err != nil { - return InvalidSchema, err + typPath := typePath(typ.Elem()) + // Only walk if the type has not been seen yet. + if _, ok := c.seen[typPath]; !ok { + // Trigger call to fromType, to recursively generate definitions for + // the slice element. + err := c.walk(typ.Elem()) + if err != nil { + return InvalidSchema, err + } } refPath := path.Join("#/$defs", typPath) @@ -290,11 +312,15 @@ func (c *constructor) fromTypeMap(typ reflect.Type) (Schema, error) { Type: ObjectType, } - // Trigger call to fromType, to recursively generate definitions for - // the map value. - typPath, err := c.walk(typ.Elem()) - if err != nil { - return InvalidSchema, err + typPath := typePath(typ.Elem()) + // Only walk if the type has not been seen yet. + if _, ok := c.seen[typPath]; !ok { + // Trigger call to fromType, to recursively generate definitions for + // the map value. + err := c.walk(typ.Elem()) + if err != nil { + return InvalidSchema, err + } } refPath := path.Join("#/$defs", typPath) diff --git a/libs/jsonschema/from_type_test.go b/libs/jsonschema/from_type_test.go index e70e75c487..24d6e99c83 100644 --- a/libs/jsonschema/from_type_test.go +++ b/libs/jsonschema/from_type_test.go @@ -1,9 +1,11 @@ package jsonschema import ( + "encoding/json" "reflect" "testing" + "github.com/databricks/cli/libs/jsonschema/test_types" "github.com/stretchr/testify/assert" ) @@ -259,3 +261,65 @@ func TestFromTypeNested(t *testing.T) { }) } } + +// TODO: Call out in the PR description that recursive Go types are supported. +func TestFromTypeRecursive(t *testing.T) { + fooRef := "#/$defs/github.com/databricks/cli/libs/jsonschema/test_types.Foo" + barRef := "#/$defs/github.com/databricks/cli/libs/jsonschema/test_types.Bar" + + expected := Schema{ + Type: "object", + Definitions: map[string]any{ + "github.com": map[string]any{ + "databricks": map[string]any{ + "cli": map[string]any{ + "libs": map[string]any{ + "jsonschema": map[string]any{ + "test_types.Bar": Schema{ + Type: "object", + Properties: map[string]*Schema{ + "foo": { + Reference: &fooRef, + }, + }, + AdditionalProperties: false, + Required: []string{}, + }, + "test_types.Foo": Schema{ + Type: "object", + Properties: map[string]*Schema{ + "bar": { + Reference: &barRef, + }, + }, + AdditionalProperties: false, + Required: []string{}, + }, + }, + }, + }, + }, + }, + }, + Properties: map[string]*Schema{ + "foo": { + Reference: &fooRef, + }, + }, + AdditionalProperties: false, + Required: []string{"foo"}, + } + + s, err := FromType(reflect.TypeOf(test_types.Outer{}), nil) + assert.NoError(t, err) + assert.Equal(t, expected, s) + + jsonSchema, err := json.MarshalIndent(s, " ", " ") + assert.NoError(t, err) + + expectedJson, err := json.MarshalIndent(expected, " ", " ") + assert.NoError(t, err) + + t.Log("[DEBUG] actual: ", string(jsonSchema)) + t.Log("[DEBUG] expected: ", string(expectedJson)) +} diff --git a/libs/jsonschema/test_types/test_types.go b/libs/jsonschema/test_types/test_types.go new file mode 100644 index 0000000000..febe5c33bd --- /dev/null +++ b/libs/jsonschema/test_types/test_types.go @@ -0,0 +1,15 @@ +package test_types + +// Recursive types cannot be defined inline without making them anonymous, +// so we define them here instead. +type Foo struct { + Bar *Bar `json:"bar,omitempty"` +} + +type Bar struct { + Foo Foo `json:"foo,omitempty"` +} + +type Outer struct { + Foo Foo `json:"foo"` +} From 7c70179091fff05ddd6f81b903c1afe895d0252f Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Thu, 22 Aug 2024 17:38:24 +0200 Subject: [PATCH 11/62] add self referential tesT --- bundle/schema/schema_test.go | 37 ++----------- libs/jsonschema/from_type_test.go | 68 +++++++++++++++++++++--- libs/jsonschema/test_types/test_types.go | 10 ++++ 3 files changed, 75 insertions(+), 40 deletions(-) diff --git a/bundle/schema/schema_test.go b/bundle/schema/schema_test.go index 9a16c89bf9..576f02e4fd 100644 --- a/bundle/schema/schema_test.go +++ b/bundle/schema/schema_test.go @@ -9,23 +9,9 @@ import ( "github.com/stretchr/testify/require" ) -// TODO: Add a test that checks the primitive overrides for reference regexs wor. - -func TestErrorWithTrace(t *testing.T) { - tracker := newTracker() - dummyType := reflect.TypeOf(struct{}{}) - err := tracker.errWithTrace("with empty trace", "root") - assert.ErrorContains(t, err, "with empty trace. traversal trace: root") - - tracker.push(dummyType, "resources") - err = tracker.errWithTrace("with depth = 1", "root") - assert.ErrorContains(t, err, "with depth = 1. traversal trace: root -> resources") - - tracker.push(dummyType, "pipelines") - tracker.push(dummyType, "datasets") - err = tracker.errWithTrace("with depth = 4", "root") - assert.ErrorContains(t, err, "with depth = 4. traversal trace: root -> resources -> pipelines -> datasets") -} +// TODO: Add a test that checks the primitive overrides for reference regexs work. +// Basically that the custom override for bundle regex works. + func TestGenericSchema(t *testing.T) { type Person struct { @@ -599,20 +585,3 @@ func TestErrorIfStructRefersToItself(t *testing.T) { _, err := New(reflect.TypeOf(elem), nil) assert.ErrorContains(t, err, "cycle detected. traversal trace: root -> my_foo") } - -func TestErrorIfStructHasLoop(t *testing.T) { - type Apple struct { - MyVal int `json:"my_val"` - MyMango struct { - MyGuava struct { - MyPapaya struct { - MyApple *Apple `json:"my_apple"` - } `json:"my_papaya"` - } `json:"my_guava"` - } `json:"my_mango"` - } - - elem := Apple{} - _, err := New(reflect.TypeOf(elem), nil) - assert.ErrorContains(t, err, "cycle detected. traversal trace: root -> my_mango -> my_guava -> my_papaya -> my_apple") -} diff --git a/libs/jsonschema/from_type_test.go b/libs/jsonschema/from_type_test.go index 24d6e99c83..493acc475c 100644 --- a/libs/jsonschema/from_type_test.go +++ b/libs/jsonschema/from_type_test.go @@ -1,7 +1,6 @@ package jsonschema import ( - "encoding/json" "reflect" "testing" @@ -314,12 +313,69 @@ func TestFromTypeRecursive(t *testing.T) { assert.NoError(t, err) assert.Equal(t, expected, s) - jsonSchema, err := json.MarshalIndent(s, " ", " ") - assert.NoError(t, err) + // jsonSchema, err := json.MarshalIndent(s, " ", " ") + // assert.NoError(t, err) + + // expectedJson, err := json.MarshalIndent(expected, " ", " ") + // assert.NoError(t, err) + + // t.Log("[DEBUG] actual: ", string(jsonSchema)) + // t.Log("[DEBUG] expected: ", string(expectedJson)) +} + +func TestFromTypeSelfReferential(t *testing.T) { + selfRef := "#/$defs/github.com/databricks/cli/libs/jsonschema/test_types.Self" + stringRef := "#/$defs/string" + + expected := Schema{ + Type: "object", + Definitions: map[string]any{ + "github.com": map[string]any{ + "databricks": map[string]any{ + "cli": map[string]any{ + "libs": map[string]any{ + "jsonschema": map[string]any{ + "test_types.Self": Schema{ + Type: "object", + Properties: map[string]*Schema{ + "self": { + Reference: &selfRef, + }, + "s": { + Reference: &stringRef, + }, + }, + AdditionalProperties: false, + Required: []string{}, + }, + }, + }, + }, + }, + }, + "string": Schema{ + Type: "string", + }, + }, + Properties: map[string]*Schema{ + "self": { + Reference: &selfRef, + }, + }, + AdditionalProperties: false, + Required: []string{}, + } - expectedJson, err := json.MarshalIndent(expected, " ", " ") + s, err := FromType(reflect.TypeOf(test_types.OuterSelf{}), nil) assert.NoError(t, err) + assert.Equal(t, expected, s) + + // jsonSchema, err := json.MarshalIndent(s, " ", " ") + // assert.NoError(t, err) + + // expectedJson, err := json.MarshalIndent(expected, " ", " ") + // assert.NoError(t, err) - t.Log("[DEBUG] actual: ", string(jsonSchema)) - t.Log("[DEBUG] expected: ", string(expectedJson)) + // t.Log("[DEBUG] actual: ", string(jsonSchema)) + // t.Log("[DEBUG] expected: ", string(expectedJson)) } diff --git a/libs/jsonschema/test_types/test_types.go b/libs/jsonschema/test_types/test_types.go index febe5c33bd..75e81595a0 100644 --- a/libs/jsonschema/test_types/test_types.go +++ b/libs/jsonschema/test_types/test_types.go @@ -13,3 +13,13 @@ type Bar struct { type Outer struct { Foo Foo `json:"foo"` } + +type Self struct { + Self *Self `json:"self,omitempty"` + + S string `json:"s,omitempty"` +} + +type OuterSelf struct { + Self Self `json:"self,omitempty"` +} From 39efb705decfb15e25f2f30125c9519012549198 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Thu, 22 Aug 2024 17:40:43 +0200 Subject: [PATCH 12/62] - --- bundle/schema/schema_test.go | 19 ------------------- libs/jsonschema/from_type_test.go | 6 ++++++ 2 files changed, 6 insertions(+), 19 deletions(-) diff --git a/bundle/schema/schema_test.go b/bundle/schema/schema_test.go index 576f02e4fd..3eab7ca704 100644 --- a/bundle/schema/schema_test.go +++ b/bundle/schema/schema_test.go @@ -566,22 +566,3 @@ func TestDocIngestionForTopLevelPrimitive(t *testing.T) { assert.Equal(t, expectedSchema, string(jsonSchema)) } - -func TestErrorOnMapWithoutStringKey(t *testing.T) { - type Foo struct { - Bar map[int]string `json:"bar"` - } - elem := Foo{} - _, err := New(reflect.TypeOf(elem), nil) - assert.ErrorContains(t, err, "only strings map keys are valid. key type: int") -} - -func TestErrorIfStructRefersToItself(t *testing.T) { - type Foo struct { - MyFoo *Foo `json:"my_foo"` - } - - elem := Foo{} - _, err := New(reflect.TypeOf(elem), nil) - assert.ErrorContains(t, err, "cycle detected. traversal trace: root -> my_foo") -} diff --git a/libs/jsonschema/from_type_test.go b/libs/jsonschema/from_type_test.go index 493acc475c..cdd26bf65e 100644 --- a/libs/jsonschema/from_type_test.go +++ b/libs/jsonschema/from_type_test.go @@ -379,3 +379,9 @@ func TestFromTypeSelfReferential(t *testing.T) { // t.Log("[DEBUG] actual: ", string(jsonSchema)) // t.Log("[DEBUG] expected: ", string(expectedJson)) } + +func TestFromTypeError(t *testing.T) { + type mapOfInts map[int]int + _, err := FromType(reflect.TypeOf(mapOfInts{}), nil) + assert.EqualError(t, err, "found map with non-string key: int") +} From 790731f9815fc238e2a6b93079b3b48489525520 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Thu, 22 Aug 2024 17:42:19 +0200 Subject: [PATCH 13/62] remove generic schema test --- bundle/schema/schema_test.go | 229 +---------------------------------- 1 file changed, 1 insertion(+), 228 deletions(-) diff --git a/bundle/schema/schema_test.go b/bundle/schema/schema_test.go index 3eab7ca704..277d5a8e4b 100644 --- a/bundle/schema/schema_test.go +++ b/bundle/schema/schema_test.go @@ -12,234 +12,7 @@ import ( // TODO: Add a test that checks the primitive overrides for reference regexs work. // Basically that the custom override for bundle regex works. - -func TestGenericSchema(t *testing.T) { - type Person struct { - Name string `json:"name"` - Age int `json:"age,omitempty"` - } - - type Plot struct { - Stakes []string `json:"stakes"` - Deaths []Person `json:"deaths"` - Murders map[string]Person `json:"murders"` - } - - type Wedding struct { - Hidden string `json:","` - Groom Person `json:"groom"` - Bride Person `json:"bride"` - Plots []Plot `json:"plots"` - } - - type Story struct { - Hero *Person `json:"hero"` - Villian Person `json:"villian,omitempty"` - Weddings []Wedding `json:"weddings"` - } - - elem := Story{} - - schema, err := New(reflect.TypeOf(elem), nil) - assert.NoError(t, err) - - jsonSchema, err := json.MarshalIndent(schema, " ", " ") - assert.NoError(t, err) - - expected := - `{ - "type": "object", - "properties": { - "hero": { - "type": "object", - "properties": { - "age": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "string", - "pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - } - ] - }, - "name": { - "type": "string" - } - }, - "additionalProperties": false, - "required": [ - "name" - ] - }, - "villian": { - "type": "object", - "properties": { - "age": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "string", - "pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - } - ] - }, - "name": { - "type": "string" - } - }, - "additionalProperties": false, - "required": [ - "name" - ] - }, - "weddings": { - "type": "array", - "items": { - "type": "object", - "properties": { - "bride": { - "type": "object", - "properties": { - "age": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "string", - "pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - } - ] - }, - "name": { - "type": "string" - } - }, - "additionalProperties": false, - "required": [ - "name" - ] - }, - "groom": { - "type": "object", - "properties": { - "age": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "string", - "pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - } - ] - }, - "name": { - "type": "string" - } - }, - "additionalProperties": false, - "required": [ - "name" - ] - }, - "plots": { - "type": "array", - "items": { - "type": "object", - "properties": { - "deaths": { - "type": "array", - "items": { - "type": "object", - "properties": { - "age": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "string", - "pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - } - ] - }, - "name": { - "type": "string" - } - }, - "additionalProperties": false, - "required": [ - "name" - ] - } - }, - "murders": { - "type": "object", - "additionalProperties": { - "type": "object", - "properties": { - "age": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "string", - "pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - } - ] - }, - "name": { - "type": "string" - } - }, - "additionalProperties": false, - "required": [ - "name" - ] - } - }, - "stakes": { - "type": "array", - "items": { - "type": "string" - } - } - }, - "additionalProperties": false, - "required": [ - "stakes", - "deaths", - "murders" - ] - } - } - }, - "additionalProperties": false, - "required": [ - "groom", - "bride", - "plots" - ] - } - } - }, - "additionalProperties": false, - "required": [ - "hero", - "weddings" - ] - }` - - t.Log("[DEBUG] actual: ", string(jsonSchema)) - t.Log("[DEBUG] expected: ", expected) - assert.Equal(t, expected, string(jsonSchema)) -} +// TODO: Add a bundle of end to end tests, that both fail and pass the schema validation. func TestDocIngestionForObject(t *testing.T) { docs := &Docs{ From 535f6708685a8f0246c738ea0eee4e9c18550084 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Thu, 22 Aug 2024 17:43:22 +0200 Subject: [PATCH 14/62] remove most of the debug helpers --- libs/jsonschema/from_type_test.go | 27 --------------------------- 1 file changed, 27 deletions(-) diff --git a/libs/jsonschema/from_type_test.go b/libs/jsonschema/from_type_test.go index cdd26bf65e..6a3ae928d5 100644 --- a/libs/jsonschema/from_type_test.go +++ b/libs/jsonschema/from_type_test.go @@ -248,15 +248,6 @@ func TestFromTypeNested(t *testing.T) { s, err := FromType(tc.typ, nil) assert.NoError(t, err) assert.Equal(t, tc.expected, s) - - // jsonSchema, err := json.MarshalIndent(s, " ", " ") - // assert.NoError(t, err) - - // expectedJson, err := json.MarshalIndent(tc.expected, " ", " ") - // assert.NoError(t, err) - - // t.Log("[DEBUG] actual: ", string(jsonSchema)) - // t.Log("[DEBUG] expected: ", string(expectedJson)) }) } } @@ -312,15 +303,6 @@ func TestFromTypeRecursive(t *testing.T) { s, err := FromType(reflect.TypeOf(test_types.Outer{}), nil) assert.NoError(t, err) assert.Equal(t, expected, s) - - // jsonSchema, err := json.MarshalIndent(s, " ", " ") - // assert.NoError(t, err) - - // expectedJson, err := json.MarshalIndent(expected, " ", " ") - // assert.NoError(t, err) - - // t.Log("[DEBUG] actual: ", string(jsonSchema)) - // t.Log("[DEBUG] expected: ", string(expectedJson)) } func TestFromTypeSelfReferential(t *testing.T) { @@ -369,15 +351,6 @@ func TestFromTypeSelfReferential(t *testing.T) { s, err := FromType(reflect.TypeOf(test_types.OuterSelf{}), nil) assert.NoError(t, err) assert.Equal(t, expected, s) - - // jsonSchema, err := json.MarshalIndent(s, " ", " ") - // assert.NoError(t, err) - - // expectedJson, err := json.MarshalIndent(expected, " ", " ") - // assert.NoError(t, err) - - // t.Log("[DEBUG] actual: ", string(jsonSchema)) - // t.Log("[DEBUG] expected: ", string(expectedJson)) } func TestFromTypeError(t *testing.T) { From 11dfdc036d17f1cbfb2ffcf20bc360db5d38197e Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 26 Aug 2024 20:16:45 +0200 Subject: [PATCH 15/62] more iteration --- bundle/internal/bundle/schema/main.go | 42 ------ bundle/internal/schema/main.go | 96 ++++++++++++++ bundle/internal/schema/parser.go | 131 ++++++++++++++++++ bundle/schema/schema.go | 43 +++--- libs/dyn/dynvar/ref.go | 4 +- libs/jsonschema/from_type.go | 182 +++++++++++++------------- libs/jsonschema/from_type_test.go | 81 +++++++++++- 7 files changed, 419 insertions(+), 160 deletions(-) delete mode 100644 bundle/internal/bundle/schema/main.go create mode 100644 bundle/internal/schema/main.go create mode 100644 bundle/internal/schema/parser.go diff --git a/bundle/internal/bundle/schema/main.go b/bundle/internal/bundle/schema/main.go deleted file mode 100644 index c9cc7cd4fb..0000000000 --- a/bundle/internal/bundle/schema/main.go +++ /dev/null @@ -1,42 +0,0 @@ -package main - -import ( - "encoding/json" - "fmt" - "log" - "os" - - "github.com/databricks/cli/bundle/schema" -) - -func main() { - if len(os.Args) != 2 { - fmt.Println("Usage: go run main.go ") - os.Exit(1) - } - - // Output file, to write the generated schema descriptions to. - outputFile := os.Args[1] - - // Input file, the databricks openapi spec. - inputFile := os.Getenv("DATABRICKS_OPENAPI_SPEC") - if inputFile == "" { - log.Fatal("DATABRICKS_OPENAPI_SPEC environment variable not set") - } - - // Generate the schema descriptions. - docs, err := schema.UpdateBundleDescriptions(inputFile) - if err != nil { - log.Fatal(err) - } - result, err := json.MarshalIndent(docs, "", " ") - if err != nil { - log.Fatal(err) - } - - // Write the schema descriptions to the output file. - err = os.WriteFile(outputFile, result, 0644) - if err != nil { - log.Fatal(err) - } -} diff --git a/bundle/internal/schema/main.go b/bundle/internal/schema/main.go new file mode 100644 index 0000000000..45d0f00a1f --- /dev/null +++ b/bundle/internal/schema/main.go @@ -0,0 +1,96 @@ +package main + +import ( + "encoding/json" + "fmt" + "log" + "os" + "reflect" + + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/libs/jsonschema" +) + +func interpolationPattern(s string) string { + return fmt.Sprintf(`\$\{(%s(\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\[[0-9]+\])*)*(\[[0-9]+\])*)\}`, s) +} + +func addInterpolationPatterns(_ reflect.Type, s jsonschema.Schema) jsonschema.Schema { + switch s.Type { + case jsonschema.ArrayType, jsonschema.ObjectType: + // arrays and objects can have complex variable values specified. + return jsonschema.Schema{ + AnyOf: []jsonschema.Schema{s, { + Type: jsonschema.StringType, + // TODO: Are multi-level complex variable references supported? + Pattern: interpolationPattern("var"), + }}, + } + case jsonschema.StringType, jsonschema.IntegerType, jsonschema.NumberType, jsonschema.BooleanType: + // primitives can have variable values, or references like ${bundle.xyz} + // or ${workspace.xyz} + // TODO: Followup, do not allow references like ${} in the schema unless + // they are of the permitted patterns? + return jsonschema.Schema{ + AnyOf: []jsonschema.Schema{s, + // TODO: Add "resources" here + {Type: jsonschema.StringType, Pattern: interpolationPattern("bundle")}, + {Type: jsonschema.StringType, Pattern: interpolationPattern("workspace")}, + {Type: jsonschema.StringType, Pattern: interpolationPattern("var")}, + }, + } + default: + return s + } +} + +// TODO: Add a couple of end to end tests that the bundle schema generated is +// correct. +// TODO: Call out in the PR description that recursive types like "for_each_task" +// are now supported. Manually test for_each_task. +// TODO: The bundle_descriptions.json file contains a bunch of custom descriptions +// as well. Make sure to pull those in. +// TODO: Add unit tests for all permutations of structs, maps and slices for the FromType +// method. + +func main() { + if len(os.Args) != 2 { + fmt.Println("Usage: go run main.go ") + os.Exit(1) + } + + // Output file, where the generated JSON schema will be written to. + outputFile := os.Args[1] + + // Input file, the databricks openapi spec. + inputFile := os.Getenv("DATABRICKS_OPENAPI_SPEC") + if inputFile == "" { + log.Fatal("DATABRICKS_OPENAPI_SPEC environment variable not set") + } + + p, err := newParser(inputFile) + if err != nil { + log.Fatal(err) + } + + // Generate the JSON schema from the bundle Go struct. + s, err := jsonschema.FromType(reflect.TypeOf(config.Root{}), []func(reflect.Type, jsonschema.Schema) jsonschema.Schema{ + p.addDescriptions, + p.addEnums, + addInterpolationPatterns, + }) + if err != nil { + log.Fatal(err) + } + + b, err := json.MarshalIndent(s, "", " ") + if err != nil { + log.Fatal(err) + } + + // Write the schema descriptions to the output file. + err = os.WriteFile(outputFile, b, 0644) + if err != nil { + log.Fatal(err) + } +} diff --git a/bundle/internal/schema/parser.go b/bundle/internal/schema/parser.go new file mode 100644 index 0000000000..dc85da3896 --- /dev/null +++ b/bundle/internal/schema/parser.go @@ -0,0 +1,131 @@ +package main + +import ( + "encoding/json" + "fmt" + "os" + "path" + "reflect" + "strings" + + "github.com/databricks/cli/libs/jsonschema" +) + +type Components struct { + Schemas map[string]jsonschema.Schema `json:"schemas,omitempty"` +} + +type Specification struct { + Components Components `json:"components"` +} + +type openapiParser struct { + ref map[string]jsonschema.Schema +} + +func newParser(path string) (*openapiParser, error) { + b, err := os.ReadFile(path) + if err != nil { + return nil, err + } + + spec := Specification{} + err = json.Unmarshal(b, &spec) + if err != nil { + return nil, err + } + + p := &openapiParser{} + p.ref = spec.Components.Schemas + return p, nil +} + +// This function finds any JSON schemas that were defined in the OpenAPI spec +// that correspond to the given Go SDK type. It looks both at the type itself +// and any embedded types within it. +func (p *openapiParser) findRef(typ reflect.Type) (jsonschema.Schema, bool) { + typs := []reflect.Type{typ} + + // If the type is a struct, the corresponding Go SDK struct might be embedded + // in it. We need to check for those as well. + if typ.Kind() == reflect.Struct { + for i := 0; i < typ.NumField(); i++ { + if !typ.Field(i).Anonymous { + continue + } + + // Deference current type if it's a pointer. + ctyp := typ.Field(i).Type + for ctyp.Kind() == reflect.Ptr { + ctyp = ctyp.Elem() + } + + typs = append(typs, ctyp) + } + } + + for _, ctyp := range typs { + // Skip if it's not a Go SDK type. + if !strings.HasPrefix(ctyp.PkgPath(), "github.com/databricks/databricks-sdk-go") { + continue + } + + pkgName := path.Base(ctyp.PkgPath()) + k := fmt.Sprintf("%s.%s", pkgName, ctyp.Name()) + + // Skip if the type is not in the openapi spec. + _, ok := p.ref[k] + if !ok { + continue + } + + // Return the first Go SDK type found in the openapi spec. + return p.ref[k], true + } + + return jsonschema.Schema{}, false +} + +// Use the OpenAPI spec to load descriptions for the given type. +func (p *openapiParser) addDescriptions(typ reflect.Type, s jsonschema.Schema) jsonschema.Schema { + ref, ok := p.findRef(typ) + if !ok { + return s + } + + s.Description = ref.Description + + // Iterate over properties to load descriptions. This is not needed for any + // OpenAPI spec generated from protobufs, which are guaranteed to be one level + // deep. + // Needed for any hand-written OpenAPI specs. + for k, v := range s.Properties { + if refProp, ok := ref.Properties[k]; ok { + v.Description = refProp.Description + } + } + + return s +} + +// Use the OpenAPI spec add enum values for the given type. +func (p *openapiParser) addEnums(typ reflect.Type, s jsonschema.Schema) jsonschema.Schema { + ref, ok := p.findRef(typ) + if !ok { + return s + } + + s.Enum = append(s.Enum, ref.Enum...) + + // Iterate over properties to load enums. This is not needed for any + // OpenAPI spec generated from protobufs, which are guaranteed to be one level + // deep. + // Needed for any hand-written OpenAPI specs. + for k, v := range s.Properties { + if refProp, ok := ref.Properties[k]; ok { + v.Enum = append(v.Enum, refProp.Enum...) + } + } + + return s +} diff --git a/bundle/schema/schema.go b/bundle/schema/schema.go index f8c77ab268..946b91725a 100644 --- a/bundle/schema/schema.go +++ b/bundle/schema/schema.go @@ -3,7 +3,6 @@ package schema import ( "reflect" - "github.com/databricks/cli/libs/dyn/dynvar" "github.com/databricks/cli/libs/jsonschema" ) @@ -42,26 +41,28 @@ import ( // for details visit: https://json-schema.org/understanding-json-schema/reference/object.html#properties func New(golangType reflect.Type, docs *Docs) (*jsonschema.Schema, error) { - s, err := jsonschema.FromType(golangType, func(s jsonschema.Schema) jsonschema.Schema { - if s.Type == jsonschema.NumberType || s.Type == jsonschema.BooleanType { - s = jsonschema.Schema{ - AnyOf: []jsonschema.Schema{ - s, - { - Type: jsonschema.StringType, - // TODO: Narrow down the scope of the regex match. - // Also likely need to rename this variable. - Pattern: dynvar.VariableRegex, - }, - }, - } - } - return s - }) - if err != nil { - return nil, err - } - return &s, nil + return nil, nil + + // s, err := jsonschema.FromType(golangType, func(s jsonschema.Schema) jsonschema.Schema { + // if s.Type == jsonschema.NumberType || s.Type == jsonschema.BooleanType { + // s = jsonschema.Schema{ + // AnyOf: []jsonschema.Schema{ + // s, + // { + // Type: jsonschema.StringType, + // // TODO: Narrow down the scope of the regex match. + // // Also likely need to rename this variable. + // Pattern: dynvar.ReferenceRegex, + // }, + // }, + // } + // } + // return s + // }) + // if err != nil { + // return nil, err + // } + // return &s, nil // tracker := newTracker() // schema, err := safeToSchema(golangType, docs, "", tracker) diff --git a/libs/dyn/dynvar/ref.go b/libs/dyn/dynvar/ref.go index bf160fa85b..f686d6779a 100644 --- a/libs/dyn/dynvar/ref.go +++ b/libs/dyn/dynvar/ref.go @@ -6,9 +6,9 @@ import ( "github.com/databricks/cli/libs/dyn" ) -const VariableRegex = `\$\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\[[0-9]+\])*)*(\[[0-9]+\])*)\}` +const ReferenceRegex = `\$\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\[[0-9]+\])*)*(\[[0-9]+\])*)\}` -var re = regexp.MustCompile(VariableRegex) +var re = regexp.MustCompile(ReferenceRegex) // ref represents a variable reference. // It is a string [dyn.Value] contained in a larger [dyn.Value]. diff --git a/libs/jsonschema/from_type.go b/libs/jsonschema/from_type.go index 2c1d4ca62f..389c166952 100644 --- a/libs/jsonschema/from_type.go +++ b/libs/jsonschema/from_type.go @@ -3,17 +3,13 @@ package jsonschema import ( "container/list" "fmt" + "maps" "path" "reflect" "slices" "strings" ) -// TODO: Maybe can be removed? -var InvalidSchema = Schema{ - Type: InvalidType, -} - // Fields tagged "readonly" should not be emitted in the schema as they are // computed at runtime, and should not be assigned a value by the bundle author. const readonlyTag = "readonly" @@ -26,19 +22,17 @@ const internalTag = "internal" // Fields tagged as "deprecated" are removed/omitted from the generated schema. const deprecatedTag = "deprecated" -// TODO: Test what happens with invalid cycles? Do integration tests fail? -// TODO: Call out in the PR description that recursive types like "for_each_task" -// are now supported. - type constructor struct { // Map of typ.PkgPath() + "." + typ.Name() to the schema for that type. // Example key: github.com/databricks/databricks-sdk-go/service/jobs.JobSettings definitions map[string]Schema - seen map[string]struct{} + // Map of typ.PkgPath() + "." + typ.Name() to the corresponding type. Used to + // track types that have been seen to avoid infinite recursion. + seen map[string]reflect.Type - // Transformation function to apply after generating a node in the schema. - fn func(s Schema) Schema + // The root type for which the schema is being generated. + root reflect.Type } // The $defs block in a JSON schema cannot contain "/", otherwise it will not be @@ -47,13 +41,19 @@ type constructor struct { // // For example: // {"a/b/c": "value"} is converted to {"a": {"b": {"c": "value"}}} -func (c *constructor) nestedDefinitions() any { - if len(c.definitions) == 0 { +func (c *constructor) Definitions() any { + defs := maps.Clone(c.definitions) + + // Remove the root type from the definitions. No need to include it in the + // definitions. + delete(defs, typePath(c.root)) + + if len(defs) == 0 { return nil } res := make(map[string]any) - for k, v := range c.definitions { + for k, v := range defs { parts := strings.Split(k, "/") cur := res for i, p := range parts { @@ -72,47 +72,74 @@ func (c *constructor) nestedDefinitions() any { return res } -// TODO: Skip generating schema for interface fields. -func FromType(typ reflect.Type, fn func(s Schema) Schema) (Schema, error) { +// FromType converts a reflect.Type to a jsonschema.Schema. Nodes in the final JSON +// schema are guaranteed to be one level deep, which is done using defining $defs +// for every Go type and referring them using $ref in the corresponding node in +// the JSON schema. +// +// fns is a list of transformation functions that will be applied to all $defs +// in the schema. +func FromType(typ reflect.Type, fns []func(typ reflect.Type, s Schema) Schema) (Schema, error) { c := constructor{ definitions: make(map[string]Schema), - seen: make(map[string]struct{}), - fn: fn, + seen: make(map[string]reflect.Type), + root: typ, } err := c.walk(typ) if err != nil { - return InvalidSchema, err + return Schema{}, err + } + + for k, v := range c.definitions { + for _, fn := range fns { + c.definitions[k] = fn(c.seen[k], v) + } } res := c.definitions[typePath(typ)] - // No need to include the root type in the definitions. - delete(c.definitions, typePath(typ)) - res.Definitions = c.nestedDefinitions() + res.Definitions = c.Definitions() return res, nil } +// typePath computes a unique string representation of the type. $ref in the generated +// JSON schema will refer to this path. See TestTypePath for examples outputs. func typePath(typ reflect.Type) string { // Pointers have a typ.Name() of "". Dereference them to get the underlying type. for typ.Kind() == reflect.Ptr { typ = typ.Elem() } - // typ.Name() resolves to "" for any type. if typ.Kind() == reflect.Interface { return "interface" } - // For built-in types, return the type name directly. - if typ.PkgPath() == "" { - return typ.Name() + // Recursively call typePath, to handle slices of slices / maps. + if typ.Kind() == reflect.Slice { + return path.Join("slice", typePath(typ.Elem())) } - return strings.Join([]string{typ.PkgPath(), typ.Name()}, ".") + if typ.Kind() == reflect.Map { + if typ.Key().Kind() != reflect.String { + panic(fmt.Sprintf("found map with non-string key: %v", typ.Key())) + } + + // Recursively call typePath, to handle maps of maps / slices. + return path.Join("map", typePath(typ.Elem())) + } + + switch { + case typ.PkgPath() != "" && typ.Name() != "": + return typ.PkgPath() + "." + typ.Name() + case typ.Name() != "": + return typ.Name() + default: + panic("unexpected empty type name for type: " + typ.String()) + } } -// TODO: would a worked based model fit better here? Is this internal API not -// the right fit? +// Walk the Go type, generating $defs for every type encountered, and populating +// the corresponding $ref in the JSON schema. func (c *constructor) walk(typ reflect.Type) error { // Dereference pointers if necessary. for typ.Kind() == reflect.Ptr { @@ -121,10 +148,11 @@ func (c *constructor) walk(typ reflect.Type) error { typPath := typePath(typ) - // Keep track of seen types to avoid infinite recursion. - if _, ok := c.seen[typPath]; !ok { - c.seen[typPath] = struct{}{} + // Return early if the type has already been seen, to avoid infinite recursion. + if _, ok := c.seen[typPath]; ok { + return nil } + c.seen[typPath] = typ // Return early directly if it's already been processed. if _, ok := c.definitions[typPath]; ok { @@ -134,7 +162,6 @@ func (c *constructor) walk(typ reflect.Type) error { var s Schema var err error - // TODO: Narrow / widen down the number of Go types handled here. switch typ.Kind() { case reflect.Struct: s, err = c.fromTypeStruct(typ) @@ -142,20 +169,20 @@ func (c *constructor) walk(typ reflect.Type) error { s, err = c.fromTypeSlice(typ) case reflect.Map: s, err = c.fromTypeMap(typ) - // TODO: Should the primitive functions below be inlined? case reflect.String: s = Schema{Type: StringType} case reflect.Bool: s = Schema{Type: BooleanType} - // TODO: Add comment about reduced coverage of primitive Go types in the code paths here. - case reflect.Int: + case reflect.Int, reflect.Int32, reflect.Int64: s = Schema{Type: IntegerType} case reflect.Float32, reflect.Float64: s = Schema{Type: NumberType} case reflect.Interface: - // An interface value can never be serialized from text, and thus is explicitly - // set to null and disallowed in the schema. - s = Schema{Type: NullType} + // Interface or any types are not serialized to JSON by the default JSON + // unmarshaller (json.Unmarshal). They likely thus are parsed by the + // dynamic configuration tree and we should support arbitary values here. + // Eg: variables.default can be anything. + s = Schema{} default: return fmt.Errorf("unsupported type: %s", typ.Kind()) } @@ -163,13 +190,7 @@ func (c *constructor) walk(typ reflect.Type) error { return err } - if c.fn != nil { - s = c.fn(s) - } - - // Store definition for the type if it's part of a Go package and not a built-in type. - // TODO: Apply transformation at the end, to all definitions instead of - // during recursive traversal? + // Store the computed JSON schema for the type. c.definitions[typPath] = s return nil } @@ -206,20 +227,15 @@ func getStructFields(typ reflect.Type) []reflect.StructField { return fields } -// TODO: get rid of the errors here and panic instead? func (c *constructor) fromTypeStruct(typ reflect.Type) (Schema, error) { if typ.Kind() != reflect.Struct { - return InvalidSchema, fmt.Errorf("expected struct, got %s", typ.Kind()) + return Schema{}, fmt.Errorf("expected struct, got %s", typ.Kind()) } res := Schema{ - Type: ObjectType, - - Properties: make(map[string]*Schema), - - // TODO: Confirm that empty arrays are not serialized. - Required: []string{}, - + Type: ObjectType, + Properties: make(map[string]*Schema), + Required: []string{}, AdditionalProperties: false, } @@ -240,6 +256,7 @@ func (c *constructor) fromTypeStruct(typ reflect.Type) (Schema, error) { if jsonTags[0] == "" || jsonTags[0] == "-" || !structField.IsExported() { continue } + // "omitempty" tags in the Go SDK structs represent fields that not are // required to be present in the API payload. Thus its absence in the // tags list indicates that the field is required. @@ -247,19 +264,16 @@ func (c *constructor) fromTypeStruct(typ reflect.Type) (Schema, error) { res.Required = append(res.Required, jsonTags[0]) } + // Walk the fields of the struct. typPath := typePath(structField.Type) - // Only walk if the type has not been seen yet. - if _, ok := c.seen[typPath]; !ok { - // Trigger call to fromType, to recursively generate definitions for - // the struct field. - err := c.walk(structField.Type) - if err != nil { - return InvalidSchema, err - } + err := c.walk(structField.Type) + if err != nil { + return Schema{}, err } + // For every property in the struct, add a $ref to the corresponding + // $defs block. refPath := path.Join("#/$defs", typPath) - // For non-built-in types, refer to the definition. res.Properties[jsonTags[0]] = &Schema{ Reference: &refPath, } @@ -268,11 +282,9 @@ func (c *constructor) fromTypeStruct(typ reflect.Type) (Schema, error) { return res, nil } -// TODO: Add comments explaining the translation between struct, map, slice and -// the JSON schema representation. func (c *constructor) fromTypeSlice(typ reflect.Type) (Schema, error) { if typ.Kind() != reflect.Slice { - return InvalidSchema, fmt.Errorf("expected slice, got %s", typ.Kind()) + return Schema{}, fmt.Errorf("expected slice, got %s", typ.Kind()) } res := Schema{ @@ -280,19 +292,16 @@ func (c *constructor) fromTypeSlice(typ reflect.Type) (Schema, error) { } typPath := typePath(typ.Elem()) - // Only walk if the type has not been seen yet. - if _, ok := c.seen[typPath]; !ok { - // Trigger call to fromType, to recursively generate definitions for - // the slice element. - err := c.walk(typ.Elem()) - if err != nil { - return InvalidSchema, err - } + + // Walk the slice element type. + err := c.walk(typ.Elem()) + if err != nil { + return Schema{}, err } refPath := path.Join("#/$defs", typPath) - // For non-built-in types, refer to the definition + // Add a $ref to the corresponding $defs block for the slice element type. res.Items = &Schema{ Reference: &refPath, } @@ -301,11 +310,11 @@ func (c *constructor) fromTypeSlice(typ reflect.Type) (Schema, error) { func (c *constructor) fromTypeMap(typ reflect.Type) (Schema, error) { if typ.Kind() != reflect.Map { - return InvalidSchema, fmt.Errorf("expected map, got %s", typ.Kind()) + return Schema{}, fmt.Errorf("expected map, got %s", typ.Kind()) } if typ.Key().Kind() != reflect.String { - return InvalidSchema, fmt.Errorf("found map with non-string key: %v", typ.Key()) + return Schema{}, fmt.Errorf("found map with non-string key: %v", typ.Key()) } res := Schema{ @@ -313,19 +322,16 @@ func (c *constructor) fromTypeMap(typ reflect.Type) (Schema, error) { } typPath := typePath(typ.Elem()) - // Only walk if the type has not been seen yet. - if _, ok := c.seen[typPath]; !ok { - // Trigger call to fromType, to recursively generate definitions for - // the map value. - err := c.walk(typ.Elem()) - if err != nil { - return InvalidSchema, err - } + + // Walk the map value type. + err := c.walk(typ.Elem()) + if err != nil { + return Schema{}, err } refPath := path.Join("#/$defs", typPath) - // For non-built-in types, refer to the definition + // Add a $ref to the corresponding $defs block for the map value type. res.AdditionalProperties = &Schema{ Reference: &refPath, } diff --git a/libs/jsonschema/from_type_test.go b/libs/jsonschema/from_type_test.go index 6a3ae928d5..e8bdd5eba6 100644 --- a/libs/jsonschema/from_type_test.go +++ b/libs/jsonschema/from_type_test.go @@ -67,9 +67,7 @@ func TestFromTypeBasic(t *testing.T) { expected: Schema{ Type: "object", Definitions: map[string]any{ - "interface": Schema{ - Type: "null", - }, + "interface": Schema{}, "string": Schema{ Type: "string", }, @@ -160,6 +158,8 @@ func TestGetStructFields(t *testing.T) { assert.Equal(t, "B", fields[2].Name) } +// TODO: Add other case coverage for all the tests below + func TestFromTypeNested(t *testing.T) { type Inner struct { S string `json:"s"` @@ -222,7 +222,7 @@ func TestFromTypeNested(t *testing.T) { }, { name: "struct as a map value", - typ: reflect.TypeOf(map[string]Inner{}), + typ: reflect.TypeOf(map[string]*Inner{}), expected: Schema{ Type: "object", Definitions: expectedDefinitions, @@ -252,7 +252,6 @@ func TestFromTypeNested(t *testing.T) { } } -// TODO: Call out in the PR description that recursive Go types are supported. func TestFromTypeRecursive(t *testing.T) { fooRef := "#/$defs/github.com/databricks/cli/libs/jsonschema/test_types.Foo" barRef := "#/$defs/github.com/databricks/cli/libs/jsonschema/test_types.Bar" @@ -353,8 +352,76 @@ func TestFromTypeSelfReferential(t *testing.T) { assert.Equal(t, expected, s) } +// TODO: Add coverage for all errors returned by FromType. func TestFromTypeError(t *testing.T) { type mapOfInts map[int]int - _, err := FromType(reflect.TypeOf(mapOfInts{}), nil) - assert.EqualError(t, err, "found map with non-string key: int") + + assert.PanicsWithValue(t, "found map with non-string key: int", func() { + FromType(reflect.TypeOf(mapOfInts{}), nil) + }) +} + +// TODO: Add test that the fn argument ot from_type works as expected. + +func TestTypePath(t *testing.T) { + type myStruct struct{} + + tcases := []struct { + typ reflect.Type + path string + }{ + { + typ: reflect.TypeOf(""), + path: "string", + }, + { + typ: reflect.TypeOf(int(0)), + path: "int", + }, + { + typ: reflect.TypeOf(true), + path: "bool", + }, + { + typ: reflect.TypeOf(float64(0)), + path: "float64", + }, + { + typ: reflect.TypeOf(myStruct{}), + path: "github.com/databricks/cli/libs/jsonschema.myStruct", + }, + { + typ: reflect.TypeOf([]int{}), + path: "slice/int", + }, + { + typ: reflect.TypeOf(map[string]int{}), + path: "map/int", + }, + { + typ: reflect.TypeOf([]myStruct{}), + path: "slice/github.com/databricks/cli/libs/jsonschema.myStruct", + }, + { + typ: reflect.TypeOf([][]map[string]map[string]myStruct{}), + path: "slice/slice/map/map/github.com/databricks/cli/libs/jsonschema.myStruct", + }, + { + typ: reflect.TypeOf(map[string]myStruct{}), + path: "map/github.com/databricks/cli/libs/jsonschema.myStruct", + }, + } + + // TODO: support arbitary depth of maps and slices. Also add validation + // in this function that non-string keys are not allowed. + for _, tc := range tcases { + t.Run(tc.typ.String(), func(t *testing.T) { + assert.Equal(t, tc.path, typePath(tc.typ)) + }) + } + + // Maps with non-string keys should panic. + assert.PanicsWithValue(t, "found map with non-string key: int", func() { + typePath(reflect.TypeOf(map[int]int{})) + }) } From 870e41912ac4c3bea803d1038bd17b3d66a07cd3 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 27 Aug 2024 11:42:33 +0200 Subject: [PATCH 16/62] from_type method is mostly complete --- libs/jsonschema/from_type.go | 6 ++- libs/jsonschema/from_type_test.go | 68 +++++++++++++++++++++++-------- 2 files changed, 56 insertions(+), 18 deletions(-) diff --git a/libs/jsonschema/from_type.go b/libs/jsonschema/from_type.go index 389c166952..d2bbb45502 100644 --- a/libs/jsonschema/from_type.go +++ b/libs/jsonschema/from_type.go @@ -91,8 +91,8 @@ func FromType(typ reflect.Type, fns []func(typ reflect.Type, s Schema) Schema) ( return Schema{}, err } - for k, v := range c.definitions { - for _, fn := range fns { + for _, fn := range fns { + for k, v := range c.definitions { c.definitions[k] = fn(c.seen[k], v) } } @@ -134,6 +134,8 @@ func typePath(typ reflect.Type) string { case typ.Name() != "": return typ.Name() default: + // Invariant. This function should return a non-empty string + // for all types. panic("unexpected empty type name for type: " + typ.String()) } } diff --git a/libs/jsonschema/from_type_test.go b/libs/jsonschema/from_type_test.go index e8bdd5eba6..29d007901a 100644 --- a/libs/jsonschema/from_type_test.go +++ b/libs/jsonschema/from_type_test.go @@ -127,15 +127,6 @@ func TestFromTypeBasic(t *testing.T) { s, err := FromType(tc.typ, nil) assert.NoError(t, err) assert.Equal(t, tc.expected, s) - - // jsonSchema, err := json.MarshalIndent(s, " ", " ") - // assert.NoError(t, err) - - // expectedJson, err := json.MarshalIndent(tc.expected, " ", " ") - // assert.NoError(t, err) - - // t.Log("[DEBUG] actual: ", string(jsonSchema)) - // t.Log("[DEBUG] expected: ", string(expectedJson)) }) } } @@ -158,8 +149,6 @@ func TestGetStructFields(t *testing.T) { assert.Equal(t, "B", fields[2].Name) } -// TODO: Add other case coverage for all the tests below - func TestFromTypeNested(t *testing.T) { type Inner struct { S string `json:"s"` @@ -352,16 +341,65 @@ func TestFromTypeSelfReferential(t *testing.T) { assert.Equal(t, expected, s) } -// TODO: Add coverage for all errors returned by FromType. func TestFromTypeError(t *testing.T) { + // Maps with non-string keys should panic. type mapOfInts map[int]int - assert.PanicsWithValue(t, "found map with non-string key: int", func() { FromType(reflect.TypeOf(mapOfInts{}), nil) }) + + // Unsupported types should return an error. + _, err := FromType(reflect.TypeOf(complex64(0)), nil) + assert.EqualError(t, err, "unsupported type: complex64") } -// TODO: Add test that the fn argument ot from_type works as expected. +func TestFromTypeFunctionsArg(t *testing.T) { + type myStruct struct { + S string `json:"s"` + } + + strRef := "#/$defs/string" + expected := Schema{ + Type: "object", + Definitions: map[string]any{ + "string": Schema{ + Type: "string", + Description: "a string", + Enum: []any{"a", "b", "c"}, + }, + }, + Properties: map[string]*Schema{ + "s": { + Reference: &strRef, + }, + }, + AdditionalProperties: false, + Required: []string{"s"}, + } + + addDescription := func(typ reflect.Type, s Schema) Schema { + if typ.Kind() != reflect.String { + return s + } + s.Description = "a string" + return s + } + + addEnums := func(typ reflect.Type, s Schema) Schema { + if typ.Kind() != reflect.String { + return s + } + s.Enum = []any{"a", "b", "c"} + return s + } + + s, err := FromType(reflect.TypeOf(myStruct{}), []func(reflect.Type, Schema) Schema{ + addDescription, + addEnums, + }) + assert.NoError(t, err) + assert.Equal(t, expected, s) +} func TestTypePath(t *testing.T) { type myStruct struct{} @@ -412,8 +450,6 @@ func TestTypePath(t *testing.T) { }, } - // TODO: support arbitary depth of maps and slices. Also add validation - // in this function that non-string keys are not allowed. for _, tc := range tcases { t.Run(tc.typ.String(), func(t *testing.T) { assert.Equal(t, tc.path, typePath(tc.typ)) From 2dea889621de8475b8fe87d6b0909c8f4bc5353d Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 27 Aug 2024 11:53:29 +0200 Subject: [PATCH 17/62] delete the old schema code --- bundle/internal/schema/main.go | 1 + bundle/schema/README.md | 18 - bundle/schema/docs.go | 109 - bundle/schema/docs/bundle_descriptions.json | 6371 ------------------- bundle/schema/docs_test.go | 62 - bundle/schema/openapi.go | 293 - bundle/schema/openapi_test.go | 493 -- bundle/schema/schema.go | 307 - bundle/schema/schema_test.go | 341 - bundle/schema/spec.go | 11 - bundle/schema/tracker.go | 53 - 11 files changed, 1 insertion(+), 8058 deletions(-) delete mode 100644 bundle/schema/README.md delete mode 100644 bundle/schema/docs.go delete mode 100644 bundle/schema/docs/bundle_descriptions.json delete mode 100644 bundle/schema/docs_test.go delete mode 100644 bundle/schema/openapi.go delete mode 100644 bundle/schema/openapi_test.go delete mode 100644 bundle/schema/schema.go delete mode 100644 bundle/schema/schema_test.go delete mode 100644 bundle/schema/spec.go delete mode 100644 bundle/schema/tracker.go diff --git a/bundle/internal/schema/main.go b/bundle/internal/schema/main.go index 45d0f00a1f..33cfbbe7f7 100644 --- a/bundle/internal/schema/main.go +++ b/bundle/internal/schema/main.go @@ -52,6 +52,7 @@ func addInterpolationPatterns(_ reflect.Type, s jsonschema.Schema) jsonschema.Sc // as well. Make sure to pull those in. // TODO: Add unit tests for all permutations of structs, maps and slices for the FromType // method. +// TODO: Note the minor regression of losing the bundle descriptions func main() { if len(os.Args) != 2 { diff --git a/bundle/schema/README.md b/bundle/schema/README.md deleted file mode 100644 index bf6b87df61..0000000000 --- a/bundle/schema/README.md +++ /dev/null @@ -1,18 +0,0 @@ -### Overview - -`docs/bundle_descriptions.json` contains both autogenerated as well as manually written -descriptions for the json schema. Specifically -1. `resources` : almost all descriptions are autogenerated from the OpenAPI spec -2. `targets` : almost all descriptions are copied over from root level entities (eg: `bundle`, `artifacts`) -3. `bundle` : manually editted -4. `include` : manually editted -5. `workspace` : manually editted -6. `artifacts` : manually editted - -These descriptions are rendered in the inline documentation in an IDE - -### SOP: Add schema descriptions for new fields in bundle config - -Manually edit bundle_descriptions.json to add your descriptions. Note that the -descriptions in `resources` block is generated from the OpenAPI spec, and thus -any changes there will be overwritten. diff --git a/bundle/schema/docs.go b/bundle/schema/docs.go deleted file mode 100644 index 6e9289f924..0000000000 --- a/bundle/schema/docs.go +++ /dev/null @@ -1,109 +0,0 @@ -package schema - -import ( - _ "embed" - "encoding/json" - "fmt" - "os" - "reflect" - - "github.com/databricks/cli/bundle/config" - "github.com/databricks/cli/libs/jsonschema" -) - -// A subset of Schema struct -type Docs struct { - Description string `json:"description"` - Properties map[string]*Docs `json:"properties,omitempty"` - Items *Docs `json:"items,omitempty"` - AdditionalProperties *Docs `json:"additionalproperties,omitempty"` -} - -//go:embed docs/bundle_descriptions.json -var bundleDocs []byte - -func (docs *Docs) refreshTargetsDocs() error { - targetsDocs, ok := docs.Properties["targets"] - if !ok || targetsDocs.AdditionalProperties == nil || - targetsDocs.AdditionalProperties.Properties == nil { - return fmt.Errorf("invalid targets descriptions") - } - targetProperties := targetsDocs.AdditionalProperties.Properties - propertiesToCopy := []string{"artifacts", "bundle", "resources", "workspace"} - for _, p := range propertiesToCopy { - targetProperties[p] = docs.Properties[p] - } - return nil -} - -func LoadBundleDescriptions() (*Docs, error) { - embedded := Docs{} - err := json.Unmarshal(bundleDocs, &embedded) - return &embedded, err -} - -func UpdateBundleDescriptions(openapiSpecPath string) (*Docs, error) { - embedded, err := LoadBundleDescriptions() - if err != nil { - return nil, err - } - - // Generate schema from the embedded descriptions, and convert it back to docs. - // This creates empty descriptions for any properties that were missing in the - // embedded descriptions. - schema, err := New(reflect.TypeOf(config.Root{}), embedded) - if err != nil { - return nil, err - } - docs := schemaToDocs(schema) - - // Load the Databricks OpenAPI spec - openapiSpec, err := os.ReadFile(openapiSpecPath) - if err != nil { - return nil, err - } - spec := &Specification{} - err = json.Unmarshal(openapiSpec, spec) - if err != nil { - return nil, err - } - openapiReader := &OpenapiReader{ - OpenapiSpec: spec, - memo: make(map[string]jsonschema.Schema), - } - - // Generate descriptions for the "resources" field - resourcesDocs, err := openapiReader.ResourcesDocs() - if err != nil { - return nil, err - } - resourceSchema, err := New(reflect.TypeOf(config.Resources{}), resourcesDocs) - if err != nil { - return nil, err - } - docs.Properties["resources"] = schemaToDocs(resourceSchema) - docs.refreshTargetsDocs() - return docs, nil -} - -// *Docs are a subset of *Schema, this function selects that subset -func schemaToDocs(jsonSchema *jsonschema.Schema) *Docs { - // terminate recursion if schema is nil - if jsonSchema == nil { - return nil - } - docs := &Docs{ - Description: jsonSchema.Description, - } - if len(jsonSchema.Properties) > 0 { - docs.Properties = make(map[string]*Docs) - } - for k, v := range jsonSchema.Properties { - docs.Properties[k] = schemaToDocs(v) - } - docs.Items = schemaToDocs(jsonSchema.Items) - if additionalProperties, ok := jsonSchema.AdditionalProperties.(*jsonschema.Schema); ok { - docs.AdditionalProperties = schemaToDocs(additionalProperties) - } - return docs -} diff --git a/bundle/schema/docs/bundle_descriptions.json b/bundle/schema/docs/bundle_descriptions.json deleted file mode 100644 index d888b36635..0000000000 --- a/bundle/schema/docs/bundle_descriptions.json +++ /dev/null @@ -1,6371 +0,0 @@ -{ - "description": "", - "properties": { - "artifacts": { - "description": "", - "additionalproperties": { - "description": "", - "properties": { - "build": { - "description": "" - }, - "executable": { - "description": "" - }, - "files": { - "description": "", - "items": { - "description": "", - "properties": { - "source": { - "description": "" - } - } - } - }, - "path": { - "description": "" - }, - "type": { - "description": "" - } - } - } - }, - "bundle": { - "description": "", - "properties": { - "compute_id": { - "description": "" - }, - "databricks_cli_version": { - "description": "" - }, - "deployment": { - "description": "", - "properties": { - "fail_on_active_runs": { - "description": "" - }, - "lock": { - "description": "", - "properties": { - "enabled": { - "description": "" - }, - "force": { - "description": "" - } - } - } - } - }, - "git": { - "description": "", - "properties": { - "branch": { - "description": "" - }, - "origin_url": { - "description": "" - } - } - }, - "name": { - "description": "" - } - } - }, - "experimental": { - "description": "", - "properties": { - "pydabs": { - "description": "", - "properties": { - "enabled": { - "description": "" - }, - "venv_path": { - "description": "" - } - } - }, - "python_wheel_wrapper": { - "description": "" - }, - "scripts": { - "description": "", - "additionalproperties": { - "description": "" - } - }, - "use_legacy_run_as": { - "description": "" - } - } - }, - "include": { - "description": "", - "items": { - "description": "" - } - }, - "permissions": { - "description": "", - "items": { - "description": "", - "properties": { - "group_name": { - "description": "" - }, - "level": { - "description": "" - }, - "service_principal_name": { - "description": "" - }, - "user_name": { - "description": "" - } - } - } - }, - "resources": { - "description": "Collection of Databricks resources to deploy.", - "properties": { - "experiments": { - "description": "List of MLflow experiments", - "additionalproperties": { - "description": "", - "properties": { - "artifact_location": { - "description": "Location where artifacts for the experiment are stored." - }, - "creation_time": { - "description": "Creation time" - }, - "experiment_id": { - "description": "Unique identifier for the experiment." - }, - "last_update_time": { - "description": "Last update time" - }, - "lifecycle_stage": { - "description": "Current life cycle stage of the experiment: \"active\" or \"deleted\".\nDeleted experiments are not returned by APIs." - }, - "name": { - "description": "Human readable name that identifies the experiment." - }, - "permissions": { - "description": "", - "items": { - "description": "", - "properties": { - "group_name": { - "description": "" - }, - "level": { - "description": "" - }, - "service_principal_name": { - "description": "" - }, - "user_name": { - "description": "" - } - } - } - }, - "tags": { - "description": "Tags: Additional metadata key-value pairs.", - "items": { - "description": "", - "properties": { - "key": { - "description": "The tag key." - }, - "value": { - "description": "The tag value." - } - } - } - } - } - } - }, - "jobs": { - "description": "List of Databricks jobs", - "additionalproperties": { - "description": "", - "properties": { - "continuous": { - "description": "An optional continuous property for this job. The continuous property will ensure that there is always one run executing. Only one of `schedule` and `continuous` can be used.", - "properties": { - "pause_status": { - "description": "Indicate whether the continuous execution of the job is paused or not. Defaults to UNPAUSED." - } - } - }, - "deployment": { - "description": "Deployment information for jobs managed by external sources.", - "properties": { - "kind": { - "description": "The kind of deployment that manages the job.\n\n* `BUNDLE`: The job is managed by Databricks Asset Bundle." - }, - "metadata_file_path": { - "description": "Path of the file that contains deployment metadata." - } - } - }, - "description": { - "description": "An optional description for the job. The maximum length is 27700 characters in UTF-8 encoding." - }, - "edit_mode": { - "description": "Edit mode of the job.\n\n* `UI_LOCKED`: The job is in a locked UI state and cannot be modified.\n* `EDITABLE`: The job is in an editable state and can be modified." - }, - "email_notifications": { - "description": "An optional set of email addresses that is notified when runs of this job begin or complete as well as when this job is deleted.", - "properties": { - "no_alert_for_skipped_runs": { - "description": "If true, do not send email to recipients specified in `on_failure` if the run is skipped." - }, - "on_duration_warning_threshold_exceeded": { - "description": "A list of email addresses to be notified when the duration of a run exceeds the threshold specified for the `RUN_DURATION_SECONDS` metric in the `health` field. If no rule for the `RUN_DURATION_SECONDS` metric is specified in the `health` field for the job, notifications are not sent.", - "items": { - "description": "" - } - }, - "on_failure": { - "description": "A list of email addresses to be notified when a run unsuccessfully completes. A run is considered to have completed unsuccessfully if it ends with an `INTERNAL_ERROR` `life_cycle_state` or a `FAILED`, or `TIMED_OUT` result_state. If this is not specified on job creation, reset, or update the list is empty, and notifications are not sent.", - "items": { - "description": "" - } - }, - "on_start": { - "description": "A list of email addresses to be notified when a run begins. If not specified on job creation, reset, or update, the list is empty, and notifications are not sent.", - "items": { - "description": "" - } - }, - "on_streaming_backlog_exceeded": { - "description": "A list of email addresses to notify when any streaming backlog thresholds are exceeded for any stream.\nStreaming backlog thresholds can be set in the `health` field using the following metrics: `STREAMING_BACKLOG_BYTES`, `STREAMING_BACKLOG_RECORDS`, `STREAMING_BACKLOG_SECONDS`, or `STREAMING_BACKLOG_FILES`.\nAlerting is based on the 10-minute average of these metrics. If the issue persists, notifications are resent every 30 minutes.", - "items": { - "description": "" - } - }, - "on_success": { - "description": "A list of email addresses to be notified when a run successfully completes. A run is considered to have completed successfully if it ends with a `TERMINATED` `life_cycle_state` and a `SUCCESS` result_state. If not specified on job creation, reset, or update, the list is empty, and notifications are not sent.", - "items": { - "description": "" - } - } - } - }, - "environments": { - "description": "A list of task execution environment specifications that can be referenced by tasks of this job.", - "items": { - "description": "", - "properties": { - "environment_key": { - "description": "The key of an environment. It has to be unique within a job." - }, - "spec": { - "description": "", - "properties": { - "client": { - "description": "Client version used by the environment\nThe client is the user-facing environment of the runtime.\nEach client comes with a specific set of pre-installed libraries.\nThe version is a string, consisting of the major client version." - }, - "dependencies": { - "description": "List of pip dependencies, as supported by the version of pip in this environment.\nEach dependency is a pip requirement file line https://pip.pypa.io/en/stable/reference/requirements-file-format/\nAllowed dependency could be \u003crequirement specifier\u003e, \u003carchive url/path\u003e, \u003clocal project path\u003e(WSFS or Volumes in Databricks), \u003cvcs project url\u003e\nE.g. dependencies: [\"foo==0.0.1\", \"-r /Workspace/test/requirements.txt\"]", - "items": { - "description": "" - } - } - } - } - } - } - }, - "format": { - "description": "Used to tell what is the format of the job. This field is ignored in Create/Update/Reset calls. When using the Jobs API 2.1 this value is always set to `\"MULTI_TASK\"`." - }, - "git_source": { - "description": "An optional specification for a remote Git repository containing the source code used by tasks. Version-controlled source code is supported by notebook, dbt, Python script, and SQL File tasks.\n\nIf `git_source` is set, these tasks retrieve the file from the remote repository by default. However, this behavior can be overridden by setting `source` to `WORKSPACE` on the task.\n\nNote: dbt and SQL File tasks support only version-controlled sources. If dbt or SQL File tasks are used, `git_source` must be defined on the job.", - "properties": { - "git_branch": { - "description": "Name of the branch to be checked out and used by this job. This field cannot be specified in conjunction with git_tag or git_commit." - }, - "git_commit": { - "description": "Commit to be checked out and used by this job. This field cannot be specified in conjunction with git_branch or git_tag." - }, - "git_provider": { - "description": "Unique identifier of the service used to host the Git repository. The value is case insensitive." - }, - "git_snapshot": { - "description": "", - "properties": { - "used_commit": { - "description": "Commit that was used to execute the run. If git_branch was specified, this points to the HEAD of the branch at the time of the run; if git_tag was specified, this points to the commit the tag points to." - } - } - }, - "git_tag": { - "description": "Name of the tag to be checked out and used by this job. This field cannot be specified in conjunction with git_branch or git_commit." - }, - "git_url": { - "description": "URL of the repository to be cloned by this job." - }, - "job_source": { - "description": "The source of the job specification in the remote repository when the job is source controlled.", - "properties": { - "dirty_state": { - "description": "Dirty state indicates the job is not fully synced with the job specification in the remote repository.\n\nPossible values are:\n* `NOT_SYNCED`: The job is not yet synced with the remote job specification. Import the remote job specification from UI to make the job fully synced.\n* `DISCONNECTED`: The job is temporary disconnected from the remote job specification and is allowed for live edit. Import the remote job specification again from UI to make the job fully synced." - }, - "import_from_git_branch": { - "description": "Name of the branch which the job is imported from." - }, - "job_config_path": { - "description": "Path of the job YAML file that contains the job specification." - } - } - } - } - }, - "health": { - "description": "", - "properties": { - "rules": { - "description": "", - "items": { - "description": "", - "properties": { - "metric": { - "description": "" - }, - "op": { - "description": "" - }, - "value": { - "description": "Specifies the threshold value that the health metric should obey to satisfy the health rule." - } - } - } - } - } - }, - "job_clusters": { - "description": "A list of job cluster specifications that can be shared and reused by tasks of this job. Libraries cannot be declared in a shared job cluster. You must declare dependent libraries in task settings.", - "items": { - "description": "", - "properties": { - "job_cluster_key": { - "description": "A unique name for the job cluster. This field is required and must be unique within the job.\n`JobTaskSettings` may refer to this field to determine which cluster to launch for the task execution." - }, - "new_cluster": { - "description": "If new_cluster, a description of a cluster that is created for each task.", - "properties": { - "apply_policy_default_values": { - "description": "When set to true, fixed and default values from the policy will be used for fields that are omitted. When set to false, only fixed values from the policy will be applied." - }, - "autoscale": { - "description": "Parameters needed in order to automatically scale clusters up and down based on load.\nNote: autoscaling works best with DB runtime versions 3.0 or later.", - "properties": { - "max_workers": { - "description": "The maximum number of workers to which the cluster can scale up when overloaded.\nNote that `max_workers` must be strictly greater than `min_workers`." - }, - "min_workers": { - "description": "The minimum number of workers to which the cluster can scale down when underutilized.\nIt is also the initial number of workers the cluster will have after creation." - } - } - }, - "autotermination_minutes": { - "description": "Automatically terminates the cluster after it is inactive for this time in minutes. If not set,\nthis cluster will not be automatically terminated. If specified, the threshold must be between\n10 and 10000 minutes.\nUsers can also set this value to 0 to explicitly disable automatic termination." - }, - "aws_attributes": { - "description": "Attributes related to clusters running on Amazon Web Services.\nIf not specified at cluster creation, a set of default values will be used.", - "properties": { - "availability": { - "description": "" - }, - "ebs_volume_count": { - "description": "The number of volumes launched for each instance. Users can choose up to 10 volumes.\nThis feature is only enabled for supported node types. Legacy node types cannot specify\ncustom EBS volumes.\nFor node types with no instance store, at least one EBS volume needs to be specified;\notherwise, cluster creation will fail.\n\nThese EBS volumes will be mounted at `/ebs0`, `/ebs1`, and etc.\nInstance store volumes will be mounted at `/local_disk0`, `/local_disk1`, and etc.\n\nIf EBS volumes are attached, Databricks will configure Spark to use only the EBS volumes for\nscratch storage because heterogenously sized scratch devices can lead to inefficient disk\nutilization. If no EBS volumes are attached, Databricks will configure Spark to use instance\nstore volumes.\n\nPlease note that if EBS volumes are specified, then the Spark configuration `spark.local.dir`\nwill be overridden." - }, - "ebs_volume_iops": { - "description": "If using gp3 volumes, what IOPS to use for the disk. If this is not set, the maximum performance of a gp2 volume with the same volume size will be used." - }, - "ebs_volume_size": { - "description": "The size of each EBS volume (in GiB) launched for each instance. For general purpose\nSSD, this value must be within the range 100 - 4096. For throughput optimized HDD,\nthis value must be within the range 500 - 4096." - }, - "ebs_volume_throughput": { - "description": "If using gp3 volumes, what throughput to use for the disk. If this is not set, the maximum performance of a gp2 volume with the same volume size will be used." - }, - "ebs_volume_type": { - "description": "" - }, - "first_on_demand": { - "description": "The first `first_on_demand` nodes of the cluster will be placed on on-demand instances.\nIf this value is greater than 0, the cluster driver node in particular will be placed on an\non-demand instance. If this value is greater than or equal to the current cluster size, all\nnodes will be placed on on-demand instances. If this value is less than the current cluster\nsize, `first_on_demand` nodes will be placed on on-demand instances and the remainder will\nbe placed on `availability` instances. Note that this value does not affect\ncluster size and cannot currently be mutated over the lifetime of a cluster." - }, - "instance_profile_arn": { - "description": "Nodes for this cluster will only be placed on AWS instances with this instance profile. If\nommitted, nodes will be placed on instances without an IAM instance profile. The instance\nprofile must have previously been added to the Databricks environment by an account\nadministrator.\n\nThis feature may only be available to certain customer plans.\n\nIf this field is ommitted, we will pull in the default from the conf if it exists." - }, - "spot_bid_price_percent": { - "description": "The bid price for AWS spot instances, as a percentage of the corresponding instance type's\non-demand price.\nFor example, if this field is set to 50, and the cluster needs a new `r3.xlarge` spot\ninstance, then the bid price is half of the price of\non-demand `r3.xlarge` instances. Similarly, if this field is set to 200, the bid price is twice\nthe price of on-demand `r3.xlarge` instances. If not specified, the default value is 100.\nWhen spot instances are requested for this cluster, only spot instances whose bid price\npercentage matches this field will be considered.\nNote that, for safety, we enforce this field to be no more than 10000.\n\nThe default value and documentation here should be kept consistent with\nCommonConf.defaultSpotBidPricePercent and CommonConf.maxSpotBidPricePercent." - }, - "zone_id": { - "description": "Identifier for the availability zone/datacenter in which the cluster resides.\nThis string will be of a form like \"us-west-2a\". The provided availability\nzone must be in the same region as the Databricks deployment. For example, \"us-west-2a\"\nis not a valid zone id if the Databricks deployment resides in the \"us-east-1\" region.\nThis is an optional field at cluster creation, and if not specified, a default zone will be used.\nIf the zone specified is \"auto\", will try to place cluster in a zone with high availability,\nand will retry placement in a different AZ if there is not enough capacity.\nThe list of available zones as well as the default value can be found by using the\n`List Zones` method." - } - } - }, - "azure_attributes": { - "description": "Attributes related to clusters running on Microsoft Azure.\nIf not specified at cluster creation, a set of default values will be used.", - "properties": { - "availability": { - "description": "" - }, - "first_on_demand": { - "description": "The first `first_on_demand` nodes of the cluster will be placed on on-demand instances.\nThis value should be greater than 0, to make sure the cluster driver node is placed on an\non-demand instance. If this value is greater than or equal to the current cluster size, all\nnodes will be placed on on-demand instances. If this value is less than the current cluster\nsize, `first_on_demand` nodes will be placed on on-demand instances and the remainder will\nbe placed on `availability` instances. Note that this value does not affect\ncluster size and cannot currently be mutated over the lifetime of a cluster." - }, - "log_analytics_info": { - "description": "Defines values necessary to configure and run Azure Log Analytics agent", - "properties": { - "log_analytics_primary_key": { - "description": "\u003cneeds content added\u003e" - }, - "log_analytics_workspace_id": { - "description": "\u003cneeds content added\u003e" - } - } - }, - "spot_bid_max_price": { - "description": "The max bid price to be used for Azure spot instances.\nThe Max price for the bid cannot be higher than the on-demand price of the instance.\nIf not specified, the default value is -1, which specifies that the instance cannot be evicted\non the basis of price, and only on the basis of availability. Further, the value should \u003e 0 or -1." - } - } - }, - "cluster_log_conf": { - "description": "The configuration for delivering spark logs to a long-term storage destination.\nTwo kinds of destinations (dbfs and s3) are supported. Only one destination can be specified\nfor one cluster. If the conf is given, the logs will be delivered to the destination every\n`5 mins`. The destination of driver logs is `$destination/$clusterId/driver`, while\nthe destination of executor logs is `$destination/$clusterId/executor`.", - "properties": { - "dbfs": { - "description": "destination needs to be provided. e.g.\n`{ \"dbfs\" : { \"destination\" : \"dbfs:/home/cluster_log\" } }`", - "properties": { - "destination": { - "description": "dbfs destination, e.g. `dbfs:/my/path`" - } - } - }, - "s3": { - "description": "destination and either the region or endpoint need to be provided. e.g.\n`{ \"s3\": { \"destination\" : \"s3://cluster_log_bucket/prefix\", \"region\" : \"us-west-2\" } }`\nCluster iam role is used to access s3, please make sure the cluster iam role in\n`instance_profile_arn` has permission to write data to the s3 destination.", - "properties": { - "canned_acl": { - "description": "(Optional) Set canned access control list for the logs, e.g. `bucket-owner-full-control`.\nIf `canned_cal` is set, please make sure the cluster iam role has `s3:PutObjectAcl` permission on\nthe destination bucket and prefix. The full list of possible canned acl can be found at\nhttp://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl.\nPlease also note that by default only the object owner gets full controls. If you are using cross account\nrole for writing data, you may want to set `bucket-owner-full-control` to make bucket owner able to\nread the logs." - }, - "destination": { - "description": "S3 destination, e.g. `s3://my-bucket/some-prefix` Note that logs will be delivered using\ncluster iam role, please make sure you set cluster iam role and the role has write access to the\ndestination. Please also note that you cannot use AWS keys to deliver logs." - }, - "enable_encryption": { - "description": "(Optional) Flag to enable server side encryption, `false` by default." - }, - "encryption_type": { - "description": "(Optional) The encryption type, it could be `sse-s3` or `sse-kms`. It will be used only when\nencryption is enabled and the default type is `sse-s3`." - }, - "endpoint": { - "description": "S3 endpoint, e.g. `https://s3-us-west-2.amazonaws.com`. Either region or endpoint needs to be set.\nIf both are set, endpoint will be used." - }, - "kms_key": { - "description": "(Optional) Kms key which will be used if encryption is enabled and encryption type is set to `sse-kms`." - }, - "region": { - "description": "S3 region, e.g. `us-west-2`. Either region or endpoint needs to be set. If both are set,\nendpoint will be used." - } - } - } - } - }, - "cluster_name": { - "description": "Cluster name requested by the user. This doesn't have to be unique.\nIf not specified at creation, the cluster name will be an empty string.\n" - }, - "custom_tags": { - "description": "Additional tags for cluster resources. Databricks will tag all cluster resources (e.g., AWS\ninstances and EBS volumes) with these tags in addition to `default_tags`. Notes:\n\n- Currently, Databricks allows at most 45 custom tags\n\n- Clusters can only reuse cloud resources if the resources' tags are a subset of the cluster tags", - "additionalproperties": { - "description": "" - } - }, - "data_security_mode": { - "description": "" - }, - "docker_image": { - "description": "", - "properties": { - "basic_auth": { - "description": "", - "properties": { - "password": { - "description": "Password of the user" - }, - "username": { - "description": "Name of the user" - } - } - }, - "url": { - "description": "URL of the docker image." - } - } - }, - "driver_instance_pool_id": { - "description": "The optional ID of the instance pool for the driver of the cluster belongs.\nThe pool cluster uses the instance pool with id (instance_pool_id) if the driver pool is not\nassigned." - }, - "driver_node_type_id": { - "description": "The node type of the Spark driver. Note that this field is optional;\nif unset, the driver node type will be set as the same value\nas `node_type_id` defined above.\n" - }, - "enable_elastic_disk": { - "description": "Autoscaling Local Storage: when enabled, this cluster will dynamically acquire additional disk\nspace when its Spark workers are running low on disk space. This feature requires specific AWS\npermissions to function correctly - refer to the User Guide for more details." - }, - "enable_local_disk_encryption": { - "description": "Whether to enable LUKS on cluster VMs' local disks" - }, - "gcp_attributes": { - "description": "Attributes related to clusters running on Google Cloud Platform.\nIf not specified at cluster creation, a set of default values will be used.", - "properties": { - "availability": { - "description": "" - }, - "boot_disk_size": { - "description": "boot disk size in GB" - }, - "google_service_account": { - "description": "If provided, the cluster will impersonate the google service account when accessing\ngcloud services (like GCS). The google service account\nmust have previously been added to the Databricks environment by an account\nadministrator." - }, - "local_ssd_count": { - "description": "If provided, each node (workers and driver) in the cluster will have this number of local SSDs attached. Each local SSD is 375GB in size. Refer to [GCP documentation](https://cloud.google.com/compute/docs/disks/local-ssd#choose_number_local_ssds) for the supported number of local SSDs for each instance type." - }, - "use_preemptible_executors": { - "description": "This field determines whether the spark executors will be scheduled to run on preemptible VMs (when set to true) versus standard compute engine VMs (when set to false; default).\nNote: Soon to be deprecated, use the availability field instead." - }, - "zone_id": { - "description": "Identifier for the availability zone in which the cluster resides.\nThis can be one of the following:\n- \"HA\" =\u003e High availability, spread nodes across availability zones for a Databricks deployment region [default]\n- \"AUTO\" =\u003e Databricks picks an availability zone to schedule the cluster on.\n- A GCP availability zone =\u003e Pick One of the available zones for (machine type + region) from https://cloud.google.com/compute/docs/regions-zones." - } - } - }, - "init_scripts": { - "description": "The configuration for storing init scripts. Any number of destinations can be specified. The scripts are executed sequentially in the order provided. If `cluster_log_conf` is specified, init script logs are sent to `\u003cdestination\u003e/\u003ccluster-ID\u003e/init_scripts`.", - "items": { - "description": "", - "properties": { - "abfss": { - "description": "destination needs to be provided. e.g.\n`{ \"abfss\" : { \"destination\" : \"abfss://\u003ccontainer-name\u003e@\u003cstorage-account-name\u003e.dfs.core.windows.net/\u003cdirectory-name\u003e\" } }", - "properties": { - "destination": { - "description": "abfss destination, e.g. `abfss://\u003ccontainer-name\u003e@\u003cstorage-account-name\u003e.dfs.core.windows.net/\u003cdirectory-name\u003e`." - } - } - }, - "dbfs": { - "description": "destination needs to be provided. e.g.\n`{ \"dbfs\" : { \"destination\" : \"dbfs:/home/cluster_log\" } }`", - "properties": { - "destination": { - "description": "dbfs destination, e.g. `dbfs:/my/path`" - } - } - }, - "file": { - "description": "destination needs to be provided. e.g.\n`{ \"file\" : { \"destination\" : \"file:/my/local/file.sh\" } }`", - "properties": { - "destination": { - "description": "local file destination, e.g. `file:/my/local/file.sh`" - } - } - }, - "gcs": { - "description": "destination needs to be provided. e.g.\n`{ \"gcs\": { \"destination\": \"gs://my-bucket/file.sh\" } }`", - "properties": { - "destination": { - "description": "GCS destination/URI, e.g. `gs://my-bucket/some-prefix`" - } - } - }, - "s3": { - "description": "destination and either the region or endpoint need to be provided. e.g.\n`{ \"s3\": { \"destination\" : \"s3://cluster_log_bucket/prefix\", \"region\" : \"us-west-2\" } }`\nCluster iam role is used to access s3, please make sure the cluster iam role in\n`instance_profile_arn` has permission to write data to the s3 destination.", - "properties": { - "canned_acl": { - "description": "(Optional) Set canned access control list for the logs, e.g. `bucket-owner-full-control`.\nIf `canned_cal` is set, please make sure the cluster iam role has `s3:PutObjectAcl` permission on\nthe destination bucket and prefix. The full list of possible canned acl can be found at\nhttp://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl.\nPlease also note that by default only the object owner gets full controls. If you are using cross account\nrole for writing data, you may want to set `bucket-owner-full-control` to make bucket owner able to\nread the logs." - }, - "destination": { - "description": "S3 destination, e.g. `s3://my-bucket/some-prefix` Note that logs will be delivered using\ncluster iam role, please make sure you set cluster iam role and the role has write access to the\ndestination. Please also note that you cannot use AWS keys to deliver logs." - }, - "enable_encryption": { - "description": "(Optional) Flag to enable server side encryption, `false` by default." - }, - "encryption_type": { - "description": "(Optional) The encryption type, it could be `sse-s3` or `sse-kms`. It will be used only when\nencryption is enabled and the default type is `sse-s3`." - }, - "endpoint": { - "description": "S3 endpoint, e.g. `https://s3-us-west-2.amazonaws.com`. Either region or endpoint needs to be set.\nIf both are set, endpoint will be used." - }, - "kms_key": { - "description": "(Optional) Kms key which will be used if encryption is enabled and encryption type is set to `sse-kms`." - }, - "region": { - "description": "S3 region, e.g. `us-west-2`. Either region or endpoint needs to be set. If both are set,\nendpoint will be used." - } - } - }, - "volumes": { - "description": "destination needs to be provided. e.g.\n`{ \"volumes\" : { \"destination\" : \"/Volumes/my-init.sh\" } }`", - "properties": { - "destination": { - "description": "Unity Catalog Volumes file destination, e.g. `/Volumes/my-init.sh`" - } - } - }, - "workspace": { - "description": "destination needs to be provided. e.g.\n`{ \"workspace\" : { \"destination\" : \"/Users/user1@databricks.com/my-init.sh\" } }`", - "properties": { - "destination": { - "description": "workspace files destination, e.g. `/Users/user1@databricks.com/my-init.sh`" - } - } - } - } - } - }, - "instance_pool_id": { - "description": "The optional ID of the instance pool to which the cluster belongs." - }, - "node_type_id": { - "description": "This field encodes, through a single value, the resources available to each of\nthe Spark nodes in this cluster. For example, the Spark nodes can be provisioned\nand optimized for memory or compute intensive workloads. A list of available node\ntypes can be retrieved by using the :method:clusters/listNodeTypes API call.\n" - }, - "num_workers": { - "description": "Number of worker nodes that this cluster should have. A cluster has one Spark Driver\nand `num_workers` Executors for a total of `num_workers` + 1 Spark nodes.\n\nNote: When reading the properties of a cluster, this field reflects the desired number\nof workers rather than the actual current number of workers. For instance, if a cluster\nis resized from 5 to 10 workers, this field will immediately be updated to reflect\nthe target size of 10 workers, whereas the workers listed in `spark_info` will gradually\nincrease from 5 to 10 as the new nodes are provisioned." - }, - "policy_id": { - "description": "The ID of the cluster policy used to create the cluster if applicable." - }, - "runtime_engine": { - "description": "" - }, - "single_user_name": { - "description": "Single user name if data_security_mode is `SINGLE_USER`" - }, - "spark_conf": { - "description": "An object containing a set of optional, user-specified Spark configuration key-value pairs.\nUsers can also pass in a string of extra JVM options to the driver and the executors via\n`spark.driver.extraJavaOptions` and `spark.executor.extraJavaOptions` respectively.\n", - "additionalproperties": { - "description": "" - } - }, - "spark_env_vars": { - "description": "An object containing a set of optional, user-specified environment variable key-value pairs.\nPlease note that key-value pair of the form (X,Y) will be exported as is (i.e.,\n`export X='Y'`) while launching the driver and workers.\n\nIn order to specify an additional set of `SPARK_DAEMON_JAVA_OPTS`, we recommend appending\nthem to `$SPARK_DAEMON_JAVA_OPTS` as shown in the example below. This ensures that all\ndefault databricks managed environmental variables are included as well.\n\nExample Spark environment variables:\n`{\"SPARK_WORKER_MEMORY\": \"28000m\", \"SPARK_LOCAL_DIRS\": \"/local_disk0\"}` or\n`{\"SPARK_DAEMON_JAVA_OPTS\": \"$SPARK_DAEMON_JAVA_OPTS -Dspark.shuffle.service.enabled=true\"}`", - "additionalproperties": { - "description": "" - } - }, - "spark_version": { - "description": "The Spark version of the cluster, e.g. `3.3.x-scala2.11`.\nA list of available Spark versions can be retrieved by using\nthe :method:clusters/sparkVersions API call.\n" - }, - "ssh_public_keys": { - "description": "SSH public key contents that will be added to each Spark node in this cluster. The\ncorresponding private keys can be used to login with the user name `ubuntu` on port `2200`.\nUp to 10 keys can be specified.", - "items": { - "description": "" - } - }, - "workload_type": { - "description": "", - "properties": { - "clients": { - "description": " defined what type of clients can use the cluster. E.g. Notebooks, Jobs", - "properties": { - "jobs": { - "description": "With jobs set, the cluster can be used for jobs" - }, - "notebooks": { - "description": "With notebooks set, this cluster can be used for notebooks" - } - } - } - } - } - } - } - } - } - }, - "max_concurrent_runs": { - "description": "An optional maximum allowed number of concurrent runs of the job.\nSet this value if you want to be able to execute multiple runs of the same job concurrently.\nThis is useful for example if you trigger your job on a frequent schedule and want to allow consecutive runs to overlap with each other, or if you want to trigger multiple runs which differ by their input parameters.\nThis setting affects only new runs. For example, suppose the job’s concurrency is 4 and there are 4 concurrent active runs. Then setting the concurrency to 3 won’t kill any of the active runs.\nHowever, from then on, new runs are skipped unless there are fewer than 3 active runs.\nThis value cannot exceed 1000. Setting this value to `0` causes all new runs to be skipped." - }, - "name": { - "description": "An optional name for the job. The maximum length is 4096 bytes in UTF-8 encoding." - }, - "notification_settings": { - "description": "Optional notification settings that are used when sending notifications to each of the `email_notifications` and `webhook_notifications` for this job.", - "properties": { - "no_alert_for_canceled_runs": { - "description": "If true, do not send notifications to recipients specified in `on_failure` if the run is canceled." - }, - "no_alert_for_skipped_runs": { - "description": "If true, do not send notifications to recipients specified in `on_failure` if the run is skipped." - } - } - }, - "parameters": { - "description": "Job-level parameter definitions", - "items": { - "description": "", - "properties": { - "default": { - "description": "Default value of the parameter." - }, - "name": { - "description": "The name of the defined parameter. May only contain alphanumeric characters, `_`, `-`, and `.`" - } - } - } - }, - "permissions": { - "description": "", - "items": { - "description": "", - "properties": { - "group_name": { - "description": "" - }, - "level": { - "description": "" - }, - "service_principal_name": { - "description": "" - }, - "user_name": { - "description": "" - } - } - } - }, - "queue": { - "description": "The queue settings of the job.", - "properties": { - "enabled": { - "description": "If true, enable queueing for the job. This is a required field." - } - } - }, - "run_as": { - "description": "", - "properties": { - "service_principal_name": { - "description": "Application ID of an active service principal. Setting this field requires the `servicePrincipal/user` role." - }, - "user_name": { - "description": "The email of an active workspace user. Non-admin users can only set this field to their own email." - } - } - }, - "schedule": { - "description": "An optional periodic schedule for this job. The default behavior is that the job only runs when triggered by clicking “Run Now” in the Jobs UI or sending an API request to `runNow`.", - "properties": { - "pause_status": { - "description": "Indicate whether this schedule is paused or not." - }, - "quartz_cron_expression": { - "description": "A Cron expression using Quartz syntax that describes the schedule for a job. See [Cron Trigger](http://www.quartz-scheduler.org/documentation/quartz-2.3.0/tutorials/crontrigger.html) for details. This field is required." - }, - "timezone_id": { - "description": "A Java timezone ID. The schedule for a job is resolved with respect to this timezone. See [Java TimeZone](https://docs.oracle.com/javase/7/docs/api/java/util/TimeZone.html) for details. This field is required." - } - } - }, - "tags": { - "description": "A map of tags associated with the job. These are forwarded to the cluster as cluster tags for jobs clusters, and are subject to the same limitations as cluster tags. A maximum of 25 tags can be added to the job.", - "additionalproperties": { - "description": "" - } - }, - "tasks": { - "description": "A list of task specifications to be executed by this job.", - "items": { - "description": "", - "properties": { - "condition_task": { - "description": "If condition_task, specifies a condition with an outcome that can be used to control the execution of other tasks. Does not require a cluster to execute and does not support retries or notifications.", - "properties": { - "left": { - "description": "The left operand of the condition task. Can be either a string value or a job state or parameter reference." - }, - "op": { - "description": "* `EQUAL_TO`, `NOT_EQUAL` operators perform string comparison of their operands. This means that `“12.0” == “12”` will evaluate to `false`.\n* `GREATER_THAN`, `GREATER_THAN_OR_EQUAL`, `LESS_THAN`, `LESS_THAN_OR_EQUAL` operators perform numeric comparison of their operands. `“12.0” \u003e= “12”` will evaluate to `true`, `“10.0” \u003e= “12”` will evaluate to `false`.\n\nThe boolean comparison to task values can be implemented with operators `EQUAL_TO`, `NOT_EQUAL`. If a task value was set to a boolean value, it will be serialized to `“true”` or `“false”` for the comparison." - }, - "right": { - "description": "The right operand of the condition task. Can be either a string value or a job state or parameter reference." - } - } - }, - "dbt_task": { - "description": "If dbt_task, indicates that this must execute a dbt task. It requires both Databricks SQL and the ability to use a serverless or a pro SQL warehouse.", - "properties": { - "catalog": { - "description": "Optional name of the catalog to use. The value is the top level in the 3-level namespace of Unity Catalog (catalog / schema / relation). The catalog value can only be specified if a warehouse_id is specified. Requires dbt-databricks \u003e= 1.1.1." - }, - "commands": { - "description": "A list of dbt commands to execute. All commands must start with `dbt`. This parameter must not be empty. A maximum of up to 10 commands can be provided.", - "items": { - "description": "" - } - }, - "profiles_directory": { - "description": "Optional (relative) path to the profiles directory. Can only be specified if no warehouse_id is specified. If no warehouse_id is specified and this folder is unset, the root directory is used." - }, - "project_directory": { - "description": "Path to the project directory. Optional for Git sourced tasks, in which\ncase if no value is provided, the root of the Git repository is used." - }, - "schema": { - "description": "Optional schema to write to. This parameter is only used when a warehouse_id is also provided. If not provided, the `default` schema is used." - }, - "source": { - "description": "Optional location type of the project directory. When set to `WORKSPACE`, the project will be retrieved\nfrom the local Databricks workspace. When set to `GIT`, the project will be retrieved from a Git repository\ndefined in `git_source`. If the value is empty, the task will use `GIT` if `git_source` is defined and `WORKSPACE` otherwise.\n\n* `WORKSPACE`: Project is located in Databricks workspace.\n* `GIT`: Project is located in cloud Git provider." - }, - "warehouse_id": { - "description": "ID of the SQL warehouse to connect to. If provided, we automatically generate and provide the profile and connection details to dbt. It can be overridden on a per-command basis by using the `--profiles-dir` command line argument." - } - } - }, - "depends_on": { - "description": "An optional array of objects specifying the dependency graph of the task. All tasks specified in this field must complete before executing this task. The task will run only if the `run_if` condition is true.\nThe key is `task_key`, and the value is the name assigned to the dependent task.", - "items": { - "description": "", - "properties": { - "outcome": { - "description": "Can only be specified on condition task dependencies. The outcome of the dependent task that must be met for this task to run." - }, - "task_key": { - "description": "The name of the task this task depends on." - } - } - } - }, - "description": { - "description": "An optional description for this task." - }, - "disable_auto_optimization": { - "description": "An option to disable auto optimization in serverless" - }, - "email_notifications": { - "description": "An optional set of email addresses that is notified when runs of this task begin or complete as well as when this task is deleted. The default behavior is to not send any emails.", - "properties": { - "no_alert_for_skipped_runs": { - "description": "If true, do not send email to recipients specified in `on_failure` if the run is skipped." - }, - "on_duration_warning_threshold_exceeded": { - "description": "A list of email addresses to be notified when the duration of a run exceeds the threshold specified for the `RUN_DURATION_SECONDS` metric in the `health` field. If no rule for the `RUN_DURATION_SECONDS` metric is specified in the `health` field for the job, notifications are not sent.", - "items": { - "description": "" - } - }, - "on_failure": { - "description": "A list of email addresses to be notified when a run unsuccessfully completes. A run is considered to have completed unsuccessfully if it ends with an `INTERNAL_ERROR` `life_cycle_state` or a `FAILED`, or `TIMED_OUT` result_state. If this is not specified on job creation, reset, or update the list is empty, and notifications are not sent.", - "items": { - "description": "" - } - }, - "on_start": { - "description": "A list of email addresses to be notified when a run begins. If not specified on job creation, reset, or update, the list is empty, and notifications are not sent.", - "items": { - "description": "" - } - }, - "on_streaming_backlog_exceeded": { - "description": "A list of email addresses to notify when any streaming backlog thresholds are exceeded for any stream.\nStreaming backlog thresholds can be set in the `health` field using the following metrics: `STREAMING_BACKLOG_BYTES`, `STREAMING_BACKLOG_RECORDS`, `STREAMING_BACKLOG_SECONDS`, or `STREAMING_BACKLOG_FILES`.\nAlerting is based on the 10-minute average of these metrics. If the issue persists, notifications are resent every 30 minutes.", - "items": { - "description": "" - } - }, - "on_success": { - "description": "A list of email addresses to be notified when a run successfully completes. A run is considered to have completed successfully if it ends with a `TERMINATED` `life_cycle_state` and a `SUCCESS` result_state. If not specified on job creation, reset, or update, the list is empty, and notifications are not sent.", - "items": { - "description": "" - } - } - } - }, - "environment_key": { - "description": "The key that references an environment spec in a job. This field is required for Python script, Python wheel and dbt tasks when using serverless compute." - }, - "existing_cluster_id": { - "description": "If existing_cluster_id, the ID of an existing cluster that is used for all runs.\nWhen running jobs or tasks on an existing cluster, you may need to manually restart\nthe cluster if it stops responding. We suggest running jobs and tasks on new clusters for\ngreater reliability" - }, - "for_each_task": { - "description": "" - }, - "health": { - "description": "", - "properties": { - "rules": { - "description": "", - "items": { - "description": "", - "properties": { - "metric": { - "description": "" - }, - "op": { - "description": "" - }, - "value": { - "description": "Specifies the threshold value that the health metric should obey to satisfy the health rule." - } - } - } - } - } - }, - "job_cluster_key": { - "description": "If job_cluster_key, this task is executed reusing the cluster specified in `job.settings.job_clusters`." - }, - "libraries": { - "description": "An optional list of libraries to be installed on the cluster.\nThe default value is an empty list.", - "items": { - "description": "", - "properties": { - "cran": { - "description": "Specification of a CRAN library to be installed as part of the library", - "properties": { - "package": { - "description": "The name of the CRAN package to install." - }, - "repo": { - "description": "The repository where the package can be found. If not specified, the default CRAN repo is used." - } - } - }, - "egg": { - "description": "Deprecated. URI of the egg library to install. Installing Python egg files is deprecated and is not supported in Databricks Runtime 14.0 and above." - }, - "jar": { - "description": "URI of the JAR library to install. Supported URIs include Workspace paths, Unity Catalog Volumes paths, and S3 URIs.\nFor example: `{ \"jar\": \"/Workspace/path/to/library.jar\" }`, `{ \"jar\" : \"/Volumes/path/to/library.jar\" }` or\n`{ \"jar\": \"s3://my-bucket/library.jar\" }`.\nIf S3 is used, please make sure the cluster has read access on the library. You may need to\nlaunch the cluster with an IAM role to access the S3 URI." - }, - "maven": { - "description": "Specification of a maven library to be installed. For example:\n`{ \"coordinates\": \"org.jsoup:jsoup:1.7.2\" }`", - "properties": { - "coordinates": { - "description": "Gradle-style maven coordinates. For example: \"org.jsoup:jsoup:1.7.2\"." - }, - "exclusions": { - "description": "List of dependences to exclude. For example: `[\"slf4j:slf4j\", \"*:hadoop-client\"]`.\n\nMaven dependency exclusions:\nhttps://maven.apache.org/guides/introduction/introduction-to-optional-and-excludes-dependencies.html.", - "items": { - "description": "" - } - }, - "repo": { - "description": "Maven repo to install the Maven package from. If omitted, both Maven Central Repository\nand Spark Packages are searched." - } - } - }, - "pypi": { - "description": "Specification of a PyPi library to be installed. For example:\n`{ \"package\": \"simplejson\" }`", - "properties": { - "package": { - "description": "The name of the pypi package to install. An optional exact version specification is also\nsupported. Examples: \"simplejson\" and \"simplejson==3.8.0\"." - }, - "repo": { - "description": "The repository where the package can be found. If not specified, the default pip index is\nused." - } - } - }, - "requirements": { - "description": "URI of the requirements.txt file to install. Only Workspace paths and Unity Catalog Volumes paths are supported.\nFor example: `{ \"requirements\": \"/Workspace/path/to/requirements.txt\" }` or `{ \"requirements\" : \"/Volumes/path/to/requirements.txt\" }`" - }, - "whl": { - "description": "URI of the wheel library to install. Supported URIs include Workspace paths, Unity Catalog Volumes paths, and S3 URIs.\nFor example: `{ \"whl\": \"/Workspace/path/to/library.whl\" }`, `{ \"whl\" : \"/Volumes/path/to/library.whl\" }` or\n`{ \"whl\": \"s3://my-bucket/library.whl\" }`.\nIf S3 is used, please make sure the cluster has read access on the library. You may need to\nlaunch the cluster with an IAM role to access the S3 URI." - } - } - } - }, - "max_retries": { - "description": "An optional maximum number of times to retry an unsuccessful run. A run is considered to be unsuccessful if it completes with the `FAILED` result_state or `INTERNAL_ERROR` `life_cycle_state`. The value `-1` means to retry indefinitely and the value `0` means to never retry." - }, - "min_retry_interval_millis": { - "description": "An optional minimal interval in milliseconds between the start of the failed run and the subsequent retry run. The default behavior is that unsuccessful runs are immediately retried." - }, - "new_cluster": { - "description": "If new_cluster, a description of a new cluster that is created for each run.", - "properties": { - "apply_policy_default_values": { - "description": "When set to true, fixed and default values from the policy will be used for fields that are omitted. When set to false, only fixed values from the policy will be applied." - }, - "autoscale": { - "description": "Parameters needed in order to automatically scale clusters up and down based on load.\nNote: autoscaling works best with DB runtime versions 3.0 or later.", - "properties": { - "max_workers": { - "description": "The maximum number of workers to which the cluster can scale up when overloaded.\nNote that `max_workers` must be strictly greater than `min_workers`." - }, - "min_workers": { - "description": "The minimum number of workers to which the cluster can scale down when underutilized.\nIt is also the initial number of workers the cluster will have after creation." - } - } - }, - "autotermination_minutes": { - "description": "Automatically terminates the cluster after it is inactive for this time in minutes. If not set,\nthis cluster will not be automatically terminated. If specified, the threshold must be between\n10 and 10000 minutes.\nUsers can also set this value to 0 to explicitly disable automatic termination." - }, - "aws_attributes": { - "description": "Attributes related to clusters running on Amazon Web Services.\nIf not specified at cluster creation, a set of default values will be used.", - "properties": { - "availability": { - "description": "" - }, - "ebs_volume_count": { - "description": "The number of volumes launched for each instance. Users can choose up to 10 volumes.\nThis feature is only enabled for supported node types. Legacy node types cannot specify\ncustom EBS volumes.\nFor node types with no instance store, at least one EBS volume needs to be specified;\notherwise, cluster creation will fail.\n\nThese EBS volumes will be mounted at `/ebs0`, `/ebs1`, and etc.\nInstance store volumes will be mounted at `/local_disk0`, `/local_disk1`, and etc.\n\nIf EBS volumes are attached, Databricks will configure Spark to use only the EBS volumes for\nscratch storage because heterogenously sized scratch devices can lead to inefficient disk\nutilization. If no EBS volumes are attached, Databricks will configure Spark to use instance\nstore volumes.\n\nPlease note that if EBS volumes are specified, then the Spark configuration `spark.local.dir`\nwill be overridden." - }, - "ebs_volume_iops": { - "description": "If using gp3 volumes, what IOPS to use for the disk. If this is not set, the maximum performance of a gp2 volume with the same volume size will be used." - }, - "ebs_volume_size": { - "description": "The size of each EBS volume (in GiB) launched for each instance. For general purpose\nSSD, this value must be within the range 100 - 4096. For throughput optimized HDD,\nthis value must be within the range 500 - 4096." - }, - "ebs_volume_throughput": { - "description": "If using gp3 volumes, what throughput to use for the disk. If this is not set, the maximum performance of a gp2 volume with the same volume size will be used." - }, - "ebs_volume_type": { - "description": "" - }, - "first_on_demand": { - "description": "The first `first_on_demand` nodes of the cluster will be placed on on-demand instances.\nIf this value is greater than 0, the cluster driver node in particular will be placed on an\non-demand instance. If this value is greater than or equal to the current cluster size, all\nnodes will be placed on on-demand instances. If this value is less than the current cluster\nsize, `first_on_demand` nodes will be placed on on-demand instances and the remainder will\nbe placed on `availability` instances. Note that this value does not affect\ncluster size and cannot currently be mutated over the lifetime of a cluster." - }, - "instance_profile_arn": { - "description": "Nodes for this cluster will only be placed on AWS instances with this instance profile. If\nommitted, nodes will be placed on instances without an IAM instance profile. The instance\nprofile must have previously been added to the Databricks environment by an account\nadministrator.\n\nThis feature may only be available to certain customer plans.\n\nIf this field is ommitted, we will pull in the default from the conf if it exists." - }, - "spot_bid_price_percent": { - "description": "The bid price for AWS spot instances, as a percentage of the corresponding instance type's\non-demand price.\nFor example, if this field is set to 50, and the cluster needs a new `r3.xlarge` spot\ninstance, then the bid price is half of the price of\non-demand `r3.xlarge` instances. Similarly, if this field is set to 200, the bid price is twice\nthe price of on-demand `r3.xlarge` instances. If not specified, the default value is 100.\nWhen spot instances are requested for this cluster, only spot instances whose bid price\npercentage matches this field will be considered.\nNote that, for safety, we enforce this field to be no more than 10000.\n\nThe default value and documentation here should be kept consistent with\nCommonConf.defaultSpotBidPricePercent and CommonConf.maxSpotBidPricePercent." - }, - "zone_id": { - "description": "Identifier for the availability zone/datacenter in which the cluster resides.\nThis string will be of a form like \"us-west-2a\". The provided availability\nzone must be in the same region as the Databricks deployment. For example, \"us-west-2a\"\nis not a valid zone id if the Databricks deployment resides in the \"us-east-1\" region.\nThis is an optional field at cluster creation, and if not specified, a default zone will be used.\nIf the zone specified is \"auto\", will try to place cluster in a zone with high availability,\nand will retry placement in a different AZ if there is not enough capacity.\nThe list of available zones as well as the default value can be found by using the\n`List Zones` method." - } - } - }, - "azure_attributes": { - "description": "Attributes related to clusters running on Microsoft Azure.\nIf not specified at cluster creation, a set of default values will be used.", - "properties": { - "availability": { - "description": "" - }, - "first_on_demand": { - "description": "The first `first_on_demand` nodes of the cluster will be placed on on-demand instances.\nThis value should be greater than 0, to make sure the cluster driver node is placed on an\non-demand instance. If this value is greater than or equal to the current cluster size, all\nnodes will be placed on on-demand instances. If this value is less than the current cluster\nsize, `first_on_demand` nodes will be placed on on-demand instances and the remainder will\nbe placed on `availability` instances. Note that this value does not affect\ncluster size and cannot currently be mutated over the lifetime of a cluster." - }, - "log_analytics_info": { - "description": "Defines values necessary to configure and run Azure Log Analytics agent", - "properties": { - "log_analytics_primary_key": { - "description": "\u003cneeds content added\u003e" - }, - "log_analytics_workspace_id": { - "description": "\u003cneeds content added\u003e" - } - } - }, - "spot_bid_max_price": { - "description": "The max bid price to be used for Azure spot instances.\nThe Max price for the bid cannot be higher than the on-demand price of the instance.\nIf not specified, the default value is -1, which specifies that the instance cannot be evicted\non the basis of price, and only on the basis of availability. Further, the value should \u003e 0 or -1." - } - } - }, - "cluster_log_conf": { - "description": "The configuration for delivering spark logs to a long-term storage destination.\nTwo kinds of destinations (dbfs and s3) are supported. Only one destination can be specified\nfor one cluster. If the conf is given, the logs will be delivered to the destination every\n`5 mins`. The destination of driver logs is `$destination/$clusterId/driver`, while\nthe destination of executor logs is `$destination/$clusterId/executor`.", - "properties": { - "dbfs": { - "description": "destination needs to be provided. e.g.\n`{ \"dbfs\" : { \"destination\" : \"dbfs:/home/cluster_log\" } }`", - "properties": { - "destination": { - "description": "dbfs destination, e.g. `dbfs:/my/path`" - } - } - }, - "s3": { - "description": "destination and either the region or endpoint need to be provided. e.g.\n`{ \"s3\": { \"destination\" : \"s3://cluster_log_bucket/prefix\", \"region\" : \"us-west-2\" } }`\nCluster iam role is used to access s3, please make sure the cluster iam role in\n`instance_profile_arn` has permission to write data to the s3 destination.", - "properties": { - "canned_acl": { - "description": "(Optional) Set canned access control list for the logs, e.g. `bucket-owner-full-control`.\nIf `canned_cal` is set, please make sure the cluster iam role has `s3:PutObjectAcl` permission on\nthe destination bucket and prefix. The full list of possible canned acl can be found at\nhttp://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl.\nPlease also note that by default only the object owner gets full controls. If you are using cross account\nrole for writing data, you may want to set `bucket-owner-full-control` to make bucket owner able to\nread the logs." - }, - "destination": { - "description": "S3 destination, e.g. `s3://my-bucket/some-prefix` Note that logs will be delivered using\ncluster iam role, please make sure you set cluster iam role and the role has write access to the\ndestination. Please also note that you cannot use AWS keys to deliver logs." - }, - "enable_encryption": { - "description": "(Optional) Flag to enable server side encryption, `false` by default." - }, - "encryption_type": { - "description": "(Optional) The encryption type, it could be `sse-s3` or `sse-kms`. It will be used only when\nencryption is enabled and the default type is `sse-s3`." - }, - "endpoint": { - "description": "S3 endpoint, e.g. `https://s3-us-west-2.amazonaws.com`. Either region or endpoint needs to be set.\nIf both are set, endpoint will be used." - }, - "kms_key": { - "description": "(Optional) Kms key which will be used if encryption is enabled and encryption type is set to `sse-kms`." - }, - "region": { - "description": "S3 region, e.g. `us-west-2`. Either region or endpoint needs to be set. If both are set,\nendpoint will be used." - } - } - } - } - }, - "cluster_name": { - "description": "Cluster name requested by the user. This doesn't have to be unique.\nIf not specified at creation, the cluster name will be an empty string.\n" - }, - "custom_tags": { - "description": "Additional tags for cluster resources. Databricks will tag all cluster resources (e.g., AWS\ninstances and EBS volumes) with these tags in addition to `default_tags`. Notes:\n\n- Currently, Databricks allows at most 45 custom tags\n\n- Clusters can only reuse cloud resources if the resources' tags are a subset of the cluster tags", - "additionalproperties": { - "description": "" - } - }, - "data_security_mode": { - "description": "" - }, - "docker_image": { - "description": "", - "properties": { - "basic_auth": { - "description": "", - "properties": { - "password": { - "description": "Password of the user" - }, - "username": { - "description": "Name of the user" - } - } - }, - "url": { - "description": "URL of the docker image." - } - } - }, - "driver_instance_pool_id": { - "description": "The optional ID of the instance pool for the driver of the cluster belongs.\nThe pool cluster uses the instance pool with id (instance_pool_id) if the driver pool is not\nassigned." - }, - "driver_node_type_id": { - "description": "The node type of the Spark driver. Note that this field is optional;\nif unset, the driver node type will be set as the same value\nas `node_type_id` defined above.\n" - }, - "enable_elastic_disk": { - "description": "Autoscaling Local Storage: when enabled, this cluster will dynamically acquire additional disk\nspace when its Spark workers are running low on disk space. This feature requires specific AWS\npermissions to function correctly - refer to the User Guide for more details." - }, - "enable_local_disk_encryption": { - "description": "Whether to enable LUKS on cluster VMs' local disks" - }, - "gcp_attributes": { - "description": "Attributes related to clusters running on Google Cloud Platform.\nIf not specified at cluster creation, a set of default values will be used.", - "properties": { - "availability": { - "description": "" - }, - "boot_disk_size": { - "description": "boot disk size in GB" - }, - "google_service_account": { - "description": "If provided, the cluster will impersonate the google service account when accessing\ngcloud services (like GCS). The google service account\nmust have previously been added to the Databricks environment by an account\nadministrator." - }, - "local_ssd_count": { - "description": "If provided, each node (workers and driver) in the cluster will have this number of local SSDs attached. Each local SSD is 375GB in size. Refer to [GCP documentation](https://cloud.google.com/compute/docs/disks/local-ssd#choose_number_local_ssds) for the supported number of local SSDs for each instance type." - }, - "use_preemptible_executors": { - "description": "This field determines whether the spark executors will be scheduled to run on preemptible VMs (when set to true) versus standard compute engine VMs (when set to false; default).\nNote: Soon to be deprecated, use the availability field instead." - }, - "zone_id": { - "description": "Identifier for the availability zone in which the cluster resides.\nThis can be one of the following:\n- \"HA\" =\u003e High availability, spread nodes across availability zones for a Databricks deployment region [default]\n- \"AUTO\" =\u003e Databricks picks an availability zone to schedule the cluster on.\n- A GCP availability zone =\u003e Pick One of the available zones for (machine type + region) from https://cloud.google.com/compute/docs/regions-zones." - } - } - }, - "init_scripts": { - "description": "The configuration for storing init scripts. Any number of destinations can be specified. The scripts are executed sequentially in the order provided. If `cluster_log_conf` is specified, init script logs are sent to `\u003cdestination\u003e/\u003ccluster-ID\u003e/init_scripts`.", - "items": { - "description": "", - "properties": { - "abfss": { - "description": "destination needs to be provided. e.g.\n`{ \"abfss\" : { \"destination\" : \"abfss://\u003ccontainer-name\u003e@\u003cstorage-account-name\u003e.dfs.core.windows.net/\u003cdirectory-name\u003e\" } }", - "properties": { - "destination": { - "description": "abfss destination, e.g. `abfss://\u003ccontainer-name\u003e@\u003cstorage-account-name\u003e.dfs.core.windows.net/\u003cdirectory-name\u003e`." - } - } - }, - "dbfs": { - "description": "destination needs to be provided. e.g.\n`{ \"dbfs\" : { \"destination\" : \"dbfs:/home/cluster_log\" } }`", - "properties": { - "destination": { - "description": "dbfs destination, e.g. `dbfs:/my/path`" - } - } - }, - "file": { - "description": "destination needs to be provided. e.g.\n`{ \"file\" : { \"destination\" : \"file:/my/local/file.sh\" } }`", - "properties": { - "destination": { - "description": "local file destination, e.g. `file:/my/local/file.sh`" - } - } - }, - "gcs": { - "description": "destination needs to be provided. e.g.\n`{ \"gcs\": { \"destination\": \"gs://my-bucket/file.sh\" } }`", - "properties": { - "destination": { - "description": "GCS destination/URI, e.g. `gs://my-bucket/some-prefix`" - } - } - }, - "s3": { - "description": "destination and either the region or endpoint need to be provided. e.g.\n`{ \"s3\": { \"destination\" : \"s3://cluster_log_bucket/prefix\", \"region\" : \"us-west-2\" } }`\nCluster iam role is used to access s3, please make sure the cluster iam role in\n`instance_profile_arn` has permission to write data to the s3 destination.", - "properties": { - "canned_acl": { - "description": "(Optional) Set canned access control list for the logs, e.g. `bucket-owner-full-control`.\nIf `canned_cal` is set, please make sure the cluster iam role has `s3:PutObjectAcl` permission on\nthe destination bucket and prefix. The full list of possible canned acl can be found at\nhttp://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl.\nPlease also note that by default only the object owner gets full controls. If you are using cross account\nrole for writing data, you may want to set `bucket-owner-full-control` to make bucket owner able to\nread the logs." - }, - "destination": { - "description": "S3 destination, e.g. `s3://my-bucket/some-prefix` Note that logs will be delivered using\ncluster iam role, please make sure you set cluster iam role and the role has write access to the\ndestination. Please also note that you cannot use AWS keys to deliver logs." - }, - "enable_encryption": { - "description": "(Optional) Flag to enable server side encryption, `false` by default." - }, - "encryption_type": { - "description": "(Optional) The encryption type, it could be `sse-s3` or `sse-kms`. It will be used only when\nencryption is enabled and the default type is `sse-s3`." - }, - "endpoint": { - "description": "S3 endpoint, e.g. `https://s3-us-west-2.amazonaws.com`. Either region or endpoint needs to be set.\nIf both are set, endpoint will be used." - }, - "kms_key": { - "description": "(Optional) Kms key which will be used if encryption is enabled and encryption type is set to `sse-kms`." - }, - "region": { - "description": "S3 region, e.g. `us-west-2`. Either region or endpoint needs to be set. If both are set,\nendpoint will be used." - } - } - }, - "volumes": { - "description": "destination needs to be provided. e.g.\n`{ \"volumes\" : { \"destination\" : \"/Volumes/my-init.sh\" } }`", - "properties": { - "destination": { - "description": "Unity Catalog Volumes file destination, e.g. `/Volumes/my-init.sh`" - } - } - }, - "workspace": { - "description": "destination needs to be provided. e.g.\n`{ \"workspace\" : { \"destination\" : \"/Users/user1@databricks.com/my-init.sh\" } }`", - "properties": { - "destination": { - "description": "workspace files destination, e.g. `/Users/user1@databricks.com/my-init.sh`" - } - } - } - } - } - }, - "instance_pool_id": { - "description": "The optional ID of the instance pool to which the cluster belongs." - }, - "node_type_id": { - "description": "This field encodes, through a single value, the resources available to each of\nthe Spark nodes in this cluster. For example, the Spark nodes can be provisioned\nand optimized for memory or compute intensive workloads. A list of available node\ntypes can be retrieved by using the :method:clusters/listNodeTypes API call.\n" - }, - "num_workers": { - "description": "Number of worker nodes that this cluster should have. A cluster has one Spark Driver\nand `num_workers` Executors for a total of `num_workers` + 1 Spark nodes.\n\nNote: When reading the properties of a cluster, this field reflects the desired number\nof workers rather than the actual current number of workers. For instance, if a cluster\nis resized from 5 to 10 workers, this field will immediately be updated to reflect\nthe target size of 10 workers, whereas the workers listed in `spark_info` will gradually\nincrease from 5 to 10 as the new nodes are provisioned." - }, - "policy_id": { - "description": "The ID of the cluster policy used to create the cluster if applicable." - }, - "runtime_engine": { - "description": "" - }, - "single_user_name": { - "description": "Single user name if data_security_mode is `SINGLE_USER`" - }, - "spark_conf": { - "description": "An object containing a set of optional, user-specified Spark configuration key-value pairs.\nUsers can also pass in a string of extra JVM options to the driver and the executors via\n`spark.driver.extraJavaOptions` and `spark.executor.extraJavaOptions` respectively.\n", - "additionalproperties": { - "description": "" - } - }, - "spark_env_vars": { - "description": "An object containing a set of optional, user-specified environment variable key-value pairs.\nPlease note that key-value pair of the form (X,Y) will be exported as is (i.e.,\n`export X='Y'`) while launching the driver and workers.\n\nIn order to specify an additional set of `SPARK_DAEMON_JAVA_OPTS`, we recommend appending\nthem to `$SPARK_DAEMON_JAVA_OPTS` as shown in the example below. This ensures that all\ndefault databricks managed environmental variables are included as well.\n\nExample Spark environment variables:\n`{\"SPARK_WORKER_MEMORY\": \"28000m\", \"SPARK_LOCAL_DIRS\": \"/local_disk0\"}` or\n`{\"SPARK_DAEMON_JAVA_OPTS\": \"$SPARK_DAEMON_JAVA_OPTS -Dspark.shuffle.service.enabled=true\"}`", - "additionalproperties": { - "description": "" - } - }, - "spark_version": { - "description": "The Spark version of the cluster, e.g. `3.3.x-scala2.11`.\nA list of available Spark versions can be retrieved by using\nthe :method:clusters/sparkVersions API call.\n" - }, - "ssh_public_keys": { - "description": "SSH public key contents that will be added to each Spark node in this cluster. The\ncorresponding private keys can be used to login with the user name `ubuntu` on port `2200`.\nUp to 10 keys can be specified.", - "items": { - "description": "" - } - }, - "workload_type": { - "description": "", - "properties": { - "clients": { - "description": " defined what type of clients can use the cluster. E.g. Notebooks, Jobs", - "properties": { - "jobs": { - "description": "With jobs set, the cluster can be used for jobs" - }, - "notebooks": { - "description": "With notebooks set, this cluster can be used for notebooks" - } - } - } - } - } - } - }, - "notebook_task": { - "description": "If notebook_task, indicates that this task must run a notebook. This field may not be specified in conjunction with spark_jar_task.", - "properties": { - "base_parameters": { - "description": "Base parameters to be used for each run of this job. If the run is initiated by a call to :method:jobs/run\nNow with parameters specified, the two parameters maps are merged. If the same key is specified in\n`base_parameters` and in `run-now`, the value from `run-now` is used.\nUse [Task parameter variables](https://docs.databricks.com/jobs.html#parameter-variables) to set parameters containing information about job runs.\n\nIf the notebook takes a parameter that is not specified in the job’s `base_parameters` or the `run-now` override parameters,\nthe default value from the notebook is used.\n\nRetrieve these parameters in a notebook using [dbutils.widgets.get](https://docs.databricks.com/dev-tools/databricks-utils.html#dbutils-widgets).\n\nThe JSON representation of this field cannot exceed 1MB.", - "additionalproperties": { - "description": "" - } - }, - "notebook_path": { - "description": "The path of the notebook to be run in the Databricks workspace or remote repository.\nFor notebooks stored in the Databricks workspace, the path must be absolute and begin with a slash.\nFor notebooks stored in a remote repository, the path must be relative. This field is required." - }, - "source": { - "description": "Optional location type of the notebook. When set to `WORKSPACE`, the notebook will be retrieved from the local Databricks workspace. When set to `GIT`, the notebook will be retrieved from a Git repository\ndefined in `git_source`. If the value is empty, the task will use `GIT` if `git_source` is defined and `WORKSPACE` otherwise.\n* `WORKSPACE`: Notebook is located in Databricks workspace.\n* `GIT`: Notebook is located in cloud Git provider." - }, - "warehouse_id": { - "description": "Optional `warehouse_id` to run the notebook on a SQL warehouse. Classic SQL warehouses are NOT supported, please use serverless or pro SQL warehouses.\n\nNote that SQL warehouses only support SQL cells; if the notebook contains non-SQL cells, the run will fail." - } - } - }, - "notification_settings": { - "description": "Optional notification settings that are used when sending notifications to each of the `email_notifications` and `webhook_notifications` for this task.", - "properties": { - "alert_on_last_attempt": { - "description": "If true, do not send notifications to recipients specified in `on_start` for the retried runs and do not send notifications to recipients specified in `on_failure` until the last retry of the run." - }, - "no_alert_for_canceled_runs": { - "description": "If true, do not send notifications to recipients specified in `on_failure` if the run is canceled." - }, - "no_alert_for_skipped_runs": { - "description": "If true, do not send notifications to recipients specified in `on_failure` if the run is skipped." - } - } - }, - "pipeline_task": { - "description": "If pipeline_task, indicates that this task must execute a Pipeline.", - "properties": { - "full_refresh": { - "description": "If true, triggers a full refresh on the delta live table." - }, - "pipeline_id": { - "description": "The full name of the pipeline task to execute." - } - } - }, - "python_wheel_task": { - "description": "If python_wheel_task, indicates that this job must execute a PythonWheel.", - "properties": { - "entry_point": { - "description": "Named entry point to use, if it does not exist in the metadata of the package it executes the function from the package directly using `$packageName.$entryPoint()`" - }, - "named_parameters": { - "description": "Command-line parameters passed to Python wheel task in the form of `[\"--name=task\", \"--data=dbfs:/path/to/data.json\"]`. Leave it empty if `parameters` is not null.", - "additionalproperties": { - "description": "" - } - }, - "package_name": { - "description": "Name of the package to execute" - }, - "parameters": { - "description": "Command-line parameters passed to Python wheel task. Leave it empty if `named_parameters` is not null.", - "items": { - "description": "" - } - } - } - }, - "retry_on_timeout": { - "description": "An optional policy to specify whether to retry a job when it times out. The default behavior\nis to not retry on timeout." - }, - "run_if": { - "description": "An optional value specifying the condition determining whether the task is run once its dependencies have been completed.\n\n* `ALL_SUCCESS`: All dependencies have executed and succeeded\n* `AT_LEAST_ONE_SUCCESS`: At least one dependency has succeeded\n* `NONE_FAILED`: None of the dependencies have failed and at least one was executed\n* `ALL_DONE`: All dependencies have been completed\n* `AT_LEAST_ONE_FAILED`: At least one dependency failed\n* `ALL_FAILED`: ALl dependencies have failed" - }, - "run_job_task": { - "description": "If run_job_task, indicates that this task must execute another job.", - "properties": { - "dbt_commands": { - "description": "An array of commands to execute for jobs with the dbt task, for example `\"dbt_commands\": [\"dbt deps\", \"dbt seed\", \"dbt deps\", \"dbt seed\", \"dbt run\"]`", - "items": { - "description": "" - } - }, - "jar_params": { - "description": "A list of parameters for jobs with Spark JAR tasks, for example `\"jar_params\": [\"john doe\", \"35\"]`.\nThe parameters are used to invoke the main function of the main class specified in the Spark JAR task.\nIf not specified upon `run-now`, it defaults to an empty list.\njar_params cannot be specified in conjunction with notebook_params.\nThe JSON representation of this field (for example `{\"jar_params\":[\"john doe\",\"35\"]}`) cannot exceed 10,000 bytes.\n\nUse [Task parameter variables](/jobs.html\\\"#parameter-variables\\\") to set parameters containing information about job runs.", - "items": { - "description": "" - } - }, - "job_id": { - "description": "ID of the job to trigger." - }, - "job_parameters": { - "description": "Job-level parameters used to trigger the job.", - "additionalproperties": { - "description": "" - } - }, - "notebook_params": { - "description": "A map from keys to values for jobs with notebook task, for example `\"notebook_params\": {\"name\": \"john doe\", \"age\": \"35\"}`.\nThe map is passed to the notebook and is accessible through the [dbutils.widgets.get](https://docs.databricks.com/dev-tools/databricks-utils.html) function.\n\nIf not specified upon `run-now`, the triggered run uses the job’s base parameters.\n\nnotebook_params cannot be specified in conjunction with jar_params.\n\nUse [Task parameter variables](https://docs.databricks.com/jobs.html#parameter-variables) to set parameters containing information about job runs.\n\nThe JSON representation of this field (for example `{\"notebook_params\":{\"name\":\"john doe\",\"age\":\"35\"}}`) cannot exceed 10,000 bytes.", - "additionalproperties": { - "description": "" - } - }, - "pipeline_params": { - "description": "", - "properties": { - "full_refresh": { - "description": "If true, triggers a full refresh on the delta live table." - } - } - }, - "python_named_params": { - "description": "", - "additionalproperties": { - "description": "" - } - }, - "python_params": { - "description": "A list of parameters for jobs with Python tasks, for example `\"python_params\": [\"john doe\", \"35\"]`.\nThe parameters are passed to Python file as command-line parameters. If specified upon `run-now`, it would overwrite\nthe parameters specified in job setting. The JSON representation of this field (for example `{\"python_params\":[\"john doe\",\"35\"]}`)\ncannot exceed 10,000 bytes.\n\nUse [Task parameter variables](https://docs.databricks.com/jobs.html#parameter-variables) to set parameters containing information about job runs.\n\nImportant\n\nThese parameters accept only Latin characters (ASCII character set). Using non-ASCII characters returns an error.\nExamples of invalid, non-ASCII characters are Chinese, Japanese kanjis, and emojis.", - "items": { - "description": "" - } - }, - "spark_submit_params": { - "description": "A list of parameters for jobs with spark submit task, for example `\"spark_submit_params\": [\"--class\", \"org.apache.spark.examples.SparkPi\"]`.\nThe parameters are passed to spark-submit script as command-line parameters. If specified upon `run-now`, it would overwrite the\nparameters specified in job setting. The JSON representation of this field (for example `{\"python_params\":[\"john doe\",\"35\"]}`)\ncannot exceed 10,000 bytes.\n\nUse [Task parameter variables](https://docs.databricks.com/jobs.html#parameter-variables) to set parameters containing information about job runs\n\nImportant\n\nThese parameters accept only Latin characters (ASCII character set). Using non-ASCII characters returns an error.\nExamples of invalid, non-ASCII characters are Chinese, Japanese kanjis, and emojis.", - "items": { - "description": "" - } - }, - "sql_params": { - "description": "A map from keys to values for jobs with SQL task, for example `\"sql_params\": {\"name\": \"john doe\", \"age\": \"35\"}`. The SQL alert task does not support custom parameters.", - "additionalproperties": { - "description": "" - } - } - } - }, - "spark_jar_task": { - "description": "If spark_jar_task, indicates that this task must run a JAR.", - "properties": { - "jar_uri": { - "description": "Deprecated since 04/2016. Provide a `jar` through the `libraries` field instead. For an example, see :method:jobs/create." - }, - "main_class_name": { - "description": "The full name of the class containing the main method to be executed. This class must be contained in a JAR provided as a library.\n\nThe code must use `SparkContext.getOrCreate` to obtain a Spark context; otherwise, runs of the job fail." - }, - "parameters": { - "description": "Parameters passed to the main method.\n\nUse [Task parameter variables](https://docs.databricks.com/jobs.html#parameter-variables) to set parameters containing information about job runs.", - "items": { - "description": "" - } - } - } - }, - "spark_python_task": { - "description": "If spark_python_task, indicates that this task must run a Python file.", - "properties": { - "parameters": { - "description": "Command line parameters passed to the Python file.\n\nUse [Task parameter variables](https://docs.databricks.com/jobs.html#parameter-variables) to set parameters containing information about job runs.", - "items": { - "description": "" - } - }, - "python_file": { - "description": "The Python file to be executed. Cloud file URIs (such as dbfs:/, s3:/, adls:/, gcs:/) and workspace paths are supported. For python files stored in the Databricks workspace, the path must be absolute and begin with `/`. For files stored in a remote repository, the path must be relative. This field is required." - }, - "source": { - "description": "Optional location type of the Python file. When set to `WORKSPACE` or not specified, the file will be retrieved from the local\nDatabricks workspace or cloud location (if the `python_file` has a URI format). When set to `GIT`,\nthe Python file will be retrieved from a Git repository defined in `git_source`.\n\n* `WORKSPACE`: The Python file is located in a Databricks workspace or at a cloud filesystem URI.\n* `GIT`: The Python file is located in a remote Git repository." - } - } - }, - "spark_submit_task": { - "description": "If `spark_submit_task`, indicates that this task must be launched by the spark submit script. This task can run only on new clusters.\n\nIn the `new_cluster` specification, `libraries` and `spark_conf` are not supported. Instead, use `--jars` and `--py-files` to add Java and Python libraries and `--conf` to set the Spark configurations.\n\n`master`, `deploy-mode`, and `executor-cores` are automatically configured by Databricks; you _cannot_ specify them in parameters.\n\nBy default, the Spark submit job uses all available memory (excluding reserved memory for Databricks services). You can set `--driver-memory`, and `--executor-memory` to a smaller value to leave some room for off-heap usage.\n\nThe `--jars`, `--py-files`, `--files` arguments support DBFS and S3 paths.", - "properties": { - "parameters": { - "description": "Command-line parameters passed to spark submit.\n\nUse [Task parameter variables](https://docs.databricks.com/jobs.html#parameter-variables) to set parameters containing information about job runs.", - "items": { - "description": "" - } - } - } - }, - "sql_task": { - "description": "If sql_task, indicates that this job must execute a SQL task.", - "properties": { - "alert": { - "description": "If alert, indicates that this job must refresh a SQL alert.", - "properties": { - "alert_id": { - "description": "The canonical identifier of the SQL alert." - }, - "pause_subscriptions": { - "description": "If true, the alert notifications are not sent to subscribers." - }, - "subscriptions": { - "description": "If specified, alert notifications are sent to subscribers.", - "items": { - "description": "", - "properties": { - "destination_id": { - "description": "The canonical identifier of the destination to receive email notification. This parameter is mutually exclusive with user_name. You cannot set both destination_id and user_name for subscription notifications." - }, - "user_name": { - "description": "The user name to receive the subscription email. This parameter is mutually exclusive with destination_id. You cannot set both destination_id and user_name for subscription notifications." - } - } - } - } - } - }, - "dashboard": { - "description": "If dashboard, indicates that this job must refresh a SQL dashboard.", - "properties": { - "custom_subject": { - "description": "Subject of the email sent to subscribers of this task." - }, - "dashboard_id": { - "description": "The canonical identifier of the SQL dashboard." - }, - "pause_subscriptions": { - "description": "If true, the dashboard snapshot is not taken, and emails are not sent to subscribers." - }, - "subscriptions": { - "description": "If specified, dashboard snapshots are sent to subscriptions.", - "items": { - "description": "", - "properties": { - "destination_id": { - "description": "The canonical identifier of the destination to receive email notification. This parameter is mutually exclusive with user_name. You cannot set both destination_id and user_name for subscription notifications." - }, - "user_name": { - "description": "The user name to receive the subscription email. This parameter is mutually exclusive with destination_id. You cannot set both destination_id and user_name for subscription notifications." - } - } - } - } - } - }, - "file": { - "description": "If file, indicates that this job runs a SQL file in a remote Git repository.", - "properties": { - "path": { - "description": "Path of the SQL file. Must be relative if the source is a remote Git repository and absolute for workspace paths." - }, - "source": { - "description": "Optional location type of the SQL file. When set to `WORKSPACE`, the SQL file will be retrieved\nfrom the local Databricks workspace. When set to `GIT`, the SQL file will be retrieved from a Git repository\ndefined in `git_source`. If the value is empty, the task will use `GIT` if `git_source` is defined and `WORKSPACE` otherwise.\n\n* `WORKSPACE`: SQL file is located in Databricks workspace.\n* `GIT`: SQL file is located in cloud Git provider." - } - } - }, - "parameters": { - "description": "Parameters to be used for each run of this job. The SQL alert task does not support custom parameters.", - "additionalproperties": { - "description": "" - } - }, - "query": { - "description": "If query, indicates that this job must execute a SQL query.", - "properties": { - "query_id": { - "description": "The canonical identifier of the SQL query." - } - } - }, - "warehouse_id": { - "description": "The canonical identifier of the SQL warehouse. Recommended to use with serverless or pro SQL warehouses. Classic SQL warehouses are only supported for SQL alert, dashboard and query tasks and are limited to scheduled single-task jobs." - } - } - }, - "task_key": { - "description": "A unique name for the task. This field is used to refer to this task from other tasks.\nThis field is required and must be unique within its parent job.\nOn Update or Reset, this field is used to reference the tasks to be updated or reset." - }, - "timeout_seconds": { - "description": "An optional timeout applied to each run of this job task. A value of `0` means no timeout." - }, - "webhook_notifications": { - "description": "A collection of system notification IDs to notify when runs of this task begin or complete. The default behavior is to not send any system notifications.", - "properties": { - "on_duration_warning_threshold_exceeded": { - "description": "An optional list of system notification IDs to call when the duration of a run exceeds the threshold specified for the `RUN_DURATION_SECONDS` metric in the `health` field. A maximum of 3 destinations can be specified for the `on_duration_warning_threshold_exceeded` property.", - "items": { - "description": "", - "properties": { - "id": { - "description": "" - } - } - } - }, - "on_failure": { - "description": "An optional list of system notification IDs to call when the run fails. A maximum of 3 destinations can be specified for the `on_failure` property.", - "items": { - "description": "", - "properties": { - "id": { - "description": "" - } - } - } - }, - "on_start": { - "description": "An optional list of system notification IDs to call when the run starts. A maximum of 3 destinations can be specified for the `on_start` property.", - "items": { - "description": "", - "properties": { - "id": { - "description": "" - } - } - } - }, - "on_streaming_backlog_exceeded": { - "description": "An optional list of system notification IDs to call when any streaming backlog thresholds are exceeded for any stream.\nStreaming backlog thresholds can be set in the `health` field using the following metrics: `STREAMING_BACKLOG_BYTES`, `STREAMING_BACKLOG_RECORDS`, `STREAMING_BACKLOG_SECONDS`, or `STREAMING_BACKLOG_FILES`.\nAlerting is based on the 10-minute average of these metrics. If the issue persists, notifications are resent every 30 minutes.\nA maximum of 3 destinations can be specified for the `on_streaming_backlog_exceeded` property.", - "items": { - "description": "", - "properties": { - "id": { - "description": "" - } - } - } - }, - "on_success": { - "description": "An optional list of system notification IDs to call when the run completes successfully. A maximum of 3 destinations can be specified for the `on_success` property.", - "items": { - "description": "", - "properties": { - "id": { - "description": "" - } - } - } - } - } - } - } - } - }, - "timeout_seconds": { - "description": "An optional timeout applied to each run of this job. A value of `0` means no timeout." - }, - "trigger": { - "description": "A configuration to trigger a run when certain conditions are met. The default behavior is that the job runs only when triggered by clicking “Run Now” in the Jobs UI or sending an API request to `runNow`.", - "properties": { - "file_arrival": { - "description": "File arrival trigger settings.", - "properties": { - "min_time_between_triggers_seconds": { - "description": "If set, the trigger starts a run only after the specified amount of time passed since\nthe last time the trigger fired. The minimum allowed value is 60 seconds" - }, - "url": { - "description": "URL to be monitored for file arrivals. The path must point to the root or a subpath of the external location." - }, - "wait_after_last_change_seconds": { - "description": "If set, the trigger starts a run only after no file activity has occurred for the specified amount of time.\nThis makes it possible to wait for a batch of incoming files to arrive before triggering a run. The\nminimum allowed value is 60 seconds." - } - } - }, - "pause_status": { - "description": "Whether this trigger is paused or not." - }, - "periodic": { - "description": "Periodic trigger settings.", - "properties": { - "interval": { - "description": "The interval at which the trigger should run." - }, - "unit": { - "description": "The unit of time for the interval." - } - } - }, - "table": { - "description": "Old table trigger settings name. Deprecated in favor of `table_update`.", - "properties": { - "condition": { - "description": "The table(s) condition based on which to trigger a job run." - }, - "min_time_between_triggers_seconds": { - "description": "If set, the trigger starts a run only after the specified amount of time has passed since\nthe last time the trigger fired. The minimum allowed value is 60 seconds." - }, - "table_names": { - "description": "A list of Delta tables to monitor for changes. The table name must be in the format `catalog_name.schema_name.table_name`.", - "items": { - "description": "" - } - }, - "wait_after_last_change_seconds": { - "description": "If set, the trigger starts a run only after no table updates have occurred for the specified time\nand can be used to wait for a series of table updates before triggering a run. The\nminimum allowed value is 60 seconds." - } - } - }, - "table_update": { - "description": "", - "properties": { - "condition": { - "description": "The table(s) condition based on which to trigger a job run." - }, - "min_time_between_triggers_seconds": { - "description": "If set, the trigger starts a run only after the specified amount of time has passed since\nthe last time the trigger fired. The minimum allowed value is 60 seconds." - }, - "table_names": { - "description": "A list of Delta tables to monitor for changes. The table name must be in the format `catalog_name.schema_name.table_name`.", - "items": { - "description": "" - } - }, - "wait_after_last_change_seconds": { - "description": "If set, the trigger starts a run only after no table updates have occurred for the specified time\nand can be used to wait for a series of table updates before triggering a run. The\nminimum allowed value is 60 seconds." - } - } - } - } - }, - "webhook_notifications": { - "description": "A collection of system notification IDs to notify when runs of this job begin or complete.", - "properties": { - "on_duration_warning_threshold_exceeded": { - "description": "An optional list of system notification IDs to call when the duration of a run exceeds the threshold specified for the `RUN_DURATION_SECONDS` metric in the `health` field. A maximum of 3 destinations can be specified for the `on_duration_warning_threshold_exceeded` property.", - "items": { - "description": "", - "properties": { - "id": { - "description": "" - } - } - } - }, - "on_failure": { - "description": "An optional list of system notification IDs to call when the run fails. A maximum of 3 destinations can be specified for the `on_failure` property.", - "items": { - "description": "", - "properties": { - "id": { - "description": "" - } - } - } - }, - "on_start": { - "description": "An optional list of system notification IDs to call when the run starts. A maximum of 3 destinations can be specified for the `on_start` property.", - "items": { - "description": "", - "properties": { - "id": { - "description": "" - } - } - } - }, - "on_streaming_backlog_exceeded": { - "description": "An optional list of system notification IDs to call when any streaming backlog thresholds are exceeded for any stream.\nStreaming backlog thresholds can be set in the `health` field using the following metrics: `STREAMING_BACKLOG_BYTES`, `STREAMING_BACKLOG_RECORDS`, `STREAMING_BACKLOG_SECONDS`, or `STREAMING_BACKLOG_FILES`.\nAlerting is based on the 10-minute average of these metrics. If the issue persists, notifications are resent every 30 minutes.\nA maximum of 3 destinations can be specified for the `on_streaming_backlog_exceeded` property.", - "items": { - "description": "", - "properties": { - "id": { - "description": "" - } - } - } - }, - "on_success": { - "description": "An optional list of system notification IDs to call when the run completes successfully. A maximum of 3 destinations can be specified for the `on_success` property.", - "items": { - "description": "", - "properties": { - "id": { - "description": "" - } - } - } - } - } - } - } - } - }, - "model_serving_endpoints": { - "description": "List of Model Serving Endpoints", - "additionalproperties": { - "description": "", - "properties": { - "config": { - "description": "The core config of the serving endpoint.", - "properties": { - "auto_capture_config": { - "description": "Configuration for Inference Tables which automatically logs requests and responses to Unity Catalog.", - "properties": { - "catalog_name": { - "description": "The name of the catalog in Unity Catalog. NOTE: On update, you cannot change the catalog name if the inference table is already enabled." - }, - "enabled": { - "description": "Indicates whether the inference table is enabled." - }, - "schema_name": { - "description": "The name of the schema in Unity Catalog. NOTE: On update, you cannot change the schema name if the inference table is already enabled." - }, - "table_name_prefix": { - "description": "The prefix of the table in Unity Catalog. NOTE: On update, you cannot change the prefix name if the inference table is already enabled." - } - } - }, - "served_entities": { - "description": "A list of served entities for the endpoint to serve. A serving endpoint can have up to 15 served entities.", - "items": { - "description": "", - "properties": { - "entity_name": { - "description": "The name of the entity to be served. The entity may be a model in the Databricks Model Registry, a model in the Unity Catalog (UC),\nor a function of type FEATURE_SPEC in the UC. If it is a UC object, the full name of the object should be given in the form of\n__catalog_name__.__schema_name__.__model_name__.\n" - }, - "entity_version": { - "description": "The version of the model in Databricks Model Registry to be served or empty if the entity is a FEATURE_SPEC." - }, - "environment_vars": { - "description": "An object containing a set of optional, user-specified environment variable key-value pairs used for serving this entity.\nNote: this is an experimental feature and subject to change. \nExample entity environment variables that refer to Databricks secrets: `{\"OPENAI_API_KEY\": \"{{secrets/my_scope/my_key}}\", \"DATABRICKS_TOKEN\": \"{{secrets/my_scope2/my_key2}}\"}`", - "additionalproperties": { - "description": "" - } - }, - "external_model": { - "description": "The external model to be served. NOTE: Only one of external_model and (entity_name, entity_version, workload_size, workload_type, and scale_to_zero_enabled)\ncan be specified with the latter set being used for custom model serving for a Databricks registered model. For an existing endpoint with external_model,\nit cannot be updated to an endpoint without external_model. If the endpoint is created without external_model, users cannot update it to add external_model later.\nThe task type of all external models within an endpoint must be the same.\n", - "properties": { - "ai21labs_config": { - "description": "AI21Labs Config. Only required if the provider is 'ai21labs'.", - "properties": { - "ai21labs_api_key": { - "description": "The Databricks secret key reference for an AI21 Labs API key. If you prefer to paste your API key directly, see `ai21labs_api_key_plaintext`. You must provide an API key using one of the following fields: `ai21labs_api_key` or `ai21labs_api_key_plaintext`." - }, - "ai21labs_api_key_plaintext": { - "description": "An AI21 Labs API key provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `ai21labs_api_key`. You must provide an API key using one of the following fields: `ai21labs_api_key` or `ai21labs_api_key_plaintext`." - } - } - }, - "amazon_bedrock_config": { - "description": "Amazon Bedrock Config. Only required if the provider is 'amazon-bedrock'.", - "properties": { - "aws_access_key_id": { - "description": "The Databricks secret key reference for an AWS access key ID with permissions to interact with Bedrock services. If you prefer to paste your API key directly, see `aws_access_key_id`. You must provide an API key using one of the following fields: `aws_access_key_id` or `aws_access_key_id_plaintext`." - }, - "aws_access_key_id_plaintext": { - "description": "An AWS access key ID with permissions to interact with Bedrock services provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `aws_access_key_id`. You must provide an API key using one of the following fields: `aws_access_key_id` or `aws_access_key_id_plaintext`." - }, - "aws_region": { - "description": "The AWS region to use. Bedrock has to be enabled there." - }, - "aws_secret_access_key": { - "description": "The Databricks secret key reference for an AWS secret access key paired with the access key ID, with permissions to interact with Bedrock services. If you prefer to paste your API key directly, see `aws_secret_access_key_plaintext`. You must provide an API key using one of the following fields: `aws_secret_access_key` or `aws_secret_access_key_plaintext`." - }, - "aws_secret_access_key_plaintext": { - "description": "An AWS secret access key paired with the access key ID, with permissions to interact with Bedrock services provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `aws_secret_access_key`. You must provide an API key using one of the following fields: `aws_secret_access_key` or `aws_secret_access_key_plaintext`." - }, - "bedrock_provider": { - "description": "The underlying provider in Amazon Bedrock. Supported values (case insensitive) include: Anthropic, Cohere, AI21Labs, Amazon." - } - } - }, - "anthropic_config": { - "description": "Anthropic Config. Only required if the provider is 'anthropic'.", - "properties": { - "anthropic_api_key": { - "description": "The Databricks secret key reference for an Anthropic API key. If you prefer to paste your API key directly, see `anthropic_api_key_plaintext`. You must provide an API key using one of the following fields: `anthropic_api_key` or `anthropic_api_key_plaintext`." - }, - "anthropic_api_key_plaintext": { - "description": "The Anthropic API key provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `anthropic_api_key`. You must provide an API key using one of the following fields: `anthropic_api_key` or `anthropic_api_key_plaintext`." - } - } - }, - "cohere_config": { - "description": "Cohere Config. Only required if the provider is 'cohere'.", - "properties": { - "cohere_api_base": { - "description": "This is an optional field to provide a customized base URL for the Cohere API. \nIf left unspecified, the standard Cohere base URL is used.\n" - }, - "cohere_api_key": { - "description": "The Databricks secret key reference for a Cohere API key. If you prefer to paste your API key directly, see `cohere_api_key_plaintext`. You must provide an API key using one of the following fields: `cohere_api_key` or `cohere_api_key_plaintext`." - }, - "cohere_api_key_plaintext": { - "description": "The Cohere API key provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `cohere_api_key`. You must provide an API key using one of the following fields: `cohere_api_key` or `cohere_api_key_plaintext`." - } - } - }, - "databricks_model_serving_config": { - "description": "Databricks Model Serving Config. Only required if the provider is 'databricks-model-serving'.", - "properties": { - "databricks_api_token": { - "description": "The Databricks secret key reference for a Databricks API token that corresponds to a user or service\nprincipal with Can Query access to the model serving endpoint pointed to by this external model.\nIf you prefer to paste your API key directly, see `databricks_api_token_plaintext`.\nYou must provide an API key using one of the following fields: `databricks_api_token` or `databricks_api_token_plaintext`.\n" - }, - "databricks_api_token_plaintext": { - "description": "The Databricks API token that corresponds to a user or service\nprincipal with Can Query access to the model serving endpoint pointed to by this external model provided as a plaintext string.\nIf you prefer to reference your key using Databricks Secrets, see `databricks_api_token`.\nYou must provide an API key using one of the following fields: `databricks_api_token` or `databricks_api_token_plaintext`.\n" - }, - "databricks_workspace_url": { - "description": "The URL of the Databricks workspace containing the model serving endpoint pointed to by this external model.\n" - } - } - }, - "google_cloud_vertex_ai_config": { - "description": "Google Cloud Vertex AI Config. Only required if the provider is 'google-cloud-vertex-ai'.", - "properties": { - "private_key": { - "description": "The Databricks secret key reference for a private key for the service account which has access to the Google Cloud Vertex AI Service. See [Best practices for managing service account keys](https://cloud.google.com/iam/docs/best-practices-for-managing-service-account-keys). If you prefer to paste your API key directly, see `private_key_plaintext`. You must provide an API key using one of the following fields: `private_key` or `private_key_plaintext`" - }, - "private_key_plaintext": { - "description": "The private key for the service account which has access to the Google Cloud Vertex AI Service provided as a plaintext secret. See [Best practices for managing service account keys](https://cloud.google.com/iam/docs/best-practices-for-managing-service-account-keys). If you prefer to reference your key using Databricks Secrets, see `private_key`. You must provide an API key using one of the following fields: `private_key` or `private_key_plaintext`." - }, - "project_id": { - "description": "This is the Google Cloud project id that the service account is associated with." - }, - "region": { - "description": "This is the region for the Google Cloud Vertex AI Service. See [supported regions](https://cloud.google.com/vertex-ai/docs/general/locations) for more details. Some models are only available in specific regions." - } - } - }, - "name": { - "description": "The name of the external model." - }, - "openai_config": { - "description": "OpenAI Config. Only required if the provider is 'openai'.", - "properties": { - "microsoft_entra_client_id": { - "description": "This field is only required for Azure AD OpenAI and is the Microsoft Entra Client ID.\n" - }, - "microsoft_entra_client_secret": { - "description": "The Databricks secret key reference for a client secret used for Microsoft Entra ID authentication.\nIf you prefer to paste your client secret directly, see `microsoft_entra_client_secret_plaintext`.\nYou must provide an API key using one of the following fields: `microsoft_entra_client_secret` or `microsoft_entra_client_secret_plaintext`.\n" - }, - "microsoft_entra_client_secret_plaintext": { - "description": "The client secret used for Microsoft Entra ID authentication provided as a plaintext string.\nIf you prefer to reference your key using Databricks Secrets, see `microsoft_entra_client_secret`.\nYou must provide an API key using one of the following fields: `microsoft_entra_client_secret` or `microsoft_entra_client_secret_plaintext`.\n" - }, - "microsoft_entra_tenant_id": { - "description": "This field is only required for Azure AD OpenAI and is the Microsoft Entra Tenant ID.\n" - }, - "openai_api_base": { - "description": "This is a field to provide a customized base URl for the OpenAI API.\nFor Azure OpenAI, this field is required, and is the base URL for the Azure OpenAI API service\nprovided by Azure.\nFor other OpenAI API types, this field is optional, and if left unspecified, the standard OpenAI base URL is used.\n" - }, - "openai_api_key": { - "description": "The Databricks secret key reference for an OpenAI API key using the OpenAI or Azure service. If you prefer to paste your API key directly, see `openai_api_key_plaintext`. You must provide an API key using one of the following fields: `openai_api_key` or `openai_api_key_plaintext`." - }, - "openai_api_key_plaintext": { - "description": "The OpenAI API key using the OpenAI or Azure service provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `openai_api_key`. You must provide an API key using one of the following fields: `openai_api_key` or `openai_api_key_plaintext`." - }, - "openai_api_type": { - "description": "This is an optional field to specify the type of OpenAI API to use.\nFor Azure OpenAI, this field is required, and adjust this parameter to represent the preferred security\naccess validation protocol. For access token validation, use azure. For authentication using Azure Active\nDirectory (Azure AD) use, azuread.\n" - }, - "openai_api_version": { - "description": "This is an optional field to specify the OpenAI API version.\nFor Azure OpenAI, this field is required, and is the version of the Azure OpenAI service to\nutilize, specified by a date.\n" - }, - "openai_deployment_name": { - "description": "This field is only required for Azure OpenAI and is the name of the deployment resource for the\nAzure OpenAI service.\n" - }, - "openai_organization": { - "description": "This is an optional field to specify the organization in OpenAI or Azure OpenAI.\n" - } - } - }, - "palm_config": { - "description": "PaLM Config. Only required if the provider is 'palm'.", - "properties": { - "palm_api_key": { - "description": "The Databricks secret key reference for a PaLM API key. If you prefer to paste your API key directly, see `palm_api_key_plaintext`. You must provide an API key using one of the following fields: `palm_api_key` or `palm_api_key_plaintext`." - }, - "palm_api_key_plaintext": { - "description": "The PaLM API key provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `palm_api_key`. You must provide an API key using one of the following fields: `palm_api_key` or `palm_api_key_plaintext`." - } - } - }, - "provider": { - "description": "The name of the provider for the external model. Currently, the supported providers are 'ai21labs', 'anthropic',\n'amazon-bedrock', 'cohere', 'databricks-model-serving', 'google-cloud-vertex-ai', 'openai', and 'palm'.\",\n" - }, - "task": { - "description": "The task type of the external model." - } - } - }, - "instance_profile_arn": { - "description": "ARN of the instance profile that the served entity uses to access AWS resources." - }, - "max_provisioned_throughput": { - "description": "The maximum tokens per second that the endpoint can scale up to." - }, - "min_provisioned_throughput": { - "description": "The minimum tokens per second that the endpoint can scale down to." - }, - "name": { - "description": "The name of a served entity. It must be unique across an endpoint. A served entity name can consist of alphanumeric characters, dashes, and underscores.\nIf not specified for an external model, this field defaults to external_model.name, with '.' and ':' replaced with '-', and if not specified for other\nentities, it defaults to \u003centity-name\u003e-\u003centity-version\u003e.\n" - }, - "scale_to_zero_enabled": { - "description": "Whether the compute resources for the served entity should scale down to zero." - }, - "workload_size": { - "description": "The workload size of the served entity. The workload size corresponds to a range of provisioned concurrency that the compute autoscales between.\nA single unit of provisioned concurrency can process one request at a time.\nValid workload sizes are \"Small\" (4 - 4 provisioned concurrency), \"Medium\" (8 - 16 provisioned concurrency), and \"Large\" (16 - 64 provisioned concurrency).\nIf scale-to-zero is enabled, the lower bound of the provisioned concurrency for each workload size is 0.\n" - }, - "workload_type": { - "description": "The workload type of the served entity. The workload type selects which type of compute to use in the endpoint. The default value for this parameter is\n\"CPU\". For deep learning workloads, GPU acceleration is available by selecting workload types like GPU_SMALL and others.\nSee the available [GPU types](https://docs.databricks.com/machine-learning/model-serving/create-manage-serving-endpoints.html#gpu-workload-types).\n" - } - } - } - }, - "served_models": { - "description": "(Deprecated, use served_entities instead) A list of served models for the endpoint to serve. A serving endpoint can have up to 15 served models.", - "items": { - "description": "", - "properties": { - "environment_vars": { - "description": "An object containing a set of optional, user-specified environment variable key-value pairs used for serving this model.\nNote: this is an experimental feature and subject to change. \nExample model environment variables that refer to Databricks secrets: `{\"OPENAI_API_KEY\": \"{{secrets/my_scope/my_key}}\", \"DATABRICKS_TOKEN\": \"{{secrets/my_scope2/my_key2}}\"}`", - "additionalproperties": { - "description": "" - } - }, - "instance_profile_arn": { - "description": "ARN of the instance profile that the served model will use to access AWS resources." - }, - "model_name": { - "description": "The name of the model in Databricks Model Registry to be served or if the model resides in Unity Catalog, the full name of model,\nin the form of __catalog_name__.__schema_name__.__model_name__.\n" - }, - "model_version": { - "description": "The version of the model in Databricks Model Registry or Unity Catalog to be served." - }, - "name": { - "description": "The name of a served model. It must be unique across an endpoint. If not specified, this field will default to \u003cmodel-name\u003e-\u003cmodel-version\u003e.\nA served model name can consist of alphanumeric characters, dashes, and underscores.\n" - }, - "scale_to_zero_enabled": { - "description": "Whether the compute resources for the served model should scale down to zero." - }, - "workload_size": { - "description": "The workload size of the served model. The workload size corresponds to a range of provisioned concurrency that the compute will autoscale between.\nA single unit of provisioned concurrency can process one request at a time.\nValid workload sizes are \"Small\" (4 - 4 provisioned concurrency), \"Medium\" (8 - 16 provisioned concurrency), and \"Large\" (16 - 64 provisioned concurrency).\nIf scale-to-zero is enabled, the lower bound of the provisioned concurrency for each workload size will be 0.\n" - }, - "workload_type": { - "description": "The workload type of the served model. The workload type selects which type of compute to use in the endpoint. The default value for this parameter is\n\"CPU\". For deep learning workloads, GPU acceleration is available by selecting workload types like GPU_SMALL and others.\nSee the available [GPU types](https://docs.databricks.com/machine-learning/model-serving/create-manage-serving-endpoints.html#gpu-workload-types).\n" - } - } - } - }, - "traffic_config": { - "description": "The traffic config defining how invocations to the serving endpoint should be routed.", - "properties": { - "routes": { - "description": "The list of routes that define traffic to each served entity.", - "items": { - "description": "", - "properties": { - "served_model_name": { - "description": "The name of the served model this route configures traffic for." - }, - "traffic_percentage": { - "description": "The percentage of endpoint traffic to send to this route. It must be an integer between 0 and 100 inclusive." - } - } - } - } - } - } - } - }, - "name": { - "description": "The name of the serving endpoint. This field is required and must be unique across a Databricks workspace.\nAn endpoint name can consist of alphanumeric characters, dashes, and underscores.\n" - }, - "permissions": { - "description": "", - "items": { - "description": "", - "properties": { - "group_name": { - "description": "" - }, - "level": { - "description": "" - }, - "service_principal_name": { - "description": "" - }, - "user_name": { - "description": "" - } - } - } - }, - "rate_limits": { - "description": "Rate limits to be applied to the serving endpoint. NOTE: only external and foundation model endpoints are supported as of now.", - "items": { - "description": "", - "properties": { - "calls": { - "description": "Used to specify how many calls are allowed for a key within the renewal_period." - }, - "key": { - "description": "Key field for a serving endpoint rate limit. Currently, only 'user' and 'endpoint' are supported, with 'endpoint' being the default if not specified." - }, - "renewal_period": { - "description": "Renewal period field for a serving endpoint rate limit. Currently, only 'minute' is supported." - } - } - } - }, - "route_optimized": { - "description": "Enable route optimization for the serving endpoint." - }, - "tags": { - "description": "Tags to be attached to the serving endpoint and automatically propagated to billing logs.", - "items": { - "description": "", - "properties": { - "key": { - "description": "Key field for a serving endpoint tag." - }, - "value": { - "description": "Optional value field for a serving endpoint tag." - } - } - } - } - } - } - }, - "models": { - "description": "List of MLflow models", - "additionalproperties": { - "description": "", - "properties": { - "creation_timestamp": { - "description": "Timestamp recorded when this `registered_model` was created." - }, - "description": { - "description": "Description of this `registered_model`." - }, - "last_updated_timestamp": { - "description": "Timestamp recorded when metadata for this `registered_model` was last updated." - }, - "latest_versions": { - "description": "Collection of latest model versions for each stage.\nOnly contains models with current `READY` status.", - "items": { - "description": "", - "properties": { - "creation_timestamp": { - "description": "Timestamp recorded when this `model_version` was created." - }, - "current_stage": { - "description": "Current stage for this `model_version`." - }, - "description": { - "description": "Description of this `model_version`." - }, - "last_updated_timestamp": { - "description": "Timestamp recorded when metadata for this `model_version` was last updated." - }, - "name": { - "description": "Unique name of the model" - }, - "run_id": { - "description": "MLflow run ID used when creating `model_version`, if `source` was generated by an\nexperiment run stored in MLflow tracking server." - }, - "run_link": { - "description": "Run Link: Direct link to the run that generated this version" - }, - "source": { - "description": "URI indicating the location of the source model artifacts, used when creating `model_version`" - }, - "status": { - "description": "Current status of `model_version`" - }, - "status_message": { - "description": "Details on current `status`, if it is pending or failed." - }, - "tags": { - "description": "Tags: Additional metadata key-value pairs for this `model_version`.", - "items": { - "description": "", - "properties": { - "key": { - "description": "The tag key." - }, - "value": { - "description": "The tag value." - } - } - } - }, - "user_id": { - "description": "User that created this `model_version`." - }, - "version": { - "description": "Model's version number." - } - } - } - }, - "name": { - "description": "Unique name for the model." - }, - "permissions": { - "description": "", - "items": { - "description": "", - "properties": { - "group_name": { - "description": "" - }, - "level": { - "description": "" - }, - "service_principal_name": { - "description": "" - }, - "user_name": { - "description": "" - } - } - } - }, - "tags": { - "description": "Tags: Additional metadata key-value pairs for this `registered_model`.", - "items": { - "description": "", - "properties": { - "key": { - "description": "The tag key." - }, - "value": { - "description": "The tag value." - } - } - } - }, - "user_id": { - "description": "User that created this `registered_model`" - } - } - } - }, - "pipelines": { - "description": "List of DLT pipelines", - "additionalproperties": { - "description": "", - "properties": { - "catalog": { - "description": "A catalog in Unity Catalog to publish data from this pipeline to. If `target` is specified, tables in this pipeline are published to a `target` schema inside `catalog` (for example, `catalog`.`target`.`table`). If `target` is not specified, no data is published to Unity Catalog." - }, - "channel": { - "description": "DLT Release Channel that specifies which version to use." - }, - "clusters": { - "description": "Cluster settings for this pipeline deployment.", - "items": { - "description": "", - "properties": { - "apply_policy_default_values": { - "description": "Note: This field won't be persisted. Only API users will check this field." - }, - "autoscale": { - "description": "Parameters needed in order to automatically scale clusters up and down based on load.\nNote: autoscaling works best with DB runtime versions 3.0 or later.", - "properties": { - "max_workers": { - "description": "The maximum number of workers to which the cluster can scale up when overloaded. `max_workers` must be strictly greater than `min_workers`." - }, - "min_workers": { - "description": "The minimum number of workers the cluster can scale down to when underutilized.\nIt is also the initial number of workers the cluster will have after creation." - }, - "mode": { - "description": "Databricks Enhanced Autoscaling optimizes cluster utilization by automatically\nallocating cluster resources based on workload volume, with minimal impact to\nthe data processing latency of your pipelines. Enhanced Autoscaling is available\nfor `updates` clusters only. The legacy autoscaling feature is used for `maintenance`\nclusters.\n" - } - } - }, - "aws_attributes": { - "description": "Attributes related to clusters running on Amazon Web Services.\nIf not specified at cluster creation, a set of default values will be used.", - "properties": { - "availability": { - "description": "" - }, - "ebs_volume_count": { - "description": "The number of volumes launched for each instance. Users can choose up to 10 volumes.\nThis feature is only enabled for supported node types. Legacy node types cannot specify\ncustom EBS volumes.\nFor node types with no instance store, at least one EBS volume needs to be specified;\notherwise, cluster creation will fail.\n\nThese EBS volumes will be mounted at `/ebs0`, `/ebs1`, and etc.\nInstance store volumes will be mounted at `/local_disk0`, `/local_disk1`, and etc.\n\nIf EBS volumes are attached, Databricks will configure Spark to use only the EBS volumes for\nscratch storage because heterogenously sized scratch devices can lead to inefficient disk\nutilization. If no EBS volumes are attached, Databricks will configure Spark to use instance\nstore volumes.\n\nPlease note that if EBS volumes are specified, then the Spark configuration `spark.local.dir`\nwill be overridden." - }, - "ebs_volume_iops": { - "description": "If using gp3 volumes, what IOPS to use for the disk. If this is not set, the maximum performance of a gp2 volume with the same volume size will be used." - }, - "ebs_volume_size": { - "description": "The size of each EBS volume (in GiB) launched for each instance. For general purpose\nSSD, this value must be within the range 100 - 4096. For throughput optimized HDD,\nthis value must be within the range 500 - 4096." - }, - "ebs_volume_throughput": { - "description": "If using gp3 volumes, what throughput to use for the disk. If this is not set, the maximum performance of a gp2 volume with the same volume size will be used." - }, - "ebs_volume_type": { - "description": "" - }, - "first_on_demand": { - "description": "The first `first_on_demand` nodes of the cluster will be placed on on-demand instances.\nIf this value is greater than 0, the cluster driver node in particular will be placed on an\non-demand instance. If this value is greater than or equal to the current cluster size, all\nnodes will be placed on on-demand instances. If this value is less than the current cluster\nsize, `first_on_demand` nodes will be placed on on-demand instances and the remainder will\nbe placed on `availability` instances. Note that this value does not affect\ncluster size and cannot currently be mutated over the lifetime of a cluster." - }, - "instance_profile_arn": { - "description": "Nodes for this cluster will only be placed on AWS instances with this instance profile. If\nommitted, nodes will be placed on instances without an IAM instance profile. The instance\nprofile must have previously been added to the Databricks environment by an account\nadministrator.\n\nThis feature may only be available to certain customer plans.\n\nIf this field is ommitted, we will pull in the default from the conf if it exists." - }, - "spot_bid_price_percent": { - "description": "The bid price for AWS spot instances, as a percentage of the corresponding instance type's\non-demand price.\nFor example, if this field is set to 50, and the cluster needs a new `r3.xlarge` spot\ninstance, then the bid price is half of the price of\non-demand `r3.xlarge` instances. Similarly, if this field is set to 200, the bid price is twice\nthe price of on-demand `r3.xlarge` instances. If not specified, the default value is 100.\nWhen spot instances are requested for this cluster, only spot instances whose bid price\npercentage matches this field will be considered.\nNote that, for safety, we enforce this field to be no more than 10000.\n\nThe default value and documentation here should be kept consistent with\nCommonConf.defaultSpotBidPricePercent and CommonConf.maxSpotBidPricePercent." - }, - "zone_id": { - "description": "Identifier for the availability zone/datacenter in which the cluster resides.\nThis string will be of a form like \"us-west-2a\". The provided availability\nzone must be in the same region as the Databricks deployment. For example, \"us-west-2a\"\nis not a valid zone id if the Databricks deployment resides in the \"us-east-1\" region.\nThis is an optional field at cluster creation, and if not specified, a default zone will be used.\nIf the zone specified is \"auto\", will try to place cluster in a zone with high availability,\nand will retry placement in a different AZ if there is not enough capacity.\nThe list of available zones as well as the default value can be found by using the\n`List Zones` method." - } - } - }, - "azure_attributes": { - "description": "Attributes related to clusters running on Microsoft Azure.\nIf not specified at cluster creation, a set of default values will be used.", - "properties": { - "availability": { - "description": "" - }, - "first_on_demand": { - "description": "The first `first_on_demand` nodes of the cluster will be placed on on-demand instances.\nThis value should be greater than 0, to make sure the cluster driver node is placed on an\non-demand instance. If this value is greater than or equal to the current cluster size, all\nnodes will be placed on on-demand instances. If this value is less than the current cluster\nsize, `first_on_demand` nodes will be placed on on-demand instances and the remainder will\nbe placed on `availability` instances. Note that this value does not affect\ncluster size and cannot currently be mutated over the lifetime of a cluster." - }, - "log_analytics_info": { - "description": "Defines values necessary to configure and run Azure Log Analytics agent", - "properties": { - "log_analytics_primary_key": { - "description": "\u003cneeds content added\u003e" - }, - "log_analytics_workspace_id": { - "description": "\u003cneeds content added\u003e" - } - } - }, - "spot_bid_max_price": { - "description": "The max bid price to be used for Azure spot instances.\nThe Max price for the bid cannot be higher than the on-demand price of the instance.\nIf not specified, the default value is -1, which specifies that the instance cannot be evicted\non the basis of price, and only on the basis of availability. Further, the value should \u003e 0 or -1." - } - } - }, - "cluster_log_conf": { - "description": "The configuration for delivering spark logs to a long-term storage destination.\nOnly dbfs destinations are supported. Only one destination can be specified\nfor one cluster. If the conf is given, the logs will be delivered to the destination every\n`5 mins`. The destination of driver logs is `$destination/$clusterId/driver`, while\nthe destination of executor logs is `$destination/$clusterId/executor`.\n", - "properties": { - "dbfs": { - "description": "destination needs to be provided. e.g.\n`{ \"dbfs\" : { \"destination\" : \"dbfs:/home/cluster_log\" } }`", - "properties": { - "destination": { - "description": "dbfs destination, e.g. `dbfs:/my/path`" - } - } - }, - "s3": { - "description": "destination and either the region or endpoint need to be provided. e.g.\n`{ \"s3\": { \"destination\" : \"s3://cluster_log_bucket/prefix\", \"region\" : \"us-west-2\" } }`\nCluster iam role is used to access s3, please make sure the cluster iam role in\n`instance_profile_arn` has permission to write data to the s3 destination.", - "properties": { - "canned_acl": { - "description": "(Optional) Set canned access control list for the logs, e.g. `bucket-owner-full-control`.\nIf `canned_cal` is set, please make sure the cluster iam role has `s3:PutObjectAcl` permission on\nthe destination bucket and prefix. The full list of possible canned acl can be found at\nhttp://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl.\nPlease also note that by default only the object owner gets full controls. If you are using cross account\nrole for writing data, you may want to set `bucket-owner-full-control` to make bucket owner able to\nread the logs." - }, - "destination": { - "description": "S3 destination, e.g. `s3://my-bucket/some-prefix` Note that logs will be delivered using\ncluster iam role, please make sure you set cluster iam role and the role has write access to the\ndestination. Please also note that you cannot use AWS keys to deliver logs." - }, - "enable_encryption": { - "description": "(Optional) Flag to enable server side encryption, `false` by default." - }, - "encryption_type": { - "description": "(Optional) The encryption type, it could be `sse-s3` or `sse-kms`. It will be used only when\nencryption is enabled and the default type is `sse-s3`." - }, - "endpoint": { - "description": "S3 endpoint, e.g. `https://s3-us-west-2.amazonaws.com`. Either region or endpoint needs to be set.\nIf both are set, endpoint will be used." - }, - "kms_key": { - "description": "(Optional) Kms key which will be used if encryption is enabled and encryption type is set to `sse-kms`." - }, - "region": { - "description": "S3 region, e.g. `us-west-2`. Either region or endpoint needs to be set. If both are set,\nendpoint will be used." - } - } - } - } - }, - "custom_tags": { - "description": "Additional tags for cluster resources. Databricks will tag all cluster resources (e.g., AWS\ninstances and EBS volumes) with these tags in addition to `default_tags`. Notes:\n\n- Currently, Databricks allows at most 45 custom tags\n\n- Clusters can only reuse cloud resources if the resources' tags are a subset of the cluster tags", - "additionalproperties": { - "description": "" - } - }, - "driver_instance_pool_id": { - "description": "The optional ID of the instance pool for the driver of the cluster belongs.\nThe pool cluster uses the instance pool with id (instance_pool_id) if the driver pool is not\nassigned." - }, - "driver_node_type_id": { - "description": "The node type of the Spark driver.\nNote that this field is optional; if unset, the driver node type will be set as the same value\nas `node_type_id` defined above." - }, - "enable_local_disk_encryption": { - "description": "Whether to enable local disk encryption for the cluster." - }, - "gcp_attributes": { - "description": "Attributes related to clusters running on Google Cloud Platform.\nIf not specified at cluster creation, a set of default values will be used.", - "properties": { - "availability": { - "description": "" - }, - "boot_disk_size": { - "description": "boot disk size in GB" - }, - "google_service_account": { - "description": "If provided, the cluster will impersonate the google service account when accessing\ngcloud services (like GCS). The google service account\nmust have previously been added to the Databricks environment by an account\nadministrator." - }, - "local_ssd_count": { - "description": "If provided, each node (workers and driver) in the cluster will have this number of local SSDs attached. Each local SSD is 375GB in size. Refer to [GCP documentation](https://cloud.google.com/compute/docs/disks/local-ssd#choose_number_local_ssds) for the supported number of local SSDs for each instance type." - }, - "use_preemptible_executors": { - "description": "This field determines whether the spark executors will be scheduled to run on preemptible VMs (when set to true) versus standard compute engine VMs (when set to false; default).\nNote: Soon to be deprecated, use the availability field instead." - }, - "zone_id": { - "description": "Identifier for the availability zone in which the cluster resides.\nThis can be one of the following:\n- \"HA\" =\u003e High availability, spread nodes across availability zones for a Databricks deployment region [default]\n- \"AUTO\" =\u003e Databricks picks an availability zone to schedule the cluster on.\n- A GCP availability zone =\u003e Pick One of the available zones for (machine type + region) from https://cloud.google.com/compute/docs/regions-zones." - } - } - }, - "init_scripts": { - "description": "The configuration for storing init scripts. Any number of destinations can be specified. The scripts are executed sequentially in the order provided. If `cluster_log_conf` is specified, init script logs are sent to `\u003cdestination\u003e/\u003ccluster-ID\u003e/init_scripts`.", - "items": { - "description": "", - "properties": { - "abfss": { - "description": "destination needs to be provided. e.g.\n`{ \"abfss\" : { \"destination\" : \"abfss://\u003ccontainer-name\u003e@\u003cstorage-account-name\u003e.dfs.core.windows.net/\u003cdirectory-name\u003e\" } }", - "properties": { - "destination": { - "description": "abfss destination, e.g. `abfss://\u003ccontainer-name\u003e@\u003cstorage-account-name\u003e.dfs.core.windows.net/\u003cdirectory-name\u003e`." - } - } - }, - "dbfs": { - "description": "destination needs to be provided. e.g.\n`{ \"dbfs\" : { \"destination\" : \"dbfs:/home/cluster_log\" } }`", - "properties": { - "destination": { - "description": "dbfs destination, e.g. `dbfs:/my/path`" - } - } - }, - "file": { - "description": "destination needs to be provided. e.g.\n`{ \"file\" : { \"destination\" : \"file:/my/local/file.sh\" } }`", - "properties": { - "destination": { - "description": "local file destination, e.g. `file:/my/local/file.sh`" - } - } - }, - "gcs": { - "description": "destination needs to be provided. e.g.\n`{ \"gcs\": { \"destination\": \"gs://my-bucket/file.sh\" } }`", - "properties": { - "destination": { - "description": "GCS destination/URI, e.g. `gs://my-bucket/some-prefix`" - } - } - }, - "s3": { - "description": "destination and either the region or endpoint need to be provided. e.g.\n`{ \"s3\": { \"destination\" : \"s3://cluster_log_bucket/prefix\", \"region\" : \"us-west-2\" } }`\nCluster iam role is used to access s3, please make sure the cluster iam role in\n`instance_profile_arn` has permission to write data to the s3 destination.", - "properties": { - "canned_acl": { - "description": "(Optional) Set canned access control list for the logs, e.g. `bucket-owner-full-control`.\nIf `canned_cal` is set, please make sure the cluster iam role has `s3:PutObjectAcl` permission on\nthe destination bucket and prefix. The full list of possible canned acl can be found at\nhttp://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl.\nPlease also note that by default only the object owner gets full controls. If you are using cross account\nrole for writing data, you may want to set `bucket-owner-full-control` to make bucket owner able to\nread the logs." - }, - "destination": { - "description": "S3 destination, e.g. `s3://my-bucket/some-prefix` Note that logs will be delivered using\ncluster iam role, please make sure you set cluster iam role and the role has write access to the\ndestination. Please also note that you cannot use AWS keys to deliver logs." - }, - "enable_encryption": { - "description": "(Optional) Flag to enable server side encryption, `false` by default." - }, - "encryption_type": { - "description": "(Optional) The encryption type, it could be `sse-s3` or `sse-kms`. It will be used only when\nencryption is enabled and the default type is `sse-s3`." - }, - "endpoint": { - "description": "S3 endpoint, e.g. `https://s3-us-west-2.amazonaws.com`. Either region or endpoint needs to be set.\nIf both are set, endpoint will be used." - }, - "kms_key": { - "description": "(Optional) Kms key which will be used if encryption is enabled and encryption type is set to `sse-kms`." - }, - "region": { - "description": "S3 region, e.g. `us-west-2`. Either region or endpoint needs to be set. If both are set,\nendpoint will be used." - } - } - }, - "volumes": { - "description": "destination needs to be provided. e.g.\n`{ \"volumes\" : { \"destination\" : \"/Volumes/my-init.sh\" } }`", - "properties": { - "destination": { - "description": "Unity Catalog Volumes file destination, e.g. `/Volumes/my-init.sh`" - } - } - }, - "workspace": { - "description": "destination needs to be provided. e.g.\n`{ \"workspace\" : { \"destination\" : \"/Users/user1@databricks.com/my-init.sh\" } }`", - "properties": { - "destination": { - "description": "workspace files destination, e.g. `/Users/user1@databricks.com/my-init.sh`" - } - } - } - } - } - }, - "instance_pool_id": { - "description": "The optional ID of the instance pool to which the cluster belongs." - }, - "label": { - "description": "A label for the cluster specification, either `default` to configure the default cluster, or `maintenance` to configure the maintenance cluster. This field is optional. The default value is `default`." - }, - "node_type_id": { - "description": "This field encodes, through a single value, the resources available to each of\nthe Spark nodes in this cluster. For example, the Spark nodes can be provisioned\nand optimized for memory or compute intensive workloads. A list of available node\ntypes can be retrieved by using the :method:clusters/listNodeTypes API call.\n" - }, - "num_workers": { - "description": "Number of worker nodes that this cluster should have. A cluster has one Spark Driver\nand `num_workers` Executors for a total of `num_workers` + 1 Spark nodes.\n\nNote: When reading the properties of a cluster, this field reflects the desired number\nof workers rather than the actual current number of workers. For instance, if a cluster\nis resized from 5 to 10 workers, this field will immediately be updated to reflect\nthe target size of 10 workers, whereas the workers listed in `spark_info` will gradually\nincrease from 5 to 10 as the new nodes are provisioned." - }, - "policy_id": { - "description": "The ID of the cluster policy used to create the cluster if applicable." - }, - "spark_conf": { - "description": "An object containing a set of optional, user-specified Spark configuration key-value pairs.\nSee :method:clusters/create for more details.\n", - "additionalproperties": { - "description": "" - } - }, - "spark_env_vars": { - "description": "An object containing a set of optional, user-specified environment variable key-value pairs.\nPlease note that key-value pair of the form (X,Y) will be exported as is (i.e.,\n`export X='Y'`) while launching the driver and workers.\n\nIn order to specify an additional set of `SPARK_DAEMON_JAVA_OPTS`, we recommend appending\nthem to `$SPARK_DAEMON_JAVA_OPTS` as shown in the example below. This ensures that all\ndefault databricks managed environmental variables are included as well.\n\nExample Spark environment variables:\n`{\"SPARK_WORKER_MEMORY\": \"28000m\", \"SPARK_LOCAL_DIRS\": \"/local_disk0\"}` or\n`{\"SPARK_DAEMON_JAVA_OPTS\": \"$SPARK_DAEMON_JAVA_OPTS -Dspark.shuffle.service.enabled=true\"}`", - "additionalproperties": { - "description": "" - } - }, - "ssh_public_keys": { - "description": "SSH public key contents that will be added to each Spark node in this cluster. The\ncorresponding private keys can be used to login with the user name `ubuntu` on port `2200`.\nUp to 10 keys can be specified.", - "items": { - "description": "" - } - } - } - } - }, - "configuration": { - "description": "String-String configuration for this pipeline execution.", - "additionalproperties": { - "description": "" - } - }, - "continuous": { - "description": "Whether the pipeline is continuous or triggered. This replaces `trigger`." - }, - "deployment": { - "description": "Deployment type of this pipeline.", - "properties": { - "kind": { - "description": "The deployment method that manages the pipeline." - }, - "metadata_file_path": { - "description": "The path to the file containing metadata about the deployment." - } - } - }, - "development": { - "description": "Whether the pipeline is in Development mode. Defaults to false." - }, - "edition": { - "description": "Pipeline product edition." - }, - "filters": { - "description": "Filters on which Pipeline packages to include in the deployed graph.", - "properties": { - "exclude": { - "description": "Paths to exclude.", - "items": { - "description": "" - } - }, - "include": { - "description": "Paths to include.", - "items": { - "description": "" - } - } - } - }, - "gateway_definition": { - "description": "The definition of a gateway pipeline to support CDC.", - "properties": { - "connection_id": { - "description": "Immutable. The Unity Catalog connection this gateway pipeline uses to communicate with the source." - }, - "gateway_storage_catalog": { - "description": "Required, Immutable. The name of the catalog for the gateway pipeline's storage location." - }, - "gateway_storage_name": { - "description": "Optional. The Unity Catalog-compatible name for the gateway storage location.\nThis is the destination to use for the data that is extracted by the gateway.\nDelta Live Tables system will automatically create the storage location under the catalog and schema.\n" - }, - "gateway_storage_schema": { - "description": "Required, Immutable. The name of the schema for the gateway pipelines's storage location." - } - } - }, - "id": { - "description": "Unique identifier for this pipeline." - }, - "ingestion_definition": { - "description": "The configuration for a managed ingestion pipeline. These settings cannot be used with the 'libraries', 'target' or 'catalog' settings.", - "properties": { - "connection_name": { - "description": "Immutable. The Unity Catalog connection this ingestion pipeline uses to communicate with the source. Specify either ingestion_gateway_id or connection_name." - }, - "ingestion_gateway_id": { - "description": "Immutable. Identifier for the ingestion gateway used by this ingestion pipeline to communicate with the source. Specify either ingestion_gateway_id or connection_name." - }, - "objects": { - "description": "Required. Settings specifying tables to replicate and the destination for the replicated tables.", - "items": { - "description": "", - "properties": { - "schema": { - "description": "Select tables from a specific source schema.", - "properties": { - "destination_catalog": { - "description": "Required. Destination catalog to store tables." - }, - "destination_schema": { - "description": "Required. Destination schema to store tables in. Tables with the same name as the source tables are created in this destination schema. The pipeline fails If a table with the same name already exists." - }, - "source_catalog": { - "description": "The source catalog name. Might be optional depending on the type of source." - }, - "source_schema": { - "description": "Required. Schema name in the source database." - }, - "table_configuration": { - "description": "Configuration settings to control the ingestion of tables. These settings are applied to all tables in this schema and override the table_configuration defined in the IngestionPipelineDefinition object.", - "properties": { - "primary_keys": { - "description": "The primary key of the table used to apply changes.", - "items": { - "description": "" - } - }, - "salesforce_include_formula_fields": { - "description": "If true, formula fields defined in the table are included in the ingestion. This setting is only valid for the Salesforce connector" - }, - "scd_type": { - "description": "The SCD type to use to ingest the table." - } - } - } - } - }, - "table": { - "description": "Select tables from a specific source table.", - "properties": { - "destination_catalog": { - "description": "Required. Destination catalog to store table." - }, - "destination_schema": { - "description": "Required. Destination schema to store table." - }, - "destination_table": { - "description": "Optional. Destination table name. The pipeline fails If a table with that name already exists. If not set, the source table name is used." - }, - "source_catalog": { - "description": "Source catalog name. Might be optional depending on the type of source." - }, - "source_schema": { - "description": "Schema name in the source database. Might be optional depending on the type of source." - }, - "source_table": { - "description": "Required. Table name in the source database." - }, - "table_configuration": { - "description": "Configuration settings to control the ingestion of tables. These settings override the table_configuration defined in the IngestionPipelineDefinition object and the SchemaSpec.", - "properties": { - "primary_keys": { - "description": "The primary key of the table used to apply changes.", - "items": { - "description": "" - } - }, - "salesforce_include_formula_fields": { - "description": "If true, formula fields defined in the table are included in the ingestion. This setting is only valid for the Salesforce connector" - }, - "scd_type": { - "description": "The SCD type to use to ingest the table." - } - } - } - } - } - } - } - }, - "table_configuration": { - "description": "Configuration settings to control the ingestion of tables. These settings are applied to all tables in the pipeline.", - "properties": { - "primary_keys": { - "description": "The primary key of the table used to apply changes.", - "items": { - "description": "" - } - }, - "salesforce_include_formula_fields": { - "description": "If true, formula fields defined in the table are included in the ingestion. This setting is only valid for the Salesforce connector" - }, - "scd_type": { - "description": "The SCD type to use to ingest the table." - } - } - } - } - }, - "libraries": { - "description": "Libraries or code needed by this deployment.", - "items": { - "description": "", - "properties": { - "file": { - "description": "The path to a file that defines a pipeline and is stored in the Databricks Repos.\n", - "properties": { - "path": { - "description": "The absolute path of the file." - } - } - }, - "jar": { - "description": "URI of the jar to be installed. Currently only DBFS is supported.\n" - }, - "maven": { - "description": "Specification of a maven library to be installed.\n", - "properties": { - "coordinates": { - "description": "Gradle-style maven coordinates. For example: \"org.jsoup:jsoup:1.7.2\"." - }, - "exclusions": { - "description": "List of dependences to exclude. For example: `[\"slf4j:slf4j\", \"*:hadoop-client\"]`.\n\nMaven dependency exclusions:\nhttps://maven.apache.org/guides/introduction/introduction-to-optional-and-excludes-dependencies.html.", - "items": { - "description": "" - } - }, - "repo": { - "description": "Maven repo to install the Maven package from. If omitted, both Maven Central Repository\nand Spark Packages are searched." - } - } - }, - "notebook": { - "description": "The path to a notebook that defines a pipeline and is stored in the Databricks workspace.\n", - "properties": { - "path": { - "description": "The absolute path of the notebook." - } - } - }, - "whl": { - "description": "URI of the whl to be installed." - } - } - } - }, - "name": { - "description": "Friendly identifier for this pipeline." - }, - "notifications": { - "description": "List of notification settings for this pipeline.", - "items": { - "description": "", - "properties": { - "alerts": { - "description": "A list of alerts that trigger the sending of notifications to the configured\ndestinations. The supported alerts are:\n\n* `on-update-success`: A pipeline update completes successfully.\n* `on-update-failure`: Each time a pipeline update fails.\n* `on-update-fatal-failure`: A pipeline update fails with a non-retryable (fatal) error.\n* `on-flow-failure`: A single data flow fails.\n", - "items": { - "description": "" - } - }, - "email_recipients": { - "description": "A list of email addresses notified when a configured alert is triggered.\n", - "items": { - "description": "" - } - } - } - } - }, - "permissions": { - "description": "", - "items": { - "description": "", - "properties": { - "group_name": { - "description": "" - }, - "level": { - "description": "" - }, - "service_principal_name": { - "description": "" - }, - "user_name": { - "description": "" - } - } - } - }, - "photon": { - "description": "Whether Photon is enabled for this pipeline." - }, - "serverless": { - "description": "Whether serverless compute is enabled for this pipeline." - }, - "storage": { - "description": "DBFS root directory for storing checkpoints and tables." - }, - "target": { - "description": "Target schema (database) to add tables in this pipeline to. If not specified, no data is published to the Hive metastore or Unity Catalog. To publish to Unity Catalog, also specify `catalog`." - }, - "trigger": { - "description": "Which pipeline trigger to use. Deprecated: Use `continuous` instead.", - "properties": { - "cron": { - "description": "", - "properties": { - "quartz_cron_schedule": { - "description": "" - }, - "timezone_id": { - "description": "" - } - } - }, - "manual": { - "description": "" - } - } - } - } - } - }, - "quality_monitors": { - "description": "", - "additionalproperties": { - "description": "", - "properties": { - "assets_dir": { - "description": "" - }, - "baseline_table_name": { - "description": "" - }, - "custom_metrics": { - "description": "", - "items": { - "description": "", - "properties": { - "definition": { - "description": "" - }, - "input_columns": { - "description": "", - "items": { - "description": "" - } - }, - "name": { - "description": "" - }, - "output_data_type": { - "description": "" - }, - "type": { - "description": "" - } - } - } - }, - "data_classification_config": { - "description": "", - "properties": { - "enabled": { - "description": "" - } - } - }, - "inference_log": { - "description": "", - "properties": { - "granularities": { - "description": "", - "items": { - "description": "" - } - }, - "label_col": { - "description": "" - }, - "model_id_col": { - "description": "" - }, - "prediction_col": { - "description": "" - }, - "prediction_proba_col": { - "description": "" - }, - "problem_type": { - "description": "" - }, - "timestamp_col": { - "description": "" - } - } - }, - "notifications": { - "description": "", - "properties": { - "on_failure": { - "description": "", - "properties": { - "email_addresses": { - "description": "", - "items": { - "description": "" - } - } - } - }, - "on_new_classification_tag_detected": { - "description": "", - "properties": { - "email_addresses": { - "description": "", - "items": { - "description": "" - } - } - } - } - } - }, - "output_schema_name": { - "description": "" - }, - "schedule": { - "description": "", - "properties": { - "pause_status": { - "description": "" - }, - "quartz_cron_expression": { - "description": "" - }, - "timezone_id": { - "description": "" - } - } - }, - "skip_builtin_dashboard": { - "description": "" - }, - "slicing_exprs": { - "description": "", - "items": { - "description": "" - } - }, - "snapshot": { - "description": "" - }, - "time_series": { - "description": "", - "properties": { - "granularities": { - "description": "", - "items": { - "description": "" - } - }, - "timestamp_col": { - "description": "" - } - } - }, - "warehouse_id": { - "description": "" - } - } - } - }, - "registered_models": { - "description": "List of Registered Models", - "additionalproperties": { - "description": "", - "properties": { - "catalog_name": { - "description": "The name of the catalog where the schema and the registered model reside" - }, - "comment": { - "description": "The comment attached to the registered model" - }, - "grants": { - "description": "", - "items": { - "description": "", - "properties": { - "principal": { - "description": "" - }, - "privileges": { - "description": "", - "items": { - "description": "" - } - } - } - } - }, - "name": { - "description": "The name of the registered model" - }, - "schema_name": { - "description": "The name of the schema where the registered model resides" - }, - "storage_location": { - "description": "The storage location on the cloud under which model version data files are stored" - } - } - } - }, - "schemas": { - "description": "", - "additionalproperties": { - "description": "", - "properties": { - "catalog_name": { - "description": "" - }, - "comment": { - "description": "" - }, - "grants": { - "description": "", - "items": { - "description": "", - "properties": { - "principal": { - "description": "" - }, - "privileges": { - "description": "", - "items": { - "description": "" - } - } - } - } - }, - "name": { - "description": "" - }, - "properties": { - "description": "", - "additionalproperties": { - "description": "" - } - }, - "storage_root": { - "description": "" - } - } - } - } - } - }, - "run_as": { - "description": "", - "properties": { - "service_principal_name": { - "description": "" - }, - "user_name": { - "description": "" - } - } - }, - "sync": { - "description": "", - "properties": { - "exclude": { - "description": "", - "items": { - "description": "" - } - }, - "include": { - "description": "", - "items": { - "description": "" - } - } - } - }, - "targets": { - "description": "", - "additionalproperties": { - "description": "", - "properties": { - "artifacts": { - "description": "", - "additionalproperties": { - "description": "", - "properties": { - "build": { - "description": "" - }, - "executable": { - "description": "" - }, - "files": { - "description": "", - "items": { - "description": "", - "properties": { - "source": { - "description": "" - } - } - } - }, - "path": { - "description": "" - }, - "type": { - "description": "" - } - } - } - }, - "bundle": { - "description": "", - "properties": { - "compute_id": { - "description": "" - }, - "databricks_cli_version": { - "description": "" - }, - "deployment": { - "description": "", - "properties": { - "fail_on_active_runs": { - "description": "" - }, - "lock": { - "description": "", - "properties": { - "enabled": { - "description": "" - }, - "force": { - "description": "" - } - } - } - } - }, - "git": { - "description": "", - "properties": { - "branch": { - "description": "" - }, - "origin_url": { - "description": "" - } - } - }, - "name": { - "description": "" - } - } - }, - "compute_id": { - "description": "" - }, - "default": { - "description": "" - }, - "git": { - "description": "", - "properties": { - "branch": { - "description": "" - }, - "origin_url": { - "description": "" - } - } - }, - "mode": { - "description": "" - }, - "permissions": { - "description": "", - "items": { - "description": "", - "properties": { - "group_name": { - "description": "" - }, - "level": { - "description": "" - }, - "service_principal_name": { - "description": "" - }, - "user_name": { - "description": "" - } - } - } - }, - "resources": { - "description": "Collection of Databricks resources to deploy.", - "properties": { - "experiments": { - "description": "List of MLflow experiments", - "additionalproperties": { - "description": "", - "properties": { - "artifact_location": { - "description": "Location where artifacts for the experiment are stored." - }, - "creation_time": { - "description": "Creation time" - }, - "experiment_id": { - "description": "Unique identifier for the experiment." - }, - "last_update_time": { - "description": "Last update time" - }, - "lifecycle_stage": { - "description": "Current life cycle stage of the experiment: \"active\" or \"deleted\".\nDeleted experiments are not returned by APIs." - }, - "name": { - "description": "Human readable name that identifies the experiment." - }, - "permissions": { - "description": "", - "items": { - "description": "", - "properties": { - "group_name": { - "description": "" - }, - "level": { - "description": "" - }, - "service_principal_name": { - "description": "" - }, - "user_name": { - "description": "" - } - } - } - }, - "tags": { - "description": "Tags: Additional metadata key-value pairs.", - "items": { - "description": "", - "properties": { - "key": { - "description": "The tag key." - }, - "value": { - "description": "The tag value." - } - } - } - } - } - } - }, - "jobs": { - "description": "List of Databricks jobs", - "additionalproperties": { - "description": "", - "properties": { - "continuous": { - "description": "An optional continuous property for this job. The continuous property will ensure that there is always one run executing. Only one of `schedule` and `continuous` can be used.", - "properties": { - "pause_status": { - "description": "Indicate whether the continuous execution of the job is paused or not. Defaults to UNPAUSED." - } - } - }, - "deployment": { - "description": "Deployment information for jobs managed by external sources.", - "properties": { - "kind": { - "description": "The kind of deployment that manages the job.\n\n* `BUNDLE`: The job is managed by Databricks Asset Bundle." - }, - "metadata_file_path": { - "description": "Path of the file that contains deployment metadata." - } - } - }, - "description": { - "description": "An optional description for the job. The maximum length is 27700 characters in UTF-8 encoding." - }, - "edit_mode": { - "description": "Edit mode of the job.\n\n* `UI_LOCKED`: The job is in a locked UI state and cannot be modified.\n* `EDITABLE`: The job is in an editable state and can be modified." - }, - "email_notifications": { - "description": "An optional set of email addresses that is notified when runs of this job begin or complete as well as when this job is deleted.", - "properties": { - "no_alert_for_skipped_runs": { - "description": "If true, do not send email to recipients specified in `on_failure` if the run is skipped." - }, - "on_duration_warning_threshold_exceeded": { - "description": "A list of email addresses to be notified when the duration of a run exceeds the threshold specified for the `RUN_DURATION_SECONDS` metric in the `health` field. If no rule for the `RUN_DURATION_SECONDS` metric is specified in the `health` field for the job, notifications are not sent.", - "items": { - "description": "" - } - }, - "on_failure": { - "description": "A list of email addresses to be notified when a run unsuccessfully completes. A run is considered to have completed unsuccessfully if it ends with an `INTERNAL_ERROR` `life_cycle_state` or a `FAILED`, or `TIMED_OUT` result_state. If this is not specified on job creation, reset, or update the list is empty, and notifications are not sent.", - "items": { - "description": "" - } - }, - "on_start": { - "description": "A list of email addresses to be notified when a run begins. If not specified on job creation, reset, or update, the list is empty, and notifications are not sent.", - "items": { - "description": "" - } - }, - "on_streaming_backlog_exceeded": { - "description": "A list of email addresses to notify when any streaming backlog thresholds are exceeded for any stream.\nStreaming backlog thresholds can be set in the `health` field using the following metrics: `STREAMING_BACKLOG_BYTES`, `STREAMING_BACKLOG_RECORDS`, `STREAMING_BACKLOG_SECONDS`, or `STREAMING_BACKLOG_FILES`.\nAlerting is based on the 10-minute average of these metrics. If the issue persists, notifications are resent every 30 minutes.", - "items": { - "description": "" - } - }, - "on_success": { - "description": "A list of email addresses to be notified when a run successfully completes. A run is considered to have completed successfully if it ends with a `TERMINATED` `life_cycle_state` and a `SUCCESS` result_state. If not specified on job creation, reset, or update, the list is empty, and notifications are not sent.", - "items": { - "description": "" - } - } - } - }, - "environments": { - "description": "A list of task execution environment specifications that can be referenced by tasks of this job.", - "items": { - "description": "", - "properties": { - "environment_key": { - "description": "The key of an environment. It has to be unique within a job." - }, - "spec": { - "description": "", - "properties": { - "client": { - "description": "Client version used by the environment\nThe client is the user-facing environment of the runtime.\nEach client comes with a specific set of pre-installed libraries.\nThe version is a string, consisting of the major client version." - }, - "dependencies": { - "description": "List of pip dependencies, as supported by the version of pip in this environment.\nEach dependency is a pip requirement file line https://pip.pypa.io/en/stable/reference/requirements-file-format/\nAllowed dependency could be \u003crequirement specifier\u003e, \u003carchive url/path\u003e, \u003clocal project path\u003e(WSFS or Volumes in Databricks), \u003cvcs project url\u003e\nE.g. dependencies: [\"foo==0.0.1\", \"-r /Workspace/test/requirements.txt\"]", - "items": { - "description": "" - } - } - } - } - } - } - }, - "format": { - "description": "Used to tell what is the format of the job. This field is ignored in Create/Update/Reset calls. When using the Jobs API 2.1 this value is always set to `\"MULTI_TASK\"`." - }, - "git_source": { - "description": "An optional specification for a remote Git repository containing the source code used by tasks. Version-controlled source code is supported by notebook, dbt, Python script, and SQL File tasks.\n\nIf `git_source` is set, these tasks retrieve the file from the remote repository by default. However, this behavior can be overridden by setting `source` to `WORKSPACE` on the task.\n\nNote: dbt and SQL File tasks support only version-controlled sources. If dbt or SQL File tasks are used, `git_source` must be defined on the job.", - "properties": { - "git_branch": { - "description": "Name of the branch to be checked out and used by this job. This field cannot be specified in conjunction with git_tag or git_commit." - }, - "git_commit": { - "description": "Commit to be checked out and used by this job. This field cannot be specified in conjunction with git_branch or git_tag." - }, - "git_provider": { - "description": "Unique identifier of the service used to host the Git repository. The value is case insensitive." - }, - "git_snapshot": { - "description": "", - "properties": { - "used_commit": { - "description": "Commit that was used to execute the run. If git_branch was specified, this points to the HEAD of the branch at the time of the run; if git_tag was specified, this points to the commit the tag points to." - } - } - }, - "git_tag": { - "description": "Name of the tag to be checked out and used by this job. This field cannot be specified in conjunction with git_branch or git_commit." - }, - "git_url": { - "description": "URL of the repository to be cloned by this job." - }, - "job_source": { - "description": "The source of the job specification in the remote repository when the job is source controlled.", - "properties": { - "dirty_state": { - "description": "Dirty state indicates the job is not fully synced with the job specification in the remote repository.\n\nPossible values are:\n* `NOT_SYNCED`: The job is not yet synced with the remote job specification. Import the remote job specification from UI to make the job fully synced.\n* `DISCONNECTED`: The job is temporary disconnected from the remote job specification and is allowed for live edit. Import the remote job specification again from UI to make the job fully synced." - }, - "import_from_git_branch": { - "description": "Name of the branch which the job is imported from." - }, - "job_config_path": { - "description": "Path of the job YAML file that contains the job specification." - } - } - } - } - }, - "health": { - "description": "", - "properties": { - "rules": { - "description": "", - "items": { - "description": "", - "properties": { - "metric": { - "description": "" - }, - "op": { - "description": "" - }, - "value": { - "description": "Specifies the threshold value that the health metric should obey to satisfy the health rule." - } - } - } - } - } - }, - "job_clusters": { - "description": "A list of job cluster specifications that can be shared and reused by tasks of this job. Libraries cannot be declared in a shared job cluster. You must declare dependent libraries in task settings.", - "items": { - "description": "", - "properties": { - "job_cluster_key": { - "description": "A unique name for the job cluster. This field is required and must be unique within the job.\n`JobTaskSettings` may refer to this field to determine which cluster to launch for the task execution." - }, - "new_cluster": { - "description": "If new_cluster, a description of a cluster that is created for each task.", - "properties": { - "apply_policy_default_values": { - "description": "When set to true, fixed and default values from the policy will be used for fields that are omitted. When set to false, only fixed values from the policy will be applied." - }, - "autoscale": { - "description": "Parameters needed in order to automatically scale clusters up and down based on load.\nNote: autoscaling works best with DB runtime versions 3.0 or later.", - "properties": { - "max_workers": { - "description": "The maximum number of workers to which the cluster can scale up when overloaded.\nNote that `max_workers` must be strictly greater than `min_workers`." - }, - "min_workers": { - "description": "The minimum number of workers to which the cluster can scale down when underutilized.\nIt is also the initial number of workers the cluster will have after creation." - } - } - }, - "autotermination_minutes": { - "description": "Automatically terminates the cluster after it is inactive for this time in minutes. If not set,\nthis cluster will not be automatically terminated. If specified, the threshold must be between\n10 and 10000 minutes.\nUsers can also set this value to 0 to explicitly disable automatic termination." - }, - "aws_attributes": { - "description": "Attributes related to clusters running on Amazon Web Services.\nIf not specified at cluster creation, a set of default values will be used.", - "properties": { - "availability": { - "description": "" - }, - "ebs_volume_count": { - "description": "The number of volumes launched for each instance. Users can choose up to 10 volumes.\nThis feature is only enabled for supported node types. Legacy node types cannot specify\ncustom EBS volumes.\nFor node types with no instance store, at least one EBS volume needs to be specified;\notherwise, cluster creation will fail.\n\nThese EBS volumes will be mounted at `/ebs0`, `/ebs1`, and etc.\nInstance store volumes will be mounted at `/local_disk0`, `/local_disk1`, and etc.\n\nIf EBS volumes are attached, Databricks will configure Spark to use only the EBS volumes for\nscratch storage because heterogenously sized scratch devices can lead to inefficient disk\nutilization. If no EBS volumes are attached, Databricks will configure Spark to use instance\nstore volumes.\n\nPlease note that if EBS volumes are specified, then the Spark configuration `spark.local.dir`\nwill be overridden." - }, - "ebs_volume_iops": { - "description": "If using gp3 volumes, what IOPS to use for the disk. If this is not set, the maximum performance of a gp2 volume with the same volume size will be used." - }, - "ebs_volume_size": { - "description": "The size of each EBS volume (in GiB) launched for each instance. For general purpose\nSSD, this value must be within the range 100 - 4096. For throughput optimized HDD,\nthis value must be within the range 500 - 4096." - }, - "ebs_volume_throughput": { - "description": "If using gp3 volumes, what throughput to use for the disk. If this is not set, the maximum performance of a gp2 volume with the same volume size will be used." - }, - "ebs_volume_type": { - "description": "" - }, - "first_on_demand": { - "description": "The first `first_on_demand` nodes of the cluster will be placed on on-demand instances.\nIf this value is greater than 0, the cluster driver node in particular will be placed on an\non-demand instance. If this value is greater than or equal to the current cluster size, all\nnodes will be placed on on-demand instances. If this value is less than the current cluster\nsize, `first_on_demand` nodes will be placed on on-demand instances and the remainder will\nbe placed on `availability` instances. Note that this value does not affect\ncluster size and cannot currently be mutated over the lifetime of a cluster." - }, - "instance_profile_arn": { - "description": "Nodes for this cluster will only be placed on AWS instances with this instance profile. If\nommitted, nodes will be placed on instances without an IAM instance profile. The instance\nprofile must have previously been added to the Databricks environment by an account\nadministrator.\n\nThis feature may only be available to certain customer plans.\n\nIf this field is ommitted, we will pull in the default from the conf if it exists." - }, - "spot_bid_price_percent": { - "description": "The bid price for AWS spot instances, as a percentage of the corresponding instance type's\non-demand price.\nFor example, if this field is set to 50, and the cluster needs a new `r3.xlarge` spot\ninstance, then the bid price is half of the price of\non-demand `r3.xlarge` instances. Similarly, if this field is set to 200, the bid price is twice\nthe price of on-demand `r3.xlarge` instances. If not specified, the default value is 100.\nWhen spot instances are requested for this cluster, only spot instances whose bid price\npercentage matches this field will be considered.\nNote that, for safety, we enforce this field to be no more than 10000.\n\nThe default value and documentation here should be kept consistent with\nCommonConf.defaultSpotBidPricePercent and CommonConf.maxSpotBidPricePercent." - }, - "zone_id": { - "description": "Identifier for the availability zone/datacenter in which the cluster resides.\nThis string will be of a form like \"us-west-2a\". The provided availability\nzone must be in the same region as the Databricks deployment. For example, \"us-west-2a\"\nis not a valid zone id if the Databricks deployment resides in the \"us-east-1\" region.\nThis is an optional field at cluster creation, and if not specified, a default zone will be used.\nIf the zone specified is \"auto\", will try to place cluster in a zone with high availability,\nand will retry placement in a different AZ if there is not enough capacity.\nThe list of available zones as well as the default value can be found by using the\n`List Zones` method." - } - } - }, - "azure_attributes": { - "description": "Attributes related to clusters running on Microsoft Azure.\nIf not specified at cluster creation, a set of default values will be used.", - "properties": { - "availability": { - "description": "" - }, - "first_on_demand": { - "description": "The first `first_on_demand` nodes of the cluster will be placed on on-demand instances.\nThis value should be greater than 0, to make sure the cluster driver node is placed on an\non-demand instance. If this value is greater than or equal to the current cluster size, all\nnodes will be placed on on-demand instances. If this value is less than the current cluster\nsize, `first_on_demand` nodes will be placed on on-demand instances and the remainder will\nbe placed on `availability` instances. Note that this value does not affect\ncluster size and cannot currently be mutated over the lifetime of a cluster." - }, - "log_analytics_info": { - "description": "Defines values necessary to configure and run Azure Log Analytics agent", - "properties": { - "log_analytics_primary_key": { - "description": "\u003cneeds content added\u003e" - }, - "log_analytics_workspace_id": { - "description": "\u003cneeds content added\u003e" - } - } - }, - "spot_bid_max_price": { - "description": "The max bid price to be used for Azure spot instances.\nThe Max price for the bid cannot be higher than the on-demand price of the instance.\nIf not specified, the default value is -1, which specifies that the instance cannot be evicted\non the basis of price, and only on the basis of availability. Further, the value should \u003e 0 or -1." - } - } - }, - "cluster_log_conf": { - "description": "The configuration for delivering spark logs to a long-term storage destination.\nTwo kinds of destinations (dbfs and s3) are supported. Only one destination can be specified\nfor one cluster. If the conf is given, the logs will be delivered to the destination every\n`5 mins`. The destination of driver logs is `$destination/$clusterId/driver`, while\nthe destination of executor logs is `$destination/$clusterId/executor`.", - "properties": { - "dbfs": { - "description": "destination needs to be provided. e.g.\n`{ \"dbfs\" : { \"destination\" : \"dbfs:/home/cluster_log\" } }`", - "properties": { - "destination": { - "description": "dbfs destination, e.g. `dbfs:/my/path`" - } - } - }, - "s3": { - "description": "destination and either the region or endpoint need to be provided. e.g.\n`{ \"s3\": { \"destination\" : \"s3://cluster_log_bucket/prefix\", \"region\" : \"us-west-2\" } }`\nCluster iam role is used to access s3, please make sure the cluster iam role in\n`instance_profile_arn` has permission to write data to the s3 destination.", - "properties": { - "canned_acl": { - "description": "(Optional) Set canned access control list for the logs, e.g. `bucket-owner-full-control`.\nIf `canned_cal` is set, please make sure the cluster iam role has `s3:PutObjectAcl` permission on\nthe destination bucket and prefix. The full list of possible canned acl can be found at\nhttp://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl.\nPlease also note that by default only the object owner gets full controls. If you are using cross account\nrole for writing data, you may want to set `bucket-owner-full-control` to make bucket owner able to\nread the logs." - }, - "destination": { - "description": "S3 destination, e.g. `s3://my-bucket/some-prefix` Note that logs will be delivered using\ncluster iam role, please make sure you set cluster iam role and the role has write access to the\ndestination. Please also note that you cannot use AWS keys to deliver logs." - }, - "enable_encryption": { - "description": "(Optional) Flag to enable server side encryption, `false` by default." - }, - "encryption_type": { - "description": "(Optional) The encryption type, it could be `sse-s3` or `sse-kms`. It will be used only when\nencryption is enabled and the default type is `sse-s3`." - }, - "endpoint": { - "description": "S3 endpoint, e.g. `https://s3-us-west-2.amazonaws.com`. Either region or endpoint needs to be set.\nIf both are set, endpoint will be used." - }, - "kms_key": { - "description": "(Optional) Kms key which will be used if encryption is enabled and encryption type is set to `sse-kms`." - }, - "region": { - "description": "S3 region, e.g. `us-west-2`. Either region or endpoint needs to be set. If both are set,\nendpoint will be used." - } - } - } - } - }, - "cluster_name": { - "description": "Cluster name requested by the user. This doesn't have to be unique.\nIf not specified at creation, the cluster name will be an empty string.\n" - }, - "custom_tags": { - "description": "Additional tags for cluster resources. Databricks will tag all cluster resources (e.g., AWS\ninstances and EBS volumes) with these tags in addition to `default_tags`. Notes:\n\n- Currently, Databricks allows at most 45 custom tags\n\n- Clusters can only reuse cloud resources if the resources' tags are a subset of the cluster tags", - "additionalproperties": { - "description": "" - } - }, - "data_security_mode": { - "description": "" - }, - "docker_image": { - "description": "", - "properties": { - "basic_auth": { - "description": "", - "properties": { - "password": { - "description": "Password of the user" - }, - "username": { - "description": "Name of the user" - } - } - }, - "url": { - "description": "URL of the docker image." - } - } - }, - "driver_instance_pool_id": { - "description": "The optional ID of the instance pool for the driver of the cluster belongs.\nThe pool cluster uses the instance pool with id (instance_pool_id) if the driver pool is not\nassigned." - }, - "driver_node_type_id": { - "description": "The node type of the Spark driver. Note that this field is optional;\nif unset, the driver node type will be set as the same value\nas `node_type_id` defined above.\n" - }, - "enable_elastic_disk": { - "description": "Autoscaling Local Storage: when enabled, this cluster will dynamically acquire additional disk\nspace when its Spark workers are running low on disk space. This feature requires specific AWS\npermissions to function correctly - refer to the User Guide for more details." - }, - "enable_local_disk_encryption": { - "description": "Whether to enable LUKS on cluster VMs' local disks" - }, - "gcp_attributes": { - "description": "Attributes related to clusters running on Google Cloud Platform.\nIf not specified at cluster creation, a set of default values will be used.", - "properties": { - "availability": { - "description": "" - }, - "boot_disk_size": { - "description": "boot disk size in GB" - }, - "google_service_account": { - "description": "If provided, the cluster will impersonate the google service account when accessing\ngcloud services (like GCS). The google service account\nmust have previously been added to the Databricks environment by an account\nadministrator." - }, - "local_ssd_count": { - "description": "If provided, each node (workers and driver) in the cluster will have this number of local SSDs attached. Each local SSD is 375GB in size. Refer to [GCP documentation](https://cloud.google.com/compute/docs/disks/local-ssd#choose_number_local_ssds) for the supported number of local SSDs for each instance type." - }, - "use_preemptible_executors": { - "description": "This field determines whether the spark executors will be scheduled to run on preemptible VMs (when set to true) versus standard compute engine VMs (when set to false; default).\nNote: Soon to be deprecated, use the availability field instead." - }, - "zone_id": { - "description": "Identifier for the availability zone in which the cluster resides.\nThis can be one of the following:\n- \"HA\" =\u003e High availability, spread nodes across availability zones for a Databricks deployment region [default]\n- \"AUTO\" =\u003e Databricks picks an availability zone to schedule the cluster on.\n- A GCP availability zone =\u003e Pick One of the available zones for (machine type + region) from https://cloud.google.com/compute/docs/regions-zones." - } - } - }, - "init_scripts": { - "description": "The configuration for storing init scripts. Any number of destinations can be specified. The scripts are executed sequentially in the order provided. If `cluster_log_conf` is specified, init script logs are sent to `\u003cdestination\u003e/\u003ccluster-ID\u003e/init_scripts`.", - "items": { - "description": "", - "properties": { - "abfss": { - "description": "destination needs to be provided. e.g.\n`{ \"abfss\" : { \"destination\" : \"abfss://\u003ccontainer-name\u003e@\u003cstorage-account-name\u003e.dfs.core.windows.net/\u003cdirectory-name\u003e\" } }", - "properties": { - "destination": { - "description": "abfss destination, e.g. `abfss://\u003ccontainer-name\u003e@\u003cstorage-account-name\u003e.dfs.core.windows.net/\u003cdirectory-name\u003e`." - } - } - }, - "dbfs": { - "description": "destination needs to be provided. e.g.\n`{ \"dbfs\" : { \"destination\" : \"dbfs:/home/cluster_log\" } }`", - "properties": { - "destination": { - "description": "dbfs destination, e.g. `dbfs:/my/path`" - } - } - }, - "file": { - "description": "destination needs to be provided. e.g.\n`{ \"file\" : { \"destination\" : \"file:/my/local/file.sh\" } }`", - "properties": { - "destination": { - "description": "local file destination, e.g. `file:/my/local/file.sh`" - } - } - }, - "gcs": { - "description": "destination needs to be provided. e.g.\n`{ \"gcs\": { \"destination\": \"gs://my-bucket/file.sh\" } }`", - "properties": { - "destination": { - "description": "GCS destination/URI, e.g. `gs://my-bucket/some-prefix`" - } - } - }, - "s3": { - "description": "destination and either the region or endpoint need to be provided. e.g.\n`{ \"s3\": { \"destination\" : \"s3://cluster_log_bucket/prefix\", \"region\" : \"us-west-2\" } }`\nCluster iam role is used to access s3, please make sure the cluster iam role in\n`instance_profile_arn` has permission to write data to the s3 destination.", - "properties": { - "canned_acl": { - "description": "(Optional) Set canned access control list for the logs, e.g. `bucket-owner-full-control`.\nIf `canned_cal` is set, please make sure the cluster iam role has `s3:PutObjectAcl` permission on\nthe destination bucket and prefix. The full list of possible canned acl can be found at\nhttp://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl.\nPlease also note that by default only the object owner gets full controls. If you are using cross account\nrole for writing data, you may want to set `bucket-owner-full-control` to make bucket owner able to\nread the logs." - }, - "destination": { - "description": "S3 destination, e.g. `s3://my-bucket/some-prefix` Note that logs will be delivered using\ncluster iam role, please make sure you set cluster iam role and the role has write access to the\ndestination. Please also note that you cannot use AWS keys to deliver logs." - }, - "enable_encryption": { - "description": "(Optional) Flag to enable server side encryption, `false` by default." - }, - "encryption_type": { - "description": "(Optional) The encryption type, it could be `sse-s3` or `sse-kms`. It will be used only when\nencryption is enabled and the default type is `sse-s3`." - }, - "endpoint": { - "description": "S3 endpoint, e.g. `https://s3-us-west-2.amazonaws.com`. Either region or endpoint needs to be set.\nIf both are set, endpoint will be used." - }, - "kms_key": { - "description": "(Optional) Kms key which will be used if encryption is enabled and encryption type is set to `sse-kms`." - }, - "region": { - "description": "S3 region, e.g. `us-west-2`. Either region or endpoint needs to be set. If both are set,\nendpoint will be used." - } - } - }, - "volumes": { - "description": "destination needs to be provided. e.g.\n`{ \"volumes\" : { \"destination\" : \"/Volumes/my-init.sh\" } }`", - "properties": { - "destination": { - "description": "Unity Catalog Volumes file destination, e.g. `/Volumes/my-init.sh`" - } - } - }, - "workspace": { - "description": "destination needs to be provided. e.g.\n`{ \"workspace\" : { \"destination\" : \"/Users/user1@databricks.com/my-init.sh\" } }`", - "properties": { - "destination": { - "description": "workspace files destination, e.g. `/Users/user1@databricks.com/my-init.sh`" - } - } - } - } - } - }, - "instance_pool_id": { - "description": "The optional ID of the instance pool to which the cluster belongs." - }, - "node_type_id": { - "description": "This field encodes, through a single value, the resources available to each of\nthe Spark nodes in this cluster. For example, the Spark nodes can be provisioned\nand optimized for memory or compute intensive workloads. A list of available node\ntypes can be retrieved by using the :method:clusters/listNodeTypes API call.\n" - }, - "num_workers": { - "description": "Number of worker nodes that this cluster should have. A cluster has one Spark Driver\nand `num_workers` Executors for a total of `num_workers` + 1 Spark nodes.\n\nNote: When reading the properties of a cluster, this field reflects the desired number\nof workers rather than the actual current number of workers. For instance, if a cluster\nis resized from 5 to 10 workers, this field will immediately be updated to reflect\nthe target size of 10 workers, whereas the workers listed in `spark_info` will gradually\nincrease from 5 to 10 as the new nodes are provisioned." - }, - "policy_id": { - "description": "The ID of the cluster policy used to create the cluster if applicable." - }, - "runtime_engine": { - "description": "" - }, - "single_user_name": { - "description": "Single user name if data_security_mode is `SINGLE_USER`" - }, - "spark_conf": { - "description": "An object containing a set of optional, user-specified Spark configuration key-value pairs.\nUsers can also pass in a string of extra JVM options to the driver and the executors via\n`spark.driver.extraJavaOptions` and `spark.executor.extraJavaOptions` respectively.\n", - "additionalproperties": { - "description": "" - } - }, - "spark_env_vars": { - "description": "An object containing a set of optional, user-specified environment variable key-value pairs.\nPlease note that key-value pair of the form (X,Y) will be exported as is (i.e.,\n`export X='Y'`) while launching the driver and workers.\n\nIn order to specify an additional set of `SPARK_DAEMON_JAVA_OPTS`, we recommend appending\nthem to `$SPARK_DAEMON_JAVA_OPTS` as shown in the example below. This ensures that all\ndefault databricks managed environmental variables are included as well.\n\nExample Spark environment variables:\n`{\"SPARK_WORKER_MEMORY\": \"28000m\", \"SPARK_LOCAL_DIRS\": \"/local_disk0\"}` or\n`{\"SPARK_DAEMON_JAVA_OPTS\": \"$SPARK_DAEMON_JAVA_OPTS -Dspark.shuffle.service.enabled=true\"}`", - "additionalproperties": { - "description": "" - } - }, - "spark_version": { - "description": "The Spark version of the cluster, e.g. `3.3.x-scala2.11`.\nA list of available Spark versions can be retrieved by using\nthe :method:clusters/sparkVersions API call.\n" - }, - "ssh_public_keys": { - "description": "SSH public key contents that will be added to each Spark node in this cluster. The\ncorresponding private keys can be used to login with the user name `ubuntu` on port `2200`.\nUp to 10 keys can be specified.", - "items": { - "description": "" - } - }, - "workload_type": { - "description": "", - "properties": { - "clients": { - "description": " defined what type of clients can use the cluster. E.g. Notebooks, Jobs", - "properties": { - "jobs": { - "description": "With jobs set, the cluster can be used for jobs" - }, - "notebooks": { - "description": "With notebooks set, this cluster can be used for notebooks" - } - } - } - } - } - } - } - } - } - }, - "max_concurrent_runs": { - "description": "An optional maximum allowed number of concurrent runs of the job.\nSet this value if you want to be able to execute multiple runs of the same job concurrently.\nThis is useful for example if you trigger your job on a frequent schedule and want to allow consecutive runs to overlap with each other, or if you want to trigger multiple runs which differ by their input parameters.\nThis setting affects only new runs. For example, suppose the job’s concurrency is 4 and there are 4 concurrent active runs. Then setting the concurrency to 3 won’t kill any of the active runs.\nHowever, from then on, new runs are skipped unless there are fewer than 3 active runs.\nThis value cannot exceed 1000. Setting this value to `0` causes all new runs to be skipped." - }, - "name": { - "description": "An optional name for the job. The maximum length is 4096 bytes in UTF-8 encoding." - }, - "notification_settings": { - "description": "Optional notification settings that are used when sending notifications to each of the `email_notifications` and `webhook_notifications` for this job.", - "properties": { - "no_alert_for_canceled_runs": { - "description": "If true, do not send notifications to recipients specified in `on_failure` if the run is canceled." - }, - "no_alert_for_skipped_runs": { - "description": "If true, do not send notifications to recipients specified in `on_failure` if the run is skipped." - } - } - }, - "parameters": { - "description": "Job-level parameter definitions", - "items": { - "description": "", - "properties": { - "default": { - "description": "Default value of the parameter." - }, - "name": { - "description": "The name of the defined parameter. May only contain alphanumeric characters, `_`, `-`, and `.`" - } - } - } - }, - "permissions": { - "description": "", - "items": { - "description": "", - "properties": { - "group_name": { - "description": "" - }, - "level": { - "description": "" - }, - "service_principal_name": { - "description": "" - }, - "user_name": { - "description": "" - } - } - } - }, - "queue": { - "description": "The queue settings of the job.", - "properties": { - "enabled": { - "description": "If true, enable queueing for the job. This is a required field." - } - } - }, - "run_as": { - "description": "", - "properties": { - "service_principal_name": { - "description": "Application ID of an active service principal. Setting this field requires the `servicePrincipal/user` role." - }, - "user_name": { - "description": "The email of an active workspace user. Non-admin users can only set this field to their own email." - } - } - }, - "schedule": { - "description": "An optional periodic schedule for this job. The default behavior is that the job only runs when triggered by clicking “Run Now” in the Jobs UI or sending an API request to `runNow`.", - "properties": { - "pause_status": { - "description": "Indicate whether this schedule is paused or not." - }, - "quartz_cron_expression": { - "description": "A Cron expression using Quartz syntax that describes the schedule for a job. See [Cron Trigger](http://www.quartz-scheduler.org/documentation/quartz-2.3.0/tutorials/crontrigger.html) for details. This field is required." - }, - "timezone_id": { - "description": "A Java timezone ID. The schedule for a job is resolved with respect to this timezone. See [Java TimeZone](https://docs.oracle.com/javase/7/docs/api/java/util/TimeZone.html) for details. This field is required." - } - } - }, - "tags": { - "description": "A map of tags associated with the job. These are forwarded to the cluster as cluster tags for jobs clusters, and are subject to the same limitations as cluster tags. A maximum of 25 tags can be added to the job.", - "additionalproperties": { - "description": "" - } - }, - "tasks": { - "description": "A list of task specifications to be executed by this job.", - "items": { - "description": "", - "properties": { - "condition_task": { - "description": "If condition_task, specifies a condition with an outcome that can be used to control the execution of other tasks. Does not require a cluster to execute and does not support retries or notifications.", - "properties": { - "left": { - "description": "The left operand of the condition task. Can be either a string value or a job state or parameter reference." - }, - "op": { - "description": "* `EQUAL_TO`, `NOT_EQUAL` operators perform string comparison of their operands. This means that `“12.0” == “12”` will evaluate to `false`.\n* `GREATER_THAN`, `GREATER_THAN_OR_EQUAL`, `LESS_THAN`, `LESS_THAN_OR_EQUAL` operators perform numeric comparison of their operands. `“12.0” \u003e= “12”` will evaluate to `true`, `“10.0” \u003e= “12”` will evaluate to `false`.\n\nThe boolean comparison to task values can be implemented with operators `EQUAL_TO`, `NOT_EQUAL`. If a task value was set to a boolean value, it will be serialized to `“true”` or `“false”` for the comparison." - }, - "right": { - "description": "The right operand of the condition task. Can be either a string value or a job state or parameter reference." - } - } - }, - "dbt_task": { - "description": "If dbt_task, indicates that this must execute a dbt task. It requires both Databricks SQL and the ability to use a serverless or a pro SQL warehouse.", - "properties": { - "catalog": { - "description": "Optional name of the catalog to use. The value is the top level in the 3-level namespace of Unity Catalog (catalog / schema / relation). The catalog value can only be specified if a warehouse_id is specified. Requires dbt-databricks \u003e= 1.1.1." - }, - "commands": { - "description": "A list of dbt commands to execute. All commands must start with `dbt`. This parameter must not be empty. A maximum of up to 10 commands can be provided.", - "items": { - "description": "" - } - }, - "profiles_directory": { - "description": "Optional (relative) path to the profiles directory. Can only be specified if no warehouse_id is specified. If no warehouse_id is specified and this folder is unset, the root directory is used." - }, - "project_directory": { - "description": "Path to the project directory. Optional for Git sourced tasks, in which\ncase if no value is provided, the root of the Git repository is used." - }, - "schema": { - "description": "Optional schema to write to. This parameter is only used when a warehouse_id is also provided. If not provided, the `default` schema is used." - }, - "source": { - "description": "Optional location type of the project directory. When set to `WORKSPACE`, the project will be retrieved\nfrom the local Databricks workspace. When set to `GIT`, the project will be retrieved from a Git repository\ndefined in `git_source`. If the value is empty, the task will use `GIT` if `git_source` is defined and `WORKSPACE` otherwise.\n\n* `WORKSPACE`: Project is located in Databricks workspace.\n* `GIT`: Project is located in cloud Git provider." - }, - "warehouse_id": { - "description": "ID of the SQL warehouse to connect to. If provided, we automatically generate and provide the profile and connection details to dbt. It can be overridden on a per-command basis by using the `--profiles-dir` command line argument." - } - } - }, - "depends_on": { - "description": "An optional array of objects specifying the dependency graph of the task. All tasks specified in this field must complete before executing this task. The task will run only if the `run_if` condition is true.\nThe key is `task_key`, and the value is the name assigned to the dependent task.", - "items": { - "description": "", - "properties": { - "outcome": { - "description": "Can only be specified on condition task dependencies. The outcome of the dependent task that must be met for this task to run." - }, - "task_key": { - "description": "The name of the task this task depends on." - } - } - } - }, - "description": { - "description": "An optional description for this task." - }, - "disable_auto_optimization": { - "description": "An option to disable auto optimization in serverless" - }, - "email_notifications": { - "description": "An optional set of email addresses that is notified when runs of this task begin or complete as well as when this task is deleted. The default behavior is to not send any emails.", - "properties": { - "no_alert_for_skipped_runs": { - "description": "If true, do not send email to recipients specified in `on_failure` if the run is skipped." - }, - "on_duration_warning_threshold_exceeded": { - "description": "A list of email addresses to be notified when the duration of a run exceeds the threshold specified for the `RUN_DURATION_SECONDS` metric in the `health` field. If no rule for the `RUN_DURATION_SECONDS` metric is specified in the `health` field for the job, notifications are not sent.", - "items": { - "description": "" - } - }, - "on_failure": { - "description": "A list of email addresses to be notified when a run unsuccessfully completes. A run is considered to have completed unsuccessfully if it ends with an `INTERNAL_ERROR` `life_cycle_state` or a `FAILED`, or `TIMED_OUT` result_state. If this is not specified on job creation, reset, or update the list is empty, and notifications are not sent.", - "items": { - "description": "" - } - }, - "on_start": { - "description": "A list of email addresses to be notified when a run begins. If not specified on job creation, reset, or update, the list is empty, and notifications are not sent.", - "items": { - "description": "" - } - }, - "on_streaming_backlog_exceeded": { - "description": "A list of email addresses to notify when any streaming backlog thresholds are exceeded for any stream.\nStreaming backlog thresholds can be set in the `health` field using the following metrics: `STREAMING_BACKLOG_BYTES`, `STREAMING_BACKLOG_RECORDS`, `STREAMING_BACKLOG_SECONDS`, or `STREAMING_BACKLOG_FILES`.\nAlerting is based on the 10-minute average of these metrics. If the issue persists, notifications are resent every 30 minutes.", - "items": { - "description": "" - } - }, - "on_success": { - "description": "A list of email addresses to be notified when a run successfully completes. A run is considered to have completed successfully if it ends with a `TERMINATED` `life_cycle_state` and a `SUCCESS` result_state. If not specified on job creation, reset, or update, the list is empty, and notifications are not sent.", - "items": { - "description": "" - } - } - } - }, - "environment_key": { - "description": "The key that references an environment spec in a job. This field is required for Python script, Python wheel and dbt tasks when using serverless compute." - }, - "existing_cluster_id": { - "description": "If existing_cluster_id, the ID of an existing cluster that is used for all runs.\nWhen running jobs or tasks on an existing cluster, you may need to manually restart\nthe cluster if it stops responding. We suggest running jobs and tasks on new clusters for\ngreater reliability" - }, - "for_each_task": { - "description": "" - }, - "health": { - "description": "", - "properties": { - "rules": { - "description": "", - "items": { - "description": "", - "properties": { - "metric": { - "description": "" - }, - "op": { - "description": "" - }, - "value": { - "description": "Specifies the threshold value that the health metric should obey to satisfy the health rule." - } - } - } - } - } - }, - "job_cluster_key": { - "description": "If job_cluster_key, this task is executed reusing the cluster specified in `job.settings.job_clusters`." - }, - "libraries": { - "description": "An optional list of libraries to be installed on the cluster.\nThe default value is an empty list.", - "items": { - "description": "", - "properties": { - "cran": { - "description": "Specification of a CRAN library to be installed as part of the library", - "properties": { - "package": { - "description": "The name of the CRAN package to install." - }, - "repo": { - "description": "The repository where the package can be found. If not specified, the default CRAN repo is used." - } - } - }, - "egg": { - "description": "Deprecated. URI of the egg library to install. Installing Python egg files is deprecated and is not supported in Databricks Runtime 14.0 and above." - }, - "jar": { - "description": "URI of the JAR library to install. Supported URIs include Workspace paths, Unity Catalog Volumes paths, and S3 URIs.\nFor example: `{ \"jar\": \"/Workspace/path/to/library.jar\" }`, `{ \"jar\" : \"/Volumes/path/to/library.jar\" }` or\n`{ \"jar\": \"s3://my-bucket/library.jar\" }`.\nIf S3 is used, please make sure the cluster has read access on the library. You may need to\nlaunch the cluster with an IAM role to access the S3 URI." - }, - "maven": { - "description": "Specification of a maven library to be installed. For example:\n`{ \"coordinates\": \"org.jsoup:jsoup:1.7.2\" }`", - "properties": { - "coordinates": { - "description": "Gradle-style maven coordinates. For example: \"org.jsoup:jsoup:1.7.2\"." - }, - "exclusions": { - "description": "List of dependences to exclude. For example: `[\"slf4j:slf4j\", \"*:hadoop-client\"]`.\n\nMaven dependency exclusions:\nhttps://maven.apache.org/guides/introduction/introduction-to-optional-and-excludes-dependencies.html.", - "items": { - "description": "" - } - }, - "repo": { - "description": "Maven repo to install the Maven package from. If omitted, both Maven Central Repository\nand Spark Packages are searched." - } - } - }, - "pypi": { - "description": "Specification of a PyPi library to be installed. For example:\n`{ \"package\": \"simplejson\" }`", - "properties": { - "package": { - "description": "The name of the pypi package to install. An optional exact version specification is also\nsupported. Examples: \"simplejson\" and \"simplejson==3.8.0\"." - }, - "repo": { - "description": "The repository where the package can be found. If not specified, the default pip index is\nused." - } - } - }, - "requirements": { - "description": "URI of the requirements.txt file to install. Only Workspace paths and Unity Catalog Volumes paths are supported.\nFor example: `{ \"requirements\": \"/Workspace/path/to/requirements.txt\" }` or `{ \"requirements\" : \"/Volumes/path/to/requirements.txt\" }`" - }, - "whl": { - "description": "URI of the wheel library to install. Supported URIs include Workspace paths, Unity Catalog Volumes paths, and S3 URIs.\nFor example: `{ \"whl\": \"/Workspace/path/to/library.whl\" }`, `{ \"whl\" : \"/Volumes/path/to/library.whl\" }` or\n`{ \"whl\": \"s3://my-bucket/library.whl\" }`.\nIf S3 is used, please make sure the cluster has read access on the library. You may need to\nlaunch the cluster with an IAM role to access the S3 URI." - } - } - } - }, - "max_retries": { - "description": "An optional maximum number of times to retry an unsuccessful run. A run is considered to be unsuccessful if it completes with the `FAILED` result_state or `INTERNAL_ERROR` `life_cycle_state`. The value `-1` means to retry indefinitely and the value `0` means to never retry." - }, - "min_retry_interval_millis": { - "description": "An optional minimal interval in milliseconds between the start of the failed run and the subsequent retry run. The default behavior is that unsuccessful runs are immediately retried." - }, - "new_cluster": { - "description": "If new_cluster, a description of a new cluster that is created for each run.", - "properties": { - "apply_policy_default_values": { - "description": "When set to true, fixed and default values from the policy will be used for fields that are omitted. When set to false, only fixed values from the policy will be applied." - }, - "autoscale": { - "description": "Parameters needed in order to automatically scale clusters up and down based on load.\nNote: autoscaling works best with DB runtime versions 3.0 or later.", - "properties": { - "max_workers": { - "description": "The maximum number of workers to which the cluster can scale up when overloaded.\nNote that `max_workers` must be strictly greater than `min_workers`." - }, - "min_workers": { - "description": "The minimum number of workers to which the cluster can scale down when underutilized.\nIt is also the initial number of workers the cluster will have after creation." - } - } - }, - "autotermination_minutes": { - "description": "Automatically terminates the cluster after it is inactive for this time in minutes. If not set,\nthis cluster will not be automatically terminated. If specified, the threshold must be between\n10 and 10000 minutes.\nUsers can also set this value to 0 to explicitly disable automatic termination." - }, - "aws_attributes": { - "description": "Attributes related to clusters running on Amazon Web Services.\nIf not specified at cluster creation, a set of default values will be used.", - "properties": { - "availability": { - "description": "" - }, - "ebs_volume_count": { - "description": "The number of volumes launched for each instance. Users can choose up to 10 volumes.\nThis feature is only enabled for supported node types. Legacy node types cannot specify\ncustom EBS volumes.\nFor node types with no instance store, at least one EBS volume needs to be specified;\notherwise, cluster creation will fail.\n\nThese EBS volumes will be mounted at `/ebs0`, `/ebs1`, and etc.\nInstance store volumes will be mounted at `/local_disk0`, `/local_disk1`, and etc.\n\nIf EBS volumes are attached, Databricks will configure Spark to use only the EBS volumes for\nscratch storage because heterogenously sized scratch devices can lead to inefficient disk\nutilization. If no EBS volumes are attached, Databricks will configure Spark to use instance\nstore volumes.\n\nPlease note that if EBS volumes are specified, then the Spark configuration `spark.local.dir`\nwill be overridden." - }, - "ebs_volume_iops": { - "description": "If using gp3 volumes, what IOPS to use for the disk. If this is not set, the maximum performance of a gp2 volume with the same volume size will be used." - }, - "ebs_volume_size": { - "description": "The size of each EBS volume (in GiB) launched for each instance. For general purpose\nSSD, this value must be within the range 100 - 4096. For throughput optimized HDD,\nthis value must be within the range 500 - 4096." - }, - "ebs_volume_throughput": { - "description": "If using gp3 volumes, what throughput to use for the disk. If this is not set, the maximum performance of a gp2 volume with the same volume size will be used." - }, - "ebs_volume_type": { - "description": "" - }, - "first_on_demand": { - "description": "The first `first_on_demand` nodes of the cluster will be placed on on-demand instances.\nIf this value is greater than 0, the cluster driver node in particular will be placed on an\non-demand instance. If this value is greater than or equal to the current cluster size, all\nnodes will be placed on on-demand instances. If this value is less than the current cluster\nsize, `first_on_demand` nodes will be placed on on-demand instances and the remainder will\nbe placed on `availability` instances. Note that this value does not affect\ncluster size and cannot currently be mutated over the lifetime of a cluster." - }, - "instance_profile_arn": { - "description": "Nodes for this cluster will only be placed on AWS instances with this instance profile. If\nommitted, nodes will be placed on instances without an IAM instance profile. The instance\nprofile must have previously been added to the Databricks environment by an account\nadministrator.\n\nThis feature may only be available to certain customer plans.\n\nIf this field is ommitted, we will pull in the default from the conf if it exists." - }, - "spot_bid_price_percent": { - "description": "The bid price for AWS spot instances, as a percentage of the corresponding instance type's\non-demand price.\nFor example, if this field is set to 50, and the cluster needs a new `r3.xlarge` spot\ninstance, then the bid price is half of the price of\non-demand `r3.xlarge` instances. Similarly, if this field is set to 200, the bid price is twice\nthe price of on-demand `r3.xlarge` instances. If not specified, the default value is 100.\nWhen spot instances are requested for this cluster, only spot instances whose bid price\npercentage matches this field will be considered.\nNote that, for safety, we enforce this field to be no more than 10000.\n\nThe default value and documentation here should be kept consistent with\nCommonConf.defaultSpotBidPricePercent and CommonConf.maxSpotBidPricePercent." - }, - "zone_id": { - "description": "Identifier for the availability zone/datacenter in which the cluster resides.\nThis string will be of a form like \"us-west-2a\". The provided availability\nzone must be in the same region as the Databricks deployment. For example, \"us-west-2a\"\nis not a valid zone id if the Databricks deployment resides in the \"us-east-1\" region.\nThis is an optional field at cluster creation, and if not specified, a default zone will be used.\nIf the zone specified is \"auto\", will try to place cluster in a zone with high availability,\nand will retry placement in a different AZ if there is not enough capacity.\nThe list of available zones as well as the default value can be found by using the\n`List Zones` method." - } - } - }, - "azure_attributes": { - "description": "Attributes related to clusters running on Microsoft Azure.\nIf not specified at cluster creation, a set of default values will be used.", - "properties": { - "availability": { - "description": "" - }, - "first_on_demand": { - "description": "The first `first_on_demand` nodes of the cluster will be placed on on-demand instances.\nThis value should be greater than 0, to make sure the cluster driver node is placed on an\non-demand instance. If this value is greater than or equal to the current cluster size, all\nnodes will be placed on on-demand instances. If this value is less than the current cluster\nsize, `first_on_demand` nodes will be placed on on-demand instances and the remainder will\nbe placed on `availability` instances. Note that this value does not affect\ncluster size and cannot currently be mutated over the lifetime of a cluster." - }, - "log_analytics_info": { - "description": "Defines values necessary to configure and run Azure Log Analytics agent", - "properties": { - "log_analytics_primary_key": { - "description": "\u003cneeds content added\u003e" - }, - "log_analytics_workspace_id": { - "description": "\u003cneeds content added\u003e" - } - } - }, - "spot_bid_max_price": { - "description": "The max bid price to be used for Azure spot instances.\nThe Max price for the bid cannot be higher than the on-demand price of the instance.\nIf not specified, the default value is -1, which specifies that the instance cannot be evicted\non the basis of price, and only on the basis of availability. Further, the value should \u003e 0 or -1." - } - } - }, - "cluster_log_conf": { - "description": "The configuration for delivering spark logs to a long-term storage destination.\nTwo kinds of destinations (dbfs and s3) are supported. Only one destination can be specified\nfor one cluster. If the conf is given, the logs will be delivered to the destination every\n`5 mins`. The destination of driver logs is `$destination/$clusterId/driver`, while\nthe destination of executor logs is `$destination/$clusterId/executor`.", - "properties": { - "dbfs": { - "description": "destination needs to be provided. e.g.\n`{ \"dbfs\" : { \"destination\" : \"dbfs:/home/cluster_log\" } }`", - "properties": { - "destination": { - "description": "dbfs destination, e.g. `dbfs:/my/path`" - } - } - }, - "s3": { - "description": "destination and either the region or endpoint need to be provided. e.g.\n`{ \"s3\": { \"destination\" : \"s3://cluster_log_bucket/prefix\", \"region\" : \"us-west-2\" } }`\nCluster iam role is used to access s3, please make sure the cluster iam role in\n`instance_profile_arn` has permission to write data to the s3 destination.", - "properties": { - "canned_acl": { - "description": "(Optional) Set canned access control list for the logs, e.g. `bucket-owner-full-control`.\nIf `canned_cal` is set, please make sure the cluster iam role has `s3:PutObjectAcl` permission on\nthe destination bucket and prefix. The full list of possible canned acl can be found at\nhttp://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl.\nPlease also note that by default only the object owner gets full controls. If you are using cross account\nrole for writing data, you may want to set `bucket-owner-full-control` to make bucket owner able to\nread the logs." - }, - "destination": { - "description": "S3 destination, e.g. `s3://my-bucket/some-prefix` Note that logs will be delivered using\ncluster iam role, please make sure you set cluster iam role and the role has write access to the\ndestination. Please also note that you cannot use AWS keys to deliver logs." - }, - "enable_encryption": { - "description": "(Optional) Flag to enable server side encryption, `false` by default." - }, - "encryption_type": { - "description": "(Optional) The encryption type, it could be `sse-s3` or `sse-kms`. It will be used only when\nencryption is enabled and the default type is `sse-s3`." - }, - "endpoint": { - "description": "S3 endpoint, e.g. `https://s3-us-west-2.amazonaws.com`. Either region or endpoint needs to be set.\nIf both are set, endpoint will be used." - }, - "kms_key": { - "description": "(Optional) Kms key which will be used if encryption is enabled and encryption type is set to `sse-kms`." - }, - "region": { - "description": "S3 region, e.g. `us-west-2`. Either region or endpoint needs to be set. If both are set,\nendpoint will be used." - } - } - } - } - }, - "cluster_name": { - "description": "Cluster name requested by the user. This doesn't have to be unique.\nIf not specified at creation, the cluster name will be an empty string.\n" - }, - "custom_tags": { - "description": "Additional tags for cluster resources. Databricks will tag all cluster resources (e.g., AWS\ninstances and EBS volumes) with these tags in addition to `default_tags`. Notes:\n\n- Currently, Databricks allows at most 45 custom tags\n\n- Clusters can only reuse cloud resources if the resources' tags are a subset of the cluster tags", - "additionalproperties": { - "description": "" - } - }, - "data_security_mode": { - "description": "" - }, - "docker_image": { - "description": "", - "properties": { - "basic_auth": { - "description": "", - "properties": { - "password": { - "description": "Password of the user" - }, - "username": { - "description": "Name of the user" - } - } - }, - "url": { - "description": "URL of the docker image." - } - } - }, - "driver_instance_pool_id": { - "description": "The optional ID of the instance pool for the driver of the cluster belongs.\nThe pool cluster uses the instance pool with id (instance_pool_id) if the driver pool is not\nassigned." - }, - "driver_node_type_id": { - "description": "The node type of the Spark driver. Note that this field is optional;\nif unset, the driver node type will be set as the same value\nas `node_type_id` defined above.\n" - }, - "enable_elastic_disk": { - "description": "Autoscaling Local Storage: when enabled, this cluster will dynamically acquire additional disk\nspace when its Spark workers are running low on disk space. This feature requires specific AWS\npermissions to function correctly - refer to the User Guide for more details." - }, - "enable_local_disk_encryption": { - "description": "Whether to enable LUKS on cluster VMs' local disks" - }, - "gcp_attributes": { - "description": "Attributes related to clusters running on Google Cloud Platform.\nIf not specified at cluster creation, a set of default values will be used.", - "properties": { - "availability": { - "description": "" - }, - "boot_disk_size": { - "description": "boot disk size in GB" - }, - "google_service_account": { - "description": "If provided, the cluster will impersonate the google service account when accessing\ngcloud services (like GCS). The google service account\nmust have previously been added to the Databricks environment by an account\nadministrator." - }, - "local_ssd_count": { - "description": "If provided, each node (workers and driver) in the cluster will have this number of local SSDs attached. Each local SSD is 375GB in size. Refer to [GCP documentation](https://cloud.google.com/compute/docs/disks/local-ssd#choose_number_local_ssds) for the supported number of local SSDs for each instance type." - }, - "use_preemptible_executors": { - "description": "This field determines whether the spark executors will be scheduled to run on preemptible VMs (when set to true) versus standard compute engine VMs (when set to false; default).\nNote: Soon to be deprecated, use the availability field instead." - }, - "zone_id": { - "description": "Identifier for the availability zone in which the cluster resides.\nThis can be one of the following:\n- \"HA\" =\u003e High availability, spread nodes across availability zones for a Databricks deployment region [default]\n- \"AUTO\" =\u003e Databricks picks an availability zone to schedule the cluster on.\n- A GCP availability zone =\u003e Pick One of the available zones for (machine type + region) from https://cloud.google.com/compute/docs/regions-zones." - } - } - }, - "init_scripts": { - "description": "The configuration for storing init scripts. Any number of destinations can be specified. The scripts are executed sequentially in the order provided. If `cluster_log_conf` is specified, init script logs are sent to `\u003cdestination\u003e/\u003ccluster-ID\u003e/init_scripts`.", - "items": { - "description": "", - "properties": { - "abfss": { - "description": "destination needs to be provided. e.g.\n`{ \"abfss\" : { \"destination\" : \"abfss://\u003ccontainer-name\u003e@\u003cstorage-account-name\u003e.dfs.core.windows.net/\u003cdirectory-name\u003e\" } }", - "properties": { - "destination": { - "description": "abfss destination, e.g. `abfss://\u003ccontainer-name\u003e@\u003cstorage-account-name\u003e.dfs.core.windows.net/\u003cdirectory-name\u003e`." - } - } - }, - "dbfs": { - "description": "destination needs to be provided. e.g.\n`{ \"dbfs\" : { \"destination\" : \"dbfs:/home/cluster_log\" } }`", - "properties": { - "destination": { - "description": "dbfs destination, e.g. `dbfs:/my/path`" - } - } - }, - "file": { - "description": "destination needs to be provided. e.g.\n`{ \"file\" : { \"destination\" : \"file:/my/local/file.sh\" } }`", - "properties": { - "destination": { - "description": "local file destination, e.g. `file:/my/local/file.sh`" - } - } - }, - "gcs": { - "description": "destination needs to be provided. e.g.\n`{ \"gcs\": { \"destination\": \"gs://my-bucket/file.sh\" } }`", - "properties": { - "destination": { - "description": "GCS destination/URI, e.g. `gs://my-bucket/some-prefix`" - } - } - }, - "s3": { - "description": "destination and either the region or endpoint need to be provided. e.g.\n`{ \"s3\": { \"destination\" : \"s3://cluster_log_bucket/prefix\", \"region\" : \"us-west-2\" } }`\nCluster iam role is used to access s3, please make sure the cluster iam role in\n`instance_profile_arn` has permission to write data to the s3 destination.", - "properties": { - "canned_acl": { - "description": "(Optional) Set canned access control list for the logs, e.g. `bucket-owner-full-control`.\nIf `canned_cal` is set, please make sure the cluster iam role has `s3:PutObjectAcl` permission on\nthe destination bucket and prefix. The full list of possible canned acl can be found at\nhttp://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl.\nPlease also note that by default only the object owner gets full controls. If you are using cross account\nrole for writing data, you may want to set `bucket-owner-full-control` to make bucket owner able to\nread the logs." - }, - "destination": { - "description": "S3 destination, e.g. `s3://my-bucket/some-prefix` Note that logs will be delivered using\ncluster iam role, please make sure you set cluster iam role and the role has write access to the\ndestination. Please also note that you cannot use AWS keys to deliver logs." - }, - "enable_encryption": { - "description": "(Optional) Flag to enable server side encryption, `false` by default." - }, - "encryption_type": { - "description": "(Optional) The encryption type, it could be `sse-s3` or `sse-kms`. It will be used only when\nencryption is enabled and the default type is `sse-s3`." - }, - "endpoint": { - "description": "S3 endpoint, e.g. `https://s3-us-west-2.amazonaws.com`. Either region or endpoint needs to be set.\nIf both are set, endpoint will be used." - }, - "kms_key": { - "description": "(Optional) Kms key which will be used if encryption is enabled and encryption type is set to `sse-kms`." - }, - "region": { - "description": "S3 region, e.g. `us-west-2`. Either region or endpoint needs to be set. If both are set,\nendpoint will be used." - } - } - }, - "volumes": { - "description": "destination needs to be provided. e.g.\n`{ \"volumes\" : { \"destination\" : \"/Volumes/my-init.sh\" } }`", - "properties": { - "destination": { - "description": "Unity Catalog Volumes file destination, e.g. `/Volumes/my-init.sh`" - } - } - }, - "workspace": { - "description": "destination needs to be provided. e.g.\n`{ \"workspace\" : { \"destination\" : \"/Users/user1@databricks.com/my-init.sh\" } }`", - "properties": { - "destination": { - "description": "workspace files destination, e.g. `/Users/user1@databricks.com/my-init.sh`" - } - } - } - } - } - }, - "instance_pool_id": { - "description": "The optional ID of the instance pool to which the cluster belongs." - }, - "node_type_id": { - "description": "This field encodes, through a single value, the resources available to each of\nthe Spark nodes in this cluster. For example, the Spark nodes can be provisioned\nand optimized for memory or compute intensive workloads. A list of available node\ntypes can be retrieved by using the :method:clusters/listNodeTypes API call.\n" - }, - "num_workers": { - "description": "Number of worker nodes that this cluster should have. A cluster has one Spark Driver\nand `num_workers` Executors for a total of `num_workers` + 1 Spark nodes.\n\nNote: When reading the properties of a cluster, this field reflects the desired number\nof workers rather than the actual current number of workers. For instance, if a cluster\nis resized from 5 to 10 workers, this field will immediately be updated to reflect\nthe target size of 10 workers, whereas the workers listed in `spark_info` will gradually\nincrease from 5 to 10 as the new nodes are provisioned." - }, - "policy_id": { - "description": "The ID of the cluster policy used to create the cluster if applicable." - }, - "runtime_engine": { - "description": "" - }, - "single_user_name": { - "description": "Single user name if data_security_mode is `SINGLE_USER`" - }, - "spark_conf": { - "description": "An object containing a set of optional, user-specified Spark configuration key-value pairs.\nUsers can also pass in a string of extra JVM options to the driver and the executors via\n`spark.driver.extraJavaOptions` and `spark.executor.extraJavaOptions` respectively.\n", - "additionalproperties": { - "description": "" - } - }, - "spark_env_vars": { - "description": "An object containing a set of optional, user-specified environment variable key-value pairs.\nPlease note that key-value pair of the form (X,Y) will be exported as is (i.e.,\n`export X='Y'`) while launching the driver and workers.\n\nIn order to specify an additional set of `SPARK_DAEMON_JAVA_OPTS`, we recommend appending\nthem to `$SPARK_DAEMON_JAVA_OPTS` as shown in the example below. This ensures that all\ndefault databricks managed environmental variables are included as well.\n\nExample Spark environment variables:\n`{\"SPARK_WORKER_MEMORY\": \"28000m\", \"SPARK_LOCAL_DIRS\": \"/local_disk0\"}` or\n`{\"SPARK_DAEMON_JAVA_OPTS\": \"$SPARK_DAEMON_JAVA_OPTS -Dspark.shuffle.service.enabled=true\"}`", - "additionalproperties": { - "description": "" - } - }, - "spark_version": { - "description": "The Spark version of the cluster, e.g. `3.3.x-scala2.11`.\nA list of available Spark versions can be retrieved by using\nthe :method:clusters/sparkVersions API call.\n" - }, - "ssh_public_keys": { - "description": "SSH public key contents that will be added to each Spark node in this cluster. The\ncorresponding private keys can be used to login with the user name `ubuntu` on port `2200`.\nUp to 10 keys can be specified.", - "items": { - "description": "" - } - }, - "workload_type": { - "description": "", - "properties": { - "clients": { - "description": " defined what type of clients can use the cluster. E.g. Notebooks, Jobs", - "properties": { - "jobs": { - "description": "With jobs set, the cluster can be used for jobs" - }, - "notebooks": { - "description": "With notebooks set, this cluster can be used for notebooks" - } - } - } - } - } - } - }, - "notebook_task": { - "description": "If notebook_task, indicates that this task must run a notebook. This field may not be specified in conjunction with spark_jar_task.", - "properties": { - "base_parameters": { - "description": "Base parameters to be used for each run of this job. If the run is initiated by a call to :method:jobs/run\nNow with parameters specified, the two parameters maps are merged. If the same key is specified in\n`base_parameters` and in `run-now`, the value from `run-now` is used.\nUse [Task parameter variables](https://docs.databricks.com/jobs.html#parameter-variables) to set parameters containing information about job runs.\n\nIf the notebook takes a parameter that is not specified in the job’s `base_parameters` or the `run-now` override parameters,\nthe default value from the notebook is used.\n\nRetrieve these parameters in a notebook using [dbutils.widgets.get](https://docs.databricks.com/dev-tools/databricks-utils.html#dbutils-widgets).\n\nThe JSON representation of this field cannot exceed 1MB.", - "additionalproperties": { - "description": "" - } - }, - "notebook_path": { - "description": "The path of the notebook to be run in the Databricks workspace or remote repository.\nFor notebooks stored in the Databricks workspace, the path must be absolute and begin with a slash.\nFor notebooks stored in a remote repository, the path must be relative. This field is required." - }, - "source": { - "description": "Optional location type of the notebook. When set to `WORKSPACE`, the notebook will be retrieved from the local Databricks workspace. When set to `GIT`, the notebook will be retrieved from a Git repository\ndefined in `git_source`. If the value is empty, the task will use `GIT` if `git_source` is defined and `WORKSPACE` otherwise.\n* `WORKSPACE`: Notebook is located in Databricks workspace.\n* `GIT`: Notebook is located in cloud Git provider." - }, - "warehouse_id": { - "description": "Optional `warehouse_id` to run the notebook on a SQL warehouse. Classic SQL warehouses are NOT supported, please use serverless or pro SQL warehouses.\n\nNote that SQL warehouses only support SQL cells; if the notebook contains non-SQL cells, the run will fail." - } - } - }, - "notification_settings": { - "description": "Optional notification settings that are used when sending notifications to each of the `email_notifications` and `webhook_notifications` for this task.", - "properties": { - "alert_on_last_attempt": { - "description": "If true, do not send notifications to recipients specified in `on_start` for the retried runs and do not send notifications to recipients specified in `on_failure` until the last retry of the run." - }, - "no_alert_for_canceled_runs": { - "description": "If true, do not send notifications to recipients specified in `on_failure` if the run is canceled." - }, - "no_alert_for_skipped_runs": { - "description": "If true, do not send notifications to recipients specified in `on_failure` if the run is skipped." - } - } - }, - "pipeline_task": { - "description": "If pipeline_task, indicates that this task must execute a Pipeline.", - "properties": { - "full_refresh": { - "description": "If true, triggers a full refresh on the delta live table." - }, - "pipeline_id": { - "description": "The full name of the pipeline task to execute." - } - } - }, - "python_wheel_task": { - "description": "If python_wheel_task, indicates that this job must execute a PythonWheel.", - "properties": { - "entry_point": { - "description": "Named entry point to use, if it does not exist in the metadata of the package it executes the function from the package directly using `$packageName.$entryPoint()`" - }, - "named_parameters": { - "description": "Command-line parameters passed to Python wheel task in the form of `[\"--name=task\", \"--data=dbfs:/path/to/data.json\"]`. Leave it empty if `parameters` is not null.", - "additionalproperties": { - "description": "" - } - }, - "package_name": { - "description": "Name of the package to execute" - }, - "parameters": { - "description": "Command-line parameters passed to Python wheel task. Leave it empty if `named_parameters` is not null.", - "items": { - "description": "" - } - } - } - }, - "retry_on_timeout": { - "description": "An optional policy to specify whether to retry a job when it times out. The default behavior\nis to not retry on timeout." - }, - "run_if": { - "description": "An optional value specifying the condition determining whether the task is run once its dependencies have been completed.\n\n* `ALL_SUCCESS`: All dependencies have executed and succeeded\n* `AT_LEAST_ONE_SUCCESS`: At least one dependency has succeeded\n* `NONE_FAILED`: None of the dependencies have failed and at least one was executed\n* `ALL_DONE`: All dependencies have been completed\n* `AT_LEAST_ONE_FAILED`: At least one dependency failed\n* `ALL_FAILED`: ALl dependencies have failed" - }, - "run_job_task": { - "description": "If run_job_task, indicates that this task must execute another job.", - "properties": { - "dbt_commands": { - "description": "An array of commands to execute for jobs with the dbt task, for example `\"dbt_commands\": [\"dbt deps\", \"dbt seed\", \"dbt deps\", \"dbt seed\", \"dbt run\"]`", - "items": { - "description": "" - } - }, - "jar_params": { - "description": "A list of parameters for jobs with Spark JAR tasks, for example `\"jar_params\": [\"john doe\", \"35\"]`.\nThe parameters are used to invoke the main function of the main class specified in the Spark JAR task.\nIf not specified upon `run-now`, it defaults to an empty list.\njar_params cannot be specified in conjunction with notebook_params.\nThe JSON representation of this field (for example `{\"jar_params\":[\"john doe\",\"35\"]}`) cannot exceed 10,000 bytes.\n\nUse [Task parameter variables](/jobs.html\\\"#parameter-variables\\\") to set parameters containing information about job runs.", - "items": { - "description": "" - } - }, - "job_id": { - "description": "ID of the job to trigger." - }, - "job_parameters": { - "description": "Job-level parameters used to trigger the job.", - "additionalproperties": { - "description": "" - } - }, - "notebook_params": { - "description": "A map from keys to values for jobs with notebook task, for example `\"notebook_params\": {\"name\": \"john doe\", \"age\": \"35\"}`.\nThe map is passed to the notebook and is accessible through the [dbutils.widgets.get](https://docs.databricks.com/dev-tools/databricks-utils.html) function.\n\nIf not specified upon `run-now`, the triggered run uses the job’s base parameters.\n\nnotebook_params cannot be specified in conjunction with jar_params.\n\nUse [Task parameter variables](https://docs.databricks.com/jobs.html#parameter-variables) to set parameters containing information about job runs.\n\nThe JSON representation of this field (for example `{\"notebook_params\":{\"name\":\"john doe\",\"age\":\"35\"}}`) cannot exceed 10,000 bytes.", - "additionalproperties": { - "description": "" - } - }, - "pipeline_params": { - "description": "", - "properties": { - "full_refresh": { - "description": "If true, triggers a full refresh on the delta live table." - } - } - }, - "python_named_params": { - "description": "", - "additionalproperties": { - "description": "" - } - }, - "python_params": { - "description": "A list of parameters for jobs with Python tasks, for example `\"python_params\": [\"john doe\", \"35\"]`.\nThe parameters are passed to Python file as command-line parameters. If specified upon `run-now`, it would overwrite\nthe parameters specified in job setting. The JSON representation of this field (for example `{\"python_params\":[\"john doe\",\"35\"]}`)\ncannot exceed 10,000 bytes.\n\nUse [Task parameter variables](https://docs.databricks.com/jobs.html#parameter-variables) to set parameters containing information about job runs.\n\nImportant\n\nThese parameters accept only Latin characters (ASCII character set). Using non-ASCII characters returns an error.\nExamples of invalid, non-ASCII characters are Chinese, Japanese kanjis, and emojis.", - "items": { - "description": "" - } - }, - "spark_submit_params": { - "description": "A list of parameters for jobs with spark submit task, for example `\"spark_submit_params\": [\"--class\", \"org.apache.spark.examples.SparkPi\"]`.\nThe parameters are passed to spark-submit script as command-line parameters. If specified upon `run-now`, it would overwrite the\nparameters specified in job setting. The JSON representation of this field (for example `{\"python_params\":[\"john doe\",\"35\"]}`)\ncannot exceed 10,000 bytes.\n\nUse [Task parameter variables](https://docs.databricks.com/jobs.html#parameter-variables) to set parameters containing information about job runs\n\nImportant\n\nThese parameters accept only Latin characters (ASCII character set). Using non-ASCII characters returns an error.\nExamples of invalid, non-ASCII characters are Chinese, Japanese kanjis, and emojis.", - "items": { - "description": "" - } - }, - "sql_params": { - "description": "A map from keys to values for jobs with SQL task, for example `\"sql_params\": {\"name\": \"john doe\", \"age\": \"35\"}`. The SQL alert task does not support custom parameters.", - "additionalproperties": { - "description": "" - } - } - } - }, - "spark_jar_task": { - "description": "If spark_jar_task, indicates that this task must run a JAR.", - "properties": { - "jar_uri": { - "description": "Deprecated since 04/2016. Provide a `jar` through the `libraries` field instead. For an example, see :method:jobs/create." - }, - "main_class_name": { - "description": "The full name of the class containing the main method to be executed. This class must be contained in a JAR provided as a library.\n\nThe code must use `SparkContext.getOrCreate` to obtain a Spark context; otherwise, runs of the job fail." - }, - "parameters": { - "description": "Parameters passed to the main method.\n\nUse [Task parameter variables](https://docs.databricks.com/jobs.html#parameter-variables) to set parameters containing information about job runs.", - "items": { - "description": "" - } - } - } - }, - "spark_python_task": { - "description": "If spark_python_task, indicates that this task must run a Python file.", - "properties": { - "parameters": { - "description": "Command line parameters passed to the Python file.\n\nUse [Task parameter variables](https://docs.databricks.com/jobs.html#parameter-variables) to set parameters containing information about job runs.", - "items": { - "description": "" - } - }, - "python_file": { - "description": "The Python file to be executed. Cloud file URIs (such as dbfs:/, s3:/, adls:/, gcs:/) and workspace paths are supported. For python files stored in the Databricks workspace, the path must be absolute and begin with `/`. For files stored in a remote repository, the path must be relative. This field is required." - }, - "source": { - "description": "Optional location type of the Python file. When set to `WORKSPACE` or not specified, the file will be retrieved from the local\nDatabricks workspace or cloud location (if the `python_file` has a URI format). When set to `GIT`,\nthe Python file will be retrieved from a Git repository defined in `git_source`.\n\n* `WORKSPACE`: The Python file is located in a Databricks workspace or at a cloud filesystem URI.\n* `GIT`: The Python file is located in a remote Git repository." - } - } - }, - "spark_submit_task": { - "description": "If `spark_submit_task`, indicates that this task must be launched by the spark submit script. This task can run only on new clusters.\n\nIn the `new_cluster` specification, `libraries` and `spark_conf` are not supported. Instead, use `--jars` and `--py-files` to add Java and Python libraries and `--conf` to set the Spark configurations.\n\n`master`, `deploy-mode`, and `executor-cores` are automatically configured by Databricks; you _cannot_ specify them in parameters.\n\nBy default, the Spark submit job uses all available memory (excluding reserved memory for Databricks services). You can set `--driver-memory`, and `--executor-memory` to a smaller value to leave some room for off-heap usage.\n\nThe `--jars`, `--py-files`, `--files` arguments support DBFS and S3 paths.", - "properties": { - "parameters": { - "description": "Command-line parameters passed to spark submit.\n\nUse [Task parameter variables](https://docs.databricks.com/jobs.html#parameter-variables) to set parameters containing information about job runs.", - "items": { - "description": "" - } - } - } - }, - "sql_task": { - "description": "If sql_task, indicates that this job must execute a SQL task.", - "properties": { - "alert": { - "description": "If alert, indicates that this job must refresh a SQL alert.", - "properties": { - "alert_id": { - "description": "The canonical identifier of the SQL alert." - }, - "pause_subscriptions": { - "description": "If true, the alert notifications are not sent to subscribers." - }, - "subscriptions": { - "description": "If specified, alert notifications are sent to subscribers.", - "items": { - "description": "", - "properties": { - "destination_id": { - "description": "The canonical identifier of the destination to receive email notification. This parameter is mutually exclusive with user_name. You cannot set both destination_id and user_name for subscription notifications." - }, - "user_name": { - "description": "The user name to receive the subscription email. This parameter is mutually exclusive with destination_id. You cannot set both destination_id and user_name for subscription notifications." - } - } - } - } - } - }, - "dashboard": { - "description": "If dashboard, indicates that this job must refresh a SQL dashboard.", - "properties": { - "custom_subject": { - "description": "Subject of the email sent to subscribers of this task." - }, - "dashboard_id": { - "description": "The canonical identifier of the SQL dashboard." - }, - "pause_subscriptions": { - "description": "If true, the dashboard snapshot is not taken, and emails are not sent to subscribers." - }, - "subscriptions": { - "description": "If specified, dashboard snapshots are sent to subscriptions.", - "items": { - "description": "", - "properties": { - "destination_id": { - "description": "The canonical identifier of the destination to receive email notification. This parameter is mutually exclusive with user_name. You cannot set both destination_id and user_name for subscription notifications." - }, - "user_name": { - "description": "The user name to receive the subscription email. This parameter is mutually exclusive with destination_id. You cannot set both destination_id and user_name for subscription notifications." - } - } - } - } - } - }, - "file": { - "description": "If file, indicates that this job runs a SQL file in a remote Git repository.", - "properties": { - "path": { - "description": "Path of the SQL file. Must be relative if the source is a remote Git repository and absolute for workspace paths." - }, - "source": { - "description": "Optional location type of the SQL file. When set to `WORKSPACE`, the SQL file will be retrieved\nfrom the local Databricks workspace. When set to `GIT`, the SQL file will be retrieved from a Git repository\ndefined in `git_source`. If the value is empty, the task will use `GIT` if `git_source` is defined and `WORKSPACE` otherwise.\n\n* `WORKSPACE`: SQL file is located in Databricks workspace.\n* `GIT`: SQL file is located in cloud Git provider." - } - } - }, - "parameters": { - "description": "Parameters to be used for each run of this job. The SQL alert task does not support custom parameters.", - "additionalproperties": { - "description": "" - } - }, - "query": { - "description": "If query, indicates that this job must execute a SQL query.", - "properties": { - "query_id": { - "description": "The canonical identifier of the SQL query." - } - } - }, - "warehouse_id": { - "description": "The canonical identifier of the SQL warehouse. Recommended to use with serverless or pro SQL warehouses. Classic SQL warehouses are only supported for SQL alert, dashboard and query tasks and are limited to scheduled single-task jobs." - } - } - }, - "task_key": { - "description": "A unique name for the task. This field is used to refer to this task from other tasks.\nThis field is required and must be unique within its parent job.\nOn Update or Reset, this field is used to reference the tasks to be updated or reset." - }, - "timeout_seconds": { - "description": "An optional timeout applied to each run of this job task. A value of `0` means no timeout." - }, - "webhook_notifications": { - "description": "A collection of system notification IDs to notify when runs of this task begin or complete. The default behavior is to not send any system notifications.", - "properties": { - "on_duration_warning_threshold_exceeded": { - "description": "An optional list of system notification IDs to call when the duration of a run exceeds the threshold specified for the `RUN_DURATION_SECONDS` metric in the `health` field. A maximum of 3 destinations can be specified for the `on_duration_warning_threshold_exceeded` property.", - "items": { - "description": "", - "properties": { - "id": { - "description": "" - } - } - } - }, - "on_failure": { - "description": "An optional list of system notification IDs to call when the run fails. A maximum of 3 destinations can be specified for the `on_failure` property.", - "items": { - "description": "", - "properties": { - "id": { - "description": "" - } - } - } - }, - "on_start": { - "description": "An optional list of system notification IDs to call when the run starts. A maximum of 3 destinations can be specified for the `on_start` property.", - "items": { - "description": "", - "properties": { - "id": { - "description": "" - } - } - } - }, - "on_streaming_backlog_exceeded": { - "description": "An optional list of system notification IDs to call when any streaming backlog thresholds are exceeded for any stream.\nStreaming backlog thresholds can be set in the `health` field using the following metrics: `STREAMING_BACKLOG_BYTES`, `STREAMING_BACKLOG_RECORDS`, `STREAMING_BACKLOG_SECONDS`, or `STREAMING_BACKLOG_FILES`.\nAlerting is based on the 10-minute average of these metrics. If the issue persists, notifications are resent every 30 minutes.\nA maximum of 3 destinations can be specified for the `on_streaming_backlog_exceeded` property.", - "items": { - "description": "", - "properties": { - "id": { - "description": "" - } - } - } - }, - "on_success": { - "description": "An optional list of system notification IDs to call when the run completes successfully. A maximum of 3 destinations can be specified for the `on_success` property.", - "items": { - "description": "", - "properties": { - "id": { - "description": "" - } - } - } - } - } - } - } - } - }, - "timeout_seconds": { - "description": "An optional timeout applied to each run of this job. A value of `0` means no timeout." - }, - "trigger": { - "description": "A configuration to trigger a run when certain conditions are met. The default behavior is that the job runs only when triggered by clicking “Run Now” in the Jobs UI or sending an API request to `runNow`.", - "properties": { - "file_arrival": { - "description": "File arrival trigger settings.", - "properties": { - "min_time_between_triggers_seconds": { - "description": "If set, the trigger starts a run only after the specified amount of time passed since\nthe last time the trigger fired. The minimum allowed value is 60 seconds" - }, - "url": { - "description": "URL to be monitored for file arrivals. The path must point to the root or a subpath of the external location." - }, - "wait_after_last_change_seconds": { - "description": "If set, the trigger starts a run only after no file activity has occurred for the specified amount of time.\nThis makes it possible to wait for a batch of incoming files to arrive before triggering a run. The\nminimum allowed value is 60 seconds." - } - } - }, - "pause_status": { - "description": "Whether this trigger is paused or not." - }, - "periodic": { - "description": "Periodic trigger settings.", - "properties": { - "interval": { - "description": "The interval at which the trigger should run." - }, - "unit": { - "description": "The unit of time for the interval." - } - } - }, - "table": { - "description": "Old table trigger settings name. Deprecated in favor of `table_update`.", - "properties": { - "condition": { - "description": "The table(s) condition based on which to trigger a job run." - }, - "min_time_between_triggers_seconds": { - "description": "If set, the trigger starts a run only after the specified amount of time has passed since\nthe last time the trigger fired. The minimum allowed value is 60 seconds." - }, - "table_names": { - "description": "A list of Delta tables to monitor for changes. The table name must be in the format `catalog_name.schema_name.table_name`.", - "items": { - "description": "" - } - }, - "wait_after_last_change_seconds": { - "description": "If set, the trigger starts a run only after no table updates have occurred for the specified time\nand can be used to wait for a series of table updates before triggering a run. The\nminimum allowed value is 60 seconds." - } - } - }, - "table_update": { - "description": "", - "properties": { - "condition": { - "description": "The table(s) condition based on which to trigger a job run." - }, - "min_time_between_triggers_seconds": { - "description": "If set, the trigger starts a run only after the specified amount of time has passed since\nthe last time the trigger fired. The minimum allowed value is 60 seconds." - }, - "table_names": { - "description": "A list of Delta tables to monitor for changes. The table name must be in the format `catalog_name.schema_name.table_name`.", - "items": { - "description": "" - } - }, - "wait_after_last_change_seconds": { - "description": "If set, the trigger starts a run only after no table updates have occurred for the specified time\nand can be used to wait for a series of table updates before triggering a run. The\nminimum allowed value is 60 seconds." - } - } - } - } - }, - "webhook_notifications": { - "description": "A collection of system notification IDs to notify when runs of this job begin or complete.", - "properties": { - "on_duration_warning_threshold_exceeded": { - "description": "An optional list of system notification IDs to call when the duration of a run exceeds the threshold specified for the `RUN_DURATION_SECONDS` metric in the `health` field. A maximum of 3 destinations can be specified for the `on_duration_warning_threshold_exceeded` property.", - "items": { - "description": "", - "properties": { - "id": { - "description": "" - } - } - } - }, - "on_failure": { - "description": "An optional list of system notification IDs to call when the run fails. A maximum of 3 destinations can be specified for the `on_failure` property.", - "items": { - "description": "", - "properties": { - "id": { - "description": "" - } - } - } - }, - "on_start": { - "description": "An optional list of system notification IDs to call when the run starts. A maximum of 3 destinations can be specified for the `on_start` property.", - "items": { - "description": "", - "properties": { - "id": { - "description": "" - } - } - } - }, - "on_streaming_backlog_exceeded": { - "description": "An optional list of system notification IDs to call when any streaming backlog thresholds are exceeded for any stream.\nStreaming backlog thresholds can be set in the `health` field using the following metrics: `STREAMING_BACKLOG_BYTES`, `STREAMING_BACKLOG_RECORDS`, `STREAMING_BACKLOG_SECONDS`, or `STREAMING_BACKLOG_FILES`.\nAlerting is based on the 10-minute average of these metrics. If the issue persists, notifications are resent every 30 minutes.\nA maximum of 3 destinations can be specified for the `on_streaming_backlog_exceeded` property.", - "items": { - "description": "", - "properties": { - "id": { - "description": "" - } - } - } - }, - "on_success": { - "description": "An optional list of system notification IDs to call when the run completes successfully. A maximum of 3 destinations can be specified for the `on_success` property.", - "items": { - "description": "", - "properties": { - "id": { - "description": "" - } - } - } - } - } - } - } - } - }, - "model_serving_endpoints": { - "description": "List of Model Serving Endpoints", - "additionalproperties": { - "description": "", - "properties": { - "config": { - "description": "The core config of the serving endpoint.", - "properties": { - "auto_capture_config": { - "description": "Configuration for Inference Tables which automatically logs requests and responses to Unity Catalog.", - "properties": { - "catalog_name": { - "description": "The name of the catalog in Unity Catalog. NOTE: On update, you cannot change the catalog name if the inference table is already enabled." - }, - "enabled": { - "description": "Indicates whether the inference table is enabled." - }, - "schema_name": { - "description": "The name of the schema in Unity Catalog. NOTE: On update, you cannot change the schema name if the inference table is already enabled." - }, - "table_name_prefix": { - "description": "The prefix of the table in Unity Catalog. NOTE: On update, you cannot change the prefix name if the inference table is already enabled." - } - } - }, - "served_entities": { - "description": "A list of served entities for the endpoint to serve. A serving endpoint can have up to 15 served entities.", - "items": { - "description": "", - "properties": { - "entity_name": { - "description": "The name of the entity to be served. The entity may be a model in the Databricks Model Registry, a model in the Unity Catalog (UC),\nor a function of type FEATURE_SPEC in the UC. If it is a UC object, the full name of the object should be given in the form of\n__catalog_name__.__schema_name__.__model_name__.\n" - }, - "entity_version": { - "description": "The version of the model in Databricks Model Registry to be served or empty if the entity is a FEATURE_SPEC." - }, - "environment_vars": { - "description": "An object containing a set of optional, user-specified environment variable key-value pairs used for serving this entity.\nNote: this is an experimental feature and subject to change. \nExample entity environment variables that refer to Databricks secrets: `{\"OPENAI_API_KEY\": \"{{secrets/my_scope/my_key}}\", \"DATABRICKS_TOKEN\": \"{{secrets/my_scope2/my_key2}}\"}`", - "additionalproperties": { - "description": "" - } - }, - "external_model": { - "description": "The external model to be served. NOTE: Only one of external_model and (entity_name, entity_version, workload_size, workload_type, and scale_to_zero_enabled)\ncan be specified with the latter set being used for custom model serving for a Databricks registered model. For an existing endpoint with external_model,\nit cannot be updated to an endpoint without external_model. If the endpoint is created without external_model, users cannot update it to add external_model later.\nThe task type of all external models within an endpoint must be the same.\n", - "properties": { - "ai21labs_config": { - "description": "AI21Labs Config. Only required if the provider is 'ai21labs'.", - "properties": { - "ai21labs_api_key": { - "description": "The Databricks secret key reference for an AI21 Labs API key. If you prefer to paste your API key directly, see `ai21labs_api_key_plaintext`. You must provide an API key using one of the following fields: `ai21labs_api_key` or `ai21labs_api_key_plaintext`." - }, - "ai21labs_api_key_plaintext": { - "description": "An AI21 Labs API key provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `ai21labs_api_key`. You must provide an API key using one of the following fields: `ai21labs_api_key` or `ai21labs_api_key_plaintext`." - } - } - }, - "amazon_bedrock_config": { - "description": "Amazon Bedrock Config. Only required if the provider is 'amazon-bedrock'.", - "properties": { - "aws_access_key_id": { - "description": "The Databricks secret key reference for an AWS access key ID with permissions to interact with Bedrock services. If you prefer to paste your API key directly, see `aws_access_key_id`. You must provide an API key using one of the following fields: `aws_access_key_id` or `aws_access_key_id_plaintext`." - }, - "aws_access_key_id_plaintext": { - "description": "An AWS access key ID with permissions to interact with Bedrock services provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `aws_access_key_id`. You must provide an API key using one of the following fields: `aws_access_key_id` or `aws_access_key_id_plaintext`." - }, - "aws_region": { - "description": "The AWS region to use. Bedrock has to be enabled there." - }, - "aws_secret_access_key": { - "description": "The Databricks secret key reference for an AWS secret access key paired with the access key ID, with permissions to interact with Bedrock services. If you prefer to paste your API key directly, see `aws_secret_access_key_plaintext`. You must provide an API key using one of the following fields: `aws_secret_access_key` or `aws_secret_access_key_plaintext`." - }, - "aws_secret_access_key_plaintext": { - "description": "An AWS secret access key paired with the access key ID, with permissions to interact with Bedrock services provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `aws_secret_access_key`. You must provide an API key using one of the following fields: `aws_secret_access_key` or `aws_secret_access_key_plaintext`." - }, - "bedrock_provider": { - "description": "The underlying provider in Amazon Bedrock. Supported values (case insensitive) include: Anthropic, Cohere, AI21Labs, Amazon." - } - } - }, - "anthropic_config": { - "description": "Anthropic Config. Only required if the provider is 'anthropic'.", - "properties": { - "anthropic_api_key": { - "description": "The Databricks secret key reference for an Anthropic API key. If you prefer to paste your API key directly, see `anthropic_api_key_plaintext`. You must provide an API key using one of the following fields: `anthropic_api_key` or `anthropic_api_key_plaintext`." - }, - "anthropic_api_key_plaintext": { - "description": "The Anthropic API key provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `anthropic_api_key`. You must provide an API key using one of the following fields: `anthropic_api_key` or `anthropic_api_key_plaintext`." - } - } - }, - "cohere_config": { - "description": "Cohere Config. Only required if the provider is 'cohere'.", - "properties": { - "cohere_api_base": { - "description": "This is an optional field to provide a customized base URL for the Cohere API. \nIf left unspecified, the standard Cohere base URL is used.\n" - }, - "cohere_api_key": { - "description": "The Databricks secret key reference for a Cohere API key. If you prefer to paste your API key directly, see `cohere_api_key_plaintext`. You must provide an API key using one of the following fields: `cohere_api_key` or `cohere_api_key_plaintext`." - }, - "cohere_api_key_plaintext": { - "description": "The Cohere API key provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `cohere_api_key`. You must provide an API key using one of the following fields: `cohere_api_key` or `cohere_api_key_plaintext`." - } - } - }, - "databricks_model_serving_config": { - "description": "Databricks Model Serving Config. Only required if the provider is 'databricks-model-serving'.", - "properties": { - "databricks_api_token": { - "description": "The Databricks secret key reference for a Databricks API token that corresponds to a user or service\nprincipal with Can Query access to the model serving endpoint pointed to by this external model.\nIf you prefer to paste your API key directly, see `databricks_api_token_plaintext`.\nYou must provide an API key using one of the following fields: `databricks_api_token` or `databricks_api_token_plaintext`.\n" - }, - "databricks_api_token_plaintext": { - "description": "The Databricks API token that corresponds to a user or service\nprincipal with Can Query access to the model serving endpoint pointed to by this external model provided as a plaintext string.\nIf you prefer to reference your key using Databricks Secrets, see `databricks_api_token`.\nYou must provide an API key using one of the following fields: `databricks_api_token` or `databricks_api_token_plaintext`.\n" - }, - "databricks_workspace_url": { - "description": "The URL of the Databricks workspace containing the model serving endpoint pointed to by this external model.\n" - } - } - }, - "google_cloud_vertex_ai_config": { - "description": "Google Cloud Vertex AI Config. Only required if the provider is 'google-cloud-vertex-ai'.", - "properties": { - "private_key": { - "description": "The Databricks secret key reference for a private key for the service account which has access to the Google Cloud Vertex AI Service. See [Best practices for managing service account keys](https://cloud.google.com/iam/docs/best-practices-for-managing-service-account-keys). If you prefer to paste your API key directly, see `private_key_plaintext`. You must provide an API key using one of the following fields: `private_key` or `private_key_plaintext`" - }, - "private_key_plaintext": { - "description": "The private key for the service account which has access to the Google Cloud Vertex AI Service provided as a plaintext secret. See [Best practices for managing service account keys](https://cloud.google.com/iam/docs/best-practices-for-managing-service-account-keys). If you prefer to reference your key using Databricks Secrets, see `private_key`. You must provide an API key using one of the following fields: `private_key` or `private_key_plaintext`." - }, - "project_id": { - "description": "This is the Google Cloud project id that the service account is associated with." - }, - "region": { - "description": "This is the region for the Google Cloud Vertex AI Service. See [supported regions](https://cloud.google.com/vertex-ai/docs/general/locations) for more details. Some models are only available in specific regions." - } - } - }, - "name": { - "description": "The name of the external model." - }, - "openai_config": { - "description": "OpenAI Config. Only required if the provider is 'openai'.", - "properties": { - "microsoft_entra_client_id": { - "description": "This field is only required for Azure AD OpenAI and is the Microsoft Entra Client ID.\n" - }, - "microsoft_entra_client_secret": { - "description": "The Databricks secret key reference for a client secret used for Microsoft Entra ID authentication.\nIf you prefer to paste your client secret directly, see `microsoft_entra_client_secret_plaintext`.\nYou must provide an API key using one of the following fields: `microsoft_entra_client_secret` or `microsoft_entra_client_secret_plaintext`.\n" - }, - "microsoft_entra_client_secret_plaintext": { - "description": "The client secret used for Microsoft Entra ID authentication provided as a plaintext string.\nIf you prefer to reference your key using Databricks Secrets, see `microsoft_entra_client_secret`.\nYou must provide an API key using one of the following fields: `microsoft_entra_client_secret` or `microsoft_entra_client_secret_plaintext`.\n" - }, - "microsoft_entra_tenant_id": { - "description": "This field is only required for Azure AD OpenAI and is the Microsoft Entra Tenant ID.\n" - }, - "openai_api_base": { - "description": "This is a field to provide a customized base URl for the OpenAI API.\nFor Azure OpenAI, this field is required, and is the base URL for the Azure OpenAI API service\nprovided by Azure.\nFor other OpenAI API types, this field is optional, and if left unspecified, the standard OpenAI base URL is used.\n" - }, - "openai_api_key": { - "description": "The Databricks secret key reference for an OpenAI API key using the OpenAI or Azure service. If you prefer to paste your API key directly, see `openai_api_key_plaintext`. You must provide an API key using one of the following fields: `openai_api_key` or `openai_api_key_plaintext`." - }, - "openai_api_key_plaintext": { - "description": "The OpenAI API key using the OpenAI or Azure service provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `openai_api_key`. You must provide an API key using one of the following fields: `openai_api_key` or `openai_api_key_plaintext`." - }, - "openai_api_type": { - "description": "This is an optional field to specify the type of OpenAI API to use.\nFor Azure OpenAI, this field is required, and adjust this parameter to represent the preferred security\naccess validation protocol. For access token validation, use azure. For authentication using Azure Active\nDirectory (Azure AD) use, azuread.\n" - }, - "openai_api_version": { - "description": "This is an optional field to specify the OpenAI API version.\nFor Azure OpenAI, this field is required, and is the version of the Azure OpenAI service to\nutilize, specified by a date.\n" - }, - "openai_deployment_name": { - "description": "This field is only required for Azure OpenAI and is the name of the deployment resource for the\nAzure OpenAI service.\n" - }, - "openai_organization": { - "description": "This is an optional field to specify the organization in OpenAI or Azure OpenAI.\n" - } - } - }, - "palm_config": { - "description": "PaLM Config. Only required if the provider is 'palm'.", - "properties": { - "palm_api_key": { - "description": "The Databricks secret key reference for a PaLM API key. If you prefer to paste your API key directly, see `palm_api_key_plaintext`. You must provide an API key using one of the following fields: `palm_api_key` or `palm_api_key_plaintext`." - }, - "palm_api_key_plaintext": { - "description": "The PaLM API key provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `palm_api_key`. You must provide an API key using one of the following fields: `palm_api_key` or `palm_api_key_plaintext`." - } - } - }, - "provider": { - "description": "The name of the provider for the external model. Currently, the supported providers are 'ai21labs', 'anthropic',\n'amazon-bedrock', 'cohere', 'databricks-model-serving', 'google-cloud-vertex-ai', 'openai', and 'palm'.\",\n" - }, - "task": { - "description": "The task type of the external model." - } - } - }, - "instance_profile_arn": { - "description": "ARN of the instance profile that the served entity uses to access AWS resources." - }, - "max_provisioned_throughput": { - "description": "The maximum tokens per second that the endpoint can scale up to." - }, - "min_provisioned_throughput": { - "description": "The minimum tokens per second that the endpoint can scale down to." - }, - "name": { - "description": "The name of a served entity. It must be unique across an endpoint. A served entity name can consist of alphanumeric characters, dashes, and underscores.\nIf not specified for an external model, this field defaults to external_model.name, with '.' and ':' replaced with '-', and if not specified for other\nentities, it defaults to \u003centity-name\u003e-\u003centity-version\u003e.\n" - }, - "scale_to_zero_enabled": { - "description": "Whether the compute resources for the served entity should scale down to zero." - }, - "workload_size": { - "description": "The workload size of the served entity. The workload size corresponds to a range of provisioned concurrency that the compute autoscales between.\nA single unit of provisioned concurrency can process one request at a time.\nValid workload sizes are \"Small\" (4 - 4 provisioned concurrency), \"Medium\" (8 - 16 provisioned concurrency), and \"Large\" (16 - 64 provisioned concurrency).\nIf scale-to-zero is enabled, the lower bound of the provisioned concurrency for each workload size is 0.\n" - }, - "workload_type": { - "description": "The workload type of the served entity. The workload type selects which type of compute to use in the endpoint. The default value for this parameter is\n\"CPU\". For deep learning workloads, GPU acceleration is available by selecting workload types like GPU_SMALL and others.\nSee the available [GPU types](https://docs.databricks.com/machine-learning/model-serving/create-manage-serving-endpoints.html#gpu-workload-types).\n" - } - } - } - }, - "served_models": { - "description": "(Deprecated, use served_entities instead) A list of served models for the endpoint to serve. A serving endpoint can have up to 15 served models.", - "items": { - "description": "", - "properties": { - "environment_vars": { - "description": "An object containing a set of optional, user-specified environment variable key-value pairs used for serving this model.\nNote: this is an experimental feature and subject to change. \nExample model environment variables that refer to Databricks secrets: `{\"OPENAI_API_KEY\": \"{{secrets/my_scope/my_key}}\", \"DATABRICKS_TOKEN\": \"{{secrets/my_scope2/my_key2}}\"}`", - "additionalproperties": { - "description": "" - } - }, - "instance_profile_arn": { - "description": "ARN of the instance profile that the served model will use to access AWS resources." - }, - "model_name": { - "description": "The name of the model in Databricks Model Registry to be served or if the model resides in Unity Catalog, the full name of model,\nin the form of __catalog_name__.__schema_name__.__model_name__.\n" - }, - "model_version": { - "description": "The version of the model in Databricks Model Registry or Unity Catalog to be served." - }, - "name": { - "description": "The name of a served model. It must be unique across an endpoint. If not specified, this field will default to \u003cmodel-name\u003e-\u003cmodel-version\u003e.\nA served model name can consist of alphanumeric characters, dashes, and underscores.\n" - }, - "scale_to_zero_enabled": { - "description": "Whether the compute resources for the served model should scale down to zero." - }, - "workload_size": { - "description": "The workload size of the served model. The workload size corresponds to a range of provisioned concurrency that the compute will autoscale between.\nA single unit of provisioned concurrency can process one request at a time.\nValid workload sizes are \"Small\" (4 - 4 provisioned concurrency), \"Medium\" (8 - 16 provisioned concurrency), and \"Large\" (16 - 64 provisioned concurrency).\nIf scale-to-zero is enabled, the lower bound of the provisioned concurrency for each workload size will be 0.\n" - }, - "workload_type": { - "description": "The workload type of the served model. The workload type selects which type of compute to use in the endpoint. The default value for this parameter is\n\"CPU\". For deep learning workloads, GPU acceleration is available by selecting workload types like GPU_SMALL and others.\nSee the available [GPU types](https://docs.databricks.com/machine-learning/model-serving/create-manage-serving-endpoints.html#gpu-workload-types).\n" - } - } - } - }, - "traffic_config": { - "description": "The traffic config defining how invocations to the serving endpoint should be routed.", - "properties": { - "routes": { - "description": "The list of routes that define traffic to each served entity.", - "items": { - "description": "", - "properties": { - "served_model_name": { - "description": "The name of the served model this route configures traffic for." - }, - "traffic_percentage": { - "description": "The percentage of endpoint traffic to send to this route. It must be an integer between 0 and 100 inclusive." - } - } - } - } - } - } - } - }, - "name": { - "description": "The name of the serving endpoint. This field is required and must be unique across a Databricks workspace.\nAn endpoint name can consist of alphanumeric characters, dashes, and underscores.\n" - }, - "permissions": { - "description": "", - "items": { - "description": "", - "properties": { - "group_name": { - "description": "" - }, - "level": { - "description": "" - }, - "service_principal_name": { - "description": "" - }, - "user_name": { - "description": "" - } - } - } - }, - "rate_limits": { - "description": "Rate limits to be applied to the serving endpoint. NOTE: only external and foundation model endpoints are supported as of now.", - "items": { - "description": "", - "properties": { - "calls": { - "description": "Used to specify how many calls are allowed for a key within the renewal_period." - }, - "key": { - "description": "Key field for a serving endpoint rate limit. Currently, only 'user' and 'endpoint' are supported, with 'endpoint' being the default if not specified." - }, - "renewal_period": { - "description": "Renewal period field for a serving endpoint rate limit. Currently, only 'minute' is supported." - } - } - } - }, - "route_optimized": { - "description": "Enable route optimization for the serving endpoint." - }, - "tags": { - "description": "Tags to be attached to the serving endpoint and automatically propagated to billing logs.", - "items": { - "description": "", - "properties": { - "key": { - "description": "Key field for a serving endpoint tag." - }, - "value": { - "description": "Optional value field for a serving endpoint tag." - } - } - } - } - } - } - }, - "models": { - "description": "List of MLflow models", - "additionalproperties": { - "description": "", - "properties": { - "creation_timestamp": { - "description": "Timestamp recorded when this `registered_model` was created." - }, - "description": { - "description": "Description of this `registered_model`." - }, - "last_updated_timestamp": { - "description": "Timestamp recorded when metadata for this `registered_model` was last updated." - }, - "latest_versions": { - "description": "Collection of latest model versions for each stage.\nOnly contains models with current `READY` status.", - "items": { - "description": "", - "properties": { - "creation_timestamp": { - "description": "Timestamp recorded when this `model_version` was created." - }, - "current_stage": { - "description": "Current stage for this `model_version`." - }, - "description": { - "description": "Description of this `model_version`." - }, - "last_updated_timestamp": { - "description": "Timestamp recorded when metadata for this `model_version` was last updated." - }, - "name": { - "description": "Unique name of the model" - }, - "run_id": { - "description": "MLflow run ID used when creating `model_version`, if `source` was generated by an\nexperiment run stored in MLflow tracking server." - }, - "run_link": { - "description": "Run Link: Direct link to the run that generated this version" - }, - "source": { - "description": "URI indicating the location of the source model artifacts, used when creating `model_version`" - }, - "status": { - "description": "Current status of `model_version`" - }, - "status_message": { - "description": "Details on current `status`, if it is pending or failed." - }, - "tags": { - "description": "Tags: Additional metadata key-value pairs for this `model_version`.", - "items": { - "description": "", - "properties": { - "key": { - "description": "The tag key." - }, - "value": { - "description": "The tag value." - } - } - } - }, - "user_id": { - "description": "User that created this `model_version`." - }, - "version": { - "description": "Model's version number." - } - } - } - }, - "name": { - "description": "Unique name for the model." - }, - "permissions": { - "description": "", - "items": { - "description": "", - "properties": { - "group_name": { - "description": "" - }, - "level": { - "description": "" - }, - "service_principal_name": { - "description": "" - }, - "user_name": { - "description": "" - } - } - } - }, - "tags": { - "description": "Tags: Additional metadata key-value pairs for this `registered_model`.", - "items": { - "description": "", - "properties": { - "key": { - "description": "The tag key." - }, - "value": { - "description": "The tag value." - } - } - } - }, - "user_id": { - "description": "User that created this `registered_model`" - } - } - } - }, - "pipelines": { - "description": "List of DLT pipelines", - "additionalproperties": { - "description": "", - "properties": { - "catalog": { - "description": "A catalog in Unity Catalog to publish data from this pipeline to. If `target` is specified, tables in this pipeline are published to a `target` schema inside `catalog` (for example, `catalog`.`target`.`table`). If `target` is not specified, no data is published to Unity Catalog." - }, - "channel": { - "description": "DLT Release Channel that specifies which version to use." - }, - "clusters": { - "description": "Cluster settings for this pipeline deployment.", - "items": { - "description": "", - "properties": { - "apply_policy_default_values": { - "description": "Note: This field won't be persisted. Only API users will check this field." - }, - "autoscale": { - "description": "Parameters needed in order to automatically scale clusters up and down based on load.\nNote: autoscaling works best with DB runtime versions 3.0 or later.", - "properties": { - "max_workers": { - "description": "The maximum number of workers to which the cluster can scale up when overloaded. `max_workers` must be strictly greater than `min_workers`." - }, - "min_workers": { - "description": "The minimum number of workers the cluster can scale down to when underutilized.\nIt is also the initial number of workers the cluster will have after creation." - }, - "mode": { - "description": "Databricks Enhanced Autoscaling optimizes cluster utilization by automatically\nallocating cluster resources based on workload volume, with minimal impact to\nthe data processing latency of your pipelines. Enhanced Autoscaling is available\nfor `updates` clusters only. The legacy autoscaling feature is used for `maintenance`\nclusters.\n" - } - } - }, - "aws_attributes": { - "description": "Attributes related to clusters running on Amazon Web Services.\nIf not specified at cluster creation, a set of default values will be used.", - "properties": { - "availability": { - "description": "" - }, - "ebs_volume_count": { - "description": "The number of volumes launched for each instance. Users can choose up to 10 volumes.\nThis feature is only enabled for supported node types. Legacy node types cannot specify\ncustom EBS volumes.\nFor node types with no instance store, at least one EBS volume needs to be specified;\notherwise, cluster creation will fail.\n\nThese EBS volumes will be mounted at `/ebs0`, `/ebs1`, and etc.\nInstance store volumes will be mounted at `/local_disk0`, `/local_disk1`, and etc.\n\nIf EBS volumes are attached, Databricks will configure Spark to use only the EBS volumes for\nscratch storage because heterogenously sized scratch devices can lead to inefficient disk\nutilization. If no EBS volumes are attached, Databricks will configure Spark to use instance\nstore volumes.\n\nPlease note that if EBS volumes are specified, then the Spark configuration `spark.local.dir`\nwill be overridden." - }, - "ebs_volume_iops": { - "description": "If using gp3 volumes, what IOPS to use for the disk. If this is not set, the maximum performance of a gp2 volume with the same volume size will be used." - }, - "ebs_volume_size": { - "description": "The size of each EBS volume (in GiB) launched for each instance. For general purpose\nSSD, this value must be within the range 100 - 4096. For throughput optimized HDD,\nthis value must be within the range 500 - 4096." - }, - "ebs_volume_throughput": { - "description": "If using gp3 volumes, what throughput to use for the disk. If this is not set, the maximum performance of a gp2 volume with the same volume size will be used." - }, - "ebs_volume_type": { - "description": "" - }, - "first_on_demand": { - "description": "The first `first_on_demand` nodes of the cluster will be placed on on-demand instances.\nIf this value is greater than 0, the cluster driver node in particular will be placed on an\non-demand instance. If this value is greater than or equal to the current cluster size, all\nnodes will be placed on on-demand instances. If this value is less than the current cluster\nsize, `first_on_demand` nodes will be placed on on-demand instances and the remainder will\nbe placed on `availability` instances. Note that this value does not affect\ncluster size and cannot currently be mutated over the lifetime of a cluster." - }, - "instance_profile_arn": { - "description": "Nodes for this cluster will only be placed on AWS instances with this instance profile. If\nommitted, nodes will be placed on instances without an IAM instance profile. The instance\nprofile must have previously been added to the Databricks environment by an account\nadministrator.\n\nThis feature may only be available to certain customer plans.\n\nIf this field is ommitted, we will pull in the default from the conf if it exists." - }, - "spot_bid_price_percent": { - "description": "The bid price for AWS spot instances, as a percentage of the corresponding instance type's\non-demand price.\nFor example, if this field is set to 50, and the cluster needs a new `r3.xlarge` spot\ninstance, then the bid price is half of the price of\non-demand `r3.xlarge` instances. Similarly, if this field is set to 200, the bid price is twice\nthe price of on-demand `r3.xlarge` instances. If not specified, the default value is 100.\nWhen spot instances are requested for this cluster, only spot instances whose bid price\npercentage matches this field will be considered.\nNote that, for safety, we enforce this field to be no more than 10000.\n\nThe default value and documentation here should be kept consistent with\nCommonConf.defaultSpotBidPricePercent and CommonConf.maxSpotBidPricePercent." - }, - "zone_id": { - "description": "Identifier for the availability zone/datacenter in which the cluster resides.\nThis string will be of a form like \"us-west-2a\". The provided availability\nzone must be in the same region as the Databricks deployment. For example, \"us-west-2a\"\nis not a valid zone id if the Databricks deployment resides in the \"us-east-1\" region.\nThis is an optional field at cluster creation, and if not specified, a default zone will be used.\nIf the zone specified is \"auto\", will try to place cluster in a zone with high availability,\nand will retry placement in a different AZ if there is not enough capacity.\nThe list of available zones as well as the default value can be found by using the\n`List Zones` method." - } - } - }, - "azure_attributes": { - "description": "Attributes related to clusters running on Microsoft Azure.\nIf not specified at cluster creation, a set of default values will be used.", - "properties": { - "availability": { - "description": "" - }, - "first_on_demand": { - "description": "The first `first_on_demand` nodes of the cluster will be placed on on-demand instances.\nThis value should be greater than 0, to make sure the cluster driver node is placed on an\non-demand instance. If this value is greater than or equal to the current cluster size, all\nnodes will be placed on on-demand instances. If this value is less than the current cluster\nsize, `first_on_demand` nodes will be placed on on-demand instances and the remainder will\nbe placed on `availability` instances. Note that this value does not affect\ncluster size and cannot currently be mutated over the lifetime of a cluster." - }, - "log_analytics_info": { - "description": "Defines values necessary to configure and run Azure Log Analytics agent", - "properties": { - "log_analytics_primary_key": { - "description": "\u003cneeds content added\u003e" - }, - "log_analytics_workspace_id": { - "description": "\u003cneeds content added\u003e" - } - } - }, - "spot_bid_max_price": { - "description": "The max bid price to be used for Azure spot instances.\nThe Max price for the bid cannot be higher than the on-demand price of the instance.\nIf not specified, the default value is -1, which specifies that the instance cannot be evicted\non the basis of price, and only on the basis of availability. Further, the value should \u003e 0 or -1." - } - } - }, - "cluster_log_conf": { - "description": "The configuration for delivering spark logs to a long-term storage destination.\nOnly dbfs destinations are supported. Only one destination can be specified\nfor one cluster. If the conf is given, the logs will be delivered to the destination every\n`5 mins`. The destination of driver logs is `$destination/$clusterId/driver`, while\nthe destination of executor logs is `$destination/$clusterId/executor`.\n", - "properties": { - "dbfs": { - "description": "destination needs to be provided. e.g.\n`{ \"dbfs\" : { \"destination\" : \"dbfs:/home/cluster_log\" } }`", - "properties": { - "destination": { - "description": "dbfs destination, e.g. `dbfs:/my/path`" - } - } - }, - "s3": { - "description": "destination and either the region or endpoint need to be provided. e.g.\n`{ \"s3\": { \"destination\" : \"s3://cluster_log_bucket/prefix\", \"region\" : \"us-west-2\" } }`\nCluster iam role is used to access s3, please make sure the cluster iam role in\n`instance_profile_arn` has permission to write data to the s3 destination.", - "properties": { - "canned_acl": { - "description": "(Optional) Set canned access control list for the logs, e.g. `bucket-owner-full-control`.\nIf `canned_cal` is set, please make sure the cluster iam role has `s3:PutObjectAcl` permission on\nthe destination bucket and prefix. The full list of possible canned acl can be found at\nhttp://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl.\nPlease also note that by default only the object owner gets full controls. If you are using cross account\nrole for writing data, you may want to set `bucket-owner-full-control` to make bucket owner able to\nread the logs." - }, - "destination": { - "description": "S3 destination, e.g. `s3://my-bucket/some-prefix` Note that logs will be delivered using\ncluster iam role, please make sure you set cluster iam role and the role has write access to the\ndestination. Please also note that you cannot use AWS keys to deliver logs." - }, - "enable_encryption": { - "description": "(Optional) Flag to enable server side encryption, `false` by default." - }, - "encryption_type": { - "description": "(Optional) The encryption type, it could be `sse-s3` or `sse-kms`. It will be used only when\nencryption is enabled and the default type is `sse-s3`." - }, - "endpoint": { - "description": "S3 endpoint, e.g. `https://s3-us-west-2.amazonaws.com`. Either region or endpoint needs to be set.\nIf both are set, endpoint will be used." - }, - "kms_key": { - "description": "(Optional) Kms key which will be used if encryption is enabled and encryption type is set to `sse-kms`." - }, - "region": { - "description": "S3 region, e.g. `us-west-2`. Either region or endpoint needs to be set. If both are set,\nendpoint will be used." - } - } - } - } - }, - "custom_tags": { - "description": "Additional tags for cluster resources. Databricks will tag all cluster resources (e.g., AWS\ninstances and EBS volumes) with these tags in addition to `default_tags`. Notes:\n\n- Currently, Databricks allows at most 45 custom tags\n\n- Clusters can only reuse cloud resources if the resources' tags are a subset of the cluster tags", - "additionalproperties": { - "description": "" - } - }, - "driver_instance_pool_id": { - "description": "The optional ID of the instance pool for the driver of the cluster belongs.\nThe pool cluster uses the instance pool with id (instance_pool_id) if the driver pool is not\nassigned." - }, - "driver_node_type_id": { - "description": "The node type of the Spark driver.\nNote that this field is optional; if unset, the driver node type will be set as the same value\nas `node_type_id` defined above." - }, - "enable_local_disk_encryption": { - "description": "Whether to enable local disk encryption for the cluster." - }, - "gcp_attributes": { - "description": "Attributes related to clusters running on Google Cloud Platform.\nIf not specified at cluster creation, a set of default values will be used.", - "properties": { - "availability": { - "description": "" - }, - "boot_disk_size": { - "description": "boot disk size in GB" - }, - "google_service_account": { - "description": "If provided, the cluster will impersonate the google service account when accessing\ngcloud services (like GCS). The google service account\nmust have previously been added to the Databricks environment by an account\nadministrator." - }, - "local_ssd_count": { - "description": "If provided, each node (workers and driver) in the cluster will have this number of local SSDs attached. Each local SSD is 375GB in size. Refer to [GCP documentation](https://cloud.google.com/compute/docs/disks/local-ssd#choose_number_local_ssds) for the supported number of local SSDs for each instance type." - }, - "use_preemptible_executors": { - "description": "This field determines whether the spark executors will be scheduled to run on preemptible VMs (when set to true) versus standard compute engine VMs (when set to false; default).\nNote: Soon to be deprecated, use the availability field instead." - }, - "zone_id": { - "description": "Identifier for the availability zone in which the cluster resides.\nThis can be one of the following:\n- \"HA\" =\u003e High availability, spread nodes across availability zones for a Databricks deployment region [default]\n- \"AUTO\" =\u003e Databricks picks an availability zone to schedule the cluster on.\n- A GCP availability zone =\u003e Pick One of the available zones for (machine type + region) from https://cloud.google.com/compute/docs/regions-zones." - } - } - }, - "init_scripts": { - "description": "The configuration for storing init scripts. Any number of destinations can be specified. The scripts are executed sequentially in the order provided. If `cluster_log_conf` is specified, init script logs are sent to `\u003cdestination\u003e/\u003ccluster-ID\u003e/init_scripts`.", - "items": { - "description": "", - "properties": { - "abfss": { - "description": "destination needs to be provided. e.g.\n`{ \"abfss\" : { \"destination\" : \"abfss://\u003ccontainer-name\u003e@\u003cstorage-account-name\u003e.dfs.core.windows.net/\u003cdirectory-name\u003e\" } }", - "properties": { - "destination": { - "description": "abfss destination, e.g. `abfss://\u003ccontainer-name\u003e@\u003cstorage-account-name\u003e.dfs.core.windows.net/\u003cdirectory-name\u003e`." - } - } - }, - "dbfs": { - "description": "destination needs to be provided. e.g.\n`{ \"dbfs\" : { \"destination\" : \"dbfs:/home/cluster_log\" } }`", - "properties": { - "destination": { - "description": "dbfs destination, e.g. `dbfs:/my/path`" - } - } - }, - "file": { - "description": "destination needs to be provided. e.g.\n`{ \"file\" : { \"destination\" : \"file:/my/local/file.sh\" } }`", - "properties": { - "destination": { - "description": "local file destination, e.g. `file:/my/local/file.sh`" - } - } - }, - "gcs": { - "description": "destination needs to be provided. e.g.\n`{ \"gcs\": { \"destination\": \"gs://my-bucket/file.sh\" } }`", - "properties": { - "destination": { - "description": "GCS destination/URI, e.g. `gs://my-bucket/some-prefix`" - } - } - }, - "s3": { - "description": "destination and either the region or endpoint need to be provided. e.g.\n`{ \"s3\": { \"destination\" : \"s3://cluster_log_bucket/prefix\", \"region\" : \"us-west-2\" } }`\nCluster iam role is used to access s3, please make sure the cluster iam role in\n`instance_profile_arn` has permission to write data to the s3 destination.", - "properties": { - "canned_acl": { - "description": "(Optional) Set canned access control list for the logs, e.g. `bucket-owner-full-control`.\nIf `canned_cal` is set, please make sure the cluster iam role has `s3:PutObjectAcl` permission on\nthe destination bucket and prefix. The full list of possible canned acl can be found at\nhttp://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl.\nPlease also note that by default only the object owner gets full controls. If you are using cross account\nrole for writing data, you may want to set `bucket-owner-full-control` to make bucket owner able to\nread the logs." - }, - "destination": { - "description": "S3 destination, e.g. `s3://my-bucket/some-prefix` Note that logs will be delivered using\ncluster iam role, please make sure you set cluster iam role and the role has write access to the\ndestination. Please also note that you cannot use AWS keys to deliver logs." - }, - "enable_encryption": { - "description": "(Optional) Flag to enable server side encryption, `false` by default." - }, - "encryption_type": { - "description": "(Optional) The encryption type, it could be `sse-s3` or `sse-kms`. It will be used only when\nencryption is enabled and the default type is `sse-s3`." - }, - "endpoint": { - "description": "S3 endpoint, e.g. `https://s3-us-west-2.amazonaws.com`. Either region or endpoint needs to be set.\nIf both are set, endpoint will be used." - }, - "kms_key": { - "description": "(Optional) Kms key which will be used if encryption is enabled and encryption type is set to `sse-kms`." - }, - "region": { - "description": "S3 region, e.g. `us-west-2`. Either region or endpoint needs to be set. If both are set,\nendpoint will be used." - } - } - }, - "volumes": { - "description": "destination needs to be provided. e.g.\n`{ \"volumes\" : { \"destination\" : \"/Volumes/my-init.sh\" } }`", - "properties": { - "destination": { - "description": "Unity Catalog Volumes file destination, e.g. `/Volumes/my-init.sh`" - } - } - }, - "workspace": { - "description": "destination needs to be provided. e.g.\n`{ \"workspace\" : { \"destination\" : \"/Users/user1@databricks.com/my-init.sh\" } }`", - "properties": { - "destination": { - "description": "workspace files destination, e.g. `/Users/user1@databricks.com/my-init.sh`" - } - } - } - } - } - }, - "instance_pool_id": { - "description": "The optional ID of the instance pool to which the cluster belongs." - }, - "label": { - "description": "A label for the cluster specification, either `default` to configure the default cluster, or `maintenance` to configure the maintenance cluster. This field is optional. The default value is `default`." - }, - "node_type_id": { - "description": "This field encodes, through a single value, the resources available to each of\nthe Spark nodes in this cluster. For example, the Spark nodes can be provisioned\nand optimized for memory or compute intensive workloads. A list of available node\ntypes can be retrieved by using the :method:clusters/listNodeTypes API call.\n" - }, - "num_workers": { - "description": "Number of worker nodes that this cluster should have. A cluster has one Spark Driver\nand `num_workers` Executors for a total of `num_workers` + 1 Spark nodes.\n\nNote: When reading the properties of a cluster, this field reflects the desired number\nof workers rather than the actual current number of workers. For instance, if a cluster\nis resized from 5 to 10 workers, this field will immediately be updated to reflect\nthe target size of 10 workers, whereas the workers listed in `spark_info` will gradually\nincrease from 5 to 10 as the new nodes are provisioned." - }, - "policy_id": { - "description": "The ID of the cluster policy used to create the cluster if applicable." - }, - "spark_conf": { - "description": "An object containing a set of optional, user-specified Spark configuration key-value pairs.\nSee :method:clusters/create for more details.\n", - "additionalproperties": { - "description": "" - } - }, - "spark_env_vars": { - "description": "An object containing a set of optional, user-specified environment variable key-value pairs.\nPlease note that key-value pair of the form (X,Y) will be exported as is (i.e.,\n`export X='Y'`) while launching the driver and workers.\n\nIn order to specify an additional set of `SPARK_DAEMON_JAVA_OPTS`, we recommend appending\nthem to `$SPARK_DAEMON_JAVA_OPTS` as shown in the example below. This ensures that all\ndefault databricks managed environmental variables are included as well.\n\nExample Spark environment variables:\n`{\"SPARK_WORKER_MEMORY\": \"28000m\", \"SPARK_LOCAL_DIRS\": \"/local_disk0\"}` or\n`{\"SPARK_DAEMON_JAVA_OPTS\": \"$SPARK_DAEMON_JAVA_OPTS -Dspark.shuffle.service.enabled=true\"}`", - "additionalproperties": { - "description": "" - } - }, - "ssh_public_keys": { - "description": "SSH public key contents that will be added to each Spark node in this cluster. The\ncorresponding private keys can be used to login with the user name `ubuntu` on port `2200`.\nUp to 10 keys can be specified.", - "items": { - "description": "" - } - } - } - } - }, - "configuration": { - "description": "String-String configuration for this pipeline execution.", - "additionalproperties": { - "description": "" - } - }, - "continuous": { - "description": "Whether the pipeline is continuous or triggered. This replaces `trigger`." - }, - "deployment": { - "description": "Deployment type of this pipeline.", - "properties": { - "kind": { - "description": "The deployment method that manages the pipeline." - }, - "metadata_file_path": { - "description": "The path to the file containing metadata about the deployment." - } - } - }, - "development": { - "description": "Whether the pipeline is in Development mode. Defaults to false." - }, - "edition": { - "description": "Pipeline product edition." - }, - "filters": { - "description": "Filters on which Pipeline packages to include in the deployed graph.", - "properties": { - "exclude": { - "description": "Paths to exclude.", - "items": { - "description": "" - } - }, - "include": { - "description": "Paths to include.", - "items": { - "description": "" - } - } - } - }, - "gateway_definition": { - "description": "The definition of a gateway pipeline to support CDC.", - "properties": { - "connection_id": { - "description": "Immutable. The Unity Catalog connection this gateway pipeline uses to communicate with the source." - }, - "gateway_storage_catalog": { - "description": "Required, Immutable. The name of the catalog for the gateway pipeline's storage location." - }, - "gateway_storage_name": { - "description": "Optional. The Unity Catalog-compatible name for the gateway storage location.\nThis is the destination to use for the data that is extracted by the gateway.\nDelta Live Tables system will automatically create the storage location under the catalog and schema.\n" - }, - "gateway_storage_schema": { - "description": "Required, Immutable. The name of the schema for the gateway pipelines's storage location." - } - } - }, - "id": { - "description": "Unique identifier for this pipeline." - }, - "ingestion_definition": { - "description": "The configuration for a managed ingestion pipeline. These settings cannot be used with the 'libraries', 'target' or 'catalog' settings.", - "properties": { - "connection_name": { - "description": "Immutable. The Unity Catalog connection this ingestion pipeline uses to communicate with the source. Specify either ingestion_gateway_id or connection_name." - }, - "ingestion_gateway_id": { - "description": "Immutable. Identifier for the ingestion gateway used by this ingestion pipeline to communicate with the source. Specify either ingestion_gateway_id or connection_name." - }, - "objects": { - "description": "Required. Settings specifying tables to replicate and the destination for the replicated tables.", - "items": { - "description": "", - "properties": { - "schema": { - "description": "Select tables from a specific source schema.", - "properties": { - "destination_catalog": { - "description": "Required. Destination catalog to store tables." - }, - "destination_schema": { - "description": "Required. Destination schema to store tables in. Tables with the same name as the source tables are created in this destination schema. The pipeline fails If a table with the same name already exists." - }, - "source_catalog": { - "description": "The source catalog name. Might be optional depending on the type of source." - }, - "source_schema": { - "description": "Required. Schema name in the source database." - }, - "table_configuration": { - "description": "Configuration settings to control the ingestion of tables. These settings are applied to all tables in this schema and override the table_configuration defined in the IngestionPipelineDefinition object.", - "properties": { - "primary_keys": { - "description": "The primary key of the table used to apply changes.", - "items": { - "description": "" - } - }, - "salesforce_include_formula_fields": { - "description": "If true, formula fields defined in the table are included in the ingestion. This setting is only valid for the Salesforce connector" - }, - "scd_type": { - "description": "The SCD type to use to ingest the table." - } - } - } - } - }, - "table": { - "description": "Select tables from a specific source table.", - "properties": { - "destination_catalog": { - "description": "Required. Destination catalog to store table." - }, - "destination_schema": { - "description": "Required. Destination schema to store table." - }, - "destination_table": { - "description": "Optional. Destination table name. The pipeline fails If a table with that name already exists. If not set, the source table name is used." - }, - "source_catalog": { - "description": "Source catalog name. Might be optional depending on the type of source." - }, - "source_schema": { - "description": "Schema name in the source database. Might be optional depending on the type of source." - }, - "source_table": { - "description": "Required. Table name in the source database." - }, - "table_configuration": { - "description": "Configuration settings to control the ingestion of tables. These settings override the table_configuration defined in the IngestionPipelineDefinition object and the SchemaSpec.", - "properties": { - "primary_keys": { - "description": "The primary key of the table used to apply changes.", - "items": { - "description": "" - } - }, - "salesforce_include_formula_fields": { - "description": "If true, formula fields defined in the table are included in the ingestion. This setting is only valid for the Salesforce connector" - }, - "scd_type": { - "description": "The SCD type to use to ingest the table." - } - } - } - } - } - } - } - }, - "table_configuration": { - "description": "Configuration settings to control the ingestion of tables. These settings are applied to all tables in the pipeline.", - "properties": { - "primary_keys": { - "description": "The primary key of the table used to apply changes.", - "items": { - "description": "" - } - }, - "salesforce_include_formula_fields": { - "description": "If true, formula fields defined in the table are included in the ingestion. This setting is only valid for the Salesforce connector" - }, - "scd_type": { - "description": "The SCD type to use to ingest the table." - } - } - } - } - }, - "libraries": { - "description": "Libraries or code needed by this deployment.", - "items": { - "description": "", - "properties": { - "file": { - "description": "The path to a file that defines a pipeline and is stored in the Databricks Repos.\n", - "properties": { - "path": { - "description": "The absolute path of the file." - } - } - }, - "jar": { - "description": "URI of the jar to be installed. Currently only DBFS is supported.\n" - }, - "maven": { - "description": "Specification of a maven library to be installed.\n", - "properties": { - "coordinates": { - "description": "Gradle-style maven coordinates. For example: \"org.jsoup:jsoup:1.7.2\"." - }, - "exclusions": { - "description": "List of dependences to exclude. For example: `[\"slf4j:slf4j\", \"*:hadoop-client\"]`.\n\nMaven dependency exclusions:\nhttps://maven.apache.org/guides/introduction/introduction-to-optional-and-excludes-dependencies.html.", - "items": { - "description": "" - } - }, - "repo": { - "description": "Maven repo to install the Maven package from. If omitted, both Maven Central Repository\nand Spark Packages are searched." - } - } - }, - "notebook": { - "description": "The path to a notebook that defines a pipeline and is stored in the Databricks workspace.\n", - "properties": { - "path": { - "description": "The absolute path of the notebook." - } - } - }, - "whl": { - "description": "URI of the whl to be installed." - } - } - } - }, - "name": { - "description": "Friendly identifier for this pipeline." - }, - "notifications": { - "description": "List of notification settings for this pipeline.", - "items": { - "description": "", - "properties": { - "alerts": { - "description": "A list of alerts that trigger the sending of notifications to the configured\ndestinations. The supported alerts are:\n\n* `on-update-success`: A pipeline update completes successfully.\n* `on-update-failure`: Each time a pipeline update fails.\n* `on-update-fatal-failure`: A pipeline update fails with a non-retryable (fatal) error.\n* `on-flow-failure`: A single data flow fails.\n", - "items": { - "description": "" - } - }, - "email_recipients": { - "description": "A list of email addresses notified when a configured alert is triggered.\n", - "items": { - "description": "" - } - } - } - } - }, - "permissions": { - "description": "", - "items": { - "description": "", - "properties": { - "group_name": { - "description": "" - }, - "level": { - "description": "" - }, - "service_principal_name": { - "description": "" - }, - "user_name": { - "description": "" - } - } - } - }, - "photon": { - "description": "Whether Photon is enabled for this pipeline." - }, - "serverless": { - "description": "Whether serverless compute is enabled for this pipeline." - }, - "storage": { - "description": "DBFS root directory for storing checkpoints and tables." - }, - "target": { - "description": "Target schema (database) to add tables in this pipeline to. If not specified, no data is published to the Hive metastore or Unity Catalog. To publish to Unity Catalog, also specify `catalog`." - }, - "trigger": { - "description": "Which pipeline trigger to use. Deprecated: Use `continuous` instead.", - "properties": { - "cron": { - "description": "", - "properties": { - "quartz_cron_schedule": { - "description": "" - }, - "timezone_id": { - "description": "" - } - } - }, - "manual": { - "description": "" - } - } - } - } - } - }, - "quality_monitors": { - "description": "", - "additionalproperties": { - "description": "", - "properties": { - "assets_dir": { - "description": "" - }, - "baseline_table_name": { - "description": "" - }, - "custom_metrics": { - "description": "", - "items": { - "description": "", - "properties": { - "definition": { - "description": "" - }, - "input_columns": { - "description": "", - "items": { - "description": "" - } - }, - "name": { - "description": "" - }, - "output_data_type": { - "description": "" - }, - "type": { - "description": "" - } - } - } - }, - "data_classification_config": { - "description": "", - "properties": { - "enabled": { - "description": "" - } - } - }, - "inference_log": { - "description": "", - "properties": { - "granularities": { - "description": "", - "items": { - "description": "" - } - }, - "label_col": { - "description": "" - }, - "model_id_col": { - "description": "" - }, - "prediction_col": { - "description": "" - }, - "prediction_proba_col": { - "description": "" - }, - "problem_type": { - "description": "" - }, - "timestamp_col": { - "description": "" - } - } - }, - "notifications": { - "description": "", - "properties": { - "on_failure": { - "description": "", - "properties": { - "email_addresses": { - "description": "", - "items": { - "description": "" - } - } - } - }, - "on_new_classification_tag_detected": { - "description": "", - "properties": { - "email_addresses": { - "description": "", - "items": { - "description": "" - } - } - } - } - } - }, - "output_schema_name": { - "description": "" - }, - "schedule": { - "description": "", - "properties": { - "pause_status": { - "description": "" - }, - "quartz_cron_expression": { - "description": "" - }, - "timezone_id": { - "description": "" - } - } - }, - "skip_builtin_dashboard": { - "description": "" - }, - "slicing_exprs": { - "description": "", - "items": { - "description": "" - } - }, - "snapshot": { - "description": "" - }, - "time_series": { - "description": "", - "properties": { - "granularities": { - "description": "", - "items": { - "description": "" - } - }, - "timestamp_col": { - "description": "" - } - } - }, - "warehouse_id": { - "description": "" - } - } - } - }, - "registered_models": { - "description": "List of Registered Models", - "additionalproperties": { - "description": "", - "properties": { - "catalog_name": { - "description": "The name of the catalog where the schema and the registered model reside" - }, - "comment": { - "description": "The comment attached to the registered model" - }, - "grants": { - "description": "", - "items": { - "description": "", - "properties": { - "principal": { - "description": "" - }, - "privileges": { - "description": "", - "items": { - "description": "" - } - } - } - } - }, - "name": { - "description": "The name of the registered model" - }, - "schema_name": { - "description": "The name of the schema where the registered model resides" - }, - "storage_location": { - "description": "The storage location on the cloud under which model version data files are stored" - } - } - } - }, - "schemas": { - "description": "", - "additionalproperties": { - "description": "", - "properties": { - "catalog_name": { - "description": "" - }, - "comment": { - "description": "" - }, - "grants": { - "description": "", - "items": { - "description": "", - "properties": { - "principal": { - "description": "" - }, - "privileges": { - "description": "", - "items": { - "description": "" - } - } - } - } - }, - "name": { - "description": "" - }, - "properties": { - "description": "", - "additionalproperties": { - "description": "" - } - }, - "storage_root": { - "description": "" - } - } - } - } - } - }, - "run_as": { - "description": "", - "properties": { - "service_principal_name": { - "description": "" - }, - "user_name": { - "description": "" - } - } - }, - "sync": { - "description": "", - "properties": { - "exclude": { - "description": "", - "items": { - "description": "" - } - }, - "include": { - "description": "", - "items": { - "description": "" - } - } - } - }, - "variables": { - "description": "", - "additionalproperties": { - "description": "", - "properties": { - "default": { - "description": "" - }, - "description": { - "description": "" - }, - "lookup": { - "description": "", - "properties": { - "alert": { - "description": "" - }, - "cluster": { - "description": "" - }, - "cluster_policy": { - "description": "" - }, - "dashboard": { - "description": "" - }, - "instance_pool": { - "description": "" - }, - "job": { - "description": "" - }, - "metastore": { - "description": "" - }, - "pipeline": { - "description": "" - }, - "query": { - "description": "" - }, - "service_principal": { - "description": "" - }, - "warehouse": { - "description": "" - } - } - }, - "type": { - "description": "" - } - } - } - }, - "workspace": { - "description": "", - "properties": { - "artifact_path": { - "description": "" - }, - "auth_type": { - "description": "" - }, - "azure_client_id": { - "description": "" - }, - "azure_environment": { - "description": "" - }, - "azure_login_app_id": { - "description": "" - }, - "azure_tenant_id": { - "description": "" - }, - "azure_use_msi": { - "description": "" - }, - "azure_workspace_resource_id": { - "description": "" - }, - "client_id": { - "description": "" - }, - "file_path": { - "description": "" - }, - "google_service_account": { - "description": "" - }, - "host": { - "description": "" - }, - "profile": { - "description": "" - }, - "root_path": { - "description": "" - }, - "state_path": { - "description": "" - } - } - } - } - } - }, - "variables": { - "description": "", - "additionalproperties": { - "description": "", - "properties": { - "default": { - "description": "" - }, - "description": { - "description": "" - }, - "lookup": { - "description": "", - "properties": { - "alert": { - "description": "" - }, - "cluster": { - "description": "" - }, - "cluster_policy": { - "description": "" - }, - "dashboard": { - "description": "" - }, - "instance_pool": { - "description": "" - }, - "job": { - "description": "" - }, - "metastore": { - "description": "" - }, - "pipeline": { - "description": "" - }, - "query": { - "description": "" - }, - "service_principal": { - "description": "" - }, - "warehouse": { - "description": "" - } - } - }, - "type": { - "description": "" - } - } - } - }, - "workspace": { - "description": "", - "properties": { - "artifact_path": { - "description": "" - }, - "auth_type": { - "description": "" - }, - "azure_client_id": { - "description": "" - }, - "azure_environment": { - "description": "" - }, - "azure_login_app_id": { - "description": "" - }, - "azure_tenant_id": { - "description": "" - }, - "azure_use_msi": { - "description": "" - }, - "azure_workspace_resource_id": { - "description": "" - }, - "client_id": { - "description": "" - }, - "file_path": { - "description": "" - }, - "google_service_account": { - "description": "" - }, - "host": { - "description": "" - }, - "profile": { - "description": "" - }, - "root_path": { - "description": "" - }, - "state_path": { - "description": "" - } - } - } - } -} \ No newline at end of file diff --git a/bundle/schema/docs_test.go b/bundle/schema/docs_test.go deleted file mode 100644 index 83ee681b03..0000000000 --- a/bundle/schema/docs_test.go +++ /dev/null @@ -1,62 +0,0 @@ -package schema - -import ( - "encoding/json" - "testing" - - "github.com/databricks/cli/libs/jsonschema" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestSchemaToDocs(t *testing.T) { - jsonSchema := &jsonschema.Schema{ - Type: "object", - Description: "root doc", - Properties: map[string]*jsonschema.Schema{ - "foo": {Type: "number", Description: "foo doc"}, - "bar": {Type: "string"}, - "octave": { - Type: "object", - AdditionalProperties: &jsonschema.Schema{Type: "number"}, - Description: "octave docs", - }, - "scales": { - Type: "object", - Description: "scale docs", - Items: &jsonschema.Schema{Type: "string"}, - }, - }, - } - docs := schemaToDocs(jsonSchema) - docsJson, err := json.MarshalIndent(docs, " ", " ") - require.NoError(t, err) - - expected := - `{ - "description": "root doc", - "properties": { - "bar": { - "description": "" - }, - "foo": { - "description": "foo doc" - }, - "octave": { - "description": "octave docs", - "additionalproperties": { - "description": "" - } - }, - "scales": { - "description": "scale docs", - "items": { - "description": "" - } - } - } - }` - t.Log("[DEBUG] actual: ", string(docsJson)) - t.Log("[DEBUG] expected: ", expected) - assert.Equal(t, expected, string(docsJson)) -} diff --git a/bundle/schema/openapi.go b/bundle/schema/openapi.go deleted file mode 100644 index 0d896b87ca..0000000000 --- a/bundle/schema/openapi.go +++ /dev/null @@ -1,293 +0,0 @@ -package schema - -import ( - "encoding/json" - "fmt" - "strings" - - "github.com/databricks/cli/libs/jsonschema" -) - -type OpenapiReader struct { - // OpenAPI spec to read schemas from. - OpenapiSpec *Specification - - // In-memory cache of schemas read from the OpenAPI spec. - memo map[string]jsonschema.Schema -} - -const SchemaPathPrefix = "#/components/schemas/" - -// Read a schema directly from the OpenAPI spec. -func (reader *OpenapiReader) readOpenapiSchema(path string) (jsonschema.Schema, error) { - schemaKey := strings.TrimPrefix(path, SchemaPathPrefix) - - // return early if we already have a computed schema - memoSchema, ok := reader.memo[schemaKey] - if ok { - return memoSchema, nil - } - - // check path is present in openapi spec - openapiSchema, ok := reader.OpenapiSpec.Components.Schemas[schemaKey] - if !ok { - return jsonschema.Schema{}, fmt.Errorf("schema with path %s not found in openapi spec", path) - } - - // convert openapi schema to the native schema struct - bytes, err := json.Marshal(*openapiSchema) - if err != nil { - return jsonschema.Schema{}, err - } - jsonSchema := jsonschema.Schema{} - err = json.Unmarshal(bytes, &jsonSchema) - if err != nil { - return jsonschema.Schema{}, err - } - - // A hack to convert a map[string]interface{} to *Schema - // We rely on the type of a AdditionalProperties in downstream functions - // to do reference interpolation - _, ok = jsonSchema.AdditionalProperties.(map[string]interface{}) - if ok { - b, err := json.Marshal(jsonSchema.AdditionalProperties) - if err != nil { - return jsonschema.Schema{}, err - } - additionalProperties := &jsonschema.Schema{} - err = json.Unmarshal(b, additionalProperties) - if err != nil { - return jsonschema.Schema{}, err - } - jsonSchema.AdditionalProperties = additionalProperties - } - - // store read schema into memo - reader.memo[schemaKey] = jsonSchema - - return jsonSchema, nil -} - -// Resolve all nested "$ref" references in the schema. This function unrolls a single -// level of "$ref" in the schema and calls into traverseSchema to resolve nested references. -// Thus this function and traverseSchema are mutually recursive. -// -// This function is safe against reference loops. If a reference loop is detected, an error -// is returned. -func (reader *OpenapiReader) safeResolveRefs(root *jsonschema.Schema, tracker *tracker) (*jsonschema.Schema, error) { - if root.Reference == nil { - return reader.traverseSchema(root, tracker) - } - key := *root.Reference - - // HACK to unblock CLI release (13th Feb 2024). This is temporary until proper - // support for recursive types is added to the docs generator. PR: https://github.com/databricks/cli/pull/1204 - if strings.Contains(key, "ForEachTask") { - return root, nil - } - - if tracker.hasCycle(key) { - // self reference loops can be supported however the logic is non-trivial because - // cross refernce loops are not allowed (see: http://json-schema.org/understanding-json-schema/structuring.html#recursion) - return nil, fmt.Errorf("references loop detected") - } - ref := *root.Reference - description := root.Description - tracker.push(ref, ref) - - // Mark reference nil, so we do not traverse this again. This is tracked - // in the memo - root.Reference = nil - - // unroll one level of reference. - selfRef, err := reader.readOpenapiSchema(ref) - if err != nil { - return nil, err - } - root = &selfRef - root.Description = description - - // traverse again to find new references - root, err = reader.traverseSchema(root, tracker) - if err != nil { - return nil, err - } - tracker.pop(ref) - return root, err -} - -// Traverse the nested properties of the schema to resolve "$ref" references. This function -// and safeResolveRefs are mutually recursive. -func (reader *OpenapiReader) traverseSchema(root *jsonschema.Schema, tracker *tracker) (*jsonschema.Schema, error) { - // case primitive (or invalid) - if root.Type != jsonschema.ObjectType && root.Type != jsonschema.ArrayType { - return root, nil - } - // only root references are resolved - if root.Reference != nil { - return reader.safeResolveRefs(root, tracker) - } - // case struct - if len(root.Properties) > 0 { - for k, v := range root.Properties { - childSchema, err := reader.safeResolveRefs(v, tracker) - if err != nil { - return nil, err - } - root.Properties[k] = childSchema - } - } - // case array - if root.Items != nil { - itemsSchema, err := reader.safeResolveRefs(root.Items, tracker) - if err != nil { - return nil, err - } - root.Items = itemsSchema - } - // case map - additionalProperties, ok := root.AdditionalProperties.(*jsonschema.Schema) - if ok && additionalProperties != nil { - valueSchema, err := reader.safeResolveRefs(additionalProperties, tracker) - if err != nil { - return nil, err - } - root.AdditionalProperties = valueSchema - } - return root, nil -} - -func (reader *OpenapiReader) readResolvedSchema(path string) (*jsonschema.Schema, error) { - root, err := reader.readOpenapiSchema(path) - if err != nil { - return nil, err - } - tracker := newTracker() - tracker.push(path, path) - resolvedRoot, err := reader.safeResolveRefs(&root, tracker) - if err != nil { - return nil, tracker.errWithTrace(err.Error(), "") - } - return resolvedRoot, nil -} - -func (reader *OpenapiReader) jobsDocs() (*Docs, error) { - jobSettingsSchema, err := reader.readResolvedSchema(SchemaPathPrefix + "jobs.JobSettings") - if err != nil { - return nil, err - } - jobDocs := schemaToDocs(jobSettingsSchema) - // TODO: add description for id if needed. - // Tracked in https://github.com/databricks/cli/issues/242 - jobsDocs := &Docs{ - Description: "List of Databricks jobs", - AdditionalProperties: jobDocs, - } - return jobsDocs, nil -} - -func (reader *OpenapiReader) pipelinesDocs() (*Docs, error) { - pipelineSpecSchema, err := reader.readResolvedSchema(SchemaPathPrefix + "pipelines.PipelineSpec") - if err != nil { - return nil, err - } - pipelineDocs := schemaToDocs(pipelineSpecSchema) - // TODO: Two fields in resources.Pipeline have the json tag id. Clarify the - // semantics and then add a description if needed. (https://github.com/databricks/cli/issues/242) - pipelinesDocs := &Docs{ - Description: "List of DLT pipelines", - AdditionalProperties: pipelineDocs, - } - return pipelinesDocs, nil -} - -func (reader *OpenapiReader) experimentsDocs() (*Docs, error) { - experimentSpecSchema, err := reader.readResolvedSchema(SchemaPathPrefix + "ml.Experiment") - if err != nil { - return nil, err - } - experimentDocs := schemaToDocs(experimentSpecSchema) - experimentsDocs := &Docs{ - Description: "List of MLflow experiments", - AdditionalProperties: experimentDocs, - } - return experimentsDocs, nil -} - -func (reader *OpenapiReader) modelsDocs() (*Docs, error) { - modelSpecSchema, err := reader.readResolvedSchema(SchemaPathPrefix + "ml.Model") - if err != nil { - return nil, err - } - modelDocs := schemaToDocs(modelSpecSchema) - modelsDocs := &Docs{ - Description: "List of MLflow models", - AdditionalProperties: modelDocs, - } - return modelsDocs, nil -} - -func (reader *OpenapiReader) modelServingEndpointsDocs() (*Docs, error) { - modelServingEndpointsSpecSchema, err := reader.readResolvedSchema(SchemaPathPrefix + "serving.CreateServingEndpoint") - if err != nil { - return nil, err - } - modelServingEndpointsDocs := schemaToDocs(modelServingEndpointsSpecSchema) - modelServingEndpointsAllDocs := &Docs{ - Description: "List of Model Serving Endpoints", - AdditionalProperties: modelServingEndpointsDocs, - } - return modelServingEndpointsAllDocs, nil -} - -func (reader *OpenapiReader) registeredModelDocs() (*Docs, error) { - registeredModelsSpecSchema, err := reader.readResolvedSchema(SchemaPathPrefix + "catalog.CreateRegisteredModelRequest") - if err != nil { - return nil, err - } - registeredModelsDocs := schemaToDocs(registeredModelsSpecSchema) - registeredModelsAllDocs := &Docs{ - Description: "List of Registered Models", - AdditionalProperties: registeredModelsDocs, - } - return registeredModelsAllDocs, nil -} - -func (reader *OpenapiReader) ResourcesDocs() (*Docs, error) { - jobsDocs, err := reader.jobsDocs() - if err != nil { - return nil, err - } - pipelinesDocs, err := reader.pipelinesDocs() - if err != nil { - return nil, err - } - experimentsDocs, err := reader.experimentsDocs() - if err != nil { - return nil, err - } - modelsDocs, err := reader.modelsDocs() - if err != nil { - return nil, err - } - modelServingEndpointsDocs, err := reader.modelServingEndpointsDocs() - if err != nil { - return nil, err - } - registeredModelsDocs, err := reader.registeredModelDocs() - if err != nil { - return nil, err - } - - return &Docs{ - Description: "Collection of Databricks resources to deploy.", - Properties: map[string]*Docs{ - "jobs": jobsDocs, - "pipelines": pipelinesDocs, - "experiments": experimentsDocs, - "models": modelsDocs, - "model_serving_endpoints": modelServingEndpointsDocs, - "registered_models": registeredModelsDocs, - }, - }, nil -} diff --git a/bundle/schema/openapi_test.go b/bundle/schema/openapi_test.go deleted file mode 100644 index 4d393cf371..0000000000 --- a/bundle/schema/openapi_test.go +++ /dev/null @@ -1,493 +0,0 @@ -package schema - -import ( - "encoding/json" - "testing" - - "github.com/databricks/cli/libs/jsonschema" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestReadSchemaForObject(t *testing.T) { - specString := ` - { - "components": { - "schemas": { - "foo": { - "type": "number" - }, - "fruits": { - "type": "object", - "description": "fruits that are cool", - "properties": { - "guava": { - "type": "string", - "description": "a guava for my schema" - }, - "mango": { - "type": "object", - "description": "a mango for my schema", - "$ref": "#/components/schemas/mango" - } - } - }, - "mango": { - "type": "object", - "properties": { - "foo": { - "$ref": "#/components/schemas/foo" - } - } - } - } - } - } - ` - spec := &Specification{} - reader := &OpenapiReader{ - OpenapiSpec: spec, - memo: make(map[string]jsonschema.Schema), - } - err := json.Unmarshal([]byte(specString), spec) - require.NoError(t, err) - - fruitsSchema, err := reader.readResolvedSchema("#/components/schemas/fruits") - require.NoError(t, err) - - fruitsSchemaJson, err := json.MarshalIndent(fruitsSchema, " ", " ") - require.NoError(t, err) - - expected := `{ - "type": "object", - "description": "fruits that are cool", - "properties": { - "guava": { - "type": "string", - "description": "a guava for my schema" - }, - "mango": { - "type": "object", - "description": "a mango for my schema", - "properties": { - "foo": { - "type": "number" - } - } - } - } - }` - - t.Log("[DEBUG] actual: ", string(fruitsSchemaJson)) - t.Log("[DEBUG] expected: ", expected) - assert.Equal(t, expected, string(fruitsSchemaJson)) -} - -func TestReadSchemaForArray(t *testing.T) { - specString := ` - { - "components": { - "schemas": { - "fruits": { - "type": "object", - "description": "fruits that are cool", - "items": { - "description": "some papayas, because papayas are fruits too", - "$ref": "#/components/schemas/papaya" - } - }, - "papaya": { - "type": "number" - } - } - } - }` - spec := &Specification{} - reader := &OpenapiReader{ - OpenapiSpec: spec, - memo: make(map[string]jsonschema.Schema), - } - err := json.Unmarshal([]byte(specString), spec) - require.NoError(t, err) - - fruitsSchema, err := reader.readResolvedSchema("#/components/schemas/fruits") - require.NoError(t, err) - - fruitsSchemaJson, err := json.MarshalIndent(fruitsSchema, " ", " ") - require.NoError(t, err) - - expected := `{ - "type": "object", - "description": "fruits that are cool", - "items": { - "type": "number", - "description": "some papayas, because papayas are fruits too" - } - }` - - t.Log("[DEBUG] actual: ", string(fruitsSchemaJson)) - t.Log("[DEBUG] expected: ", expected) - assert.Equal(t, expected, string(fruitsSchemaJson)) -} - -func TestReadSchemaForMap(t *testing.T) { - specString := `{ - "components": { - "schemas": { - "fruits": { - "type": "object", - "description": "fruits that are meh", - "additionalProperties": { - "description": "watermelons. watermelons.", - "$ref": "#/components/schemas/watermelon" - } - }, - "watermelon": { - "type": "number" - } - } - } - }` - spec := &Specification{} - reader := &OpenapiReader{ - OpenapiSpec: spec, - memo: make(map[string]jsonschema.Schema), - } - err := json.Unmarshal([]byte(specString), spec) - require.NoError(t, err) - - fruitsSchema, err := reader.readResolvedSchema("#/components/schemas/fruits") - require.NoError(t, err) - - fruitsSchemaJson, err := json.MarshalIndent(fruitsSchema, " ", " ") - require.NoError(t, err) - - expected := `{ - "type": "object", - "description": "fruits that are meh", - "additionalProperties": { - "type": "number", - "description": "watermelons. watermelons." - } - }` - - t.Log("[DEBUG] actual: ", string(fruitsSchemaJson)) - t.Log("[DEBUG] expected: ", expected) - assert.Equal(t, expected, string(fruitsSchemaJson)) -} - -func TestRootReferenceIsResolved(t *testing.T) { - specString := `{ - "components": { - "schemas": { - "foo": { - "type": "object", - "description": "this description is ignored", - "properties": { - "abc": { - "type": "string" - } - } - }, - "fruits": { - "type": "object", - "description": "foo fighters fighting fruits", - "$ref": "#/components/schemas/foo" - } - } - } - }` - spec := &Specification{} - reader := &OpenapiReader{ - OpenapiSpec: spec, - memo: make(map[string]jsonschema.Schema), - } - err := json.Unmarshal([]byte(specString), spec) - require.NoError(t, err) - - schema, err := reader.readResolvedSchema("#/components/schemas/fruits") - require.NoError(t, err) - fruitsSchemaJson, err := json.MarshalIndent(schema, " ", " ") - require.NoError(t, err) - - expected := `{ - "type": "object", - "description": "foo fighters fighting fruits", - "properties": { - "abc": { - "type": "string" - } - } - }` - - t.Log("[DEBUG] actual: ", string(fruitsSchemaJson)) - t.Log("[DEBUG] expected: ", expected) - assert.Equal(t, expected, string(fruitsSchemaJson)) -} - -func TestSelfReferenceLoopErrors(t *testing.T) { - specString := `{ - "components": { - "schemas": { - "foo": { - "type": "object", - "description": "this description is ignored", - "properties": { - "bar": { - "type": "object", - "$ref": "#/components/schemas/foo" - } - } - }, - "fruits": { - "type": "object", - "description": "foo fighters fighting fruits", - "$ref": "#/components/schemas/foo" - } - } - } - }` - spec := &Specification{} - reader := &OpenapiReader{ - OpenapiSpec: spec, - memo: make(map[string]jsonschema.Schema), - } - err := json.Unmarshal([]byte(specString), spec) - require.NoError(t, err) - - _, err = reader.readResolvedSchema("#/components/schemas/fruits") - assert.ErrorContains(t, err, "references loop detected. traversal trace: -> #/components/schemas/fruits -> #/components/schemas/foo") -} - -func TestCrossReferenceLoopErrors(t *testing.T) { - specString := `{ - "components": { - "schemas": { - "foo": { - "type": "object", - "description": "this description is ignored", - "properties": { - "bar": { - "type": "object", - "$ref": "#/components/schemas/fruits" - } - } - }, - "fruits": { - "type": "object", - "description": "foo fighters fighting fruits", - "$ref": "#/components/schemas/foo" - } - } - } - }` - spec := &Specification{} - reader := &OpenapiReader{ - OpenapiSpec: spec, - memo: make(map[string]jsonschema.Schema), - } - err := json.Unmarshal([]byte(specString), spec) - require.NoError(t, err) - - _, err = reader.readResolvedSchema("#/components/schemas/fruits") - assert.ErrorContains(t, err, "references loop detected. traversal trace: -> #/components/schemas/fruits -> #/components/schemas/foo") -} - -func TestReferenceResolutionForMapInObject(t *testing.T) { - specString := ` - { - "components": { - "schemas": { - "foo": { - "type": "number" - }, - "fruits": { - "type": "object", - "description": "fruits that are cool", - "properties": { - "guava": { - "type": "string", - "description": "a guava for my schema" - }, - "mangos": { - "type": "object", - "description": "multiple mangos", - "$ref": "#/components/schemas/mango" - } - } - }, - "mango": { - "type": "object", - "additionalProperties": { - "description": "a single mango", - "$ref": "#/components/schemas/foo" - } - } - } - } - }` - spec := &Specification{} - reader := &OpenapiReader{ - OpenapiSpec: spec, - memo: make(map[string]jsonschema.Schema), - } - err := json.Unmarshal([]byte(specString), spec) - require.NoError(t, err) - - fruitsSchema, err := reader.readResolvedSchema("#/components/schemas/fruits") - require.NoError(t, err) - - fruitsSchemaJson, err := json.MarshalIndent(fruitsSchema, " ", " ") - require.NoError(t, err) - - expected := `{ - "type": "object", - "description": "fruits that are cool", - "properties": { - "guava": { - "type": "string", - "description": "a guava for my schema" - }, - "mangos": { - "type": "object", - "description": "multiple mangos", - "additionalProperties": { - "type": "number", - "description": "a single mango" - } - } - } - }` - - t.Log("[DEBUG] actual: ", string(fruitsSchemaJson)) - t.Log("[DEBUG] expected: ", expected) - assert.Equal(t, expected, string(fruitsSchemaJson)) -} - -func TestReferenceResolutionForArrayInObject(t *testing.T) { - specString := `{ - "components": { - "schemas": { - "foo": { - "type": "number" - }, - "fruits": { - "type": "object", - "description": "fruits that are cool", - "properties": { - "guava": { - "type": "string", - "description": "a guava for my schema" - }, - "mangos": { - "type": "object", - "description": "multiple mangos", - "$ref": "#/components/schemas/mango" - } - } - }, - "mango": { - "type": "object", - "items": { - "description": "a single mango", - "$ref": "#/components/schemas/foo" - } - } - } - } - }` - spec := &Specification{} - reader := &OpenapiReader{ - OpenapiSpec: spec, - memo: make(map[string]jsonschema.Schema), - } - err := json.Unmarshal([]byte(specString), spec) - require.NoError(t, err) - - fruitsSchema, err := reader.readResolvedSchema("#/components/schemas/fruits") - require.NoError(t, err) - - fruitsSchemaJson, err := json.MarshalIndent(fruitsSchema, " ", " ") - require.NoError(t, err) - - expected := `{ - "type": "object", - "description": "fruits that are cool", - "properties": { - "guava": { - "type": "string", - "description": "a guava for my schema" - }, - "mangos": { - "type": "object", - "description": "multiple mangos", - "items": { - "type": "number", - "description": "a single mango" - } - } - } - }` - - t.Log("[DEBUG] actual: ", string(fruitsSchemaJson)) - t.Log("[DEBUG] expected: ", expected) - assert.Equal(t, expected, string(fruitsSchemaJson)) -} - -func TestReferenceResolutionDoesNotOverwriteDescriptions(t *testing.T) { - specString := `{ - "components": { - "schemas": { - "foo": { - "type": "number" - }, - "fruits": { - "type": "object", - "properties": { - "guava": { - "type": "object", - "description": "Guava is a fruit", - "$ref": "#/components/schemas/foo" - }, - "mango": { - "type": "object", - "description": "What is a mango?", - "$ref": "#/components/schemas/foo" - } - } - } - } - } - }` - spec := &Specification{} - reader := &OpenapiReader{ - OpenapiSpec: spec, - memo: make(map[string]jsonschema.Schema), - } - err := json.Unmarshal([]byte(specString), spec) - require.NoError(t, err) - - fruitsSchema, err := reader.readResolvedSchema("#/components/schemas/fruits") - require.NoError(t, err) - - fruitsSchemaJson, err := json.MarshalIndent(fruitsSchema, " ", " ") - require.NoError(t, err) - - expected := `{ - "type": "object", - "properties": { - "guava": { - "type": "number", - "description": "Guava is a fruit" - }, - "mango": { - "type": "number", - "description": "What is a mango?" - } - } - }` - - t.Log("[DEBUG] actual: ", string(fruitsSchemaJson)) - t.Log("[DEBUG] expected: ", expected) - assert.Equal(t, expected, string(fruitsSchemaJson)) -} diff --git a/bundle/schema/schema.go b/bundle/schema/schema.go deleted file mode 100644 index 946b91725a..0000000000 --- a/bundle/schema/schema.go +++ /dev/null @@ -1,307 +0,0 @@ -package schema - -import ( - "reflect" - - "github.com/databricks/cli/libs/jsonschema" -) - -// // Fields tagged "readonly" should not be emitted in the schema as they are -// // computed at runtime, and should not be assigned a value by the bundle author. -// const readonlyTag = "readonly" - -// // Annotation for internal bundle fields that should not be exposed to customers. -// // Fields can be tagged as "internal" to remove them from the generated schema. -// const internalTag = "internal" - -// // Annotation for bundle fields that have been deprecated. -// // Fields tagged as "deprecated" are removed/omitted from the generated schema. -// const deprecatedTag = "deprecated" - -// This function translates golang types into json schema. Here is the mapping -// between json schema types and golang types -// -// - GolangType -> Javascript type / Json Schema2 -// -// - bool -> boolean -// -// - string -> string -// -// - int (all variants) -> number -// -// - float (all variants) -> number -// -// - map[string]MyStruct -> { type: object, additionalProperties: {}} -// for details visit: https://json-schema.org/understanding-json-schema/reference/object.html#additional-properties -// -// - []MyStruct -> {type: array, items: {}} -// for details visit: https://json-schema.org/understanding-json-schema/reference/array.html#items -// -// - []MyStruct -> {type: object, properties: {}, additionalProperties: false} -// for details visit: https://json-schema.org/understanding-json-schema/reference/object.html#properties -func New(golangType reflect.Type, docs *Docs) (*jsonschema.Schema, error) { - - return nil, nil - - // s, err := jsonschema.FromType(golangType, func(s jsonschema.Schema) jsonschema.Schema { - // if s.Type == jsonschema.NumberType || s.Type == jsonschema.BooleanType { - // s = jsonschema.Schema{ - // AnyOf: []jsonschema.Schema{ - // s, - // { - // Type: jsonschema.StringType, - // // TODO: Narrow down the scope of the regex match. - // // Also likely need to rename this variable. - // Pattern: dynvar.ReferenceRegex, - // }, - // }, - // } - // } - // return s - // }) - // if err != nil { - // return nil, err - // } - // return &s, nil - - // tracker := newTracker() - // schema, err := safeToSchema(golangType, docs, "", tracker) - // if err != nil { - // return nil, tracker.errWithTrace(err.Error(), "root") - // } - // return schema, nil -} - -// func jsonSchemaType(golangType reflect.Type) (jsonschema.Type, error) { -// switch golangType.Kind() { -// case reflect.Bool: -// return jsonschema.BooleanType, nil -// case reflect.String: -// return jsonschema.StringType, nil -// case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, -// reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, -// reflect.Float32, reflect.Float64: - -// return jsonschema.NumberType, nil -// case reflect.Struct: -// return jsonschema.ObjectType, nil -// case reflect.Map: -// if golangType.Key().Kind() != reflect.String { -// return jsonschema.InvalidType, fmt.Errorf("only strings map keys are valid. key type: %v", golangType.Key().Kind()) -// } -// return jsonschema.ObjectType, nil -// case reflect.Array, reflect.Slice: -// return jsonschema.ArrayType, nil -// default: -// return jsonschema.InvalidType, fmt.Errorf("unhandled golang type: %s", golangType) -// } -// } - -// A wrapper over toSchema function to: -// 1. Detect cycles in the bundle config struct. -// 2. Update tracker -// -// params: -// -// - golangType: Golang type to generate json schema for -// -// - docs: Contains documentation to be injected into the generated json schema -// -// - traceId: An identifier for the current type, to trace recursive traversal. -// Its value is the first json tag in case of struct fields and "" in other cases -// like array, map or no json tags -// -// - tracker: Keeps track of types / traceIds seen during recursive traversal -// func safeToSchema(golangType reflect.Type, docs *Docs, traceId string, tracker *tracker) (*jsonschema.Schema, error) { -// // HACK to unblock CLI release (13th Feb 2024). This is temporary until proper -// // support for recursive types is added to the schema generator. PR: https://github.com/databricks/cli/pull/1204 -// if traceId == "for_each_task" { -// return &jsonschema.Schema{ -// Type: jsonschema.ObjectType, -// }, nil -// } - -// // WE ERROR OUT IF THERE ARE CYCLES IN THE JSON SCHEMA -// // There are mechanisms to deal with cycles though recursive identifiers in json -// // schema. However if we use them, we would need to make sure we are able to detect -// // cycles where two properties (directly or indirectly) pointing to each other -// // -// // see: https://json-schema.org/understanding-json-schema/structuring.html#recursion -// // for details -// if tracker.hasCycle(golangType) { -// return nil, fmt.Errorf("cycle detected") -// } - -// tracker.push(golangType, traceId) -// props, err := toSchema(golangType, docs, tracker) -// if err != nil { -// return nil, err -// } -// tracker.pop(golangType) -// return props, nil -// } - -// This function returns all member fields of the provided type. -// If the type has embedded (aka anonymous) fields, this function traverses -// those in a breadth first manner -// func getStructFields(golangType reflect.Type) []reflect.StructField { -// fields := []reflect.StructField{} -// bfsQueue := list.New() - -// for i := 0; i < golangType.NumField(); i++ { -// bfsQueue.PushBack(golangType.Field(i)) -// } -// for bfsQueue.Len() > 0 { -// front := bfsQueue.Front() -// field := front.Value.(reflect.StructField) -// bfsQueue.Remove(front) - -// if !field.Anonymous { -// fields = append(fields, field) -// continue -// } - -// fieldType := field.Type -// if fieldType.Kind() == reflect.Pointer { -// fieldType = fieldType.Elem() -// } - -// for i := 0; i < fieldType.NumField(); i++ { -// bfsQueue.PushBack(fieldType.Field(i)) -// } -// } -// return fields -// } - -// func toSchema(golangType reflect.Type, docs *Docs, tracker *tracker) (*jsonschema.Schema, error) { -// // *Struct and Struct generate identical json schemas -// if golangType.Kind() == reflect.Pointer { -// return safeToSchema(golangType.Elem(), docs, "", tracker) -// } -// if golangType.Kind() == reflect.Interface { -// return &jsonschema.Schema{}, nil -// } - -// rootJavascriptType, err := jsonSchemaType(golangType) -// if err != nil { -// return nil, err -// } -// jsonSchema := &jsonschema.Schema{Type: rootJavascriptType} - -// // If the type is a non-string primitive, then we allow it to be a string -// // provided it's a pure variable reference (ie only a single variable reference). -// if rootJavascriptType == jsonschema.BooleanType || rootJavascriptType == jsonschema.NumberType { -// jsonSchema = &jsonschema.Schema{ -// AnyOf: []*jsonschema.Schema{ -// { -// Type: rootJavascriptType, -// }, -// { -// Type: jsonschema.StringType, -// Pattern: dynvar.VariableRegex, -// }, -// }, -// } -// } - -// if docs != nil { -// jsonSchema.Description = docs.Description -// } - -// // case array/slice -// if golangType.Kind() == reflect.Array || golangType.Kind() == reflect.Slice { -// elemGolangType := golangType.Elem() -// elemJavascriptType, err := jsonSchemaType(elemGolangType) -// if err != nil { -// return nil, err -// } -// var childDocs *Docs -// if docs != nil { -// childDocs = docs.Items -// } -// elemProps, err := safeToSchema(elemGolangType, childDocs, "", tracker) -// if err != nil { -// return nil, err -// } -// jsonSchema.Items = &jsonschema.Schema{ -// Type: elemJavascriptType, -// Properties: elemProps.Properties, -// AdditionalProperties: elemProps.AdditionalProperties, -// Items: elemProps.Items, -// Required: elemProps.Required, -// } -// } - -// // case map -// if golangType.Kind() == reflect.Map { -// if golangType.Key().Kind() != reflect.String { -// return nil, fmt.Errorf("only string keyed maps allowed") -// } -// var childDocs *Docs -// if docs != nil { -// childDocs = docs.AdditionalProperties -// } -// jsonSchema.AdditionalProperties, err = safeToSchema(golangType.Elem(), childDocs, "", tracker) -// if err != nil { -// return nil, err -// } -// } - -// // case struct -// if golangType.Kind() == reflect.Struct { -// children := getStructFields(golangType) -// properties := map[string]*jsonschema.Schema{} -// required := []string{} -// for _, child := range children { -// bundleTag := child.Tag.Get("bundle") -// // Fields marked as "readonly", "internal" or "deprecated" are skipped -// // while generating the schema -// if bundleTag == readonlyTag || bundleTag == internalTag || bundleTag == deprecatedTag { -// continue -// } - -// // get child json tags -// childJsonTag := strings.Split(child.Tag.Get("json"), ",") -// childName := childJsonTag[0] - -// // skip children that have no json tags, the first json tag is "" -// // or the first json tag is "-" -// if childName == "" || childName == "-" { -// continue -// } - -// // get docs for the child if they exist -// var childDocs *Docs -// if docs != nil { -// if val, ok := docs.Properties[childName]; ok { -// childDocs = val -// } -// } - -// // compute if the child is a required field. Determined by the -// // presence of "omitempty" in the json tags -// hasOmitEmptyTag := false -// for i := 1; i < len(childJsonTag); i++ { -// if childJsonTag[i] == "omitempty" { -// hasOmitEmptyTag = true -// } -// } -// if !hasOmitEmptyTag { -// required = append(required, childName) -// } - -// // compute Schema.Properties for the child recursively -// fieldProps, err := safeToSchema(child.Type, childDocs, childName, tracker) -// if err != nil { -// return nil, err -// } -// properties[childName] = fieldProps -// } - -// jsonSchema.AdditionalProperties = false -// jsonSchema.Properties = properties -// jsonSchema.Required = required -// } - -// return jsonSchema, nil -// } diff --git a/bundle/schema/schema_test.go b/bundle/schema/schema_test.go deleted file mode 100644 index 277d5a8e4b..0000000000 --- a/bundle/schema/schema_test.go +++ /dev/null @@ -1,341 +0,0 @@ -package schema - -import ( - "encoding/json" - "reflect" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -// TODO: Add a test that checks the primitive overrides for reference regexs work. -// Basically that the custom override for bundle regex works. - -// TODO: Add a bundle of end to end tests, that both fail and pass the schema validation. - -func TestDocIngestionForObject(t *testing.T) { - docs := &Docs{ - Description: "docs for root", - Properties: map[string]*Docs{ - "my_struct": { - Description: "docs for my struct", - Properties: map[string]*Docs{ - "a": { - Description: "docs for a", - }, - "c": { - Description: "docs for c which does not exist on my_struct", - }, - }, - }, - }, - } - - type MyStruct struct { - A string `json:"a"` - B int `json:"b"` - } - - type Root struct { - MyStruct *MyStruct `json:"my_struct"` - } - - elem := Root{} - - schema, err := New(reflect.TypeOf(elem), docs) - require.NoError(t, err) - - jsonSchema, err := json.MarshalIndent(schema, " ", " ") - assert.NoError(t, err) - - expectedSchema := - `{ - "type": "object", - "description": "docs for root", - "properties": { - "my_struct": { - "type": "object", - "description": "docs for my struct", - "properties": { - "a": { - "type": "string", - "description": "docs for a" - }, - "b": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "string", - "pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - } - ] - } - }, - "additionalProperties": false, - "required": [ - "a", - "b" - ] - } - }, - "additionalProperties": false, - "required": [ - "my_struct" - ] - }` - - t.Log("[DEBUG] actual: ", string(jsonSchema)) - t.Log("[DEBUG] expected: ", expectedSchema) - - assert.Equal(t, expectedSchema, string(jsonSchema)) -} - -func TestDocIngestionForSlice(t *testing.T) { - docs := &Docs{ - Description: "docs for root", - Properties: map[string]*Docs{ - "my_slice": { - Description: "docs for my slice", - Items: &Docs{ - Properties: map[string]*Docs{ - "guava": { - Description: "docs for guava", - }, - "pineapple": { - Description: "docs for pineapple", - }, - "watermelon": { - Description: "docs for watermelon which does not exist in schema", - }, - }, - }, - }, - }, - } - - type Bar struct { - Guava int `json:"guava"` - Pineapple int `json:"pineapple"` - } - - type Root struct { - MySlice []Bar `json:"my_slice"` - } - - elem := Root{} - - schema, err := New(reflect.TypeOf(elem), docs) - require.NoError(t, err) - - jsonSchema, err := json.MarshalIndent(schema, " ", " ") - assert.NoError(t, err) - - expectedSchema := - `{ - "type": "object", - "description": "docs for root", - "properties": { - "my_slice": { - "type": "array", - "description": "docs for my slice", - "items": { - "type": "object", - "properties": { - "guava": { - "description": "docs for guava", - "anyOf": [ - { - "type": "number" - }, - { - "type": "string", - "pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - } - ] - }, - "pineapple": { - "description": "docs for pineapple", - "anyOf": [ - { - "type": "number" - }, - { - "type": "string", - "pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - } - ] - } - }, - "additionalProperties": false, - "required": [ - "guava", - "pineapple" - ] - } - } - }, - "additionalProperties": false, - "required": [ - "my_slice" - ] - }` - - t.Log("[DEBUG] actual: ", string(jsonSchema)) - t.Log("[DEBUG] expected: ", expectedSchema) - - assert.Equal(t, expectedSchema, string(jsonSchema)) -} - -func TestDocIngestionForMap(t *testing.T) { - docs := &Docs{ - Description: "docs for root", - Properties: map[string]*Docs{ - "my_map": { - Description: "docs for my map", - AdditionalProperties: &Docs{ - Properties: map[string]*Docs{ - "apple": { - Description: "docs for apple", - }, - "mango": { - Description: "docs for mango", - }, - "watermelon": { - Description: "docs for watermelon which does not exist in schema", - }, - "papaya": { - Description: "docs for papaya which does not exist in schema", - }, - }, - }, - }, - }, - } - - type Foo struct { - Apple int `json:"apple"` - Mango int `json:"mango"` - } - - type Root struct { - MyMap map[string]*Foo `json:"my_map"` - } - - elem := Root{} - - schema, err := New(reflect.TypeOf(elem), docs) - require.NoError(t, err) - - jsonSchema, err := json.MarshalIndent(schema, " ", " ") - assert.NoError(t, err) - - expectedSchema := - `{ - "type": "object", - "description": "docs for root", - "properties": { - "my_map": { - "type": "object", - "description": "docs for my map", - "additionalProperties": { - "type": "object", - "properties": { - "apple": { - "description": "docs for apple", - "anyOf": [ - { - "type": "number" - }, - { - "type": "string", - "pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - } - ] - }, - "mango": { - "description": "docs for mango", - "anyOf": [ - { - "type": "number" - }, - { - "type": "string", - "pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - } - ] - } - }, - "additionalProperties": false, - "required": [ - "apple", - "mango" - ] - } - } - }, - "additionalProperties": false, - "required": [ - "my_map" - ] - }` - - t.Log("[DEBUG] actual: ", string(jsonSchema)) - t.Log("[DEBUG] expected: ", expectedSchema) - - assert.Equal(t, expectedSchema, string(jsonSchema)) -} - -func TestDocIngestionForTopLevelPrimitive(t *testing.T) { - docs := &Docs{ - Description: "docs for root", - Properties: map[string]*Docs{ - "my_val": { - Description: "docs for my val", - }, - }, - } - - type Root struct { - MyVal int `json:"my_val"` - } - - elem := Root{} - - schema, err := New(reflect.TypeOf(elem), docs) - require.NoError(t, err) - - jsonSchema, err := json.MarshalIndent(schema, " ", " ") - assert.NoError(t, err) - - expectedSchema := - `{ - "type": "object", - "description": "docs for root", - "properties": { - "my_val": { - "description": "docs for my val", - "anyOf": [ - { - "type": "number" - }, - { - "type": "string", - "pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - } - ] - } - }, - "additionalProperties": false, - "required": [ - "my_val" - ] - }` - - t.Log("[DEBUG] actual: ", string(jsonSchema)) - t.Log("[DEBUG] expected: ", expectedSchema) - - assert.Equal(t, expectedSchema, string(jsonSchema)) -} diff --git a/bundle/schema/spec.go b/bundle/schema/spec.go deleted file mode 100644 index fdc31a4ca5..0000000000 --- a/bundle/schema/spec.go +++ /dev/null @@ -1,11 +0,0 @@ -package schema - -import "github.com/databricks/cli/libs/jsonschema" - -type Specification struct { - Components *Components `json:"components"` -} - -type Components struct { - Schemas map[string]*jsonschema.Schema `json:"schemas,omitempty"` -} diff --git a/bundle/schema/tracker.go b/bundle/schema/tracker.go deleted file mode 100644 index ace6559bc4..0000000000 --- a/bundle/schema/tracker.go +++ /dev/null @@ -1,53 +0,0 @@ -package schema - -import ( - "container/list" - "fmt" -) - -type tracker struct { - // Nodes encountered in current path during the recursive traversal. Used to - // check for cycles - seenNodes map[interface{}]struct{} - - // List of node names encountered in order in current path during the recursive traversal. - // Used to hydrate errors with path to the exact node where error occured. - // - // NOTE: node and node names can be the same - listOfNodes *list.List -} - -func newTracker() *tracker { - return &tracker{ - seenNodes: map[interface{}]struct{}{}, - listOfNodes: list.New(), - } -} - -func (t *tracker) errWithTrace(prefix string, initTrace string) error { - traceString := initTrace - curr := t.listOfNodes.Front() - for curr != nil { - if curr.Value.(string) != "" { - traceString += " -> " + curr.Value.(string) - } - curr = curr.Next() - } - return fmt.Errorf(prefix + ". traversal trace: " + traceString) -} - -func (t *tracker) hasCycle(node interface{}) bool { - _, ok := t.seenNodes[node] - return ok -} - -func (t *tracker) push(node interface{}, name string) { - t.seenNodes[node] = struct{}{} - t.listOfNodes.PushBack(name) -} - -func (t *tracker) pop(nodeType interface{}) { - back := t.listOfNodes.Back() - t.listOfNodes.Remove(back) - delete(t.seenNodes, nodeType) -} From fcdccb359a566190ed205ad4dfc39c62be0f6c82 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 27 Aug 2024 12:47:49 +0200 Subject: [PATCH 18/62] cleanu unused code --- bundle/internal/schema/main.go | 3 +- cmd/bundle/schema.go | 56 +++++++++++++++------------------- libs/jsonschema/schema.go | 45 +-------------------------- libs/jsonschema/schema_test.go | 56 ---------------------------------- libs/template/config_test.go | 2 +- 5 files changed, 29 insertions(+), 133 deletions(-) diff --git a/bundle/internal/schema/main.go b/bundle/internal/schema/main.go index 33cfbbe7f7..e59db91e05 100644 --- a/bundle/internal/schema/main.go +++ b/bundle/internal/schema/main.go @@ -52,7 +52,8 @@ func addInterpolationPatterns(_ reflect.Type, s jsonschema.Schema) jsonschema.Sc // as well. Make sure to pull those in. // TODO: Add unit tests for all permutations of structs, maps and slices for the FromType // method. -// TODO: Note the minor regression of losing the bundle descriptions +// TODO: Note the minor regression of losing the bundle descriptions. +// TODO: Ensure descriptions work for target resources section. func main() { if len(os.Args) != 2 { diff --git a/cmd/bundle/schema.go b/cmd/bundle/schema.go index 813aebbae3..9938ee14f8 100644 --- a/cmd/bundle/schema.go +++ b/cmd/bundle/schema.go @@ -1,13 +1,7 @@ package bundle import ( - "encoding/json" - "reflect" - - "github.com/databricks/cli/bundle/config" - "github.com/databricks/cli/bundle/schema" "github.com/databricks/cli/cmd/root" - "github.com/databricks/cli/libs/jsonschema" "github.com/spf13/cobra" ) @@ -19,31 +13,31 @@ func newSchemaCommand() *cobra.Command { } cmd.RunE = func(cmd *cobra.Command, args []string) error { - // Load embedded schema descriptions. - docs, err := schema.LoadBundleDescriptions() - if err != nil { - return err - } - - // Generate the JSON schema from the bundle configuration struct in Go. - schema, err := schema.New(reflect.TypeOf(config.Root{}), docs) - if err != nil { - return err - } - - // Target variable value overrides can be primitives, maps or sequences. - // Set an empty schema for them. - err = schema.SetByPath("targets.*.variables.*", jsonschema.Schema{}) - if err != nil { - return err - } - - // Print the JSON schema to stdout. - result, err := json.MarshalIndent(schema, "", " ") - if err != nil { - return err - } - cmd.OutOrStdout().Write(result) + // // Load embedded schema descriptions. + // docs, err := schema.LoadBundleDescriptions() + // if err != nil { + // return err + // } + + // // Generate the JSON schema from the bundle configuration struct in Go. + // schema, err := schema.New(reflect.TypeOf(config.Root{}), docs) + // if err != nil { + // return err + // } + + // // Target variable value overrides can be primitives, maps or sequences. + // // Set an empty schema for them. + // err = schema.SetByPath("targets.*.variables.*", jsonschema.Schema{}) + // if err != nil { + // return err + // } + + // // Print the JSON schema to stdout. + // result, err := json.MarshalIndent(schema, "", " ") + // if err != nil { + // return err + // } + // cmd.OutOrStdout().Write(result) return nil } diff --git a/libs/jsonschema/schema.go b/libs/jsonschema/schema.go index 44cc3b9c89..b364c82768 100644 --- a/libs/jsonschema/schema.go +++ b/libs/jsonschema/schema.go @@ -6,20 +6,12 @@ import ( "os" "regexp" "slices" - "strings" "github.com/databricks/cli/internal/build" "golang.org/x/mod/semver" ) // defines schema for a json object -// TODO: Remove pointers from properties and AnyOf. -// TODO: Can / should we emulate dyn.V here in having a readonly model for the data -// structure? Makes it easier to reason about. -// -// Any performance issues can be addressed by storing the schema -// -// as an embedded file. type Schema struct { // TODO: Comments for this field Definitions any `json:"$defs,omitempty"` @@ -38,6 +30,7 @@ type Schema struct { // Schemas for the fields of an struct. The keys are the first json tag. // The values are the schema for the type of the field + // TODO: Followup: Make this a map[string]Schema Properties map[string]*Schema `json:"properties,omitempty"` // The schema for all values of an array @@ -95,48 +88,12 @@ func (s *Schema) ParseString(v string) (any, error) { return fromString(v, s.Type) } -func (s *Schema) getByPath(path string) (*Schema, error) { - p := strings.Split(path, ".") - - res := s - for _, node := range p { - if node == "*" { - res = res.AdditionalProperties.(*Schema) - continue - } - var ok bool - res, ok = res.Properties[node] - if !ok { - return nil, fmt.Errorf("property %q not found in schema. Query path: %s", node, path) - } - } - return res, nil -} - -func (s *Schema) GetByPath(path string) (Schema, error) { - v, err := s.getByPath(path) - if err != nil { - return Schema{}, err - } - return *v, nil -} - -func (s *Schema) SetByPath(path string, v Schema) error { - dst, err := s.getByPath(path) - if err != nil { - return err - } - *dst = v - return nil -} - type Type string const ( // Default zero value of a schema. This does not correspond to a type in the // JSON schema spec and is an internal type defined for convenience. InvalidType Type = "invalid" - NullType Type = "null" BooleanType Type = "boolean" StringType Type = "string" NumberType Type = "number" diff --git a/libs/jsonschema/schema_test.go b/libs/jsonschema/schema_test.go index c365cf2352..46f4db6b1c 100644 --- a/libs/jsonschema/schema_test.go +++ b/libs/jsonschema/schema_test.go @@ -4,7 +4,6 @@ import ( "testing" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" ) func TestSchemaValidateTypeNames(t *testing.T) { @@ -340,58 +339,3 @@ func testSchema() *Schema { } } - -func TestSchemaGetByPath(t *testing.T) { - s := testSchema() - - ss, err := s.GetByPath("int_val") - require.NoError(t, err) - assert.Equal(t, Schema{ - Type: IntegerType, - Default: int64(123), - }, ss) - - ss, err = s.GetByPath("string_val") - require.NoError(t, err) - assert.Equal(t, Schema{ - Type: StringType, - }, ss) - - ss, err = s.GetByPath("object_val.bar") - require.NoError(t, err) - assert.Equal(t, Schema{ - Type: StringType, - Default: "baz", - }, ss) - - ss, err = s.GetByPath("object_val.*.foo") - require.NoError(t, err) - assert.Equal(t, Schema{ - Type: StringType, - Default: "zab", - }, ss) -} - -func TestSchemaSetByPath(t *testing.T) { - s := testSchema() - - err := s.SetByPath("int_val", Schema{ - Type: IntegerType, - Default: int64(456), - }) - require.NoError(t, err) - assert.Equal(t, int64(456), s.Properties["int_val"].Default) - - err = s.SetByPath("object_val.*.foo", Schema{ - Type: StringType, - Default: "zooby", - }) - require.NoError(t, err) - - ns, err := s.GetByPath("object_val.*.foo") - require.NoError(t, err) - assert.Equal(t, Schema{ - Type: StringType, - Default: "zooby", - }, ns) -} diff --git a/libs/template/config_test.go b/libs/template/config_test.go index 73b47f2891..ab9dbeb5f9 100644 --- a/libs/template/config_test.go +++ b/libs/template/config_test.go @@ -461,7 +461,7 @@ func TestPromptIsSkippedAnyOf(t *testing.T) { Default: "hello-world", Extension: jsonschema.Extension{ SkipPromptIf: &jsonschema.Schema{ - AnyOf: []*jsonschema.Schema{ + AnyOf: []jsonschema.Schema{ { Properties: map[string]*jsonschema.Schema{ "abc": { From e7fd063e8adc864d146fa8f74fd9971cdbb13a83 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 27 Aug 2024 13:10:13 +0200 Subject: [PATCH 19/62] - --- .codegen.json | 4 +- .gitattributes | 2 +- bundle/internal/schema/main.go | 10 +- bundle/internal/schema/schema.generated.json | 6264 ++++++++++++++++++ 4 files changed, 6275 insertions(+), 5 deletions(-) create mode 100644 bundle/internal/schema/schema.generated.json diff --git a/.codegen.json b/.codegen.json index 8cb42b415c..5604f6a863 100644 --- a/.codegen.json +++ b/.codegen.json @@ -11,10 +11,10 @@ "toolchain": { "required": ["go"], "post_generate": [ - "go run ./bundle/internal/bundle/schema/main.go ./bundle/schema/docs/bundle_descriptions.json", + "go run ./bundle/internal/schema/*.go ./bundle/internal/schema/schema.generated.json", "echo 'bundle/internal/tf/schema/\\*.go linguist-generated=true' >> ./.gitattributes", "echo 'go.sum linguist-generated=true' >> ./.gitattributes", - "echo 'bundle/schema/docs/bundle_descriptions.json linguist-generated=true' >> ./.gitattributes" + "echo 'bundle/schema/docs/schema.generated.json linguist-generated=true' >> ./.gitattributes" ] } } diff --git a/.gitattributes b/.gitattributes index d82ab7696d..7c7e9e9eef 100755 --- a/.gitattributes +++ b/.gitattributes @@ -120,4 +120,4 @@ cmd/workspace/workspace-conf/workspace-conf.go linguist-generated=true cmd/workspace/workspace/workspace.go linguist-generated=true bundle/internal/tf/schema/\*.go linguist-generated=true go.sum linguist-generated=true -bundle/schema/docs/bundle_descriptions.json linguist-generated=true +bundle/schema/docs/schema.generated.json linguist-generated=true diff --git a/bundle/internal/schema/main.go b/bundle/internal/schema/main.go index e59db91e05..93210ca1ee 100644 --- a/bundle/internal/schema/main.go +++ b/bundle/internal/schema/main.go @@ -8,6 +8,7 @@ import ( "reflect" "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/config/variable" "github.com/databricks/cli/libs/jsonschema" ) @@ -15,7 +16,11 @@ func interpolationPattern(s string) string { return fmt.Sprintf(`\$\{(%s(\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\[[0-9]+\])*)*(\[[0-9]+\])*)\}`, s) } -func addInterpolationPatterns(_ reflect.Type, s jsonschema.Schema) jsonschema.Schema { +func addInterpolationPatterns(typ reflect.Type, s jsonschema.Schema) jsonschema.Schema { + if typ == reflect.TypeOf(config.Root{}) || typ == reflect.TypeOf(variable.Variable{}) { + return s + } + switch s.Type { case jsonschema.ArrayType, jsonschema.ObjectType: // arrays and objects can have complex variable values specified. @@ -33,7 +38,8 @@ func addInterpolationPatterns(_ reflect.Type, s jsonschema.Schema) jsonschema.Sc // they are of the permitted patterns? return jsonschema.Schema{ AnyOf: []jsonschema.Schema{s, - // TODO: Add "resources" here + // TODO: Is it only resource IDs or is it resources in general? + {Type: jsonschema.StringType, Pattern: interpolationPattern("resources")}, {Type: jsonschema.StringType, Pattern: interpolationPattern("bundle")}, {Type: jsonschema.StringType, Pattern: interpolationPattern("workspace")}, {Type: jsonschema.StringType, Pattern: interpolationPattern("var")}, diff --git a/bundle/internal/schema/schema.generated.json b/bundle/internal/schema/schema.generated.json new file mode 100644 index 0000000000..337867b28f --- /dev/null +++ b/bundle/internal/schema/schema.generated.json @@ -0,0 +1,6264 @@ +{ + "$defs": { + "bool": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "string", + "pattern": "\\$\\{(resources(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(bundle(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(workspace(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "float64": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "string", + "pattern": "\\$\\{(resources(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(bundle(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(workspace(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "github.com": { + "databricks": { + "cli": { + "bundle": { + "config": { + "resources.Grant": { + "anyOf": [ + { + "type": "object", + "properties": { + "principal": { + "$ref": "#/$defs/string" + }, + "privileges": { + "$ref": "#/$defs/slice/string" + } + }, + "additionalProperties": false, + "required": [ + "privileges", + "principal" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "resources.Job": { + "anyOf": [ + { + "type": "object", + "properties": { + "continuous": { + "description": "An optional continuous property for this job. The continuous property will ensure that there is always one run executing. Only one of `schedule` and `continuous` can be used.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.Continuous" + }, + "deployment": { + "description": "Deployment information for jobs managed by external sources.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.JobDeployment" + }, + "description": { + "description": "An optional description for the job. The maximum length is 27700 characters in UTF-8 encoding.", + "$ref": "#/$defs/string" + }, + "edit_mode": { + "description": "Edit mode of the job.\n\n* `UI_LOCKED`: The job is in a locked UI state and cannot be modified.\n* `EDITABLE`: The job is in an editable state and can be modified.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.JobEditMode" + }, + "email_notifications": { + "description": "An optional set of email addresses that is notified when runs of this job begin or complete as well as when this job is deleted.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.JobEmailNotifications" + }, + "environments": { + "description": "A list of task execution environment specifications that can be referenced by tasks of this job.", + "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/jobs.JobEnvironment" + }, + "format": { + "description": "Used to tell what is the format of the job. This field is ignored in Create/Update/Reset calls. When using the Jobs API 2.1 this value is always set to `\"MULTI_TASK\"`.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.Format" + }, + "git_source": { + "description": "An optional specification for a remote Git repository containing the source code used by tasks. Version-controlled source code is supported by notebook, dbt, Python script, and SQL File tasks.\n\nIf `git_source` is set, these tasks retrieve the file from the remote repository by default. However, this behavior can be overridden by setting `source` to `WORKSPACE` on the task.\n\nNote: dbt and SQL File tasks support only version-controlled sources. If dbt or SQL File tasks are used, `git_source` must be defined on the job.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.GitSource" + }, + "health": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.JobsHealthRules" + }, + "job_clusters": { + "description": "A list of job cluster specifications that can be shared and reused by tasks of this job. Libraries cannot be declared in a shared job cluster. You must declare dependent libraries in task settings.", + "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/jobs.JobCluster" + }, + "max_concurrent_runs": { + "description": "An optional maximum allowed number of concurrent runs of the job.\nSet this value if you want to be able to execute multiple runs of the same job concurrently.\nThis is useful for example if you trigger your job on a frequent schedule and want to allow consecutive runs to overlap with each other, or if you want to trigger multiple runs which differ by their input parameters.\nThis setting affects only new runs. For example, suppose the job’s concurrency is 4 and there are 4 concurrent active runs. Then setting the concurrency to 3 won’t kill any of the active runs.\nHowever, from then on, new runs are skipped unless there are fewer than 3 active runs.\nThis value cannot exceed 1000. Setting this value to `0` causes all new runs to be skipped.", + "$ref": "#/$defs/int" + }, + "name": { + "description": "An optional name for the job. The maximum length is 4096 bytes in UTF-8 encoding.", + "$ref": "#/$defs/string" + }, + "notification_settings": { + "description": "Optional notification settings that are used when sending notifications to each of the `email_notifications` and `webhook_notifications` for this job.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.JobNotificationSettings" + }, + "parameters": { + "description": "Job-level parameter definitions", + "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/jobs.JobParameterDefinition" + }, + "permissions": { + "$ref": "#/$defs/slice/github.com/databricks/cli/bundle/config/resources.Permission" + }, + "queue": { + "description": "The queue settings of the job.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.QueueSettings" + }, + "run_as": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.JobRunAs" + }, + "schedule": { + "description": "An optional periodic schedule for this job. The default behavior is that the job only runs when triggered by clicking “Run Now” in the Jobs UI or sending an API request to `runNow`.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.CronSchedule" + }, + "tags": { + "description": "A map of tags associated with the job. These are forwarded to the cluster as cluster tags for jobs clusters, and are subject to the same limitations as cluster tags. A maximum of 25 tags can be added to the job.", + "$ref": "#/$defs/map/string" + }, + "tasks": { + "description": "A list of task specifications to be executed by this job.", + "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/jobs.Task" + }, + "timeout_seconds": { + "description": "An optional timeout applied to each run of this job. A value of `0` means no timeout.", + "$ref": "#/$defs/int" + }, + "trigger": { + "description": "A configuration to trigger a run when certain conditions are met. The default behavior is that the job runs only when triggered by clicking “Run Now” in the Jobs UI or sending an API request to `runNow`.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.TriggerSettings" + }, + "webhook_notifications": { + "description": "A collection of system notification IDs to notify when runs of this job begin or complete.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.WebhookNotifications" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "resources.MlflowExperiment": { + "anyOf": [ + { + "type": "object", + "properties": { + "artifact_location": { + "description": "Location where artifacts for the experiment are stored.", + "$ref": "#/$defs/string" + }, + "creation_time": { + "description": "Creation time", + "$ref": "#/$defs/int64" + }, + "experiment_id": { + "description": "Unique identifier for the experiment.", + "$ref": "#/$defs/string" + }, + "last_update_time": { + "description": "Last update time", + "$ref": "#/$defs/int64" + }, + "lifecycle_stage": { + "description": "Current life cycle stage of the experiment: \"active\" or \"deleted\".\nDeleted experiments are not returned by APIs.", + "$ref": "#/$defs/string" + }, + "name": { + "description": "Human readable name that identifies the experiment.", + "$ref": "#/$defs/string" + }, + "permissions": { + "$ref": "#/$defs/slice/github.com/databricks/cli/bundle/config/resources.Permission" + }, + "tags": { + "description": "Tags: Additional metadata key-value pairs.", + "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/ml.ExperimentTag" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "resources.MlflowModel": { + "anyOf": [ + { + "type": "object", + "properties": { + "creation_timestamp": { + "description": "Timestamp recorded when this `registered_model` was created.", + "$ref": "#/$defs/int64" + }, + "description": { + "description": "Description of this `registered_model`.", + "$ref": "#/$defs/string" + }, + "last_updated_timestamp": { + "description": "Timestamp recorded when metadata for this `registered_model` was last updated.", + "$ref": "#/$defs/int64" + }, + "latest_versions": { + "description": "Collection of latest model versions for each stage.\nOnly contains models with current `READY` status.", + "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/ml.ModelVersion" + }, + "name": { + "description": "Unique name for the model.", + "$ref": "#/$defs/string" + }, + "permissions": { + "$ref": "#/$defs/slice/github.com/databricks/cli/bundle/config/resources.Permission" + }, + "tags": { + "description": "Tags: Additional metadata key-value pairs for this `registered_model`.", + "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/ml.ModelTag" + }, + "user_id": { + "description": "User that created this `registered_model`", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "resources.ModelServingEndpoint": { + "anyOf": [ + { + "type": "object", + "properties": { + "config": { + "description": "The core config of the serving endpoint.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.EndpointCoreConfigInput" + }, + "name": { + "description": "The name of the serving endpoint. This field is required and must be unique across a Databricks workspace.\nAn endpoint name can consist of alphanumeric characters, dashes, and underscores.\n", + "$ref": "#/$defs/string" + }, + "permissions": { + "$ref": "#/$defs/slice/github.com/databricks/cli/bundle/config/resources.Permission" + }, + "rate_limits": { + "description": "Rate limits to be applied to the serving endpoint. NOTE: only external and foundation model endpoints are supported as of now.", + "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/serving.RateLimit" + }, + "route_optimized": { + "description": "Enable route optimization for the serving endpoint.", + "$ref": "#/$defs/bool" + }, + "tags": { + "description": "Tags to be attached to the serving endpoint and automatically propagated to billing logs.", + "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/serving.EndpointTag" + } + }, + "additionalProperties": false, + "required": [ + "config", + "name" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "resources.Permission": { + "anyOf": [ + { + "type": "object", + "properties": { + "group_name": { + "$ref": "#/$defs/string" + }, + "level": { + "$ref": "#/$defs/string" + }, + "service_principal_name": { + "$ref": "#/$defs/string" + }, + "user_name": { + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "level" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "resources.Pipeline": { + "anyOf": [ + { + "type": "object", + "properties": { + "catalog": { + "description": "A catalog in Unity Catalog to publish data from this pipeline to. If `target` is specified, tables in this pipeline are published to a `target` schema inside `catalog` (for example, `catalog`.`target`.`table`). If `target` is not specified, no data is published to Unity Catalog.", + "$ref": "#/$defs/string" + }, + "channel": { + "description": "DLT Release Channel that specifies which version to use.", + "$ref": "#/$defs/string" + }, + "clusters": { + "description": "Cluster settings for this pipeline deployment.", + "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/pipelines.PipelineCluster" + }, + "configuration": { + "description": "String-String configuration for this pipeline execution.", + "$ref": "#/$defs/map/string" + }, + "continuous": { + "description": "Whether the pipeline is continuous or triggered. This replaces `trigger`.", + "$ref": "#/$defs/bool" + }, + "deployment": { + "description": "Deployment type of this pipeline.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.PipelineDeployment" + }, + "development": { + "description": "Whether the pipeline is in Development mode. Defaults to false.", + "$ref": "#/$defs/bool" + }, + "edition": { + "description": "Pipeline product edition.", + "$ref": "#/$defs/string" + }, + "filters": { + "description": "Filters on which Pipeline packages to include in the deployed graph.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.Filters" + }, + "gateway_definition": { + "description": "The definition of a gateway pipeline to support CDC.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.IngestionGatewayPipelineDefinition" + }, + "id": { + "description": "Unique identifier for this pipeline.", + "$ref": "#/$defs/string" + }, + "ingestion_definition": { + "description": "The configuration for a managed ingestion pipeline. These settings cannot be used with the 'libraries', 'target' or 'catalog' settings.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.IngestionPipelineDefinition" + }, + "libraries": { + "description": "Libraries or code needed by this deployment.", + "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/pipelines.PipelineLibrary" + }, + "name": { + "description": "Friendly identifier for this pipeline.", + "$ref": "#/$defs/string" + }, + "notifications": { + "description": "List of notification settings for this pipeline.", + "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/pipelines.Notifications" + }, + "permissions": { + "$ref": "#/$defs/slice/github.com/databricks/cli/bundle/config/resources.Permission" + }, + "photon": { + "description": "Whether Photon is enabled for this pipeline.", + "$ref": "#/$defs/bool" + }, + "serverless": { + "description": "Whether serverless compute is enabled for this pipeline.", + "$ref": "#/$defs/bool" + }, + "storage": { + "description": "DBFS root directory for storing checkpoints and tables.", + "$ref": "#/$defs/string" + }, + "target": { + "description": "Target schema (database) to add tables in this pipeline to. If not specified, no data is published to the Hive metastore or Unity Catalog. To publish to Unity Catalog, also specify `catalog`.", + "$ref": "#/$defs/string" + }, + "trigger": { + "description": "Which pipeline trigger to use. Deprecated: Use `continuous` instead.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.PipelineTrigger" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "resources.QualityMonitor": { + "anyOf": [ + { + "type": "object", + "properties": { + "assets_dir": { + "description": "The directory to store monitoring assets (e.g. dashboard, metric tables).", + "$ref": "#/$defs/string" + }, + "baseline_table_name": { + "description": "Name of the baseline table from which drift metrics are computed from.\nColumns in the monitored table should also be present in the baseline table.\n", + "$ref": "#/$defs/string" + }, + "custom_metrics": { + "description": "Custom metrics to compute on the monitored table. These can be aggregate metrics, derived\nmetrics (from already computed aggregate metrics), or drift metrics (comparing metrics across time\nwindows).\n", + "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/catalog.MonitorMetric" + }, + "data_classification_config": { + "description": "The data classification config for the monitor.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/catalog.MonitorDataClassificationConfig" + }, + "inference_log": { + "description": "Configuration for monitoring inference logs.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/catalog.MonitorInferenceLog" + }, + "notifications": { + "description": "The notification settings for the monitor.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/catalog.MonitorNotifications" + }, + "output_schema_name": { + "description": "Schema where output metric tables are created.", + "$ref": "#/$defs/string" + }, + "schedule": { + "description": "The schedule for automatically updating and refreshing metric tables.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/catalog.MonitorCronSchedule" + }, + "skip_builtin_dashboard": { + "description": "Whether to skip creating a default dashboard summarizing data quality metrics.", + "$ref": "#/$defs/bool" + }, + "slicing_exprs": { + "description": "List of column expressions to slice data with for targeted analysis. The data is grouped by\neach expression independently, resulting in a separate slice for each predicate and its\ncomplements. For high-cardinality columns, only the top 100 unique values by frequency will\ngenerate slices.\n", + "$ref": "#/$defs/slice/string" + }, + "snapshot": { + "description": "Configuration for monitoring snapshot tables.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/catalog.MonitorSnapshot" + }, + "time_series": { + "description": "Configuration for monitoring time series tables.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/catalog.MonitorTimeSeries" + }, + "warehouse_id": { + "description": "Optional argument to specify the warehouse for dashboard creation. If not specified, the first running\nwarehouse will be used.\n", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "assets_dir", + "output_schema_name" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "resources.RegisteredModel": { + "anyOf": [ + { + "type": "object", + "properties": { + "catalog_name": { + "description": "The name of the catalog where the schema and the registered model reside", + "$ref": "#/$defs/string" + }, + "comment": { + "description": "The comment attached to the registered model", + "$ref": "#/$defs/string" + }, + "grants": { + "$ref": "#/$defs/slice/github.com/databricks/cli/bundle/config/resources.Grant" + }, + "name": { + "description": "The name of the registered model", + "$ref": "#/$defs/string" + }, + "schema_name": { + "description": "The name of the schema where the registered model resides", + "$ref": "#/$defs/string" + }, + "storage_location": { + "description": "The storage location on the cloud under which model version data files are stored", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "catalog_name", + "name", + "schema_name" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "resources.Schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "catalog_name": { + "description": "Name of parent catalog.", + "$ref": "#/$defs/string" + }, + "comment": { + "description": "User-provided free-form text description.", + "$ref": "#/$defs/string" + }, + "grants": { + "$ref": "#/$defs/slice/github.com/databricks/cli/bundle/config/resources.Grant" + }, + "name": { + "description": "Name of schema, relative to parent catalog.", + "$ref": "#/$defs/string" + }, + "properties": { + "$ref": "#/$defs/map/string" + }, + "storage_root": { + "description": "Storage root URL for managed tables within schema.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "catalog_name", + "name" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "variable.Lookup": { + "anyOf": [ + { + "type": "object", + "properties": { + "alert": { + "$ref": "#/$defs/string" + }, + "cluster": { + "$ref": "#/$defs/string" + }, + "cluster_policy": { + "$ref": "#/$defs/string" + }, + "dashboard": { + "$ref": "#/$defs/string" + }, + "instance_pool": { + "$ref": "#/$defs/string" + }, + "job": { + "$ref": "#/$defs/string" + }, + "metastore": { + "$ref": "#/$defs/string" + }, + "pipeline": { + "$ref": "#/$defs/string" + }, + "query": { + "$ref": "#/$defs/string" + }, + "service_principal": { + "$ref": "#/$defs/string" + }, + "warehouse": { + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "variable.Variable": { + "type": "object", + "properties": { + "default": { + "$ref": "#/$defs/interface" + }, + "description": { + "$ref": "#/$defs/string" + }, + "lookup": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/variable.Lookup" + }, + "type": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/variable.VariableType" + } + }, + "additionalProperties": false + }, + "variable.VariableType": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "string", + "pattern": "\\$\\{(resources(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(bundle(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(workspace(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + } + }, + "config.Artifact": { + "anyOf": [ + { + "type": "object", + "properties": { + "build": { + "$ref": "#/$defs/string" + }, + "executable": { + "$ref": "#/$defs/github.com/databricks/cli/libs/exec.ExecutableType" + }, + "files": { + "$ref": "#/$defs/slice/github.com/databricks/cli/bundle/config.ArtifactFile" + }, + "path": { + "$ref": "#/$defs/string" + }, + "type": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config.ArtifactType" + } + }, + "additionalProperties": false, + "required": [ + "type" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "config.ArtifactFile": { + "anyOf": [ + { + "type": "object", + "properties": { + "source": { + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "source" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "config.ArtifactType": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "string", + "pattern": "\\$\\{(resources(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(bundle(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(workspace(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "config.Bundle": { + "anyOf": [ + { + "type": "object", + "properties": { + "compute_id": { + "$ref": "#/$defs/string" + }, + "databricks_cli_version": { + "$ref": "#/$defs/string" + }, + "deployment": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config.Deployment" + }, + "git": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config.Git" + }, + "name": { + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "name" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "config.Command": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "string", + "pattern": "\\$\\{(resources(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(bundle(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(workspace(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "config.Deployment": { + "anyOf": [ + { + "type": "object", + "properties": { + "fail_on_active_runs": { + "$ref": "#/$defs/bool" + }, + "lock": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config.Lock" + } + }, + "additionalProperties": false, + "required": [ + "lock" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "config.Experimental": { + "anyOf": [ + { + "type": "object", + "properties": { + "pydabs": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config.PyDABs" + }, + "python_wheel_wrapper": { + "$ref": "#/$defs/bool" + }, + "scripts": { + "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config.Command" + }, + "use_legacy_run_as": { + "$ref": "#/$defs/bool" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "config.Git": { + "anyOf": [ + { + "type": "object", + "properties": { + "branch": { + "$ref": "#/$defs/string" + }, + "origin_url": { + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "config.Lock": { + "anyOf": [ + { + "type": "object", + "properties": { + "enabled": { + "$ref": "#/$defs/bool" + }, + "force": { + "$ref": "#/$defs/bool" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "config.Mode": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "string", + "pattern": "\\$\\{(resources(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(bundle(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(workspace(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "config.Presets": { + "anyOf": [ + { + "type": "object", + "properties": { + "jobs_max_concurrent_runs": { + "$ref": "#/$defs/int" + }, + "name_prefix": { + "$ref": "#/$defs/string" + }, + "pipelines_development": { + "$ref": "#/$defs/bool" + }, + "tags": { + "$ref": "#/$defs/map/string" + }, + "trigger_pause_status": { + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "config.PyDABs": { + "anyOf": [ + { + "type": "object", + "properties": { + "enabled": { + "$ref": "#/$defs/bool" + }, + "import": { + "$ref": "#/$defs/slice/string" + }, + "venv_path": { + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "config.Resources": { + "anyOf": [ + { + "type": "object", + "properties": { + "experiments": { + "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.MlflowExperiment" + }, + "jobs": { + "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.Job" + }, + "model_serving_endpoints": { + "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.ModelServingEndpoint" + }, + "models": { + "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.MlflowModel" + }, + "pipelines": { + "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.Pipeline" + }, + "quality_monitors": { + "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.QualityMonitor" + }, + "registered_models": { + "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.RegisteredModel" + }, + "schemas": { + "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.Schema" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "config.Sync": { + "anyOf": [ + { + "type": "object", + "properties": { + "exclude": { + "$ref": "#/$defs/slice/string" + }, + "include": { + "$ref": "#/$defs/slice/string" + }, + "paths": { + "$ref": "#/$defs/slice/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "config.Target": { + "anyOf": [ + { + "type": "object", + "properties": { + "artifacts": { + "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config.Artifact" + }, + "bundle": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config.Bundle" + }, + "compute_id": { + "$ref": "#/$defs/string" + }, + "default": { + "$ref": "#/$defs/bool" + }, + "git": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config.Git" + }, + "mode": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config.Mode" + }, + "permissions": { + "$ref": "#/$defs/slice/github.com/databricks/cli/bundle/config/resources.Permission" + }, + "presets": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config.Presets" + }, + "resources": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config.Resources" + }, + "run_as": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.JobRunAs" + }, + "sync": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config.Sync" + }, + "variables": { + "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/variable.Variable" + }, + "workspace": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config.Workspace" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "config.Workspace": { + "anyOf": [ + { + "type": "object", + "properties": { + "artifact_path": { + "$ref": "#/$defs/string" + }, + "auth_type": { + "$ref": "#/$defs/string" + }, + "azure_client_id": { + "$ref": "#/$defs/string" + }, + "azure_environment": { + "$ref": "#/$defs/string" + }, + "azure_login_app_id": { + "$ref": "#/$defs/string" + }, + "azure_tenant_id": { + "$ref": "#/$defs/string" + }, + "azure_use_msi": { + "$ref": "#/$defs/bool" + }, + "azure_workspace_resource_id": { + "$ref": "#/$defs/string" + }, + "client_id": { + "$ref": "#/$defs/string" + }, + "file_path": { + "$ref": "#/$defs/string" + }, + "google_service_account": { + "$ref": "#/$defs/string" + }, + "host": { + "$ref": "#/$defs/string" + }, + "profile": { + "$ref": "#/$defs/string" + }, + "root_path": { + "$ref": "#/$defs/string" + }, + "state_path": { + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + } + }, + "libs": { + "exec.ExecutableType": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "string", + "pattern": "\\$\\{(resources(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(bundle(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(workspace(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + } + } + }, + "databricks-sdk-go": { + "service": { + "catalog.MonitorCronSchedule": { + "anyOf": [ + { + "type": "object", + "properties": { + "pause_status": { + "description": "Read only field that indicates whether a schedule is paused or not.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/catalog.MonitorCronSchedulePauseStatus", + "enum": [ + "UNPAUSED", + "PAUSED" + ] + }, + "quartz_cron_expression": { + "description": "The expression that determines when to run the monitor. See [examples](https://www.quartz-scheduler.org/documentation/quartz-2.3.0/tutorials/crontrigger.html).\n", + "$ref": "#/$defs/string" + }, + "timezone_id": { + "description": "The timezone id (e.g., ``\"PST\"``) in which to evaluate the quartz expression.\n", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "quartz_cron_expression", + "timezone_id" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "catalog.MonitorCronSchedulePauseStatus": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "string", + "pattern": "\\$\\{(resources(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(bundle(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(workspace(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "catalog.MonitorDataClassificationConfig": { + "anyOf": [ + { + "type": "object", + "properties": { + "enabled": { + "description": "Whether data classification is enabled.", + "$ref": "#/$defs/bool" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "catalog.MonitorDestination": { + "anyOf": [ + { + "type": "object", + "properties": { + "email_addresses": { + "description": "The list of email addresses to send the notification to. A maximum of 5 email addresses is supported.", + "$ref": "#/$defs/slice/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "catalog.MonitorInferenceLog": { + "anyOf": [ + { + "type": "object", + "properties": { + "granularities": { + "description": "Granularities for aggregating data into time windows based on their timestamp. Currently the following static\ngranularities are supported:\n{``\"5 minutes\"``, ``\"30 minutes\"``, ``\"1 hour\"``, ``\"1 day\"``, ``\"\u003cn\u003e week(s)\"``, ``\"1 month\"``, ``\"1 year\"``}.\n", + "$ref": "#/$defs/slice/string" + }, + "label_col": { + "description": "Optional column that contains the ground truth for the prediction.", + "$ref": "#/$defs/string" + }, + "model_id_col": { + "description": "Column that contains the id of the model generating the predictions. Metrics will be computed per model id by\ndefault, and also across all model ids.\n", + "$ref": "#/$defs/string" + }, + "prediction_col": { + "description": "Column that contains the output/prediction from the model.", + "$ref": "#/$defs/string" + }, + "prediction_proba_col": { + "description": "Optional column that contains the prediction probabilities for each class in a classification problem type.\nThe values in this column should be a map, mapping each class label to the prediction probability for a given\nsample. The map should be of PySpark MapType().\n", + "$ref": "#/$defs/string" + }, + "problem_type": { + "description": "Problem type the model aims to solve. Determines the type of model-quality metrics that will be computed.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/catalog.MonitorInferenceLogProblemType", + "enum": [ + "PROBLEM_TYPE_CLASSIFICATION", + "PROBLEM_TYPE_REGRESSION" + ] + }, + "timestamp_col": { + "description": "Column that contains the timestamps of requests. The column must be one of the following:\n- A ``TimestampType`` column\n- A column whose values can be converted to timestamps through the pyspark\n ``to_timestamp`` [function](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.to_timestamp.html).\n", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "granularities", + "model_id_col", + "prediction_col", + "problem_type", + "timestamp_col" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "catalog.MonitorInferenceLogProblemType": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "string", + "pattern": "\\$\\{(resources(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(bundle(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(workspace(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "catalog.MonitorMetric": { + "anyOf": [ + { + "type": "object", + "properties": { + "definition": { + "description": "Jinja template for a SQL expression that specifies how to compute the metric. See [create metric definition](https://docs.databricks.com/en/lakehouse-monitoring/custom-metrics.html#create-definition).", + "$ref": "#/$defs/string" + }, + "input_columns": { + "description": "A list of column names in the input table the metric should be computed for.\nCan use ``\":table\"`` to indicate that the metric needs information from multiple columns.\n", + "$ref": "#/$defs/slice/string" + }, + "name": { + "description": "Name of the metric in the output tables.", + "$ref": "#/$defs/string" + }, + "output_data_type": { + "description": "The output type of the custom metric.", + "$ref": "#/$defs/string" + }, + "type": { + "description": "Can only be one of ``\"CUSTOM_METRIC_TYPE_AGGREGATE\"``, ``\"CUSTOM_METRIC_TYPE_DERIVED\"``, or ``\"CUSTOM_METRIC_TYPE_DRIFT\"``.\nThe ``\"CUSTOM_METRIC_TYPE_AGGREGATE\"`` and ``\"CUSTOM_METRIC_TYPE_DERIVED\"`` metrics\nare computed on a single table, whereas the ``\"CUSTOM_METRIC_TYPE_DRIFT\"`` compare metrics across\nbaseline and input table, or across the two consecutive time windows.\n- CUSTOM_METRIC_TYPE_AGGREGATE: only depend on the existing columns in your table\n- CUSTOM_METRIC_TYPE_DERIVED: depend on previously computed aggregate metrics\n- CUSTOM_METRIC_TYPE_DRIFT: depend on previously computed aggregate or derived metrics\n", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/catalog.MonitorMetricType", + "enum": [ + "CUSTOM_METRIC_TYPE_AGGREGATE", + "CUSTOM_METRIC_TYPE_DERIVED", + "CUSTOM_METRIC_TYPE_DRIFT" + ] + } + }, + "additionalProperties": false, + "required": [ + "definition", + "input_columns", + "name", + "output_data_type", + "type" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "catalog.MonitorMetricType": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "string", + "pattern": "\\$\\{(resources(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(bundle(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(workspace(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "catalog.MonitorNotifications": { + "anyOf": [ + { + "type": "object", + "properties": { + "on_failure": { + "description": "Who to send notifications to on monitor failure.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/catalog.MonitorDestination" + }, + "on_new_classification_tag_detected": { + "description": "Who to send notifications to when new data classification tags are detected.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/catalog.MonitorDestination" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "catalog.MonitorSnapshot": { + "anyOf": [ + { + "type": "object", + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "catalog.MonitorTimeSeries": { + "anyOf": [ + { + "type": "object", + "properties": { + "granularities": { + "description": "Granularities for aggregating data into time windows based on their timestamp. Currently the following static\ngranularities are supported:\n{``\"5 minutes\"``, ``\"30 minutes\"``, ``\"1 hour\"``, ``\"1 day\"``, ``\"\u003cn\u003e week(s)\"``, ``\"1 month\"``, ``\"1 year\"``}.\n", + "$ref": "#/$defs/slice/string" + }, + "timestamp_col": { + "description": "Column that contains the timestamps of requests. The column must be one of the following:\n- A ``TimestampType`` column\n- A column whose values can be converted to timestamps through the pyspark\n ``to_timestamp`` [function](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.functions.to_timestamp.html).\n", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "granularities", + "timestamp_col" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "compute.Adlsgen2Info": { + "anyOf": [ + { + "type": "object", + "properties": { + "destination": { + "description": "abfss destination, e.g. `abfss://\u003ccontainer-name\u003e@\u003cstorage-account-name\u003e.dfs.core.windows.net/\u003cdirectory-name\u003e`.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "destination" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "compute.AutoScale": { + "anyOf": [ + { + "type": "object", + "properties": { + "max_workers": { + "description": "The maximum number of workers to which the cluster can scale up when overloaded.\nNote that `max_workers` must be strictly greater than `min_workers`.", + "$ref": "#/$defs/int" + }, + "min_workers": { + "description": "The minimum number of workers to which the cluster can scale down when underutilized.\nIt is also the initial number of workers the cluster will have after creation.", + "$ref": "#/$defs/int" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "compute.AwsAttributes": { + "anyOf": [ + { + "type": "object", + "properties": { + "availability": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.AwsAvailability" + }, + "ebs_volume_count": { + "description": "The number of volumes launched for each instance. Users can choose up to 10 volumes.\nThis feature is only enabled for supported node types. Legacy node types cannot specify\ncustom EBS volumes.\nFor node types with no instance store, at least one EBS volume needs to be specified;\notherwise, cluster creation will fail.\n\nThese EBS volumes will be mounted at `/ebs0`, `/ebs1`, and etc.\nInstance store volumes will be mounted at `/local_disk0`, `/local_disk1`, and etc.\n\nIf EBS volumes are attached, Databricks will configure Spark to use only the EBS volumes for\nscratch storage because heterogenously sized scratch devices can lead to inefficient disk\nutilization. If no EBS volumes are attached, Databricks will configure Spark to use instance\nstore volumes.\n\nPlease note that if EBS volumes are specified, then the Spark configuration `spark.local.dir`\nwill be overridden.", + "$ref": "#/$defs/int" + }, + "ebs_volume_iops": { + "description": "If using gp3 volumes, what IOPS to use for the disk. If this is not set, the maximum performance of a gp2 volume with the same volume size will be used.", + "$ref": "#/$defs/int" + }, + "ebs_volume_size": { + "description": "The size of each EBS volume (in GiB) launched for each instance. For general purpose\nSSD, this value must be within the range 100 - 4096. For throughput optimized HDD,\nthis value must be within the range 500 - 4096.", + "$ref": "#/$defs/int" + }, + "ebs_volume_throughput": { + "description": "If using gp3 volumes, what throughput to use for the disk. If this is not set, the maximum performance of a gp2 volume with the same volume size will be used.", + "$ref": "#/$defs/int" + }, + "ebs_volume_type": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.EbsVolumeType" + }, + "first_on_demand": { + "description": "The first `first_on_demand` nodes of the cluster will be placed on on-demand instances.\nIf this value is greater than 0, the cluster driver node in particular will be placed on an\non-demand instance. If this value is greater than or equal to the current cluster size, all\nnodes will be placed on on-demand instances. If this value is less than the current cluster\nsize, `first_on_demand` nodes will be placed on on-demand instances and the remainder will\nbe placed on `availability` instances. Note that this value does not affect\ncluster size and cannot currently be mutated over the lifetime of a cluster.", + "$ref": "#/$defs/int" + }, + "instance_profile_arn": { + "description": "Nodes for this cluster will only be placed on AWS instances with this instance profile. If\nommitted, nodes will be placed on instances without an IAM instance profile. The instance\nprofile must have previously been added to the Databricks environment by an account\nadministrator.\n\nThis feature may only be available to certain customer plans.\n\nIf this field is ommitted, we will pull in the default from the conf if it exists.", + "$ref": "#/$defs/string" + }, + "spot_bid_price_percent": { + "description": "The bid price for AWS spot instances, as a percentage of the corresponding instance type's\non-demand price.\nFor example, if this field is set to 50, and the cluster needs a new `r3.xlarge` spot\ninstance, then the bid price is half of the price of\non-demand `r3.xlarge` instances. Similarly, if this field is set to 200, the bid price is twice\nthe price of on-demand `r3.xlarge` instances. If not specified, the default value is 100.\nWhen spot instances are requested for this cluster, only spot instances whose bid price\npercentage matches this field will be considered.\nNote that, for safety, we enforce this field to be no more than 10000.\n\nThe default value and documentation here should be kept consistent with\nCommonConf.defaultSpotBidPricePercent and CommonConf.maxSpotBidPricePercent.", + "$ref": "#/$defs/int" + }, + "zone_id": { + "description": "Identifier for the availability zone/datacenter in which the cluster resides.\nThis string will be of a form like \"us-west-2a\". The provided availability\nzone must be in the same region as the Databricks deployment. For example, \"us-west-2a\"\nis not a valid zone id if the Databricks deployment resides in the \"us-east-1\" region.\nThis is an optional field at cluster creation, and if not specified, a default zone will be used.\nIf the zone specified is \"auto\", will try to place cluster in a zone with high availability,\nand will retry placement in a different AZ if there is not enough capacity.\nThe list of available zones as well as the default value can be found by using the\n`List Zones` method.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "compute.AwsAvailability": { + "anyOf": [ + { + "type": "string", + "description": "Availability type used for all subsequent nodes past the `first_on_demand` ones.\n\nNote: If `first_on_demand` is zero, this availability type will be used for the entire cluster.\n", + "enum": [ + "SPOT", + "ON_DEMAND", + "SPOT_WITH_FALLBACK" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(resources(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(bundle(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(workspace(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "compute.AzureAttributes": { + "anyOf": [ + { + "type": "object", + "properties": { + "availability": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.AzureAvailability" + }, + "first_on_demand": { + "description": "The first `first_on_demand` nodes of the cluster will be placed on on-demand instances.\nThis value should be greater than 0, to make sure the cluster driver node is placed on an\non-demand instance. If this value is greater than or equal to the current cluster size, all\nnodes will be placed on on-demand instances. If this value is less than the current cluster\nsize, `first_on_demand` nodes will be placed on on-demand instances and the remainder will\nbe placed on `availability` instances. Note that this value does not affect\ncluster size and cannot currently be mutated over the lifetime of a cluster.", + "$ref": "#/$defs/int" + }, + "log_analytics_info": { + "description": "Defines values necessary to configure and run Azure Log Analytics agent", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.LogAnalyticsInfo" + }, + "spot_bid_max_price": { + "description": "The max bid price to be used for Azure spot instances.\nThe Max price for the bid cannot be higher than the on-demand price of the instance.\nIf not specified, the default value is -1, which specifies that the instance cannot be evicted\non the basis of price, and only on the basis of availability. Further, the value should \u003e 0 or -1.", + "$ref": "#/$defs/float64" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "compute.AzureAvailability": { + "anyOf": [ + { + "type": "string", + "description": "Availability type used for all subsequent nodes past the `first_on_demand` ones.\nNote: If `first_on_demand` is zero (which only happens on pool clusters), this availability\ntype will be used for the entire cluster.", + "enum": [ + "SPOT_AZURE", + "ON_DEMAND_AZURE", + "SPOT_WITH_FALLBACK_AZURE" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(resources(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(bundle(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(workspace(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "compute.ClientsTypes": { + "anyOf": [ + { + "type": "object", + "properties": { + "jobs": { + "description": "With jobs set, the cluster can be used for jobs", + "$ref": "#/$defs/bool" + }, + "notebooks": { + "description": "With notebooks set, this cluster can be used for notebooks", + "$ref": "#/$defs/bool" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "compute.ClusterLogConf": { + "anyOf": [ + { + "type": "object", + "properties": { + "dbfs": { + "description": "destination needs to be provided. e.g.\n`{ \"dbfs\" : { \"destination\" : \"dbfs:/home/cluster_log\" } }`", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.DbfsStorageInfo" + }, + "s3": { + "description": "destination and either the region or endpoint need to be provided. e.g.\n`{ \"s3\": { \"destination\" : \"s3://cluster_log_bucket/prefix\", \"region\" : \"us-west-2\" } }`\nCluster iam role is used to access s3, please make sure the cluster iam role in\n`instance_profile_arn` has permission to write data to the s3 destination.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.S3StorageInfo" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "compute.ClusterSpec": { + "anyOf": [ + { + "type": "object", + "properties": { + "apply_policy_default_values": { + "description": "When set to true, fixed and default values from the policy will be used for fields that are omitted. When set to false, only fixed values from the policy will be applied.", + "$ref": "#/$defs/bool" + }, + "autoscale": { + "description": "Parameters needed in order to automatically scale clusters up and down based on load.\nNote: autoscaling works best with DB runtime versions 3.0 or later.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.AutoScale" + }, + "autotermination_minutes": { + "description": "Automatically terminates the cluster after it is inactive for this time in minutes. If not set,\nthis cluster will not be automatically terminated. If specified, the threshold must be between\n10 and 10000 minutes.\nUsers can also set this value to 0 to explicitly disable automatic termination.", + "$ref": "#/$defs/int" + }, + "aws_attributes": { + "description": "Attributes related to clusters running on Amazon Web Services.\nIf not specified at cluster creation, a set of default values will be used.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.AwsAttributes" + }, + "azure_attributes": { + "description": "Attributes related to clusters running on Microsoft Azure.\nIf not specified at cluster creation, a set of default values will be used.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.AzureAttributes" + }, + "cluster_log_conf": { + "description": "The configuration for delivering spark logs to a long-term storage destination.\nTwo kinds of destinations (dbfs and s3) are supported. Only one destination can be specified\nfor one cluster. If the conf is given, the logs will be delivered to the destination every\n`5 mins`. The destination of driver logs is `$destination/$clusterId/driver`, while\nthe destination of executor logs is `$destination/$clusterId/executor`.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.ClusterLogConf" + }, + "cluster_name": { + "description": "Cluster name requested by the user. This doesn't have to be unique.\nIf not specified at creation, the cluster name will be an empty string.\n", + "$ref": "#/$defs/string" + }, + "custom_tags": { + "description": "Additional tags for cluster resources. Databricks will tag all cluster resources (e.g., AWS\ninstances and EBS volumes) with these tags in addition to `default_tags`. Notes:\n\n- Currently, Databricks allows at most 45 custom tags\n\n- Clusters can only reuse cloud resources if the resources' tags are a subset of the cluster tags", + "$ref": "#/$defs/map/string" + }, + "data_security_mode": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.DataSecurityMode" + }, + "docker_image": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.DockerImage" + }, + "driver_instance_pool_id": { + "description": "The optional ID of the instance pool for the driver of the cluster belongs.\nThe pool cluster uses the instance pool with id (instance_pool_id) if the driver pool is not\nassigned.", + "$ref": "#/$defs/string" + }, + "driver_node_type_id": { + "description": "The node type of the Spark driver. Note that this field is optional;\nif unset, the driver node type will be set as the same value\nas `node_type_id` defined above.\n", + "$ref": "#/$defs/string" + }, + "enable_elastic_disk": { + "description": "Autoscaling Local Storage: when enabled, this cluster will dynamically acquire additional disk\nspace when its Spark workers are running low on disk space. This feature requires specific AWS\npermissions to function correctly - refer to the User Guide for more details.", + "$ref": "#/$defs/bool" + }, + "enable_local_disk_encryption": { + "description": "Whether to enable LUKS on cluster VMs' local disks", + "$ref": "#/$defs/bool" + }, + "gcp_attributes": { + "description": "Attributes related to clusters running on Google Cloud Platform.\nIf not specified at cluster creation, a set of default values will be used.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.GcpAttributes" + }, + "init_scripts": { + "description": "The configuration for storing init scripts. Any number of destinations can be specified. The scripts are executed sequentially in the order provided. If `cluster_log_conf` is specified, init script logs are sent to `\u003cdestination\u003e/\u003ccluster-ID\u003e/init_scripts`.", + "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/compute.InitScriptInfo" + }, + "instance_pool_id": { + "description": "The optional ID of the instance pool to which the cluster belongs.", + "$ref": "#/$defs/string" + }, + "node_type_id": { + "description": "This field encodes, through a single value, the resources available to each of\nthe Spark nodes in this cluster. For example, the Spark nodes can be provisioned\nand optimized for memory or compute intensive workloads. A list of available node\ntypes can be retrieved by using the :method:clusters/listNodeTypes API call.\n", + "$ref": "#/$defs/string" + }, + "num_workers": { + "description": "Number of worker nodes that this cluster should have. A cluster has one Spark Driver\nand `num_workers` Executors for a total of `num_workers` + 1 Spark nodes.\n\nNote: When reading the properties of a cluster, this field reflects the desired number\nof workers rather than the actual current number of workers. For instance, if a cluster\nis resized from 5 to 10 workers, this field will immediately be updated to reflect\nthe target size of 10 workers, whereas the workers listed in `spark_info` will gradually\nincrease from 5 to 10 as the new nodes are provisioned.", + "$ref": "#/$defs/int" + }, + "policy_id": { + "description": "The ID of the cluster policy used to create the cluster if applicable.", + "$ref": "#/$defs/string" + }, + "runtime_engine": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.RuntimeEngine" + }, + "single_user_name": { + "description": "Single user name if data_security_mode is `SINGLE_USER`", + "$ref": "#/$defs/string" + }, + "spark_conf": { + "description": "An object containing a set of optional, user-specified Spark configuration key-value pairs.\nUsers can also pass in a string of extra JVM options to the driver and the executors via\n`spark.driver.extraJavaOptions` and `spark.executor.extraJavaOptions` respectively.\n", + "$ref": "#/$defs/map/string" + }, + "spark_env_vars": { + "description": "An object containing a set of optional, user-specified environment variable key-value pairs.\nPlease note that key-value pair of the form (X,Y) will be exported as is (i.e.,\n`export X='Y'`) while launching the driver and workers.\n\nIn order to specify an additional set of `SPARK_DAEMON_JAVA_OPTS`, we recommend appending\nthem to `$SPARK_DAEMON_JAVA_OPTS` as shown in the example below. This ensures that all\ndefault databricks managed environmental variables are included as well.\n\nExample Spark environment variables:\n`{\"SPARK_WORKER_MEMORY\": \"28000m\", \"SPARK_LOCAL_DIRS\": \"/local_disk0\"}` or\n`{\"SPARK_DAEMON_JAVA_OPTS\": \"$SPARK_DAEMON_JAVA_OPTS -Dspark.shuffle.service.enabled=true\"}`", + "$ref": "#/$defs/map/string" + }, + "spark_version": { + "description": "The Spark version of the cluster, e.g. `3.3.x-scala2.11`.\nA list of available Spark versions can be retrieved by using\nthe :method:clusters/sparkVersions API call.\n", + "$ref": "#/$defs/string" + }, + "ssh_public_keys": { + "description": "SSH public key contents that will be added to each Spark node in this cluster. The\ncorresponding private keys can be used to login with the user name `ubuntu` on port `2200`.\nUp to 10 keys can be specified.", + "$ref": "#/$defs/slice/string" + }, + "workload_type": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.WorkloadType" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "compute.DataSecurityMode": { + "anyOf": [ + { + "type": "string", + "description": "Data security mode decides what data governance model to use when accessing data\nfrom a cluster.\n\n* `NONE`: No security isolation for multiple users sharing the cluster. Data governance features are not available in this mode.\n* `SINGLE_USER`: A secure cluster that can only be exclusively used by a single user specified in `single_user_name`. Most programming languages, cluster features and data governance features are available in this mode.\n* `USER_ISOLATION`: A secure cluster that can be shared by multiple users. Cluster users are fully isolated so that they cannot see each other's data and credentials. Most data governance features are supported in this mode. But programming languages and cluster features might be limited.\n\nThe following modes are deprecated starting with Databricks Runtime 15.0 and\nwill be removed for future Databricks Runtime versions:\n\n* `LEGACY_TABLE_ACL`: This mode is for users migrating from legacy Table ACL clusters.\n* `LEGACY_PASSTHROUGH`: This mode is for users migrating from legacy Passthrough on high concurrency clusters.\n* `LEGACY_SINGLE_USER`: This mode is for users migrating from legacy Passthrough on standard clusters.\n* `LEGACY_SINGLE_USER_STANDARD`: This mode provides a way that doesn’t have UC nor passthrough enabled.\n", + "enum": [ + "NONE", + "SINGLE_USER", + "USER_ISOLATION", + "LEGACY_TABLE_ACL", + "LEGACY_PASSTHROUGH", + "LEGACY_SINGLE_USER", + "LEGACY_SINGLE_USER_STANDARD" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(resources(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(bundle(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(workspace(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "compute.DbfsStorageInfo": { + "anyOf": [ + { + "type": "object", + "properties": { + "destination": { + "description": "dbfs destination, e.g. `dbfs:/my/path`", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "destination" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "compute.DockerBasicAuth": { + "anyOf": [ + { + "type": "object", + "properties": { + "password": { + "description": "Password of the user", + "$ref": "#/$defs/string" + }, + "username": { + "description": "Name of the user", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "compute.DockerImage": { + "anyOf": [ + { + "type": "object", + "properties": { + "basic_auth": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.DockerBasicAuth" + }, + "url": { + "description": "URL of the docker image.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "compute.EbsVolumeType": { + "anyOf": [ + { + "type": "string", + "description": "The type of EBS volumes that will be launched with this cluster.", + "enum": [ + "GENERAL_PURPOSE_SSD", + "THROUGHPUT_OPTIMIZED_HDD" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(resources(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(bundle(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(workspace(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "compute.Environment": { + "anyOf": [ + { + "type": "object", + "description": "The environment entity used to preserve serverless environment side panel and jobs' environment for non-notebook task.\nIn this minimal environment spec, only pip dependencies are supported.", + "properties": { + "client": { + "description": "Client version used by the environment\nThe client is the user-facing environment of the runtime.\nEach client comes with a specific set of pre-installed libraries.\nThe version is a string, consisting of the major client version.", + "$ref": "#/$defs/string" + }, + "dependencies": { + "description": "List of pip dependencies, as supported by the version of pip in this environment.\nEach dependency is a pip requirement file line https://pip.pypa.io/en/stable/reference/requirements-file-format/\nAllowed dependency could be \u003crequirement specifier\u003e, \u003carchive url/path\u003e, \u003clocal project path\u003e(WSFS or Volumes in Databricks), \u003cvcs project url\u003e\nE.g. dependencies: [\"foo==0.0.1\", \"-r /Workspace/test/requirements.txt\"]", + "$ref": "#/$defs/slice/string" + } + }, + "additionalProperties": false, + "required": [ + "client" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "compute.GcpAttributes": { + "anyOf": [ + { + "type": "object", + "properties": { + "availability": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.GcpAvailability" + }, + "boot_disk_size": { + "description": "boot disk size in GB", + "$ref": "#/$defs/int" + }, + "google_service_account": { + "description": "If provided, the cluster will impersonate the google service account when accessing\ngcloud services (like GCS). The google service account\nmust have previously been added to the Databricks environment by an account\nadministrator.", + "$ref": "#/$defs/string" + }, + "local_ssd_count": { + "description": "If provided, each node (workers and driver) in the cluster will have this number of local SSDs attached. Each local SSD is 375GB in size. Refer to [GCP documentation](https://cloud.google.com/compute/docs/disks/local-ssd#choose_number_local_ssds) for the supported number of local SSDs for each instance type.", + "$ref": "#/$defs/int" + }, + "use_preemptible_executors": { + "description": "This field determines whether the spark executors will be scheduled to run on preemptible VMs (when set to true) versus standard compute engine VMs (when set to false; default).\nNote: Soon to be deprecated, use the availability field instead.", + "$ref": "#/$defs/bool" + }, + "zone_id": { + "description": "Identifier for the availability zone in which the cluster resides.\nThis can be one of the following:\n- \"HA\" =\u003e High availability, spread nodes across availability zones for a Databricks deployment region [default]\n- \"AUTO\" =\u003e Databricks picks an availability zone to schedule the cluster on.\n- A GCP availability zone =\u003e Pick One of the available zones for (machine type + region) from https://cloud.google.com/compute/docs/regions-zones.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "compute.GcpAvailability": { + "anyOf": [ + { + "type": "string", + "description": "This field determines whether the instance pool will contain preemptible\nVMs, on-demand VMs, or preemptible VMs with a fallback to on-demand VMs if the former is unavailable.", + "enum": [ + "PREEMPTIBLE_GCP", + "ON_DEMAND_GCP", + "PREEMPTIBLE_WITH_FALLBACK_GCP" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(resources(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(bundle(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(workspace(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "compute.GcsStorageInfo": { + "anyOf": [ + { + "type": "object", + "properties": { + "destination": { + "description": "GCS destination/URI, e.g. `gs://my-bucket/some-prefix`", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "destination" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "compute.InitScriptInfo": { + "anyOf": [ + { + "type": "object", + "properties": { + "abfss": { + "description": "destination needs to be provided. e.g.\n`{ \"abfss\" : { \"destination\" : \"abfss://\u003ccontainer-name\u003e@\u003cstorage-account-name\u003e.dfs.core.windows.net/\u003cdirectory-name\u003e\" } }", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.Adlsgen2Info" + }, + "dbfs": { + "description": "destination needs to be provided. e.g.\n`{ \"dbfs\" : { \"destination\" : \"dbfs:/home/cluster_log\" } }`", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.DbfsStorageInfo" + }, + "file": { + "description": "destination needs to be provided. e.g.\n`{ \"file\" : { \"destination\" : \"file:/my/local/file.sh\" } }`", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.LocalFileInfo" + }, + "gcs": { + "description": "destination needs to be provided. e.g.\n`{ \"gcs\": { \"destination\": \"gs://my-bucket/file.sh\" } }`", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.GcsStorageInfo" + }, + "s3": { + "description": "destination and either the region or endpoint need to be provided. e.g.\n`{ \"s3\": { \"destination\" : \"s3://cluster_log_bucket/prefix\", \"region\" : \"us-west-2\" } }`\nCluster iam role is used to access s3, please make sure the cluster iam role in\n`instance_profile_arn` has permission to write data to the s3 destination.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.S3StorageInfo" + }, + "volumes": { + "description": "destination needs to be provided. e.g.\n`{ \"volumes\" : { \"destination\" : \"/Volumes/my-init.sh\" } }`", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.VolumesStorageInfo" + }, + "workspace": { + "description": "destination needs to be provided. e.g.\n`{ \"workspace\" : { \"destination\" : \"/Users/user1@databricks.com/my-init.sh\" } }`", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.WorkspaceStorageInfo" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "compute.Library": { + "anyOf": [ + { + "type": "object", + "properties": { + "cran": { + "description": "Specification of a CRAN library to be installed as part of the library", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.RCranLibrary" + }, + "egg": { + "description": "Deprecated. URI of the egg library to install. Installing Python egg files is deprecated and is not supported in Databricks Runtime 14.0 and above.", + "$ref": "#/$defs/string" + }, + "jar": { + "description": "URI of the JAR library to install. Supported URIs include Workspace paths, Unity Catalog Volumes paths, and S3 URIs.\nFor example: `{ \"jar\": \"/Workspace/path/to/library.jar\" }`, `{ \"jar\" : \"/Volumes/path/to/library.jar\" }` or\n`{ \"jar\": \"s3://my-bucket/library.jar\" }`.\nIf S3 is used, please make sure the cluster has read access on the library. You may need to\nlaunch the cluster with an IAM role to access the S3 URI.", + "$ref": "#/$defs/string" + }, + "maven": { + "description": "Specification of a maven library to be installed. For example:\n`{ \"coordinates\": \"org.jsoup:jsoup:1.7.2\" }`", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.MavenLibrary" + }, + "pypi": { + "description": "Specification of a PyPi library to be installed. For example:\n`{ \"package\": \"simplejson\" }`", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.PythonPyPiLibrary" + }, + "requirements": { + "description": "URI of the requirements.txt file to install. Only Workspace paths and Unity Catalog Volumes paths are supported.\nFor example: `{ \"requirements\": \"/Workspace/path/to/requirements.txt\" }` or `{ \"requirements\" : \"/Volumes/path/to/requirements.txt\" }`", + "$ref": "#/$defs/string" + }, + "whl": { + "description": "URI of the wheel library to install. Supported URIs include Workspace paths, Unity Catalog Volumes paths, and S3 URIs.\nFor example: `{ \"whl\": \"/Workspace/path/to/library.whl\" }`, `{ \"whl\" : \"/Volumes/path/to/library.whl\" }` or\n`{ \"whl\": \"s3://my-bucket/library.whl\" }`.\nIf S3 is used, please make sure the cluster has read access on the library. You may need to\nlaunch the cluster with an IAM role to access the S3 URI.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "compute.LocalFileInfo": { + "anyOf": [ + { + "type": "object", + "properties": { + "destination": { + "description": "local file destination, e.g. `file:/my/local/file.sh`", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "destination" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "compute.LogAnalyticsInfo": { + "anyOf": [ + { + "type": "object", + "properties": { + "log_analytics_primary_key": { + "description": "\u003cneeds content added\u003e", + "$ref": "#/$defs/string" + }, + "log_analytics_workspace_id": { + "description": "\u003cneeds content added\u003e", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "compute.MavenLibrary": { + "anyOf": [ + { + "type": "object", + "properties": { + "coordinates": { + "description": "Gradle-style maven coordinates. For example: \"org.jsoup:jsoup:1.7.2\".", + "$ref": "#/$defs/string" + }, + "exclusions": { + "description": "List of dependences to exclude. For example: `[\"slf4j:slf4j\", \"*:hadoop-client\"]`.\n\nMaven dependency exclusions:\nhttps://maven.apache.org/guides/introduction/introduction-to-optional-and-excludes-dependencies.html.", + "$ref": "#/$defs/slice/string" + }, + "repo": { + "description": "Maven repo to install the Maven package from. If omitted, both Maven Central Repository\nand Spark Packages are searched.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "coordinates" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "compute.PythonPyPiLibrary": { + "anyOf": [ + { + "type": "object", + "properties": { + "package": { + "description": "The name of the pypi package to install. An optional exact version specification is also\nsupported. Examples: \"simplejson\" and \"simplejson==3.8.0\".", + "$ref": "#/$defs/string" + }, + "repo": { + "description": "The repository where the package can be found. If not specified, the default pip index is\nused.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "package" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "compute.RCranLibrary": { + "anyOf": [ + { + "type": "object", + "properties": { + "package": { + "description": "The name of the CRAN package to install.", + "$ref": "#/$defs/string" + }, + "repo": { + "description": "The repository where the package can be found. If not specified, the default CRAN repo is used.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "package" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "compute.RuntimeEngine": { + "anyOf": [ + { + "type": "string", + "description": "Decides which runtime engine to be use, e.g. Standard vs. Photon. If unspecified, the runtime\nengine is inferred from spark_version.", + "enum": [ + "NULL", + "STANDARD", + "PHOTON" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(resources(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(bundle(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(workspace(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "compute.S3StorageInfo": { + "anyOf": [ + { + "type": "object", + "properties": { + "canned_acl": { + "description": "(Optional) Set canned access control list for the logs, e.g. `bucket-owner-full-control`.\nIf `canned_cal` is set, please make sure the cluster iam role has `s3:PutObjectAcl` permission on\nthe destination bucket and prefix. The full list of possible canned acl can be found at\nhttp://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl.\nPlease also note that by default only the object owner gets full controls. If you are using cross account\nrole for writing data, you may want to set `bucket-owner-full-control` to make bucket owner able to\nread the logs.", + "$ref": "#/$defs/string" + }, + "destination": { + "description": "S3 destination, e.g. `s3://my-bucket/some-prefix` Note that logs will be delivered using\ncluster iam role, please make sure you set cluster iam role and the role has write access to the\ndestination. Please also note that you cannot use AWS keys to deliver logs.", + "$ref": "#/$defs/string" + }, + "enable_encryption": { + "description": "(Optional) Flag to enable server side encryption, `false` by default.", + "$ref": "#/$defs/bool" + }, + "encryption_type": { + "description": "(Optional) The encryption type, it could be `sse-s3` or `sse-kms`. It will be used only when\nencryption is enabled and the default type is `sse-s3`.", + "$ref": "#/$defs/string" + }, + "endpoint": { + "description": "S3 endpoint, e.g. `https://s3-us-west-2.amazonaws.com`. Either region or endpoint needs to be set.\nIf both are set, endpoint will be used.", + "$ref": "#/$defs/string" + }, + "kms_key": { + "description": "(Optional) Kms key which will be used if encryption is enabled and encryption type is set to `sse-kms`.", + "$ref": "#/$defs/string" + }, + "region": { + "description": "S3 region, e.g. `us-west-2`. Either region or endpoint needs to be set. If both are set,\nendpoint will be used.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "destination" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "compute.VolumesStorageInfo": { + "anyOf": [ + { + "type": "object", + "properties": { + "destination": { + "description": "Unity Catalog Volumes file destination, e.g. `/Volumes/my-init.sh`", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "destination" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "compute.WorkloadType": { + "anyOf": [ + { + "type": "object", + "properties": { + "clients": { + "description": " defined what type of clients can use the cluster. E.g. Notebooks, Jobs", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.ClientsTypes" + } + }, + "additionalProperties": false, + "required": [ + "clients" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "compute.WorkspaceStorageInfo": { + "anyOf": [ + { + "type": "object", + "properties": { + "destination": { + "description": "workspace files destination, e.g. `/Users/user1@databricks.com/my-init.sh`", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "destination" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "jobs.Condition": { + "anyOf": [ + { + "type": "string", + "enum": [ + "ANY_UPDATED", + "ALL_UPDATED" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(resources(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(bundle(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(workspace(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "jobs.ConditionTask": { + "anyOf": [ + { + "type": "object", + "properties": { + "left": { + "description": "The left operand of the condition task. Can be either a string value or a job state or parameter reference.", + "$ref": "#/$defs/string" + }, + "op": { + "description": "* `EQUAL_TO`, `NOT_EQUAL` operators perform string comparison of their operands. This means that `“12.0” == “12”` will evaluate to `false`.\n* `GREATER_THAN`, `GREATER_THAN_OR_EQUAL`, `LESS_THAN`, `LESS_THAN_OR_EQUAL` operators perform numeric comparison of their operands. `“12.0” \u003e= “12”` will evaluate to `true`, `“10.0” \u003e= “12”` will evaluate to `false`.\n\nThe boolean comparison to task values can be implemented with operators `EQUAL_TO`, `NOT_EQUAL`. If a task value was set to a boolean value, it will be serialized to `“true”` or `“false”` for the comparison.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.ConditionTaskOp" + }, + "right": { + "description": "The right operand of the condition task. Can be either a string value or a job state or parameter reference.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "left", + "op", + "right" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "jobs.ConditionTaskOp": { + "anyOf": [ + { + "type": "string", + "description": "* `EQUAL_TO`, `NOT_EQUAL` operators perform string comparison of their operands. This means that `“12.0” == “12”` will evaluate to `false`.\n* `GREATER_THAN`, `GREATER_THAN_OR_EQUAL`, `LESS_THAN`, `LESS_THAN_OR_EQUAL` operators perform numeric comparison of their operands. `“12.0” \u003e= “12”` will evaluate to `true`, `“10.0” \u003e= “12”` will evaluate to `false`.\n\nThe boolean comparison to task values can be implemented with operators `EQUAL_TO`, `NOT_EQUAL`. If a task value was set to a boolean value, it will be serialized to `“true”` or `“false”` for the comparison.", + "enum": [ + "EQUAL_TO", + "GREATER_THAN", + "GREATER_THAN_OR_EQUAL", + "LESS_THAN", + "LESS_THAN_OR_EQUAL", + "NOT_EQUAL" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(resources(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(bundle(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(workspace(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "jobs.Continuous": { + "anyOf": [ + { + "type": "object", + "properties": { + "pause_status": { + "description": "Indicate whether the continuous execution of the job is paused or not. Defaults to UNPAUSED.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.PauseStatus" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "jobs.CronSchedule": { + "anyOf": [ + { + "type": "object", + "properties": { + "pause_status": { + "description": "Indicate whether this schedule is paused or not.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.PauseStatus" + }, + "quartz_cron_expression": { + "description": "A Cron expression using Quartz syntax that describes the schedule for a job. See [Cron Trigger](http://www.quartz-scheduler.org/documentation/quartz-2.3.0/tutorials/crontrigger.html) for details. This field is required.", + "$ref": "#/$defs/string" + }, + "timezone_id": { + "description": "A Java timezone ID. The schedule for a job is resolved with respect to this timezone. See [Java TimeZone](https://docs.oracle.com/javase/7/docs/api/java/util/TimeZone.html) for details. This field is required.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "quartz_cron_expression", + "timezone_id" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "jobs.DbtTask": { + "anyOf": [ + { + "type": "object", + "properties": { + "catalog": { + "description": "Optional name of the catalog to use. The value is the top level in the 3-level namespace of Unity Catalog (catalog / schema / relation). The catalog value can only be specified if a warehouse_id is specified. Requires dbt-databricks \u003e= 1.1.1.", + "$ref": "#/$defs/string" + }, + "commands": { + "description": "A list of dbt commands to execute. All commands must start with `dbt`. This parameter must not be empty. A maximum of up to 10 commands can be provided.", + "$ref": "#/$defs/slice/string" + }, + "profiles_directory": { + "description": "Optional (relative) path to the profiles directory. Can only be specified if no warehouse_id is specified. If no warehouse_id is specified and this folder is unset, the root directory is used.", + "$ref": "#/$defs/string" + }, + "project_directory": { + "description": "Path to the project directory. Optional for Git sourced tasks, in which\ncase if no value is provided, the root of the Git repository is used.", + "$ref": "#/$defs/string" + }, + "schema": { + "description": "Optional schema to write to. This parameter is only used when a warehouse_id is also provided. If not provided, the `default` schema is used.", + "$ref": "#/$defs/string" + }, + "source": { + "description": "Optional location type of the project directory. When set to `WORKSPACE`, the project will be retrieved\nfrom the local Databricks workspace. When set to `GIT`, the project will be retrieved from a Git repository\ndefined in `git_source`. If the value is empty, the task will use `GIT` if `git_source` is defined and `WORKSPACE` otherwise.\n\n* `WORKSPACE`: Project is located in Databricks workspace.\n* `GIT`: Project is located in cloud Git provider.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.Source" + }, + "warehouse_id": { + "description": "ID of the SQL warehouse to connect to. If provided, we automatically generate and provide the profile and connection details to dbt. It can be overridden on a per-command basis by using the `--profiles-dir` command line argument.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "commands" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "jobs.FileArrivalTriggerConfiguration": { + "anyOf": [ + { + "type": "object", + "properties": { + "min_time_between_triggers_seconds": { + "description": "If set, the trigger starts a run only after the specified amount of time passed since\nthe last time the trigger fired. The minimum allowed value is 60 seconds", + "$ref": "#/$defs/int" + }, + "url": { + "description": "URL to be monitored for file arrivals. The path must point to the root or a subpath of the external location.", + "$ref": "#/$defs/string" + }, + "wait_after_last_change_seconds": { + "description": "If set, the trigger starts a run only after no file activity has occurred for the specified amount of time.\nThis makes it possible to wait for a batch of incoming files to arrive before triggering a run. The\nminimum allowed value is 60 seconds.", + "$ref": "#/$defs/int" + } + }, + "additionalProperties": false, + "required": [ + "url" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "jobs.ForEachTask": { + "anyOf": [ + { + "type": "object", + "properties": { + "concurrency": { + "description": "An optional maximum allowed number of concurrent runs of the task.\nSet this value if you want to be able to execute multiple runs of the task concurrently.", + "$ref": "#/$defs/int" + }, + "inputs": { + "description": "Array for task to iterate on. This can be a JSON string or a reference to\nan array parameter.", + "$ref": "#/$defs/string" + }, + "task": { + "description": "Configuration for the task that will be run for each element in the array", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.Task" + } + }, + "additionalProperties": false, + "required": [ + "inputs", + "task" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "jobs.Format": { + "anyOf": [ + { + "type": "string", + "enum": [ + "SINGLE_TASK", + "MULTI_TASK" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(resources(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(bundle(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(workspace(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "jobs.GitProvider": { + "anyOf": [ + { + "type": "string", + "enum": [ + "gitHub", + "bitbucketCloud", + "azureDevOpsServices", + "gitHubEnterprise", + "bitbucketServer", + "gitLab", + "gitLabEnterpriseEdition", + "awsCodeCommit" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(resources(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(bundle(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(workspace(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "jobs.GitSnapshot": { + "anyOf": [ + { + "type": "object", + "description": "Read-only state of the remote repository at the time the job was run. This field is only included on job runs.", + "properties": { + "used_commit": { + "description": "Commit that was used to execute the run. If git_branch was specified, this points to the HEAD of the branch at the time of the run; if git_tag was specified, this points to the commit the tag points to.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "jobs.GitSource": { + "anyOf": [ + { + "type": "object", + "description": "An optional specification for a remote Git repository containing the source code used by tasks. Version-controlled source code is supported by notebook, dbt, Python script, and SQL File tasks.\n\nIf `git_source` is set, these tasks retrieve the file from the remote repository by default. However, this behavior can be overridden by setting `source` to `WORKSPACE` on the task.\n\nNote: dbt and SQL File tasks support only version-controlled sources. If dbt or SQL File tasks are used, `git_source` must be defined on the job.", + "properties": { + "git_branch": { + "description": "Name of the branch to be checked out and used by this job. This field cannot be specified in conjunction with git_tag or git_commit.", + "$ref": "#/$defs/string" + }, + "git_commit": { + "description": "Commit to be checked out and used by this job. This field cannot be specified in conjunction with git_branch or git_tag.", + "$ref": "#/$defs/string" + }, + "git_provider": { + "description": "Unique identifier of the service used to host the Git repository. The value is case insensitive.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.GitProvider" + }, + "git_snapshot": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.GitSnapshot" + }, + "git_tag": { + "description": "Name of the tag to be checked out and used by this job. This field cannot be specified in conjunction with git_branch or git_commit.", + "$ref": "#/$defs/string" + }, + "git_url": { + "description": "URL of the repository to be cloned by this job.", + "$ref": "#/$defs/string" + }, + "job_source": { + "description": "The source of the job specification in the remote repository when the job is source controlled.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.JobSource" + } + }, + "additionalProperties": false, + "required": [ + "git_provider", + "git_url" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "jobs.JobCluster": { + "anyOf": [ + { + "type": "object", + "properties": { + "job_cluster_key": { + "description": "A unique name for the job cluster. This field is required and must be unique within the job.\n`JobTaskSettings` may refer to this field to determine which cluster to launch for the task execution.", + "$ref": "#/$defs/string" + }, + "new_cluster": { + "description": "If new_cluster, a description of a cluster that is created for each task.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.ClusterSpec" + } + }, + "additionalProperties": false, + "required": [ + "job_cluster_key", + "new_cluster" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "jobs.JobDeployment": { + "anyOf": [ + { + "type": "object", + "properties": { + "kind": { + "description": "The kind of deployment that manages the job.\n\n* `BUNDLE`: The job is managed by Databricks Asset Bundle.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.JobDeploymentKind" + }, + "metadata_file_path": { + "description": "Path of the file that contains deployment metadata.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "kind" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "jobs.JobDeploymentKind": { + "anyOf": [ + { + "type": "string", + "description": "* `BUNDLE`: The job is managed by Databricks Asset Bundle.", + "enum": [ + "BUNDLE" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(resources(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(bundle(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(workspace(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "jobs.JobEditMode": { + "anyOf": [ + { + "type": "string", + "description": "Edit mode of the job.\n\n* `UI_LOCKED`: The job is in a locked UI state and cannot be modified.\n* `EDITABLE`: The job is in an editable state and can be modified.", + "enum": [ + "UI_LOCKED", + "EDITABLE" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(resources(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(bundle(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(workspace(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "jobs.JobEmailNotifications": { + "anyOf": [ + { + "type": "object", + "properties": { + "no_alert_for_skipped_runs": { + "description": "If true, do not send email to recipients specified in `on_failure` if the run is skipped.", + "$ref": "#/$defs/bool" + }, + "on_duration_warning_threshold_exceeded": { + "description": "A list of email addresses to be notified when the duration of a run exceeds the threshold specified for the `RUN_DURATION_SECONDS` metric in the `health` field. If no rule for the `RUN_DURATION_SECONDS` metric is specified in the `health` field for the job, notifications are not sent.", + "$ref": "#/$defs/slice/string" + }, + "on_failure": { + "description": "A list of email addresses to be notified when a run unsuccessfully completes. A run is considered to have completed unsuccessfully if it ends with an `INTERNAL_ERROR` `life_cycle_state` or a `FAILED`, or `TIMED_OUT` result_state. If this is not specified on job creation, reset, or update the list is empty, and notifications are not sent.", + "$ref": "#/$defs/slice/string" + }, + "on_start": { + "description": "A list of email addresses to be notified when a run begins. If not specified on job creation, reset, or update, the list is empty, and notifications are not sent.", + "$ref": "#/$defs/slice/string" + }, + "on_streaming_backlog_exceeded": { + "description": "A list of email addresses to notify when any streaming backlog thresholds are exceeded for any stream.\nStreaming backlog thresholds can be set in the `health` field using the following metrics: `STREAMING_BACKLOG_BYTES`, `STREAMING_BACKLOG_RECORDS`, `STREAMING_BACKLOG_SECONDS`, or `STREAMING_BACKLOG_FILES`.\nAlerting is based on the 10-minute average of these metrics. If the issue persists, notifications are resent every 30 minutes.", + "$ref": "#/$defs/slice/string" + }, + "on_success": { + "description": "A list of email addresses to be notified when a run successfully completes. A run is considered to have completed successfully if it ends with a `TERMINATED` `life_cycle_state` and a `SUCCESS` result_state. If not specified on job creation, reset, or update, the list is empty, and notifications are not sent.", + "$ref": "#/$defs/slice/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "jobs.JobEnvironment": { + "anyOf": [ + { + "type": "object", + "properties": { + "environment_key": { + "description": "The key of an environment. It has to be unique within a job.", + "$ref": "#/$defs/string" + }, + "spec": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.Environment" + } + }, + "additionalProperties": false, + "required": [ + "environment_key" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "jobs.JobNotificationSettings": { + "anyOf": [ + { + "type": "object", + "properties": { + "no_alert_for_canceled_runs": { + "description": "If true, do not send notifications to recipients specified in `on_failure` if the run is canceled.", + "$ref": "#/$defs/bool" + }, + "no_alert_for_skipped_runs": { + "description": "If true, do not send notifications to recipients specified in `on_failure` if the run is skipped.", + "$ref": "#/$defs/bool" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "jobs.JobParameterDefinition": { + "anyOf": [ + { + "type": "object", + "properties": { + "default": { + "description": "Default value of the parameter.", + "$ref": "#/$defs/string" + }, + "name": { + "description": "The name of the defined parameter. May only contain alphanumeric characters, `_`, `-`, and `.`", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "default", + "name" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "jobs.JobRunAs": { + "anyOf": [ + { + "type": "object", + "description": "Write-only setting, available only in Create/Update/Reset and Submit calls. Specifies the user or service principal that the job runs as. If not specified, the job runs as the user who created the job.\n\nOnly `user_name` or `service_principal_name` can be specified. If both are specified, an error is thrown.", + "properties": { + "service_principal_name": { + "description": "Application ID of an active service principal. Setting this field requires the `servicePrincipal/user` role.", + "$ref": "#/$defs/string" + }, + "user_name": { + "description": "The email of an active workspace user. Non-admin users can only set this field to their own email.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "jobs.JobSource": { + "anyOf": [ + { + "type": "object", + "description": "The source of the job specification in the remote repository when the job is source controlled.", + "properties": { + "dirty_state": { + "description": "Dirty state indicates the job is not fully synced with the job specification in the remote repository.\n\nPossible values are:\n* `NOT_SYNCED`: The job is not yet synced with the remote job specification. Import the remote job specification from UI to make the job fully synced.\n* `DISCONNECTED`: The job is temporary disconnected from the remote job specification and is allowed for live edit. Import the remote job specification again from UI to make the job fully synced.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.JobSourceDirtyState" + }, + "import_from_git_branch": { + "description": "Name of the branch which the job is imported from.", + "$ref": "#/$defs/string" + }, + "job_config_path": { + "description": "Path of the job YAML file that contains the job specification.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "import_from_git_branch", + "job_config_path" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "jobs.JobSourceDirtyState": { + "anyOf": [ + { + "type": "string", + "description": "Dirty state indicates the job is not fully synced with the job specification\nin the remote repository.\n\nPossible values are:\n* `NOT_SYNCED`: The job is not yet synced with the remote job specification. Import the remote job specification from UI to make the job fully synced.\n* `DISCONNECTED`: The job is temporary disconnected from the remote job specification and is allowed for live edit. Import the remote job specification again from UI to make the job fully synced.", + "enum": [ + "NOT_SYNCED", + "DISCONNECTED" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(resources(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(bundle(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(workspace(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "jobs.JobsHealthMetric": { + "anyOf": [ + { + "type": "string", + "description": "Specifies the health metric that is being evaluated for a particular health rule.\n\n* `RUN_DURATION_SECONDS`: Expected total time for a run in seconds.\n* `STREAMING_BACKLOG_BYTES`: An estimate of the maximum bytes of data waiting to be consumed across all streams. This metric is in Private Preview.\n* `STREAMING_BACKLOG_RECORDS`: An estimate of the maximum offset lag across all streams. This metric is in Private Preview.\n* `STREAMING_BACKLOG_SECONDS`: An estimate of the maximum consumer delay across all streams. This metric is in Private Preview.\n* `STREAMING_BACKLOG_FILES`: An estimate of the maximum number of outstanding files across all streams. This metric is in Private Preview.", + "enum": [ + "RUN_DURATION_SECONDS", + "STREAMING_BACKLOG_BYTES", + "STREAMING_BACKLOG_RECORDS", + "STREAMING_BACKLOG_SECONDS", + "STREAMING_BACKLOG_FILES" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(resources(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(bundle(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(workspace(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "jobs.JobsHealthOperator": { + "anyOf": [ + { + "type": "string", + "description": "Specifies the operator used to compare the health metric value with the specified threshold.", + "enum": [ + "GREATER_THAN" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(resources(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(bundle(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(workspace(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "jobs.JobsHealthRule": { + "anyOf": [ + { + "type": "object", + "properties": { + "metric": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.JobsHealthMetric" + }, + "op": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.JobsHealthOperator" + }, + "value": { + "description": "Specifies the threshold value that the health metric should obey to satisfy the health rule.", + "$ref": "#/$defs/int64" + } + }, + "additionalProperties": false, + "required": [ + "metric", + "op", + "value" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "jobs.JobsHealthRules": { + "anyOf": [ + { + "type": "object", + "description": "An optional set of health rules that can be defined for this job.", + "properties": { + "rules": { + "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/jobs.JobsHealthRule" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "jobs.NotebookTask": { + "anyOf": [ + { + "type": "object", + "properties": { + "base_parameters": { + "description": "Base parameters to be used for each run of this job. If the run is initiated by a call to :method:jobs/run\nNow with parameters specified, the two parameters maps are merged. If the same key is specified in\n`base_parameters` and in `run-now`, the value from `run-now` is used.\nUse [Task parameter variables](https://docs.databricks.com/jobs.html#parameter-variables) to set parameters containing information about job runs.\n\nIf the notebook takes a parameter that is not specified in the job’s `base_parameters` or the `run-now` override parameters,\nthe default value from the notebook is used.\n\nRetrieve these parameters in a notebook using [dbutils.widgets.get](https://docs.databricks.com/dev-tools/databricks-utils.html#dbutils-widgets).\n\nThe JSON representation of this field cannot exceed 1MB.", + "$ref": "#/$defs/map/string" + }, + "notebook_path": { + "description": "The path of the notebook to be run in the Databricks workspace or remote repository.\nFor notebooks stored in the Databricks workspace, the path must be absolute and begin with a slash.\nFor notebooks stored in a remote repository, the path must be relative. This field is required.", + "$ref": "#/$defs/string" + }, + "source": { + "description": "Optional location type of the notebook. When set to `WORKSPACE`, the notebook will be retrieved from the local Databricks workspace. When set to `GIT`, the notebook will be retrieved from a Git repository\ndefined in `git_source`. If the value is empty, the task will use `GIT` if `git_source` is defined and `WORKSPACE` otherwise.\n* `WORKSPACE`: Notebook is located in Databricks workspace.\n* `GIT`: Notebook is located in cloud Git provider.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.Source" + }, + "warehouse_id": { + "description": "Optional `warehouse_id` to run the notebook on a SQL warehouse. Classic SQL warehouses are NOT supported, please use serverless or pro SQL warehouses.\n\nNote that SQL warehouses only support SQL cells; if the notebook contains non-SQL cells, the run will fail.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "notebook_path" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "jobs.PauseStatus": { + "anyOf": [ + { + "type": "string", + "enum": [ + "UNPAUSED", + "PAUSED" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(resources(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(bundle(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(workspace(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "jobs.PeriodicTriggerConfiguration": { + "anyOf": [ + { + "type": "object", + "properties": { + "interval": { + "description": "The interval at which the trigger should run.", + "$ref": "#/$defs/int" + }, + "unit": { + "description": "The unit of time for the interval.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.PeriodicTriggerConfigurationTimeUnit" + } + }, + "additionalProperties": false, + "required": [ + "interval", + "unit" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "jobs.PeriodicTriggerConfigurationTimeUnit": { + "anyOf": [ + { + "type": "string", + "enum": [ + "HOURS", + "DAYS", + "WEEKS" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(resources(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(bundle(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(workspace(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "jobs.PipelineParams": { + "anyOf": [ + { + "type": "object", + "properties": { + "full_refresh": { + "description": "If true, triggers a full refresh on the delta live table.", + "$ref": "#/$defs/bool" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "jobs.PipelineTask": { + "anyOf": [ + { + "type": "object", + "properties": { + "full_refresh": { + "description": "If true, triggers a full refresh on the delta live table.", + "$ref": "#/$defs/bool" + }, + "pipeline_id": { + "description": "The full name of the pipeline task to execute.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "pipeline_id" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "jobs.PythonWheelTask": { + "anyOf": [ + { + "type": "object", + "properties": { + "entry_point": { + "description": "Named entry point to use, if it does not exist in the metadata of the package it executes the function from the package directly using `$packageName.$entryPoint()`", + "$ref": "#/$defs/string" + }, + "named_parameters": { + "description": "Command-line parameters passed to Python wheel task in the form of `[\"--name=task\", \"--data=dbfs:/path/to/data.json\"]`. Leave it empty if `parameters` is not null.", + "$ref": "#/$defs/map/string" + }, + "package_name": { + "description": "Name of the package to execute", + "$ref": "#/$defs/string" + }, + "parameters": { + "description": "Command-line parameters passed to Python wheel task. Leave it empty if `named_parameters` is not null.", + "$ref": "#/$defs/slice/string" + } + }, + "additionalProperties": false, + "required": [ + "entry_point", + "package_name" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "jobs.QueueSettings": { + "anyOf": [ + { + "type": "object", + "properties": { + "enabled": { + "description": "If true, enable queueing for the job. This is a required field.", + "$ref": "#/$defs/bool" + } + }, + "additionalProperties": false, + "required": [ + "enabled" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "jobs.RunIf": { + "anyOf": [ + { + "type": "string", + "description": "An optional value indicating the condition that determines whether the task should be run once its dependencies have been completed. When omitted, defaults to `ALL_SUCCESS`.\n\nPossible values are:\n* `ALL_SUCCESS`: All dependencies have executed and succeeded\n* `AT_LEAST_ONE_SUCCESS`: At least one dependency has succeeded\n* `NONE_FAILED`: None of the dependencies have failed and at least one was executed\n* `ALL_DONE`: All dependencies have been completed\n* `AT_LEAST_ONE_FAILED`: At least one dependency failed\n* `ALL_FAILED`: ALl dependencies have failed", + "enum": [ + "ALL_SUCCESS", + "ALL_DONE", + "NONE_FAILED", + "AT_LEAST_ONE_SUCCESS", + "ALL_FAILED", + "AT_LEAST_ONE_FAILED" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(resources(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(bundle(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(workspace(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "jobs.RunJobTask": { + "anyOf": [ + { + "type": "object", + "properties": { + "dbt_commands": { + "description": "An array of commands to execute for jobs with the dbt task, for example `\"dbt_commands\": [\"dbt deps\", \"dbt seed\", \"dbt deps\", \"dbt seed\", \"dbt run\"]`", + "$ref": "#/$defs/slice/string" + }, + "jar_params": { + "description": "A list of parameters for jobs with Spark JAR tasks, for example `\"jar_params\": [\"john doe\", \"35\"]`.\nThe parameters are used to invoke the main function of the main class specified in the Spark JAR task.\nIf not specified upon `run-now`, it defaults to an empty list.\njar_params cannot be specified in conjunction with notebook_params.\nThe JSON representation of this field (for example `{\"jar_params\":[\"john doe\",\"35\"]}`) cannot exceed 10,000 bytes.\n\nUse [Task parameter variables](/jobs.html\\\"#parameter-variables\\\") to set parameters containing information about job runs.", + "$ref": "#/$defs/slice/string" + }, + "job_id": { + "description": "ID of the job to trigger.", + "$ref": "#/$defs/int64" + }, + "job_parameters": { + "description": "Job-level parameters used to trigger the job.", + "$ref": "#/$defs/map/string" + }, + "notebook_params": { + "description": "A map from keys to values for jobs with notebook task, for example `\"notebook_params\": {\"name\": \"john doe\", \"age\": \"35\"}`.\nThe map is passed to the notebook and is accessible through the [dbutils.widgets.get](https://docs.databricks.com/dev-tools/databricks-utils.html) function.\n\nIf not specified upon `run-now`, the triggered run uses the job’s base parameters.\n\nnotebook_params cannot be specified in conjunction with jar_params.\n\nUse [Task parameter variables](https://docs.databricks.com/jobs.html#parameter-variables) to set parameters containing information about job runs.\n\nThe JSON representation of this field (for example `{\"notebook_params\":{\"name\":\"john doe\",\"age\":\"35\"}}`) cannot exceed 10,000 bytes.", + "$ref": "#/$defs/map/string" + }, + "pipeline_params": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.PipelineParams" + }, + "python_named_params": { + "$ref": "#/$defs/map/string" + }, + "python_params": { + "description": "A list of parameters for jobs with Python tasks, for example `\"python_params\": [\"john doe\", \"35\"]`.\nThe parameters are passed to Python file as command-line parameters. If specified upon `run-now`, it would overwrite\nthe parameters specified in job setting. The JSON representation of this field (for example `{\"python_params\":[\"john doe\",\"35\"]}`)\ncannot exceed 10,000 bytes.\n\nUse [Task parameter variables](https://docs.databricks.com/jobs.html#parameter-variables) to set parameters containing information about job runs.\n\nImportant\n\nThese parameters accept only Latin characters (ASCII character set). Using non-ASCII characters returns an error.\nExamples of invalid, non-ASCII characters are Chinese, Japanese kanjis, and emojis.", + "$ref": "#/$defs/slice/string" + }, + "spark_submit_params": { + "description": "A list of parameters for jobs with spark submit task, for example `\"spark_submit_params\": [\"--class\", \"org.apache.spark.examples.SparkPi\"]`.\nThe parameters are passed to spark-submit script as command-line parameters. If specified upon `run-now`, it would overwrite the\nparameters specified in job setting. The JSON representation of this field (for example `{\"python_params\":[\"john doe\",\"35\"]}`)\ncannot exceed 10,000 bytes.\n\nUse [Task parameter variables](https://docs.databricks.com/jobs.html#parameter-variables) to set parameters containing information about job runs\n\nImportant\n\nThese parameters accept only Latin characters (ASCII character set). Using non-ASCII characters returns an error.\nExamples of invalid, non-ASCII characters are Chinese, Japanese kanjis, and emojis.", + "$ref": "#/$defs/slice/string" + }, + "sql_params": { + "description": "A map from keys to values for jobs with SQL task, for example `\"sql_params\": {\"name\": \"john doe\", \"age\": \"35\"}`. The SQL alert task does not support custom parameters.", + "$ref": "#/$defs/map/string" + } + }, + "additionalProperties": false, + "required": [ + "job_id" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "jobs.Source": { + "anyOf": [ + { + "type": "string", + "description": "Optional location type of the SQL file. When set to `WORKSPACE`, the SQL file will be retrieved\\\nfrom the local Databricks workspace. When set to `GIT`, the SQL file will be retrieved from a Git repository\ndefined in `git_source`. If the value is empty, the task will use `GIT` if `git_source` is defined and `WORKSPACE` otherwise.\n\n* `WORKSPACE`: SQL file is located in Databricks workspace.\n* `GIT`: SQL file is located in cloud Git provider.", + "enum": [ + "WORKSPACE", + "GIT" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(resources(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(bundle(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(workspace(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "jobs.SparkJarTask": { + "anyOf": [ + { + "type": "object", + "properties": { + "jar_uri": { + "description": "Deprecated since 04/2016. Provide a `jar` through the `libraries` field instead. For an example, see :method:jobs/create.", + "$ref": "#/$defs/string" + }, + "main_class_name": { + "description": "The full name of the class containing the main method to be executed. This class must be contained in a JAR provided as a library.\n\nThe code must use `SparkContext.getOrCreate` to obtain a Spark context; otherwise, runs of the job fail.", + "$ref": "#/$defs/string" + }, + "parameters": { + "description": "Parameters passed to the main method.\n\nUse [Task parameter variables](https://docs.databricks.com/jobs.html#parameter-variables) to set parameters containing information about job runs.", + "$ref": "#/$defs/slice/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "jobs.SparkPythonTask": { + "anyOf": [ + { + "type": "object", + "properties": { + "parameters": { + "description": "Command line parameters passed to the Python file.\n\nUse [Task parameter variables](https://docs.databricks.com/jobs.html#parameter-variables) to set parameters containing information about job runs.", + "$ref": "#/$defs/slice/string" + }, + "python_file": { + "description": "The Python file to be executed. Cloud file URIs (such as dbfs:/, s3:/, adls:/, gcs:/) and workspace paths are supported. For python files stored in the Databricks workspace, the path must be absolute and begin with `/`. For files stored in a remote repository, the path must be relative. This field is required.", + "$ref": "#/$defs/string" + }, + "source": { + "description": "Optional location type of the Python file. When set to `WORKSPACE` or not specified, the file will be retrieved from the local\nDatabricks workspace or cloud location (if the `python_file` has a URI format). When set to `GIT`,\nthe Python file will be retrieved from a Git repository defined in `git_source`.\n\n* `WORKSPACE`: The Python file is located in a Databricks workspace or at a cloud filesystem URI.\n* `GIT`: The Python file is located in a remote Git repository.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.Source" + } + }, + "additionalProperties": false, + "required": [ + "python_file" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "jobs.SparkSubmitTask": { + "anyOf": [ + { + "type": "object", + "properties": { + "parameters": { + "description": "Command-line parameters passed to spark submit.\n\nUse [Task parameter variables](https://docs.databricks.com/jobs.html#parameter-variables) to set parameters containing information about job runs.", + "$ref": "#/$defs/slice/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "jobs.SqlTask": { + "anyOf": [ + { + "type": "object", + "properties": { + "alert": { + "description": "If alert, indicates that this job must refresh a SQL alert.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.SqlTaskAlert" + }, + "dashboard": { + "description": "If dashboard, indicates that this job must refresh a SQL dashboard.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.SqlTaskDashboard" + }, + "file": { + "description": "If file, indicates that this job runs a SQL file in a remote Git repository.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.SqlTaskFile" + }, + "parameters": { + "description": "Parameters to be used for each run of this job. The SQL alert task does not support custom parameters.", + "$ref": "#/$defs/map/string" + }, + "query": { + "description": "If query, indicates that this job must execute a SQL query.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.SqlTaskQuery" + }, + "warehouse_id": { + "description": "The canonical identifier of the SQL warehouse. Recommended to use with serverless or pro SQL warehouses. Classic SQL warehouses are only supported for SQL alert, dashboard and query tasks and are limited to scheduled single-task jobs.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "warehouse_id" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "jobs.SqlTaskAlert": { + "anyOf": [ + { + "type": "object", + "properties": { + "alert_id": { + "description": "The canonical identifier of the SQL alert.", + "$ref": "#/$defs/string" + }, + "pause_subscriptions": { + "description": "If true, the alert notifications are not sent to subscribers.", + "$ref": "#/$defs/bool" + }, + "subscriptions": { + "description": "If specified, alert notifications are sent to subscribers.", + "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/jobs.SqlTaskSubscription" + } + }, + "additionalProperties": false, + "required": [ + "alert_id" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "jobs.SqlTaskDashboard": { + "anyOf": [ + { + "type": "object", + "properties": { + "custom_subject": { + "description": "Subject of the email sent to subscribers of this task.", + "$ref": "#/$defs/string" + }, + "dashboard_id": { + "description": "The canonical identifier of the SQL dashboard.", + "$ref": "#/$defs/string" + }, + "pause_subscriptions": { + "description": "If true, the dashboard snapshot is not taken, and emails are not sent to subscribers.", + "$ref": "#/$defs/bool" + }, + "subscriptions": { + "description": "If specified, dashboard snapshots are sent to subscriptions.", + "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/jobs.SqlTaskSubscription" + } + }, + "additionalProperties": false, + "required": [ + "dashboard_id" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "jobs.SqlTaskFile": { + "anyOf": [ + { + "type": "object", + "properties": { + "path": { + "description": "Path of the SQL file. Must be relative if the source is a remote Git repository and absolute for workspace paths.", + "$ref": "#/$defs/string" + }, + "source": { + "description": "Optional location type of the SQL file. When set to `WORKSPACE`, the SQL file will be retrieved\nfrom the local Databricks workspace. When set to `GIT`, the SQL file will be retrieved from a Git repository\ndefined in `git_source`. If the value is empty, the task will use `GIT` if `git_source` is defined and `WORKSPACE` otherwise.\n\n* `WORKSPACE`: SQL file is located in Databricks workspace.\n* `GIT`: SQL file is located in cloud Git provider.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.Source" + } + }, + "additionalProperties": false, + "required": [ + "path" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "jobs.SqlTaskQuery": { + "anyOf": [ + { + "type": "object", + "properties": { + "query_id": { + "description": "The canonical identifier of the SQL query.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "query_id" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "jobs.SqlTaskSubscription": { + "anyOf": [ + { + "type": "object", + "properties": { + "destination_id": { + "description": "The canonical identifier of the destination to receive email notification. This parameter is mutually exclusive with user_name. You cannot set both destination_id and user_name for subscription notifications.", + "$ref": "#/$defs/string" + }, + "user_name": { + "description": "The user name to receive the subscription email. This parameter is mutually exclusive with destination_id. You cannot set both destination_id and user_name for subscription notifications.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "jobs.TableUpdateTriggerConfiguration": { + "anyOf": [ + { + "type": "object", + "properties": { + "condition": { + "description": "The table(s) condition based on which to trigger a job run.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.Condition" + }, + "min_time_between_triggers_seconds": { + "description": "If set, the trigger starts a run only after the specified amount of time has passed since\nthe last time the trigger fired. The minimum allowed value is 60 seconds.", + "$ref": "#/$defs/int" + }, + "table_names": { + "description": "A list of Delta tables to monitor for changes. The table name must be in the format `catalog_name.schema_name.table_name`.", + "$ref": "#/$defs/slice/string" + }, + "wait_after_last_change_seconds": { + "description": "If set, the trigger starts a run only after no table updates have occurred for the specified time\nand can be used to wait for a series of table updates before triggering a run. The\nminimum allowed value is 60 seconds.", + "$ref": "#/$defs/int" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "jobs.Task": { + "anyOf": [ + { + "type": "object", + "properties": { + "condition_task": { + "description": "If condition_task, specifies a condition with an outcome that can be used to control the execution of other tasks. Does not require a cluster to execute and does not support retries or notifications.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.ConditionTask" + }, + "dbt_task": { + "description": "If dbt_task, indicates that this must execute a dbt task. It requires both Databricks SQL and the ability to use a serverless or a pro SQL warehouse.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.DbtTask" + }, + "depends_on": { + "description": "An optional array of objects specifying the dependency graph of the task. All tasks specified in this field must complete before executing this task. The task will run only if the `run_if` condition is true.\nThe key is `task_key`, and the value is the name assigned to the dependent task.", + "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/jobs.TaskDependency" + }, + "description": { + "description": "An optional description for this task.", + "$ref": "#/$defs/string" + }, + "disable_auto_optimization": { + "description": "An option to disable auto optimization in serverless", + "$ref": "#/$defs/bool" + }, + "email_notifications": { + "description": "An optional set of email addresses that is notified when runs of this task begin or complete as well as when this task is deleted. The default behavior is to not send any emails.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.TaskEmailNotifications" + }, + "environment_key": { + "description": "The key that references an environment spec in a job. This field is required for Python script, Python wheel and dbt tasks when using serverless compute.", + "$ref": "#/$defs/string" + }, + "existing_cluster_id": { + "description": "If existing_cluster_id, the ID of an existing cluster that is used for all runs.\nWhen running jobs or tasks on an existing cluster, you may need to manually restart\nthe cluster if it stops responding. We suggest running jobs and tasks on new clusters for\ngreater reliability", + "$ref": "#/$defs/string" + }, + "for_each_task": { + "description": "If for_each_task, indicates that this task must execute the nested task within it.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.ForEachTask" + }, + "health": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.JobsHealthRules" + }, + "job_cluster_key": { + "description": "If job_cluster_key, this task is executed reusing the cluster specified in `job.settings.job_clusters`.", + "$ref": "#/$defs/string" + }, + "libraries": { + "description": "An optional list of libraries to be installed on the cluster.\nThe default value is an empty list.", + "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/compute.Library" + }, + "max_retries": { + "description": "An optional maximum number of times to retry an unsuccessful run. A run is considered to be unsuccessful if it completes with the `FAILED` result_state or `INTERNAL_ERROR` `life_cycle_state`. The value `-1` means to retry indefinitely and the value `0` means to never retry.", + "$ref": "#/$defs/int" + }, + "min_retry_interval_millis": { + "description": "An optional minimal interval in milliseconds between the start of the failed run and the subsequent retry run. The default behavior is that unsuccessful runs are immediately retried.", + "$ref": "#/$defs/int" + }, + "new_cluster": { + "description": "If new_cluster, a description of a new cluster that is created for each run.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.ClusterSpec" + }, + "notebook_task": { + "description": "If notebook_task, indicates that this task must run a notebook. This field may not be specified in conjunction with spark_jar_task.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.NotebookTask" + }, + "notification_settings": { + "description": "Optional notification settings that are used when sending notifications to each of the `email_notifications` and `webhook_notifications` for this task.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.TaskNotificationSettings" + }, + "pipeline_task": { + "description": "If pipeline_task, indicates that this task must execute a Pipeline.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.PipelineTask" + }, + "python_wheel_task": { + "description": "If python_wheel_task, indicates that this job must execute a PythonWheel.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.PythonWheelTask" + }, + "retry_on_timeout": { + "description": "An optional policy to specify whether to retry a job when it times out. The default behavior\nis to not retry on timeout.", + "$ref": "#/$defs/bool" + }, + "run_if": { + "description": "An optional value specifying the condition determining whether the task is run once its dependencies have been completed.\n\n* `ALL_SUCCESS`: All dependencies have executed and succeeded\n* `AT_LEAST_ONE_SUCCESS`: At least one dependency has succeeded\n* `NONE_FAILED`: None of the dependencies have failed and at least one was executed\n* `ALL_DONE`: All dependencies have been completed\n* `AT_LEAST_ONE_FAILED`: At least one dependency failed\n* `ALL_FAILED`: ALl dependencies have failed", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.RunIf" + }, + "run_job_task": { + "description": "If run_job_task, indicates that this task must execute another job.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.RunJobTask" + }, + "spark_jar_task": { + "description": "If spark_jar_task, indicates that this task must run a JAR.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.SparkJarTask" + }, + "spark_python_task": { + "description": "If spark_python_task, indicates that this task must run a Python file.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.SparkPythonTask" + }, + "spark_submit_task": { + "description": "If `spark_submit_task`, indicates that this task must be launched by the spark submit script. This task can run only on new clusters.\n\nIn the `new_cluster` specification, `libraries` and `spark_conf` are not supported. Instead, use `--jars` and `--py-files` to add Java and Python libraries and `--conf` to set the Spark configurations.\n\n`master`, `deploy-mode`, and `executor-cores` are automatically configured by Databricks; you _cannot_ specify them in parameters.\n\nBy default, the Spark submit job uses all available memory (excluding reserved memory for Databricks services). You can set `--driver-memory`, and `--executor-memory` to a smaller value to leave some room for off-heap usage.\n\nThe `--jars`, `--py-files`, `--files` arguments support DBFS and S3 paths.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.SparkSubmitTask" + }, + "sql_task": { + "description": "If sql_task, indicates that this job must execute a SQL task.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.SqlTask" + }, + "task_key": { + "description": "A unique name for the task. This field is used to refer to this task from other tasks.\nThis field is required and must be unique within its parent job.\nOn Update or Reset, this field is used to reference the tasks to be updated or reset.", + "$ref": "#/$defs/string" + }, + "timeout_seconds": { + "description": "An optional timeout applied to each run of this job task. A value of `0` means no timeout.", + "$ref": "#/$defs/int" + }, + "webhook_notifications": { + "description": "A collection of system notification IDs to notify when runs of this task begin or complete. The default behavior is to not send any system notifications.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.WebhookNotifications" + } + }, + "additionalProperties": false, + "required": [ + "task_key" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "jobs.TaskDependency": { + "anyOf": [ + { + "type": "object", + "properties": { + "outcome": { + "description": "Can only be specified on condition task dependencies. The outcome of the dependent task that must be met for this task to run.", + "$ref": "#/$defs/string" + }, + "task_key": { + "description": "The name of the task this task depends on.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "task_key" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "jobs.TaskEmailNotifications": { + "anyOf": [ + { + "type": "object", + "properties": { + "no_alert_for_skipped_runs": { + "description": "If true, do not send email to recipients specified in `on_failure` if the run is skipped.", + "$ref": "#/$defs/bool" + }, + "on_duration_warning_threshold_exceeded": { + "description": "A list of email addresses to be notified when the duration of a run exceeds the threshold specified for the `RUN_DURATION_SECONDS` metric in the `health` field. If no rule for the `RUN_DURATION_SECONDS` metric is specified in the `health` field for the job, notifications are not sent.", + "$ref": "#/$defs/slice/string" + }, + "on_failure": { + "description": "A list of email addresses to be notified when a run unsuccessfully completes. A run is considered to have completed unsuccessfully if it ends with an `INTERNAL_ERROR` `life_cycle_state` or a `FAILED`, or `TIMED_OUT` result_state. If this is not specified on job creation, reset, or update the list is empty, and notifications are not sent.", + "$ref": "#/$defs/slice/string" + }, + "on_start": { + "description": "A list of email addresses to be notified when a run begins. If not specified on job creation, reset, or update, the list is empty, and notifications are not sent.", + "$ref": "#/$defs/slice/string" + }, + "on_streaming_backlog_exceeded": { + "description": "A list of email addresses to notify when any streaming backlog thresholds are exceeded for any stream.\nStreaming backlog thresholds can be set in the `health` field using the following metrics: `STREAMING_BACKLOG_BYTES`, `STREAMING_BACKLOG_RECORDS`, `STREAMING_BACKLOG_SECONDS`, or `STREAMING_BACKLOG_FILES`.\nAlerting is based on the 10-minute average of these metrics. If the issue persists, notifications are resent every 30 minutes.", + "$ref": "#/$defs/slice/string" + }, + "on_success": { + "description": "A list of email addresses to be notified when a run successfully completes. A run is considered to have completed successfully if it ends with a `TERMINATED` `life_cycle_state` and a `SUCCESS` result_state. If not specified on job creation, reset, or update, the list is empty, and notifications are not sent.", + "$ref": "#/$defs/slice/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "jobs.TaskNotificationSettings": { + "anyOf": [ + { + "type": "object", + "properties": { + "alert_on_last_attempt": { + "description": "If true, do not send notifications to recipients specified in `on_start` for the retried runs and do not send notifications to recipients specified in `on_failure` until the last retry of the run.", + "$ref": "#/$defs/bool" + }, + "no_alert_for_canceled_runs": { + "description": "If true, do not send notifications to recipients specified in `on_failure` if the run is canceled.", + "$ref": "#/$defs/bool" + }, + "no_alert_for_skipped_runs": { + "description": "If true, do not send notifications to recipients specified in `on_failure` if the run is skipped.", + "$ref": "#/$defs/bool" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "jobs.TriggerSettings": { + "anyOf": [ + { + "type": "object", + "properties": { + "file_arrival": { + "description": "File arrival trigger settings.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.FileArrivalTriggerConfiguration" + }, + "pause_status": { + "description": "Whether this trigger is paused or not.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.PauseStatus" + }, + "periodic": { + "description": "Periodic trigger settings.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.PeriodicTriggerConfiguration" + }, + "table": { + "description": "Old table trigger settings name. Deprecated in favor of `table_update`.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.TableUpdateTriggerConfiguration" + }, + "table_update": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.TableUpdateTriggerConfiguration" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "jobs.Webhook": { + "anyOf": [ + { + "type": "object", + "properties": { + "id": { + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "id" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "jobs.WebhookNotifications": { + "anyOf": [ + { + "type": "object", + "properties": { + "on_duration_warning_threshold_exceeded": { + "description": "An optional list of system notification IDs to call when the duration of a run exceeds the threshold specified for the `RUN_DURATION_SECONDS` metric in the `health` field. A maximum of 3 destinations can be specified for the `on_duration_warning_threshold_exceeded` property.", + "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/jobs.Webhook" + }, + "on_failure": { + "description": "An optional list of system notification IDs to call when the run fails. A maximum of 3 destinations can be specified for the `on_failure` property.", + "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/jobs.Webhook" + }, + "on_start": { + "description": "An optional list of system notification IDs to call when the run starts. A maximum of 3 destinations can be specified for the `on_start` property.", + "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/jobs.Webhook" + }, + "on_streaming_backlog_exceeded": { + "description": "An optional list of system notification IDs to call when any streaming backlog thresholds are exceeded for any stream.\nStreaming backlog thresholds can be set in the `health` field using the following metrics: `STREAMING_BACKLOG_BYTES`, `STREAMING_BACKLOG_RECORDS`, `STREAMING_BACKLOG_SECONDS`, or `STREAMING_BACKLOG_FILES`.\nAlerting is based on the 10-minute average of these metrics. If the issue persists, notifications are resent every 30 minutes.\nA maximum of 3 destinations can be specified for the `on_streaming_backlog_exceeded` property.", + "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/jobs.Webhook" + }, + "on_success": { + "description": "An optional list of system notification IDs to call when the run completes successfully. A maximum of 3 destinations can be specified for the `on_success` property.", + "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/jobs.Webhook" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "ml.ExperimentTag": { + "anyOf": [ + { + "type": "object", + "properties": { + "key": { + "description": "The tag key.", + "$ref": "#/$defs/string" + }, + "value": { + "description": "The tag value.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "ml.ModelTag": { + "anyOf": [ + { + "type": "object", + "properties": { + "key": { + "description": "The tag key.", + "$ref": "#/$defs/string" + }, + "value": { + "description": "The tag value.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "ml.ModelVersion": { + "anyOf": [ + { + "type": "object", + "properties": { + "creation_timestamp": { + "description": "Timestamp recorded when this `model_version` was created.", + "$ref": "#/$defs/int64" + }, + "current_stage": { + "description": "Current stage for this `model_version`.", + "$ref": "#/$defs/string" + }, + "description": { + "description": "Description of this `model_version`.", + "$ref": "#/$defs/string" + }, + "last_updated_timestamp": { + "description": "Timestamp recorded when metadata for this `model_version` was last updated.", + "$ref": "#/$defs/int64" + }, + "name": { + "description": "Unique name of the model", + "$ref": "#/$defs/string" + }, + "run_id": { + "description": "MLflow run ID used when creating `model_version`, if `source` was generated by an\nexperiment run stored in MLflow tracking server.", + "$ref": "#/$defs/string" + }, + "run_link": { + "description": "Run Link: Direct link to the run that generated this version", + "$ref": "#/$defs/string" + }, + "source": { + "description": "URI indicating the location of the source model artifacts, used when creating `model_version`", + "$ref": "#/$defs/string" + }, + "status": { + "description": "Current status of `model_version`", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/ml.ModelVersionStatus", + "enum": [ + "PENDING_REGISTRATION", + "FAILED_REGISTRATION", + "READY" + ] + }, + "status_message": { + "description": "Details on current `status`, if it is pending or failed.", + "$ref": "#/$defs/string" + }, + "tags": { + "description": "Tags: Additional metadata key-value pairs for this `model_version`.", + "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/ml.ModelVersionTag" + }, + "user_id": { + "description": "User that created this `model_version`.", + "$ref": "#/$defs/string" + }, + "version": { + "description": "Model's version number.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "ml.ModelVersionStatus": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "string", + "pattern": "\\$\\{(resources(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(bundle(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(workspace(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "ml.ModelVersionTag": { + "anyOf": [ + { + "type": "object", + "properties": { + "key": { + "description": "The tag key.", + "$ref": "#/$defs/string" + }, + "value": { + "description": "The tag value.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "pipelines.CronTrigger": { + "anyOf": [ + { + "type": "object", + "properties": { + "quartz_cron_schedule": { + "$ref": "#/$defs/string" + }, + "timezone_id": { + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "pipelines.DeploymentKind": { + "anyOf": [ + { + "type": "string", + "description": "The deployment method that manages the pipeline:\n- BUNDLE: The pipeline is managed by a Databricks Asset Bundle.\n", + "enum": [ + "BUNDLE" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(resources(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(bundle(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(workspace(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "pipelines.FileLibrary": { + "anyOf": [ + { + "type": "object", + "properties": { + "path": { + "description": "The absolute path of the file.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "pipelines.Filters": { + "anyOf": [ + { + "type": "object", + "properties": { + "exclude": { + "description": "Paths to exclude.", + "$ref": "#/$defs/slice/string" + }, + "include": { + "description": "Paths to include.", + "$ref": "#/$defs/slice/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "pipelines.IngestionConfig": { + "anyOf": [ + { + "type": "object", + "properties": { + "schema": { + "description": "Select tables from a specific source schema.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.SchemaSpec" + }, + "table": { + "description": "Select tables from a specific source table.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.TableSpec" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "pipelines.IngestionGatewayPipelineDefinition": { + "anyOf": [ + { + "type": "object", + "properties": { + "connection_id": { + "description": "Immutable. The Unity Catalog connection this gateway pipeline uses to communicate with the source.", + "$ref": "#/$defs/string" + }, + "gateway_storage_catalog": { + "description": "Required, Immutable. The name of the catalog for the gateway pipeline's storage location.", + "$ref": "#/$defs/string" + }, + "gateway_storage_name": { + "description": "Optional. The Unity Catalog-compatible name for the gateway storage location.\nThis is the destination to use for the data that is extracted by the gateway.\nDelta Live Tables system will automatically create the storage location under the catalog and schema.\n", + "$ref": "#/$defs/string" + }, + "gateway_storage_schema": { + "description": "Required, Immutable. The name of the schema for the gateway pipelines's storage location.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "pipelines.IngestionPipelineDefinition": { + "anyOf": [ + { + "type": "object", + "properties": { + "connection_name": { + "description": "Immutable. The Unity Catalog connection this ingestion pipeline uses to communicate with the source. Specify either ingestion_gateway_id or connection_name.", + "$ref": "#/$defs/string" + }, + "ingestion_gateway_id": { + "description": "Immutable. Identifier for the ingestion gateway used by this ingestion pipeline to communicate with the source. Specify either ingestion_gateway_id or connection_name.", + "$ref": "#/$defs/string" + }, + "objects": { + "description": "Required. Settings specifying tables to replicate and the destination for the replicated tables.", + "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/pipelines.IngestionConfig" + }, + "table_configuration": { + "description": "Configuration settings to control the ingestion of tables. These settings are applied to all tables in the pipeline.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.TableSpecificConfig" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "pipelines.ManualTrigger": { + "anyOf": [ + { + "type": "object", + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "pipelines.NotebookLibrary": { + "anyOf": [ + { + "type": "object", + "properties": { + "path": { + "description": "The absolute path of the notebook.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "pipelines.Notifications": { + "anyOf": [ + { + "type": "object", + "properties": { + "alerts": { + "description": "A list of alerts that trigger the sending of notifications to the configured\ndestinations. The supported alerts are:\n\n* `on-update-success`: A pipeline update completes successfully.\n* `on-update-failure`: Each time a pipeline update fails.\n* `on-update-fatal-failure`: A pipeline update fails with a non-retryable (fatal) error.\n* `on-flow-failure`: A single data flow fails.\n", + "$ref": "#/$defs/slice/string" + }, + "email_recipients": { + "description": "A list of email addresses notified when a configured alert is triggered.\n", + "$ref": "#/$defs/slice/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "pipelines.PipelineCluster": { + "anyOf": [ + { + "type": "object", + "properties": { + "apply_policy_default_values": { + "description": "Note: This field won't be persisted. Only API users will check this field.", + "$ref": "#/$defs/bool" + }, + "autoscale": { + "description": "Parameters needed in order to automatically scale clusters up and down based on load.\nNote: autoscaling works best with DB runtime versions 3.0 or later.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.PipelineClusterAutoscale" + }, + "aws_attributes": { + "description": "Attributes related to clusters running on Amazon Web Services.\nIf not specified at cluster creation, a set of default values will be used.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.AwsAttributes" + }, + "azure_attributes": { + "description": "Attributes related to clusters running on Microsoft Azure.\nIf not specified at cluster creation, a set of default values will be used.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.AzureAttributes" + }, + "cluster_log_conf": { + "description": "The configuration for delivering spark logs to a long-term storage destination.\nOnly dbfs destinations are supported. Only one destination can be specified\nfor one cluster. If the conf is given, the logs will be delivered to the destination every\n`5 mins`. The destination of driver logs is `$destination/$clusterId/driver`, while\nthe destination of executor logs is `$destination/$clusterId/executor`.\n", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.ClusterLogConf" + }, + "custom_tags": { + "description": "Additional tags for cluster resources. Databricks will tag all cluster resources (e.g., AWS\ninstances and EBS volumes) with these tags in addition to `default_tags`. Notes:\n\n- Currently, Databricks allows at most 45 custom tags\n\n- Clusters can only reuse cloud resources if the resources' tags are a subset of the cluster tags", + "$ref": "#/$defs/map/string" + }, + "driver_instance_pool_id": { + "description": "The optional ID of the instance pool for the driver of the cluster belongs.\nThe pool cluster uses the instance pool with id (instance_pool_id) if the driver pool is not\nassigned.", + "$ref": "#/$defs/string" + }, + "driver_node_type_id": { + "description": "The node type of the Spark driver.\nNote that this field is optional; if unset, the driver node type will be set as the same value\nas `node_type_id` defined above.", + "$ref": "#/$defs/string" + }, + "enable_local_disk_encryption": { + "description": "Whether to enable local disk encryption for the cluster.", + "$ref": "#/$defs/bool" + }, + "gcp_attributes": { + "description": "Attributes related to clusters running on Google Cloud Platform.\nIf not specified at cluster creation, a set of default values will be used.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.GcpAttributes" + }, + "init_scripts": { + "description": "The configuration for storing init scripts. Any number of destinations can be specified. The scripts are executed sequentially in the order provided. If `cluster_log_conf` is specified, init script logs are sent to `\u003cdestination\u003e/\u003ccluster-ID\u003e/init_scripts`.", + "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/compute.InitScriptInfo" + }, + "instance_pool_id": { + "description": "The optional ID of the instance pool to which the cluster belongs.", + "$ref": "#/$defs/string" + }, + "label": { + "description": "A label for the cluster specification, either `default` to configure the default cluster, or `maintenance` to configure the maintenance cluster. This field is optional. The default value is `default`.", + "$ref": "#/$defs/string" + }, + "node_type_id": { + "description": "This field encodes, through a single value, the resources available to each of\nthe Spark nodes in this cluster. For example, the Spark nodes can be provisioned\nand optimized for memory or compute intensive workloads. A list of available node\ntypes can be retrieved by using the :method:clusters/listNodeTypes API call.\n", + "$ref": "#/$defs/string" + }, + "num_workers": { + "description": "Number of worker nodes that this cluster should have. A cluster has one Spark Driver\nand `num_workers` Executors for a total of `num_workers` + 1 Spark nodes.\n\nNote: When reading the properties of a cluster, this field reflects the desired number\nof workers rather than the actual current number of workers. For instance, if a cluster\nis resized from 5 to 10 workers, this field will immediately be updated to reflect\nthe target size of 10 workers, whereas the workers listed in `spark_info` will gradually\nincrease from 5 to 10 as the new nodes are provisioned.", + "$ref": "#/$defs/int" + }, + "policy_id": { + "description": "The ID of the cluster policy used to create the cluster if applicable.", + "$ref": "#/$defs/string" + }, + "spark_conf": { + "description": "An object containing a set of optional, user-specified Spark configuration key-value pairs.\nSee :method:clusters/create for more details.\n", + "$ref": "#/$defs/map/string" + }, + "spark_env_vars": { + "description": "An object containing a set of optional, user-specified environment variable key-value pairs.\nPlease note that key-value pair of the form (X,Y) will be exported as is (i.e.,\n`export X='Y'`) while launching the driver and workers.\n\nIn order to specify an additional set of `SPARK_DAEMON_JAVA_OPTS`, we recommend appending\nthem to `$SPARK_DAEMON_JAVA_OPTS` as shown in the example below. This ensures that all\ndefault databricks managed environmental variables are included as well.\n\nExample Spark environment variables:\n`{\"SPARK_WORKER_MEMORY\": \"28000m\", \"SPARK_LOCAL_DIRS\": \"/local_disk0\"}` or\n`{\"SPARK_DAEMON_JAVA_OPTS\": \"$SPARK_DAEMON_JAVA_OPTS -Dspark.shuffle.service.enabled=true\"}`", + "$ref": "#/$defs/map/string" + }, + "ssh_public_keys": { + "description": "SSH public key contents that will be added to each Spark node in this cluster. The\ncorresponding private keys can be used to login with the user name `ubuntu` on port `2200`.\nUp to 10 keys can be specified.", + "$ref": "#/$defs/slice/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "pipelines.PipelineClusterAutoscale": { + "anyOf": [ + { + "type": "object", + "properties": { + "max_workers": { + "description": "The maximum number of workers to which the cluster can scale up when overloaded. `max_workers` must be strictly greater than `min_workers`.", + "$ref": "#/$defs/int" + }, + "min_workers": { + "description": "The minimum number of workers the cluster can scale down to when underutilized.\nIt is also the initial number of workers the cluster will have after creation.", + "$ref": "#/$defs/int" + }, + "mode": { + "description": "Databricks Enhanced Autoscaling optimizes cluster utilization by automatically\nallocating cluster resources based on workload volume, with minimal impact to\nthe data processing latency of your pipelines. Enhanced Autoscaling is available\nfor `updates` clusters only. The legacy autoscaling feature is used for `maintenance`\nclusters.\n", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.PipelineClusterAutoscaleMode", + "enum": [ + "ENHANCED", + "LEGACY" + ] + } + }, + "additionalProperties": false, + "required": [ + "max_workers", + "min_workers" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "pipelines.PipelineClusterAutoscaleMode": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "string", + "pattern": "\\$\\{(resources(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(bundle(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(workspace(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "pipelines.PipelineDeployment": { + "anyOf": [ + { + "type": "object", + "properties": { + "kind": { + "description": "The deployment method that manages the pipeline.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.DeploymentKind" + }, + "metadata_file_path": { + "description": "The path to the file containing metadata about the deployment.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "pipelines.PipelineLibrary": { + "anyOf": [ + { + "type": "object", + "properties": { + "file": { + "description": "The path to a file that defines a pipeline and is stored in the Databricks Repos.\n", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.FileLibrary" + }, + "jar": { + "description": "URI of the jar to be installed. Currently only DBFS is supported.\n", + "$ref": "#/$defs/string" + }, + "maven": { + "description": "Specification of a maven library to be installed.\n", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.MavenLibrary" + }, + "notebook": { + "description": "The path to a notebook that defines a pipeline and is stored in the Databricks workspace.\n", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.NotebookLibrary" + }, + "whl": { + "description": "URI of the whl to be installed.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "pipelines.PipelineTrigger": { + "anyOf": [ + { + "type": "object", + "properties": { + "cron": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.CronTrigger" + }, + "manual": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.ManualTrigger" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "pipelines.SchemaSpec": { + "anyOf": [ + { + "type": "object", + "properties": { + "destination_catalog": { + "description": "Required. Destination catalog to store tables.", + "$ref": "#/$defs/string" + }, + "destination_schema": { + "description": "Required. Destination schema to store tables in. Tables with the same name as the source tables are created in this destination schema. The pipeline fails If a table with the same name already exists.", + "$ref": "#/$defs/string" + }, + "source_catalog": { + "description": "The source catalog name. Might be optional depending on the type of source.", + "$ref": "#/$defs/string" + }, + "source_schema": { + "description": "Required. Schema name in the source database.", + "$ref": "#/$defs/string" + }, + "table_configuration": { + "description": "Configuration settings to control the ingestion of tables. These settings are applied to all tables in this schema and override the table_configuration defined in the IngestionPipelineDefinition object.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.TableSpecificConfig" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "pipelines.TableSpec": { + "anyOf": [ + { + "type": "object", + "properties": { + "destination_catalog": { + "description": "Required. Destination catalog to store table.", + "$ref": "#/$defs/string" + }, + "destination_schema": { + "description": "Required. Destination schema to store table.", + "$ref": "#/$defs/string" + }, + "destination_table": { + "description": "Optional. Destination table name. The pipeline fails If a table with that name already exists. If not set, the source table name is used.", + "$ref": "#/$defs/string" + }, + "source_catalog": { + "description": "Source catalog name. Might be optional depending on the type of source.", + "$ref": "#/$defs/string" + }, + "source_schema": { + "description": "Schema name in the source database. Might be optional depending on the type of source.", + "$ref": "#/$defs/string" + }, + "source_table": { + "description": "Required. Table name in the source database.", + "$ref": "#/$defs/string" + }, + "table_configuration": { + "description": "Configuration settings to control the ingestion of tables. These settings override the table_configuration defined in the IngestionPipelineDefinition object and the SchemaSpec.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.TableSpecificConfig" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "pipelines.TableSpecificConfig": { + "anyOf": [ + { + "type": "object", + "properties": { + "primary_keys": { + "description": "The primary key of the table used to apply changes.", + "$ref": "#/$defs/slice/string" + }, + "salesforce_include_formula_fields": { + "description": "If true, formula fields defined in the table are included in the ingestion. This setting is only valid for the Salesforce connector", + "$ref": "#/$defs/bool" + }, + "scd_type": { + "description": "The SCD type to use to ingest the table.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.TableSpecificConfigScdType", + "enum": [ + "SCD_TYPE_1", + "SCD_TYPE_2" + ] + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "pipelines.TableSpecificConfigScdType": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "string", + "pattern": "\\$\\{(resources(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(bundle(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(workspace(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "serving.Ai21LabsConfig": { + "anyOf": [ + { + "type": "object", + "properties": { + "ai21labs_api_key": { + "$ref": "#/$defs/string" + }, + "ai21labs_api_key_plaintext": { + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "serving.AmazonBedrockConfig": { + "anyOf": [ + { + "type": "object", + "properties": { + "aws_access_key_id": { + "description": "The Databricks secret key reference for an AWS access key ID with permissions to interact with Bedrock services. If you prefer to paste your API key directly, see `aws_access_key_id`. You must provide an API key using one of the following fields: `aws_access_key_id` or `aws_access_key_id_plaintext`.", + "$ref": "#/$defs/string" + }, + "aws_access_key_id_plaintext": { + "description": "An AWS access key ID with permissions to interact with Bedrock services provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `aws_access_key_id`. You must provide an API key using one of the following fields: `aws_access_key_id` or `aws_access_key_id_plaintext`.", + "$ref": "#/$defs/string" + }, + "aws_region": { + "description": "The AWS region to use. Bedrock has to be enabled there.", + "$ref": "#/$defs/string" + }, + "aws_secret_access_key": { + "description": "The Databricks secret key reference for an AWS secret access key paired with the access key ID, with permissions to interact with Bedrock services. If you prefer to paste your API key directly, see `aws_secret_access_key_plaintext`. You must provide an API key using one of the following fields: `aws_secret_access_key` or `aws_secret_access_key_plaintext`.", + "$ref": "#/$defs/string" + }, + "aws_secret_access_key_plaintext": { + "description": "An AWS secret access key paired with the access key ID, with permissions to interact with Bedrock services provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `aws_secret_access_key`. You must provide an API key using one of the following fields: `aws_secret_access_key` or `aws_secret_access_key_plaintext`.", + "$ref": "#/$defs/string" + }, + "bedrock_provider": { + "description": "The underlying provider in Amazon Bedrock. Supported values (case insensitive) include: Anthropic, Cohere, AI21Labs, Amazon.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.AmazonBedrockConfigBedrockProvider", + "enum": [ + "anthropic", + "cohere", + "ai21labs", + "amazon" + ] + } + }, + "additionalProperties": false, + "required": [ + "aws_region", + "bedrock_provider" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "serving.AmazonBedrockConfigBedrockProvider": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "string", + "pattern": "\\$\\{(resources(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(bundle(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(workspace(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "serving.AnthropicConfig": { + "anyOf": [ + { + "type": "object", + "properties": { + "anthropic_api_key": { + "description": "The Databricks secret key reference for an Anthropic API key. If you prefer to paste your API key directly, see `anthropic_api_key_plaintext`. You must provide an API key using one of the following fields: `anthropic_api_key` or `anthropic_api_key_plaintext`.", + "$ref": "#/$defs/string" + }, + "anthropic_api_key_plaintext": { + "description": "The Anthropic API key provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `anthropic_api_key`. You must provide an API key using one of the following fields: `anthropic_api_key` or `anthropic_api_key_plaintext`.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "serving.AutoCaptureConfigInput": { + "anyOf": [ + { + "type": "object", + "properties": { + "catalog_name": { + "description": "The name of the catalog in Unity Catalog. NOTE: On update, you cannot change the catalog name if the inference table is already enabled.", + "$ref": "#/$defs/string" + }, + "enabled": { + "description": "Indicates whether the inference table is enabled.", + "$ref": "#/$defs/bool" + }, + "schema_name": { + "description": "The name of the schema in Unity Catalog. NOTE: On update, you cannot change the schema name if the inference table is already enabled.", + "$ref": "#/$defs/string" + }, + "table_name_prefix": { + "description": "The prefix of the table in Unity Catalog. NOTE: On update, you cannot change the prefix name if the inference table is already enabled.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "serving.CohereConfig": { + "anyOf": [ + { + "type": "object", + "properties": { + "cohere_api_base": { + "description": "This is an optional field to provide a customized base URL for the Cohere API. \nIf left unspecified, the standard Cohere base URL is used.\n", + "$ref": "#/$defs/string" + }, + "cohere_api_key": { + "description": "The Databricks secret key reference for a Cohere API key. If you prefer to paste your API key directly, see `cohere_api_key_plaintext`. You must provide an API key using one of the following fields: `cohere_api_key` or `cohere_api_key_plaintext`.", + "$ref": "#/$defs/string" + }, + "cohere_api_key_plaintext": { + "description": "The Cohere API key provided as a plaintext string. If you prefer to reference your key using Databricks Secrets, see `cohere_api_key`. You must provide an API key using one of the following fields: `cohere_api_key` or `cohere_api_key_plaintext`.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "serving.DatabricksModelServingConfig": { + "anyOf": [ + { + "type": "object", + "properties": { + "databricks_api_token": { + "description": "The Databricks secret key reference for a Databricks API token that corresponds to a user or service\nprincipal with Can Query access to the model serving endpoint pointed to by this external model.\nIf you prefer to paste your API key directly, see `databricks_api_token_plaintext`.\nYou must provide an API key using one of the following fields: `databricks_api_token` or `databricks_api_token_plaintext`.\n", + "$ref": "#/$defs/string" + }, + "databricks_api_token_plaintext": { + "description": "The Databricks API token that corresponds to a user or service\nprincipal with Can Query access to the model serving endpoint pointed to by this external model provided as a plaintext string.\nIf you prefer to reference your key using Databricks Secrets, see `databricks_api_token`.\nYou must provide an API key using one of the following fields: `databricks_api_token` or `databricks_api_token_plaintext`.\n", + "$ref": "#/$defs/string" + }, + "databricks_workspace_url": { + "description": "The URL of the Databricks workspace containing the model serving endpoint pointed to by this external model.\n", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "databricks_workspace_url" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "serving.EndpointCoreConfigInput": { + "anyOf": [ + { + "type": "object", + "properties": { + "auto_capture_config": { + "description": "Configuration for Inference Tables which automatically logs requests and responses to Unity Catalog.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.AutoCaptureConfigInput" + }, + "served_entities": { + "description": "A list of served entities for the endpoint to serve. A serving endpoint can have up to 15 served entities.", + "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/serving.ServedEntityInput" + }, + "served_models": { + "description": "(Deprecated, use served_entities instead) A list of served models for the endpoint to serve. A serving endpoint can have up to 15 served models.", + "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/serving.ServedModelInput" + }, + "traffic_config": { + "description": "The traffic config defining how invocations to the serving endpoint should be routed.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.TrafficConfig" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "serving.EndpointTag": { + "anyOf": [ + { + "type": "object", + "properties": { + "key": { + "description": "Key field for a serving endpoint tag.", + "$ref": "#/$defs/string" + }, + "value": { + "description": "Optional value field for a serving endpoint tag.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "key" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "serving.ExternalModel": { + "anyOf": [ + { + "type": "object", + "properties": { + "ai21labs_config": { + "description": "AI21Labs Config. Only required if the provider is 'ai21labs'.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.Ai21LabsConfig" + }, + "amazon_bedrock_config": { + "description": "Amazon Bedrock Config. Only required if the provider is 'amazon-bedrock'.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.AmazonBedrockConfig" + }, + "anthropic_config": { + "description": "Anthropic Config. Only required if the provider is 'anthropic'.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.AnthropicConfig" + }, + "cohere_config": { + "description": "Cohere Config. Only required if the provider is 'cohere'.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.CohereConfig" + }, + "databricks_model_serving_config": { + "description": "Databricks Model Serving Config. Only required if the provider is 'databricks-model-serving'.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.DatabricksModelServingConfig" + }, + "google_cloud_vertex_ai_config": { + "description": "Google Cloud Vertex AI Config. Only required if the provider is 'google-cloud-vertex-ai'.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.GoogleCloudVertexAiConfig" + }, + "name": { + "description": "The name of the external model.", + "$ref": "#/$defs/string" + }, + "openai_config": { + "description": "OpenAI Config. Only required if the provider is 'openai'.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.OpenAiConfig" + }, + "palm_config": { + "description": "PaLM Config. Only required if the provider is 'palm'.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.PaLmConfig" + }, + "provider": { + "description": "The name of the provider for the external model. Currently, the supported providers are 'ai21labs', 'anthropic',\n'amazon-bedrock', 'cohere', 'databricks-model-serving', 'google-cloud-vertex-ai', 'openai', and 'palm'.\",\n", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.ExternalModelProvider", + "enum": [ + "ai21labs", + "anthropic", + "amazon-bedrock", + "cohere", + "databricks-model-serving", + "google-cloud-vertex-ai", + "openai", + "palm" + ] + }, + "task": { + "description": "The task type of the external model.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "name", + "provider", + "task" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "serving.ExternalModelProvider": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "string", + "pattern": "\\$\\{(resources(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(bundle(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(workspace(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "serving.GoogleCloudVertexAiConfig": { + "anyOf": [ + { + "type": "object", + "properties": { + "private_key": { + "$ref": "#/$defs/string" + }, + "private_key_plaintext": { + "$ref": "#/$defs/string" + }, + "project_id": { + "$ref": "#/$defs/string" + }, + "region": { + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "serving.OpenAiConfig": { + "anyOf": [ + { + "type": "object", + "properties": { + "microsoft_entra_client_id": { + "$ref": "#/$defs/string" + }, + "microsoft_entra_client_secret": { + "$ref": "#/$defs/string" + }, + "microsoft_entra_client_secret_plaintext": { + "$ref": "#/$defs/string" + }, + "microsoft_entra_tenant_id": { + "$ref": "#/$defs/string" + }, + "openai_api_base": { + "$ref": "#/$defs/string" + }, + "openai_api_key": { + "$ref": "#/$defs/string" + }, + "openai_api_key_plaintext": { + "$ref": "#/$defs/string" + }, + "openai_api_type": { + "$ref": "#/$defs/string" + }, + "openai_api_version": { + "$ref": "#/$defs/string" + }, + "openai_deployment_name": { + "$ref": "#/$defs/string" + }, + "openai_organization": { + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "serving.PaLmConfig": { + "anyOf": [ + { + "type": "object", + "properties": { + "palm_api_key": { + "$ref": "#/$defs/string" + }, + "palm_api_key_plaintext": { + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "serving.RateLimit": { + "anyOf": [ + { + "type": "object", + "properties": { + "calls": { + "description": "Used to specify how many calls are allowed for a key within the renewal_period.", + "$ref": "#/$defs/int" + }, + "key": { + "description": "Key field for a serving endpoint rate limit. Currently, only 'user' and 'endpoint' are supported, with 'endpoint' being the default if not specified.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.RateLimitKey", + "enum": [ + "user", + "endpoint" + ] + }, + "renewal_period": { + "description": "Renewal period field for a serving endpoint rate limit. Currently, only 'minute' is supported.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.RateLimitRenewalPeriod", + "enum": [ + "minute" + ] + } + }, + "additionalProperties": false, + "required": [ + "calls", + "renewal_period" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "serving.RateLimitKey": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "string", + "pattern": "\\$\\{(resources(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(bundle(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(workspace(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "serving.RateLimitRenewalPeriod": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "string", + "pattern": "\\$\\{(resources(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(bundle(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(workspace(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "serving.Route": { + "anyOf": [ + { + "type": "object", + "properties": { + "served_model_name": { + "description": "The name of the served model this route configures traffic for.", + "$ref": "#/$defs/string" + }, + "traffic_percentage": { + "description": "The percentage of endpoint traffic to send to this route. It must be an integer between 0 and 100 inclusive.", + "$ref": "#/$defs/int" + } + }, + "additionalProperties": false, + "required": [ + "served_model_name", + "traffic_percentage" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "serving.ServedEntityInput": { + "anyOf": [ + { + "type": "object", + "properties": { + "entity_name": { + "description": "The name of the entity to be served. The entity may be a model in the Databricks Model Registry, a model in the Unity Catalog (UC),\nor a function of type FEATURE_SPEC in the UC. If it is a UC object, the full name of the object should be given in the form of\n__catalog_name__.__schema_name__.__model_name__.\n", + "$ref": "#/$defs/string" + }, + "entity_version": { + "description": "The version of the model in Databricks Model Registry to be served or empty if the entity is a FEATURE_SPEC.", + "$ref": "#/$defs/string" + }, + "environment_vars": { + "description": "An object containing a set of optional, user-specified environment variable key-value pairs used for serving this entity.\nNote: this is an experimental feature and subject to change. \nExample entity environment variables that refer to Databricks secrets: `{\"OPENAI_API_KEY\": \"{{secrets/my_scope/my_key}}\", \"DATABRICKS_TOKEN\": \"{{secrets/my_scope2/my_key2}}\"}`", + "$ref": "#/$defs/map/string" + }, + "external_model": { + "description": "The external model to be served. NOTE: Only one of external_model and (entity_name, entity_version, workload_size, workload_type, and scale_to_zero_enabled)\ncan be specified with the latter set being used for custom model serving for a Databricks registered model. For an existing endpoint with external_model,\nit cannot be updated to an endpoint without external_model. If the endpoint is created without external_model, users cannot update it to add external_model later.\nThe task type of all external models within an endpoint must be the same.\n", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.ExternalModel" + }, + "instance_profile_arn": { + "description": "ARN of the instance profile that the served entity uses to access AWS resources.", + "$ref": "#/$defs/string" + }, + "max_provisioned_throughput": { + "description": "The maximum tokens per second that the endpoint can scale up to.", + "$ref": "#/$defs/int" + }, + "min_provisioned_throughput": { + "description": "The minimum tokens per second that the endpoint can scale down to.", + "$ref": "#/$defs/int" + }, + "name": { + "description": "The name of a served entity. It must be unique across an endpoint. A served entity name can consist of alphanumeric characters, dashes, and underscores.\nIf not specified for an external model, this field defaults to external_model.name, with '.' and ':' replaced with '-', and if not specified for other\nentities, it defaults to \u003centity-name\u003e-\u003centity-version\u003e.\n", + "$ref": "#/$defs/string" + }, + "scale_to_zero_enabled": { + "description": "Whether the compute resources for the served entity should scale down to zero.", + "$ref": "#/$defs/bool" + }, + "workload_size": { + "description": "The workload size of the served entity. The workload size corresponds to a range of provisioned concurrency that the compute autoscales between.\nA single unit of provisioned concurrency can process one request at a time.\nValid workload sizes are \"Small\" (4 - 4 provisioned concurrency), \"Medium\" (8 - 16 provisioned concurrency), and \"Large\" (16 - 64 provisioned concurrency).\nIf scale-to-zero is enabled, the lower bound of the provisioned concurrency for each workload size is 0.\n", + "$ref": "#/$defs/string" + }, + "workload_type": { + "description": "The workload type of the served entity. The workload type selects which type of compute to use in the endpoint. The default value for this parameter is\n\"CPU\". For deep learning workloads, GPU acceleration is available by selecting workload types like GPU_SMALL and others.\nSee the available [GPU types](https://docs.databricks.com/machine-learning/model-serving/create-manage-serving-endpoints.html#gpu-workload-types).\n", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "serving.ServedModelInput": { + "anyOf": [ + { + "type": "object", + "properties": { + "environment_vars": { + "description": "An object containing a set of optional, user-specified environment variable key-value pairs used for serving this model.\nNote: this is an experimental feature and subject to change. \nExample model environment variables that refer to Databricks secrets: `{\"OPENAI_API_KEY\": \"{{secrets/my_scope/my_key}}\", \"DATABRICKS_TOKEN\": \"{{secrets/my_scope2/my_key2}}\"}`", + "$ref": "#/$defs/map/string" + }, + "instance_profile_arn": { + "description": "ARN of the instance profile that the served model will use to access AWS resources.", + "$ref": "#/$defs/string" + }, + "model_name": { + "description": "The name of the model in Databricks Model Registry to be served or if the model resides in Unity Catalog, the full name of model,\nin the form of __catalog_name__.__schema_name__.__model_name__.\n", + "$ref": "#/$defs/string" + }, + "model_version": { + "description": "The version of the model in Databricks Model Registry or Unity Catalog to be served.", + "$ref": "#/$defs/string" + }, + "name": { + "description": "The name of a served model. It must be unique across an endpoint. If not specified, this field will default to \u003cmodel-name\u003e-\u003cmodel-version\u003e.\nA served model name can consist of alphanumeric characters, dashes, and underscores.\n", + "$ref": "#/$defs/string" + }, + "scale_to_zero_enabled": { + "description": "Whether the compute resources for the served model should scale down to zero.", + "$ref": "#/$defs/bool" + }, + "workload_size": { + "description": "The workload size of the served model. The workload size corresponds to a range of provisioned concurrency that the compute will autoscale between.\nA single unit of provisioned concurrency can process one request at a time.\nValid workload sizes are \"Small\" (4 - 4 provisioned concurrency), \"Medium\" (8 - 16 provisioned concurrency), and \"Large\" (16 - 64 provisioned concurrency).\nIf scale-to-zero is enabled, the lower bound of the provisioned concurrency for each workload size will be 0.\n", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.ServedModelInputWorkloadSize", + "enum": [ + "Small", + "Medium", + "Large" + ] + }, + "workload_type": { + "description": "The workload type of the served model. The workload type selects which type of compute to use in the endpoint. The default value for this parameter is\n\"CPU\". For deep learning workloads, GPU acceleration is available by selecting workload types like GPU_SMALL and others.\nSee the available [GPU types](https://docs.databricks.com/machine-learning/model-serving/create-manage-serving-endpoints.html#gpu-workload-types).\n", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.ServedModelInputWorkloadType", + "enum": [ + "CPU", + "GPU_SMALL", + "GPU_MEDIUM", + "GPU_LARGE", + "MULTIGPU_MEDIUM" + ] + } + }, + "additionalProperties": false, + "required": [ + "model_name", + "model_version", + "scale_to_zero_enabled", + "workload_size" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "serving.ServedModelInputWorkloadSize": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "string", + "pattern": "\\$\\{(resources(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(bundle(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(workspace(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "serving.ServedModelInputWorkloadType": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "string", + "pattern": "\\$\\{(resources(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(bundle(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(workspace(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "serving.TrafficConfig": { + "anyOf": [ + { + "type": "object", + "properties": { + "routes": { + "description": "The list of routes that define traffic to each served entity.", + "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/serving.Route" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + } + } + } + } + }, + "int": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "string", + "pattern": "\\$\\{(resources(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(bundle(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(workspace(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "int64": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "string", + "pattern": "\\$\\{(resources(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(bundle(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(workspace(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "interface": {}, + "map": { + "github.com": { + "databricks": { + "cli": { + "bundle": { + "config": { + "resources.Job": { + "anyOf": [ + { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.Job" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "resources.MlflowExperiment": { + "anyOf": [ + { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.MlflowExperiment" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "resources.MlflowModel": { + "anyOf": [ + { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.MlflowModel" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "resources.ModelServingEndpoint": { + "anyOf": [ + { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.ModelServingEndpoint" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "resources.Pipeline": { + "anyOf": [ + { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.Pipeline" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "resources.QualityMonitor": { + "anyOf": [ + { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.QualityMonitor" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "resources.RegisteredModel": { + "anyOf": [ + { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.RegisteredModel" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "resources.Schema": { + "anyOf": [ + { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.Schema" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "variable.Variable": { + "anyOf": [ + { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/variable.Variable" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + } + }, + "config.Artifact": { + "anyOf": [ + { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config.Artifact" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "config.Command": { + "anyOf": [ + { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config.Command" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "config.Target": { + "anyOf": [ + { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config.Target" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + } + } + } + } + }, + "string": { + "anyOf": [ + { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/string" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + } + }, + "slice": { + "github.com": { + "databricks": { + "cli": { + "bundle": { + "config": { + "resources.Grant": { + "anyOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.Grant" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "resources.Permission": { + "anyOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.Permission" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + } + }, + "config.ArtifactFile": { + "anyOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config.ArtifactFile" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + } + } + }, + "databricks-sdk-go": { + "service": { + "catalog.MonitorMetric": { + "anyOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/catalog.MonitorMetric" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "compute.InitScriptInfo": { + "anyOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.InitScriptInfo" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "compute.Library": { + "anyOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.Library" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "jobs.JobCluster": { + "anyOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.JobCluster" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "jobs.JobEnvironment": { + "anyOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.JobEnvironment" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "jobs.JobParameterDefinition": { + "anyOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.JobParameterDefinition" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "jobs.JobsHealthRule": { + "anyOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.JobsHealthRule" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "jobs.SqlTaskSubscription": { + "anyOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.SqlTaskSubscription" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "jobs.Task": { + "anyOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.Task" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "jobs.TaskDependency": { + "anyOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.TaskDependency" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "jobs.Webhook": { + "anyOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.Webhook" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "ml.ExperimentTag": { + "anyOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/ml.ExperimentTag" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "ml.ModelTag": { + "anyOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/ml.ModelTag" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "ml.ModelVersion": { + "anyOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/ml.ModelVersion" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "ml.ModelVersionTag": { + "anyOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/ml.ModelVersionTag" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "pipelines.IngestionConfig": { + "anyOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.IngestionConfig" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "pipelines.Notifications": { + "anyOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.Notifications" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "pipelines.PipelineCluster": { + "anyOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.PipelineCluster" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "pipelines.PipelineLibrary": { + "anyOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.PipelineLibrary" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "serving.EndpointTag": { + "anyOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.EndpointTag" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "serving.RateLimit": { + "anyOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.RateLimit" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "serving.Route": { + "anyOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.Route" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "serving.ServedEntityInput": { + "anyOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.ServedEntityInput" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, + "serving.ServedModelInput": { + "anyOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.ServedModelInput" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + } + } + } + } + }, + "string": { + "anyOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/string" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + } + }, + "string": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "string", + "pattern": "\\$\\{(resources(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(bundle(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(workspace(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + } + }, + "type": "object", + "properties": { + "artifacts": { + "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config.Artifact" + }, + "bundle": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config.Bundle" + }, + "experimental": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config.Experimental" + }, + "include": { + "$ref": "#/$defs/slice/string" + }, + "permissions": { + "$ref": "#/$defs/slice/github.com/databricks/cli/bundle/config/resources.Permission" + }, + "presets": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config.Presets" + }, + "resources": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config.Resources" + }, + "run_as": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.JobRunAs" + }, + "sync": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config.Sync" + }, + "targets": { + "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config.Target" + }, + "variables": { + "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/variable.Variable" + }, + "workspace": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config.Workspace" + } + }, + "additionalProperties": false +} \ No newline at end of file From 5b7974757ddad627eaf659c7e9706a84aaebef83 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 27 Aug 2024 13:28:03 +0200 Subject: [PATCH 20/62] - --- libs/jsonschema/schema_test.go | 34 ---------------------------------- 1 file changed, 34 deletions(-) diff --git a/libs/jsonschema/schema_test.go b/libs/jsonschema/schema_test.go index 46f4db6b1c..cf1f127672 100644 --- a/libs/jsonschema/schema_test.go +++ b/libs/jsonschema/schema_test.go @@ -305,37 +305,3 @@ func TestValidateSchemaSkippedPropertiesHaveDefaults(t *testing.T) { err = s.validate() assert.NoError(t, err) } - -func testSchema() *Schema { - return &Schema{ - Type: "object", - Properties: map[string]*Schema{ - "int_val": { - Type: "integer", - Default: int64(123), - }, - "string_val": { - Type: "string", - }, - "object_val": { - Type: "object", - Properties: map[string]*Schema{ - "bar": { - Type: "string", - Default: "baz", - }, - }, - AdditionalProperties: &Schema{ - Type: "object", - Properties: map[string]*Schema{ - "foo": { - Type: "string", - Default: "zab", - }, - }, - }, - }, - }, - } - -} From ac60163c19ed7c050a6ee4d3884883f78976f0a8 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 27 Aug 2024 13:35:56 +0200 Subject: [PATCH 21/62] move gen file --- .codegen.json | 4 ++-- .../bundle/_generated/jsonschema.json | 0 cmd/bundle/schema.go | 2 ++ 3 files changed, 4 insertions(+), 2 deletions(-) rename bundle/internal/schema/schema.generated.json => cmd/bundle/_generated/jsonschema.json (100%) diff --git a/.codegen.json b/.codegen.json index 5604f6a863..61a2fe8514 100644 --- a/.codegen.json +++ b/.codegen.json @@ -11,10 +11,10 @@ "toolchain": { "required": ["go"], "post_generate": [ - "go run ./bundle/internal/schema/*.go ./bundle/internal/schema/schema.generated.json", + "go run ./bundle/internal/schema/*.go ./cmd/bundle/_generated/jsonschema.json", "echo 'bundle/internal/tf/schema/\\*.go linguist-generated=true' >> ./.gitattributes", "echo 'go.sum linguist-generated=true' >> ./.gitattributes", - "echo 'bundle/schema/docs/schema.generated.json linguist-generated=true' >> ./.gitattributes" + "echo 'cmd/bundle/_generated linguist-generated=true' >> ./.gitattributes" ] } } diff --git a/bundle/internal/schema/schema.generated.json b/cmd/bundle/_generated/jsonschema.json similarity index 100% rename from bundle/internal/schema/schema.generated.json rename to cmd/bundle/_generated/jsonschema.json diff --git a/cmd/bundle/schema.go b/cmd/bundle/schema.go index 9938ee14f8..6d1bf90caa 100644 --- a/cmd/bundle/schema.go +++ b/cmd/bundle/schema.go @@ -13,6 +13,8 @@ func newSchemaCommand() *cobra.Command { } cmd.RunE = func(cmd *cobra.Command, args []string) error { + // os.ReadFile() + // // Load embedded schema descriptions. // docs, err := schema.LoadBundleDescriptions() // if err != nil { From 2c30cfa5a202e07e5f7ae5cbf4a7a5f156329383 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 27 Aug 2024 13:38:42 +0200 Subject: [PATCH 22/62] - --- .gitattributes | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitattributes b/.gitattributes index 7c7e9e9eef..78f2d55ef8 100755 --- a/.gitattributes +++ b/.gitattributes @@ -120,4 +120,4 @@ cmd/workspace/workspace-conf/workspace-conf.go linguist-generated=true cmd/workspace/workspace/workspace.go linguist-generated=true bundle/internal/tf/schema/\*.go linguist-generated=true go.sum linguist-generated=true -bundle/schema/docs/schema.generated.json linguist-generated=true +cmd/bundle/_generated/\* linguist-generated=true From 483480f4a9afa132a4bb37d634fd73946e69398a Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 27 Aug 2024 13:39:23 +0200 Subject: [PATCH 23/62] - --- .gitattributes | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitattributes b/.gitattributes index 78f2d55ef8..d77729c10d 100755 --- a/.gitattributes +++ b/.gitattributes @@ -120,4 +120,4 @@ cmd/workspace/workspace-conf/workspace-conf.go linguist-generated=true cmd/workspace/workspace/workspace.go linguist-generated=true bundle/internal/tf/schema/\*.go linguist-generated=true go.sum linguist-generated=true -cmd/bundle/_generated/\* linguist-generated=true +cmd/bundle/_generated linguist-generated=true From 727036bb408c61f05b0d397b2715abb760a448f7 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 27 Aug 2024 13:46:52 +0200 Subject: [PATCH 24/62] bundle schema command is OK now --- cmd/bundle/schema.go | 35 +++++++---------------------------- 1 file changed, 7 insertions(+), 28 deletions(-) diff --git a/cmd/bundle/schema.go b/cmd/bundle/schema.go index 6d1bf90caa..425be5e12e 100644 --- a/cmd/bundle/schema.go +++ b/cmd/bundle/schema.go @@ -1,10 +1,15 @@ package bundle import ( + _ "embed" + "github.com/databricks/cli/cmd/root" "github.com/spf13/cobra" ) +//go:embed _generated/jsonschema.json +var b []byte + func newSchemaCommand() *cobra.Command { cmd := &cobra.Command{ Use: "schema", @@ -13,34 +18,8 @@ func newSchemaCommand() *cobra.Command { } cmd.RunE = func(cmd *cobra.Command, args []string) error { - // os.ReadFile() - - // // Load embedded schema descriptions. - // docs, err := schema.LoadBundleDescriptions() - // if err != nil { - // return err - // } - - // // Generate the JSON schema from the bundle configuration struct in Go. - // schema, err := schema.New(reflect.TypeOf(config.Root{}), docs) - // if err != nil { - // return err - // } - - // // Target variable value overrides can be primitives, maps or sequences. - // // Set an empty schema for them. - // err = schema.SetByPath("targets.*.variables.*", jsonschema.Schema{}) - // if err != nil { - // return err - // } - - // // Print the JSON schema to stdout. - // result, err := json.MarshalIndent(schema, "", " ") - // if err != nil { - // return err - // } - // cmd.OutOrStdout().Write(result) - return nil + _, err := cmd.OutOrStdout().Write(b) + return err } return cmd From acc43092ad12ed9af1b7edcf8e54f493062e27b7 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 27 Aug 2024 14:17:04 +0200 Subject: [PATCH 25/62] fix lint --- libs/jsonschema/from_type_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/libs/jsonschema/from_type_test.go b/libs/jsonschema/from_type_test.go index 29d007901a..ab2f1def8c 100644 --- a/libs/jsonschema/from_type_test.go +++ b/libs/jsonschema/from_type_test.go @@ -17,7 +17,6 @@ func TestFromTypeBasic(t *testing.T) { // These fields should be ignored in the resulting schema. NotAnnotated string DashedTag string `json:"-"` - notExported string `json:"not_exported"` InternalTagged string `json:"internal_tagged" bundle:"internal"` DeprecatedTagged string `json:"deprecated_tagged" bundle:"deprecated"` ReadOnlyTagged string `json:"readonly_tagged" bundle:"readonly"` From f194a5b86aed1545cb2ba14eaa1ef041310c47f0 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 27 Aug 2024 17:36:12 +0200 Subject: [PATCH 26/62] skip adding patterns for string schemas --- bundle/internal/schema/main.go | 2 +- cmd/bundle/_generated/jsonschema.json | 1056 ++++--------------------- 2 files changed, 149 insertions(+), 909 deletions(-) diff --git a/bundle/internal/schema/main.go b/bundle/internal/schema/main.go index 93210ca1ee..68a7f13b22 100644 --- a/bundle/internal/schema/main.go +++ b/bundle/internal/schema/main.go @@ -31,7 +31,7 @@ func addInterpolationPatterns(typ reflect.Type, s jsonschema.Schema) jsonschema. Pattern: interpolationPattern("var"), }}, } - case jsonschema.StringType, jsonschema.IntegerType, jsonschema.NumberType, jsonschema.BooleanType: + case jsonschema.IntegerType, jsonschema.NumberType, jsonschema.BooleanType: // primitives can have variable values, or references like ${bundle.xyz} // or ${workspace.xyz} // TODO: Followup, do not allow references like ${} in the schema unless diff --git a/cmd/bundle/_generated/jsonschema.json b/cmd/bundle/_generated/jsonschema.json index 337867b28f..114a05aaf2 100644 --- a/cmd/bundle/_generated/jsonschema.json +++ b/cmd/bundle/_generated/jsonschema.json @@ -653,27 +653,7 @@ "additionalProperties": false }, "variable.VariableType": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "string", - "pattern": "\\$\\{(resources(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - }, - { - "type": "string", - "pattern": "\\$\\{(bundle(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - }, - { - "type": "string", - "pattern": "\\$\\{(workspace(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - }, - { - "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - } - ] + "type": "string" } }, "config.Artifact": { @@ -729,27 +709,7 @@ ] }, "config.ArtifactType": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "string", - "pattern": "\\$\\{(resources(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - }, - { - "type": "string", - "pattern": "\\$\\{(bundle(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - }, - { - "type": "string", - "pattern": "\\$\\{(workspace(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - }, - { - "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - } - ] + "type": "string" }, "config.Bundle": { "anyOf": [ @@ -784,27 +744,7 @@ ] }, "config.Command": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "string", - "pattern": "\\$\\{(resources(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - }, - { - "type": "string", - "pattern": "\\$\\{(bundle(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - }, - { - "type": "string", - "pattern": "\\$\\{(workspace(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - }, - { - "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - } - ] + "type": "string" }, "config.Deployment": { "anyOf": [ @@ -896,27 +836,7 @@ ] }, "config.Mode": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "string", - "pattern": "\\$\\{(resources(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - }, - { - "type": "string", - "pattern": "\\$\\{(bundle(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - }, - { - "type": "string", - "pattern": "\\$\\{(workspace(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - }, - { - "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - } - ] + "type": "string" }, "config.Presets": { "anyOf": [ @@ -1146,27 +1066,7 @@ }, "libs": { "exec.ExecutableType": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "string", - "pattern": "\\$\\{(resources(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - }, - { - "type": "string", - "pattern": "\\$\\{(bundle(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - }, - { - "type": "string", - "pattern": "\\$\\{(workspace(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - }, - { - "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - } - ] + "type": "string" } } }, @@ -1207,27 +1107,7 @@ ] }, "catalog.MonitorCronSchedulePauseStatus": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "string", - "pattern": "\\$\\{(resources(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - }, - { - "type": "string", - "pattern": "\\$\\{(bundle(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - }, - { - "type": "string", - "pattern": "\\$\\{(workspace(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - }, - { - "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - } - ] + "type": "string" }, "catalog.MonitorDataClassificationConfig": { "anyOf": [ @@ -1319,27 +1199,7 @@ ] }, "catalog.MonitorInferenceLogProblemType": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "string", - "pattern": "\\$\\{(resources(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - }, - { - "type": "string", - "pattern": "\\$\\{(bundle(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - }, - { - "type": "string", - "pattern": "\\$\\{(workspace(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - }, - { - "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - } - ] + "type": "string" }, "catalog.MonitorMetric": { "anyOf": [ @@ -1388,27 +1248,7 @@ ] }, "catalog.MonitorMetricType": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "string", - "pattern": "\\$\\{(resources(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - }, - { - "type": "string", - "pattern": "\\$\\{(bundle(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - }, - { - "type": "string", - "pattern": "\\$\\{(workspace(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - }, - { - "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - } - ] + "type": "string" }, "catalog.MonitorNotifications": { "anyOf": [ @@ -1566,32 +1406,12 @@ ] }, "compute.AwsAvailability": { - "anyOf": [ - { - "type": "string", - "description": "Availability type used for all subsequent nodes past the `first_on_demand` ones.\n\nNote: If `first_on_demand` is zero, this availability type will be used for the entire cluster.\n", - "enum": [ - "SPOT", - "ON_DEMAND", - "SPOT_WITH_FALLBACK" - ] - }, - { - "type": "string", - "pattern": "\\$\\{(resources(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - }, - { - "type": "string", - "pattern": "\\$\\{(bundle(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - }, - { - "type": "string", - "pattern": "\\$\\{(workspace(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - }, - { - "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - } + "type": "string", + "description": "Availability type used for all subsequent nodes past the `first_on_demand` ones.\n\nNote: If `first_on_demand` is zero, this availability type will be used for the entire cluster.\n", + "enum": [ + "SPOT", + "ON_DEMAND", + "SPOT_WITH_FALLBACK" ] }, "compute.AzureAttributes": { @@ -1624,32 +1444,12 @@ ] }, "compute.AzureAvailability": { - "anyOf": [ - { - "type": "string", - "description": "Availability type used for all subsequent nodes past the `first_on_demand` ones.\nNote: If `first_on_demand` is zero (which only happens on pool clusters), this availability\ntype will be used for the entire cluster.", - "enum": [ - "SPOT_AZURE", - "ON_DEMAND_AZURE", - "SPOT_WITH_FALLBACK_AZURE" - ] - }, - { - "type": "string", - "pattern": "\\$\\{(resources(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - }, - { - "type": "string", - "pattern": "\\$\\{(bundle(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - }, - { - "type": "string", - "pattern": "\\$\\{(workspace(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - }, - { - "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - } + "type": "string", + "description": "Availability type used for all subsequent nodes past the `first_on_demand` ones.\nNote: If `first_on_demand` is zero (which only happens on pool clusters), this availability\ntype will be used for the entire cluster.", + "enum": [ + "SPOT_AZURE", + "ON_DEMAND_AZURE", + "SPOT_WITH_FALLBACK_AZURE" ] }, "compute.ClientsTypes": { @@ -1815,36 +1615,16 @@ ] }, "compute.DataSecurityMode": { - "anyOf": [ - { - "type": "string", - "description": "Data security mode decides what data governance model to use when accessing data\nfrom a cluster.\n\n* `NONE`: No security isolation for multiple users sharing the cluster. Data governance features are not available in this mode.\n* `SINGLE_USER`: A secure cluster that can only be exclusively used by a single user specified in `single_user_name`. Most programming languages, cluster features and data governance features are available in this mode.\n* `USER_ISOLATION`: A secure cluster that can be shared by multiple users. Cluster users are fully isolated so that they cannot see each other's data and credentials. Most data governance features are supported in this mode. But programming languages and cluster features might be limited.\n\nThe following modes are deprecated starting with Databricks Runtime 15.0 and\nwill be removed for future Databricks Runtime versions:\n\n* `LEGACY_TABLE_ACL`: This mode is for users migrating from legacy Table ACL clusters.\n* `LEGACY_PASSTHROUGH`: This mode is for users migrating from legacy Passthrough on high concurrency clusters.\n* `LEGACY_SINGLE_USER`: This mode is for users migrating from legacy Passthrough on standard clusters.\n* `LEGACY_SINGLE_USER_STANDARD`: This mode provides a way that doesn’t have UC nor passthrough enabled.\n", - "enum": [ - "NONE", - "SINGLE_USER", - "USER_ISOLATION", - "LEGACY_TABLE_ACL", - "LEGACY_PASSTHROUGH", - "LEGACY_SINGLE_USER", - "LEGACY_SINGLE_USER_STANDARD" - ] - }, - { - "type": "string", - "pattern": "\\$\\{(resources(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - }, - { - "type": "string", - "pattern": "\\$\\{(bundle(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - }, - { - "type": "string", - "pattern": "\\$\\{(workspace(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - }, - { - "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - } + "type": "string", + "description": "Data security mode decides what data governance model to use when accessing data\nfrom a cluster.\n\n* `NONE`: No security isolation for multiple users sharing the cluster. Data governance features are not available in this mode.\n* `SINGLE_USER`: A secure cluster that can only be exclusively used by a single user specified in `single_user_name`. Most programming languages, cluster features and data governance features are available in this mode.\n* `USER_ISOLATION`: A secure cluster that can be shared by multiple users. Cluster users are fully isolated so that they cannot see each other's data and credentials. Most data governance features are supported in this mode. But programming languages and cluster features might be limited.\n\nThe following modes are deprecated starting with Databricks Runtime 15.0 and\nwill be removed for future Databricks Runtime versions:\n\n* `LEGACY_TABLE_ACL`: This mode is for users migrating from legacy Table ACL clusters.\n* `LEGACY_PASSTHROUGH`: This mode is for users migrating from legacy Passthrough on high concurrency clusters.\n* `LEGACY_SINGLE_USER`: This mode is for users migrating from legacy Passthrough on standard clusters.\n* `LEGACY_SINGLE_USER_STANDARD`: This mode provides a way that doesn’t have UC nor passthrough enabled.\n", + "enum": [ + "NONE", + "SINGLE_USER", + "USER_ISOLATION", + "LEGACY_TABLE_ACL", + "LEGACY_PASSTHROUGH", + "LEGACY_SINGLE_USER", + "LEGACY_SINGLE_USER_STANDARD" ] }, "compute.DbfsStorageInfo": { @@ -1912,31 +1692,11 @@ ] }, "compute.EbsVolumeType": { - "anyOf": [ - { - "type": "string", - "description": "The type of EBS volumes that will be launched with this cluster.", - "enum": [ - "GENERAL_PURPOSE_SSD", - "THROUGHPUT_OPTIMIZED_HDD" - ] - }, - { - "type": "string", - "pattern": "\\$\\{(resources(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - }, - { - "type": "string", - "pattern": "\\$\\{(bundle(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - }, - { - "type": "string", - "pattern": "\\$\\{(workspace(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - }, - { - "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - } + "type": "string", + "description": "The type of EBS volumes that will be launched with this cluster.", + "enum": [ + "GENERAL_PURPOSE_SSD", + "THROUGHPUT_OPTIMIZED_HDD" ] }, "compute.Environment": { @@ -2003,32 +1763,12 @@ ] }, "compute.GcpAvailability": { - "anyOf": [ - { - "type": "string", - "description": "This field determines whether the instance pool will contain preemptible\nVMs, on-demand VMs, or preemptible VMs with a fallback to on-demand VMs if the former is unavailable.", - "enum": [ - "PREEMPTIBLE_GCP", - "ON_DEMAND_GCP", - "PREEMPTIBLE_WITH_FALLBACK_GCP" - ] - }, - { - "type": "string", - "pattern": "\\$\\{(resources(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - }, - { - "type": "string", - "pattern": "\\$\\{(bundle(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - }, - { - "type": "string", - "pattern": "\\$\\{(workspace(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - }, - { - "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - } + "type": "string", + "description": "This field determines whether the instance pool will contain preemptible\nVMs, on-demand VMs, or preemptible VMs with a fallback to on-demand VMs if the former is unavailable.", + "enum": [ + "PREEMPTIBLE_GCP", + "ON_DEMAND_GCP", + "PREEMPTIBLE_WITH_FALLBACK_GCP" ] }, "compute.GcsStorageInfo": { @@ -2259,32 +1999,12 @@ ] }, "compute.RuntimeEngine": { - "anyOf": [ - { - "type": "string", - "description": "Decides which runtime engine to be use, e.g. Standard vs. Photon. If unspecified, the runtime\nengine is inferred from spark_version.", - "enum": [ - "NULL", - "STANDARD", - "PHOTON" - ] - }, - { - "type": "string", - "pattern": "\\$\\{(resources(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - }, - { - "type": "string", - "pattern": "\\$\\{(bundle(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - }, - { - "type": "string", - "pattern": "\\$\\{(workspace(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - }, - { - "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - } + "type": "string", + "description": "Decides which runtime engine to be use, e.g. Standard vs. Photon. If unspecified, the runtime\nengine is inferred from spark_version.", + "enum": [ + "NULL", + "STANDARD", + "PHOTON" ] }, "compute.S3StorageInfo": { @@ -2396,30 +2116,10 @@ ] }, "jobs.Condition": { - "anyOf": [ - { - "type": "string", - "enum": [ - "ANY_UPDATED", - "ALL_UPDATED" - ] - }, - { - "type": "string", - "pattern": "\\$\\{(resources(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - }, - { - "type": "string", - "pattern": "\\$\\{(bundle(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - }, - { - "type": "string", - "pattern": "\\$\\{(workspace(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - }, - { - "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - } + "type": "string", + "enum": [ + "ANY_UPDATED", + "ALL_UPDATED" ] }, "jobs.ConditionTask": { @@ -2454,35 +2154,15 @@ ] }, "jobs.ConditionTaskOp": { - "anyOf": [ - { - "type": "string", - "description": "* `EQUAL_TO`, `NOT_EQUAL` operators perform string comparison of their operands. This means that `“12.0” == “12”` will evaluate to `false`.\n* `GREATER_THAN`, `GREATER_THAN_OR_EQUAL`, `LESS_THAN`, `LESS_THAN_OR_EQUAL` operators perform numeric comparison of their operands. `“12.0” \u003e= “12”` will evaluate to `true`, `“10.0” \u003e= “12”` will evaluate to `false`.\n\nThe boolean comparison to task values can be implemented with operators `EQUAL_TO`, `NOT_EQUAL`. If a task value was set to a boolean value, it will be serialized to `“true”` or `“false”` for the comparison.", - "enum": [ - "EQUAL_TO", - "GREATER_THAN", - "GREATER_THAN_OR_EQUAL", - "LESS_THAN", - "LESS_THAN_OR_EQUAL", - "NOT_EQUAL" - ] - }, - { - "type": "string", - "pattern": "\\$\\{(resources(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - }, - { - "type": "string", - "pattern": "\\$\\{(bundle(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - }, - { - "type": "string", - "pattern": "\\$\\{(workspace(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - }, - { - "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - } + "type": "string", + "description": "* `EQUAL_TO`, `NOT_EQUAL` operators perform string comparison of their operands. This means that `“12.0” == “12”` will evaluate to `false`.\n* `GREATER_THAN`, `GREATER_THAN_OR_EQUAL`, `LESS_THAN`, `LESS_THAN_OR_EQUAL` operators perform numeric comparison of their operands. `“12.0” \u003e= “12”` will evaluate to `true`, `“10.0” \u003e= “12”` will evaluate to `false`.\n\nThe boolean comparison to task values can be implemented with operators `EQUAL_TO`, `NOT_EQUAL`. If a task value was set to a boolean value, it will be serialized to `“true”` or `“false”` for the comparison.", + "enum": [ + "EQUAL_TO", + "GREATER_THAN", + "GREATER_THAN_OR_EQUAL", + "LESS_THAN", + "LESS_THAN_OR_EQUAL", + "NOT_EQUAL" ] }, "jobs.Continuous": { @@ -2638,63 +2318,23 @@ ] }, "jobs.Format": { - "anyOf": [ - { - "type": "string", - "enum": [ - "SINGLE_TASK", - "MULTI_TASK" - ] - }, - { - "type": "string", - "pattern": "\\$\\{(resources(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - }, - { - "type": "string", - "pattern": "\\$\\{(bundle(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - }, - { - "type": "string", - "pattern": "\\$\\{(workspace(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - }, - { - "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - } + "type": "string", + "enum": [ + "SINGLE_TASK", + "MULTI_TASK" ] }, "jobs.GitProvider": { - "anyOf": [ - { - "type": "string", - "enum": [ - "gitHub", - "bitbucketCloud", - "azureDevOpsServices", - "gitHubEnterprise", - "bitbucketServer", - "gitLab", - "gitLabEnterpriseEdition", - "awsCodeCommit" - ] - }, - { - "type": "string", - "pattern": "\\$\\{(resources(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - }, - { - "type": "string", - "pattern": "\\$\\{(bundle(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - }, - { - "type": "string", - "pattern": "\\$\\{(workspace(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - }, - { - "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - } + "type": "string", + "enum": [ + "gitHub", + "bitbucketCloud", + "azureDevOpsServices", + "gitHubEnterprise", + "bitbucketServer", + "gitLab", + "gitLabEnterpriseEdition", + "awsCodeCommit" ] }, "jobs.GitSnapshot": { @@ -2814,58 +2454,18 @@ ] }, "jobs.JobDeploymentKind": { - "anyOf": [ - { - "type": "string", - "description": "* `BUNDLE`: The job is managed by Databricks Asset Bundle.", - "enum": [ - "BUNDLE" - ] - }, - { - "type": "string", - "pattern": "\\$\\{(resources(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - }, - { - "type": "string", - "pattern": "\\$\\{(bundle(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - }, - { - "type": "string", - "pattern": "\\$\\{(workspace(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - }, - { - "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - } + "type": "string", + "description": "* `BUNDLE`: The job is managed by Databricks Asset Bundle.", + "enum": [ + "BUNDLE" ] }, "jobs.JobEditMode": { - "anyOf": [ - { - "type": "string", - "description": "Edit mode of the job.\n\n* `UI_LOCKED`: The job is in a locked UI state and cannot be modified.\n* `EDITABLE`: The job is in an editable state and can be modified.", - "enum": [ - "UI_LOCKED", - "EDITABLE" - ] - }, - { - "type": "string", - "pattern": "\\$\\{(resources(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - }, - { - "type": "string", - "pattern": "\\$\\{(bundle(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - }, - { - "type": "string", - "pattern": "\\$\\{(workspace(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - }, - { - "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - } + "type": "string", + "description": "Edit mode of the job.\n\n* `UI_LOCKED`: The job is in a locked UI state and cannot be modified.\n* `EDITABLE`: The job is in an editable state and can be modified.", + "enum": [ + "UI_LOCKED", + "EDITABLE" ] }, "jobs.JobEmailNotifications": { @@ -3033,89 +2633,29 @@ ] }, "jobs.JobSourceDirtyState": { - "anyOf": [ - { - "type": "string", - "description": "Dirty state indicates the job is not fully synced with the job specification\nin the remote repository.\n\nPossible values are:\n* `NOT_SYNCED`: The job is not yet synced with the remote job specification. Import the remote job specification from UI to make the job fully synced.\n* `DISCONNECTED`: The job is temporary disconnected from the remote job specification and is allowed for live edit. Import the remote job specification again from UI to make the job fully synced.", - "enum": [ - "NOT_SYNCED", - "DISCONNECTED" - ] - }, - { - "type": "string", - "pattern": "\\$\\{(resources(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - }, - { - "type": "string", - "pattern": "\\$\\{(bundle(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - }, - { - "type": "string", - "pattern": "\\$\\{(workspace(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - }, - { - "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - } + "type": "string", + "description": "Dirty state indicates the job is not fully synced with the job specification\nin the remote repository.\n\nPossible values are:\n* `NOT_SYNCED`: The job is not yet synced with the remote job specification. Import the remote job specification from UI to make the job fully synced.\n* `DISCONNECTED`: The job is temporary disconnected from the remote job specification and is allowed for live edit. Import the remote job specification again from UI to make the job fully synced.", + "enum": [ + "NOT_SYNCED", + "DISCONNECTED" ] }, "jobs.JobsHealthMetric": { - "anyOf": [ - { - "type": "string", - "description": "Specifies the health metric that is being evaluated for a particular health rule.\n\n* `RUN_DURATION_SECONDS`: Expected total time for a run in seconds.\n* `STREAMING_BACKLOG_BYTES`: An estimate of the maximum bytes of data waiting to be consumed across all streams. This metric is in Private Preview.\n* `STREAMING_BACKLOG_RECORDS`: An estimate of the maximum offset lag across all streams. This metric is in Private Preview.\n* `STREAMING_BACKLOG_SECONDS`: An estimate of the maximum consumer delay across all streams. This metric is in Private Preview.\n* `STREAMING_BACKLOG_FILES`: An estimate of the maximum number of outstanding files across all streams. This metric is in Private Preview.", - "enum": [ - "RUN_DURATION_SECONDS", - "STREAMING_BACKLOG_BYTES", - "STREAMING_BACKLOG_RECORDS", - "STREAMING_BACKLOG_SECONDS", - "STREAMING_BACKLOG_FILES" - ] - }, - { - "type": "string", - "pattern": "\\$\\{(resources(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - }, - { - "type": "string", - "pattern": "\\$\\{(bundle(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - }, - { - "type": "string", - "pattern": "\\$\\{(workspace(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - }, - { - "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - } + "type": "string", + "description": "Specifies the health metric that is being evaluated for a particular health rule.\n\n* `RUN_DURATION_SECONDS`: Expected total time for a run in seconds.\n* `STREAMING_BACKLOG_BYTES`: An estimate of the maximum bytes of data waiting to be consumed across all streams. This metric is in Private Preview.\n* `STREAMING_BACKLOG_RECORDS`: An estimate of the maximum offset lag across all streams. This metric is in Private Preview.\n* `STREAMING_BACKLOG_SECONDS`: An estimate of the maximum consumer delay across all streams. This metric is in Private Preview.\n* `STREAMING_BACKLOG_FILES`: An estimate of the maximum number of outstanding files across all streams. This metric is in Private Preview.", + "enum": [ + "RUN_DURATION_SECONDS", + "STREAMING_BACKLOG_BYTES", + "STREAMING_BACKLOG_RECORDS", + "STREAMING_BACKLOG_SECONDS", + "STREAMING_BACKLOG_FILES" ] }, "jobs.JobsHealthOperator": { - "anyOf": [ - { - "type": "string", - "description": "Specifies the operator used to compare the health metric value with the specified threshold.", - "enum": [ - "GREATER_THAN" - ] - }, - { - "type": "string", - "pattern": "\\$\\{(resources(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - }, - { - "type": "string", - "pattern": "\\$\\{(bundle(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - }, - { - "type": "string", - "pattern": "\\$\\{(workspace(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - }, - { - "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - } + "type": "string", + "description": "Specifies the operator used to compare the health metric value with the specified threshold.", + "enum": [ + "GREATER_THAN" ] }, "jobs.JobsHealthRule": { @@ -3183,41 +2723,14 @@ "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.Source" }, "warehouse_id": { - "description": "Optional `warehouse_id` to run the notebook on a SQL warehouse. Classic SQL warehouses are NOT supported, please use serverless or pro SQL warehouses.\n\nNote that SQL warehouses only support SQL cells; if the notebook contains non-SQL cells, the run will fail.", - "$ref": "#/$defs/string" - } - }, - "additionalProperties": false, - "required": [ - "notebook_path" - ] - }, - { - "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - } - ] - }, - "jobs.PauseStatus": { - "anyOf": [ - { - "type": "string", - "enum": [ - "UNPAUSED", - "PAUSED" - ] - }, - { - "type": "string", - "pattern": "\\$\\{(resources(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - }, - { - "type": "string", - "pattern": "\\$\\{(bundle(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - }, - { - "type": "string", - "pattern": "\\$\\{(workspace(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "description": "Optional `warehouse_id` to run the notebook on a SQL warehouse. Classic SQL warehouses are NOT supported, please use serverless or pro SQL warehouses.\n\nNote that SQL warehouses only support SQL cells; if the notebook contains non-SQL cells, the run will fail.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "notebook_path" + ] }, { "type": "string", @@ -3225,6 +2738,13 @@ } ] }, + "jobs.PauseStatus": { + "type": "string", + "enum": [ + "UNPAUSED", + "PAUSED" + ] + }, "jobs.PeriodicTriggerConfiguration": { "anyOf": [ { @@ -3252,31 +2772,11 @@ ] }, "jobs.PeriodicTriggerConfigurationTimeUnit": { - "anyOf": [ - { - "type": "string", - "enum": [ - "HOURS", - "DAYS", - "WEEKS" - ] - }, - { - "type": "string", - "pattern": "\\$\\{(resources(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - }, - { - "type": "string", - "pattern": "\\$\\{(bundle(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - }, - { - "type": "string", - "pattern": "\\$\\{(workspace(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - }, - { - "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - } + "type": "string", + "enum": [ + "HOURS", + "DAYS", + "WEEKS" ] }, "jobs.PipelineParams": { @@ -3378,35 +2878,15 @@ ] }, "jobs.RunIf": { - "anyOf": [ - { - "type": "string", - "description": "An optional value indicating the condition that determines whether the task should be run once its dependencies have been completed. When omitted, defaults to `ALL_SUCCESS`.\n\nPossible values are:\n* `ALL_SUCCESS`: All dependencies have executed and succeeded\n* `AT_LEAST_ONE_SUCCESS`: At least one dependency has succeeded\n* `NONE_FAILED`: None of the dependencies have failed and at least one was executed\n* `ALL_DONE`: All dependencies have been completed\n* `AT_LEAST_ONE_FAILED`: At least one dependency failed\n* `ALL_FAILED`: ALl dependencies have failed", - "enum": [ - "ALL_SUCCESS", - "ALL_DONE", - "NONE_FAILED", - "AT_LEAST_ONE_SUCCESS", - "ALL_FAILED", - "AT_LEAST_ONE_FAILED" - ] - }, - { - "type": "string", - "pattern": "\\$\\{(resources(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - }, - { - "type": "string", - "pattern": "\\$\\{(bundle(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - }, - { - "type": "string", - "pattern": "\\$\\{(workspace(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - }, - { - "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - } + "type": "string", + "description": "An optional value indicating the condition that determines whether the task should be run once its dependencies have been completed. When omitted, defaults to `ALL_SUCCESS`.\n\nPossible values are:\n* `ALL_SUCCESS`: All dependencies have executed and succeeded\n* `AT_LEAST_ONE_SUCCESS`: At least one dependency has succeeded\n* `NONE_FAILED`: None of the dependencies have failed and at least one was executed\n* `ALL_DONE`: All dependencies have been completed\n* `AT_LEAST_ONE_FAILED`: At least one dependency failed\n* `ALL_FAILED`: ALl dependencies have failed", + "enum": [ + "ALL_SUCCESS", + "ALL_DONE", + "NONE_FAILED", + "AT_LEAST_ONE_SUCCESS", + "ALL_FAILED", + "AT_LEAST_ONE_FAILED" ] }, "jobs.RunJobTask": { @@ -3465,31 +2945,11 @@ ] }, "jobs.Source": { - "anyOf": [ - { - "type": "string", - "description": "Optional location type of the SQL file. When set to `WORKSPACE`, the SQL file will be retrieved\\\nfrom the local Databricks workspace. When set to `GIT`, the SQL file will be retrieved from a Git repository\ndefined in `git_source`. If the value is empty, the task will use `GIT` if `git_source` is defined and `WORKSPACE` otherwise.\n\n* `WORKSPACE`: SQL file is located in Databricks workspace.\n* `GIT`: SQL file is located in cloud Git provider.", - "enum": [ - "WORKSPACE", - "GIT" - ] - }, - { - "type": "string", - "pattern": "\\$\\{(resources(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - }, - { - "type": "string", - "pattern": "\\$\\{(bundle(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - }, - { - "type": "string", - "pattern": "\\$\\{(workspace(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - }, - { - "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - } + "type": "string", + "description": "Optional location type of the SQL file. When set to `WORKSPACE`, the SQL file will be retrieved\\\nfrom the local Databricks workspace. When set to `GIT`, the SQL file will be retrieved from a Git repository\ndefined in `git_source`. If the value is empty, the task will use `GIT` if `git_source` is defined and `WORKSPACE` otherwise.\n\n* `WORKSPACE`: SQL file is located in Databricks workspace.\n* `GIT`: SQL file is located in cloud Git provider.", + "enum": [ + "WORKSPACE", + "GIT" ] }, "jobs.SparkJarTask": { @@ -4190,27 +3650,7 @@ ] }, "ml.ModelVersionStatus": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "string", - "pattern": "\\$\\{(resources(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - }, - { - "type": "string", - "pattern": "\\$\\{(bundle(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - }, - { - "type": "string", - "pattern": "\\$\\{(workspace(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - }, - { - "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - } - ] + "type": "string" }, "ml.ModelVersionTag": { "anyOf": [ @@ -4255,30 +3695,10 @@ ] }, "pipelines.DeploymentKind": { - "anyOf": [ - { - "type": "string", - "description": "The deployment method that manages the pipeline:\n- BUNDLE: The pipeline is managed by a Databricks Asset Bundle.\n", - "enum": [ - "BUNDLE" - ] - }, - { - "type": "string", - "pattern": "\\$\\{(resources(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - }, - { - "type": "string", - "pattern": "\\$\\{(bundle(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - }, - { - "type": "string", - "pattern": "\\$\\{(workspace(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - }, - { - "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - } + "type": "string", + "description": "The deployment method that manages the pipeline:\n- BUNDLE: The pipeline is managed by a Databricks Asset Bundle.\n", + "enum": [ + "BUNDLE" ] }, "pipelines.FileLibrary": { @@ -4580,27 +4000,7 @@ ] }, "pipelines.PipelineClusterAutoscaleMode": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "string", - "pattern": "\\$\\{(resources(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - }, - { - "type": "string", - "pattern": "\\$\\{(bundle(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - }, - { - "type": "string", - "pattern": "\\$\\{(workspace(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - }, - { - "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - } - ] + "type": "string" }, "pipelines.PipelineDeployment": { "anyOf": [ @@ -4785,27 +4185,7 @@ ] }, "pipelines.TableSpecificConfigScdType": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "string", - "pattern": "\\$\\{(resources(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - }, - { - "type": "string", - "pattern": "\\$\\{(bundle(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - }, - { - "type": "string", - "pattern": "\\$\\{(workspace(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - }, - { - "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - } - ] + "type": "string" }, "serving.Ai21LabsConfig": { "anyOf": [ @@ -4876,27 +4256,7 @@ ] }, "serving.AmazonBedrockConfigBedrockProvider": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "string", - "pattern": "\\$\\{(resources(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - }, - { - "type": "string", - "pattern": "\\$\\{(bundle(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - }, - { - "type": "string", - "pattern": "\\$\\{(workspace(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - }, - { - "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - } - ] + "type": "string" }, "serving.AnthropicConfig": { "anyOf": [ @@ -5134,27 +4494,7 @@ ] }, "serving.ExternalModelProvider": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "string", - "pattern": "\\$\\{(resources(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - }, - { - "type": "string", - "pattern": "\\$\\{(bundle(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - }, - { - "type": "string", - "pattern": "\\$\\{(workspace(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - }, - { - "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - } - ] + "type": "string" }, "serving.GoogleCloudVertexAiConfig": { "anyOf": [ @@ -5287,50 +4627,10 @@ ] }, "serving.RateLimitKey": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "string", - "pattern": "\\$\\{(resources(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - }, - { - "type": "string", - "pattern": "\\$\\{(bundle(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - }, - { - "type": "string", - "pattern": "\\$\\{(workspace(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - }, - { - "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - } - ] + "type": "string" }, "serving.RateLimitRenewalPeriod": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "string", - "pattern": "\\$\\{(resources(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - }, - { - "type": "string", - "pattern": "\\$\\{(bundle(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - }, - { - "type": "string", - "pattern": "\\$\\{(workspace(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - }, - { - "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - } - ] + "type": "string" }, "serving.Route": { "anyOf": [ @@ -5481,50 +4781,10 @@ ] }, "serving.ServedModelInputWorkloadSize": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "string", - "pattern": "\\$\\{(resources(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - }, - { - "type": "string", - "pattern": "\\$\\{(bundle(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - }, - { - "type": "string", - "pattern": "\\$\\{(workspace(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - }, - { - "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - } - ] + "type": "string" }, "serving.ServedModelInputWorkloadType": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "string", - "pattern": "\\$\\{(resources(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - }, - { - "type": "string", - "pattern": "\\$\\{(bundle(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - }, - { - "type": "string", - "pattern": "\\$\\{(workspace(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - }, - { - "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - } - ] + "type": "string" }, "serving.TrafficConfig": { "anyOf": [ @@ -6198,27 +5458,7 @@ } }, "string": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "string", - "pattern": "\\$\\{(resources(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - }, - { - "type": "string", - "pattern": "\\$\\{(bundle(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - }, - { - "type": "string", - "pattern": "\\$\\{(workspace(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - }, - { - "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - } - ] + "type": "string" } }, "type": "object", From ad7503a1762e6e72dbc2d2d0defd053f6ab36f28 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 27 Aug 2024 17:40:36 +0200 Subject: [PATCH 27/62] cleanup todos --- bundle/internal/schema/main.go | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/bundle/internal/schema/main.go b/bundle/internal/schema/main.go index 68a7f13b22..c35caf1d98 100644 --- a/bundle/internal/schema/main.go +++ b/bundle/internal/schema/main.go @@ -34,8 +34,6 @@ func addInterpolationPatterns(typ reflect.Type, s jsonschema.Schema) jsonschema. case jsonschema.IntegerType, jsonschema.NumberType, jsonschema.BooleanType: // primitives can have variable values, or references like ${bundle.xyz} // or ${workspace.xyz} - // TODO: Followup, do not allow references like ${} in the schema unless - // they are of the permitted patterns? return jsonschema.Schema{ AnyOf: []jsonschema.Schema{s, // TODO: Is it only resource IDs or is it resources in general? @@ -50,16 +48,7 @@ func addInterpolationPatterns(typ reflect.Type, s jsonschema.Schema) jsonschema. } } -// TODO: Add a couple of end to end tests that the bundle schema generated is -// correct. -// TODO: Call out in the PR description that recursive types like "for_each_task" -// are now supported. Manually test for_each_task. -// TODO: The bundle_descriptions.json file contains a bunch of custom descriptions -// as well. Make sure to pull those in. -// TODO: Add unit tests for all permutations of structs, maps and slices for the FromType -// method. -// TODO: Note the minor regression of losing the bundle descriptions. -// TODO: Ensure descriptions work for target resources section. +// TODO: Add enum overrides for DABs enum values. Maybe use reflection? func main() { if len(os.Args) != 2 { From 40f4d35c4af10d266e91ca0f8c60faa96b81d4c4 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 27 Aug 2024 17:45:20 +0200 Subject: [PATCH 28/62] cleanup todos --- bundle/internal/schema/main.go | 2 -- libs/jsonschema/schema.go | 3 ++- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/bundle/internal/schema/main.go b/bundle/internal/schema/main.go index c35caf1d98..e9d0f4dc09 100644 --- a/bundle/internal/schema/main.go +++ b/bundle/internal/schema/main.go @@ -27,7 +27,6 @@ func addInterpolationPatterns(typ reflect.Type, s jsonschema.Schema) jsonschema. return jsonschema.Schema{ AnyOf: []jsonschema.Schema{s, { Type: jsonschema.StringType, - // TODO: Are multi-level complex variable references supported? Pattern: interpolationPattern("var"), }}, } @@ -36,7 +35,6 @@ func addInterpolationPatterns(typ reflect.Type, s jsonschema.Schema) jsonschema. // or ${workspace.xyz} return jsonschema.Schema{ AnyOf: []jsonschema.Schema{s, - // TODO: Is it only resource IDs or is it resources in general? {Type: jsonschema.StringType, Pattern: interpolationPattern("resources")}, {Type: jsonschema.StringType, Pattern: interpolationPattern("bundle")}, {Type: jsonschema.StringType, Pattern: interpolationPattern("workspace")}, diff --git a/libs/jsonschema/schema.go b/libs/jsonschema/schema.go index b364c82768..aaa075ed64 100644 --- a/libs/jsonschema/schema.go +++ b/libs/jsonschema/schema.go @@ -13,7 +13,8 @@ import ( // defines schema for a json object type Schema struct { - // TODO: Comments for this field + // Definitions that can be reused and referenced throughout the schema. The + // syntax for a reference is $ref: #/$defs/ Definitions any `json:"$defs,omitempty"` // Type of the object From aac6687900af69485b224d2cd11a3a5856c64b34 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 27 Aug 2024 17:46:57 +0200 Subject: [PATCH 29/62] remove more todos --- libs/jsonschema/schema.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/libs/jsonschema/schema.go b/libs/jsonschema/schema.go index aaa075ed64..5b58d81d55 100644 --- a/libs/jsonschema/schema.go +++ b/libs/jsonschema/schema.go @@ -26,12 +26,10 @@ type Schema struct { // Expected value for the JSON object. The object value must be equal to this // field if it's specified in the schema. - // TODO: Generics here? OR maybe a type from the reflection package. Const any `json:"const,omitempty"` // Schemas for the fields of an struct. The keys are the first json tag. // The values are the schema for the type of the field - // TODO: Followup: Make this a map[string]Schema Properties map[string]*Schema `json:"properties,omitempty"` // The schema for all values of an array @@ -44,7 +42,6 @@ type Schema struct { // properties in the config have been defined in the json schema as properties // // Its type during runtime will either be Schema or bool - // TODO: Generics to represent either a Schema{} or a bool. AdditionalProperties any `json:"additionalProperties,omitempty"` // Required properties for the object. Any fields missing the "omitempty" @@ -52,7 +49,6 @@ type Schema struct { Required []string `json:"required,omitempty"` // URI to a json schema - // TODO: Would be nice to make items as well as this a non-pointer. Reference *string `json:"$ref,omitempty"` // Default value for the property / object From 4141f4ea349f4de37f13e2c03e7902436ebe3cad Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 27 Aug 2024 17:49:09 +0200 Subject: [PATCH 30/62] - --- bundle/internal/schema/main.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/bundle/internal/schema/main.go b/bundle/internal/schema/main.go index e9d0f4dc09..0faa8d09b7 100644 --- a/bundle/internal/schema/main.go +++ b/bundle/internal/schema/main.go @@ -26,7 +26,7 @@ func addInterpolationPatterns(typ reflect.Type, s jsonschema.Schema) jsonschema. // arrays and objects can have complex variable values specified. return jsonschema.Schema{ AnyOf: []jsonschema.Schema{s, { - Type: jsonschema.StringType, + Type: jsonschema.StringType, Pattern: interpolationPattern("var"), }}, } @@ -46,8 +46,6 @@ func addInterpolationPatterns(typ reflect.Type, s jsonschema.Schema) jsonschema. } } -// TODO: Add enum overrides for DABs enum values. Maybe use reflection? - func main() { if len(os.Args) != 2 { fmt.Println("Usage: go run main.go ") From cb8d6a9f60ca4b360e0d24291fbfbc67fd924361 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 27 Aug 2024 18:34:33 +0200 Subject: [PATCH 31/62] self pass 1 --- bundle/internal/schema/parser.go | 22 +++++++--------------- cmd/bundle/schema.go | 4 ++-- libs/dyn/dynvar/ref.go | 4 +--- libs/dyn/path.go | 4 ++-- libs/dyn/visit.go | 8 ++++---- libs/dyn/visit_set.go | 4 ++-- libs/jsonschema/from_type.go | 20 +++++++------------- libs/jsonschema/schema.go | 4 +--- 8 files changed, 26 insertions(+), 44 deletions(-) diff --git a/bundle/internal/schema/parser.go b/bundle/internal/schema/parser.go index dc85da3896..ef3d6e719b 100644 --- a/bundle/internal/schema/parser.go +++ b/bundle/internal/schema/parser.go @@ -40,14 +40,16 @@ func newParser(path string) (*openapiParser, error) { return p, nil } -// This function finds any JSON schemas that were defined in the OpenAPI spec -// that correspond to the given Go SDK type. It looks both at the type itself -// and any embedded types within it. +// This function checks if the input type: +// 1. Is a Databricks Go SDK type. +// 2. Has a Databricks Go SDK type embedded in it. +// +// If the above conditions are met, the function returns the JSON schema +// corresponding to the Databricks Go SDK type from the OpenAPI spec. func (p *openapiParser) findRef(typ reflect.Type) (jsonschema.Schema, bool) { typs := []reflect.Type{typ} - // If the type is a struct, the corresponding Go SDK struct might be embedded - // in it. We need to check for those as well. + // Check for embedded Databricks Go SDK types. if typ.Kind() == reflect.Struct { for i := 0; i < typ.NumField(); i++ { if !typ.Field(i).Anonymous { @@ -94,11 +96,6 @@ func (p *openapiParser) addDescriptions(typ reflect.Type, s jsonschema.Schema) j } s.Description = ref.Description - - // Iterate over properties to load descriptions. This is not needed for any - // OpenAPI spec generated from protobufs, which are guaranteed to be one level - // deep. - // Needed for any hand-written OpenAPI specs. for k, v := range s.Properties { if refProp, ok := ref.Properties[k]; ok { v.Description = refProp.Description @@ -116,11 +113,6 @@ func (p *openapiParser) addEnums(typ reflect.Type, s jsonschema.Schema) jsonsche } s.Enum = append(s.Enum, ref.Enum...) - - // Iterate over properties to load enums. This is not needed for any - // OpenAPI spec generated from protobufs, which are guaranteed to be one level - // deep. - // Needed for any hand-written OpenAPI specs. for k, v := range s.Properties { if refProp, ok := ref.Properties[k]; ok { v.Enum = append(v.Enum, refProp.Enum...) diff --git a/cmd/bundle/schema.go b/cmd/bundle/schema.go index 425be5e12e..d33714c9c2 100644 --- a/cmd/bundle/schema.go +++ b/cmd/bundle/schema.go @@ -8,7 +8,7 @@ import ( ) //go:embed _generated/jsonschema.json -var b []byte +var bundleSchemaBytes []byte func newSchemaCommand() *cobra.Command { cmd := &cobra.Command{ @@ -18,7 +18,7 @@ func newSchemaCommand() *cobra.Command { } cmd.RunE = func(cmd *cobra.Command, args []string) error { - _, err := cmd.OutOrStdout().Write(b) + _, err := cmd.OutOrStdout().Write(bundleSchemaBytes) return err } diff --git a/libs/dyn/dynvar/ref.go b/libs/dyn/dynvar/ref.go index f686d6779a..338ac8ce6c 100644 --- a/libs/dyn/dynvar/ref.go +++ b/libs/dyn/dynvar/ref.go @@ -6,9 +6,7 @@ import ( "github.com/databricks/cli/libs/dyn" ) -const ReferenceRegex = `\$\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\[[0-9]+\])*)*(\[[0-9]+\])*)\}` - -var re = regexp.MustCompile(ReferenceRegex) +var re = regexp.MustCompile(`\$\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\[[0-9]+\])*)*(\[[0-9]+\])*)\}`) // ref represents a variable reference. // It is a string [dyn.Value] contained in a larger [dyn.Value]. diff --git a/libs/dyn/path.go b/libs/dyn/path.go index 1d0d4afabd..76377e2dce 100644 --- a/libs/dyn/path.go +++ b/libs/dyn/path.go @@ -18,11 +18,11 @@ func (c pathComponent) Index() int { return c.index } -func (c pathComponent) IsKey() bool { +func (c pathComponent) isKey() bool { return c.key != "" } -func (c pathComponent) IsIndex() bool { +func (c pathComponent) isIndex() bool { return c.key == "" } diff --git a/libs/dyn/visit.go b/libs/dyn/visit.go index ee30227d7d..4d3cf50142 100644 --- a/libs/dyn/visit.go +++ b/libs/dyn/visit.go @@ -14,9 +14,9 @@ type cannotTraverseNilError struct { func (e cannotTraverseNilError) Error() string { component := e.p[len(e.p)-1] switch { - case component.IsKey(): + case component.isKey(): return fmt.Sprintf("expected a map to index %q, found nil", e.p) - case component.IsIndex(): + case component.isIndex(): return fmt.Sprintf("expected a sequence to index %q, found nil", e.p) default: panic("invalid component") @@ -90,7 +90,7 @@ func (component pathComponent) visit(v Value, prefix Path, suffix Pattern, opts path := append(prefix, component) switch { - case component.IsKey(): + case component.isKey(): // Expect a map to be set if this is a key. switch v.Kind() { case KindMap: @@ -129,7 +129,7 @@ func (component pathComponent) visit(v Value, prefix Path, suffix Pattern, opts l: v.l, }, nil - case component.IsIndex(): + case component.isIndex(): // Expect a sequence to be set if this is an index. switch v.Kind() { case KindSequence: diff --git a/libs/dyn/visit_set.go b/libs/dyn/visit_set.go index cf3b3b3127..b086fb8a91 100644 --- a/libs/dyn/visit_set.go +++ b/libs/dyn/visit_set.go @@ -32,7 +32,7 @@ func SetByPath(v Value, p Path, nv Value) (Value, error) { path := append(prefix, component) switch { - case component.IsKey(): + case component.isKey(): // Expect a map to be set if this is a key. m, ok := v.AsMap() if !ok { @@ -48,7 +48,7 @@ func SetByPath(v Value, p Path, nv Value) (Value, error) { l: v.l, }, nil - case component.IsIndex(): + case component.isIndex(): // Expect a sequence to be set if this is an index. s, ok := v.AsSequence() if !ok { diff --git a/libs/jsonschema/from_type.go b/libs/jsonschema/from_type.go index d2bbb45502..f916caa0cb 100644 --- a/libs/jsonschema/from_type.go +++ b/libs/jsonschema/from_type.go @@ -19,7 +19,7 @@ const readonlyTag = "readonly" const internalTag = "internal" // Annotation for bundle fields that have been deprecated. -// Fields tagged as "deprecated" are removed/omitted from the generated schema. +// Fields tagged as "deprecated" are omitted from the generated schema. const deprecatedTag = "deprecated" type constructor struct { @@ -36,8 +36,8 @@ type constructor struct { } // The $defs block in a JSON schema cannot contain "/", otherwise it will not be -// correctly parsed by a JSON schema validator. So we replace "/" with an additional -// level of nesting in the output map. +// correctly parsed by a JSON schema validator (like the Red Hat YAML extension for VSCode). +// So we replace "/" with an additional level of nesting in the output map. // // For example: // {"a/b/c": "value"} is converted to {"a": {"b": {"c": "value"}}} @@ -57,11 +57,14 @@ func (c *constructor) Definitions() any { parts := strings.Split(k, "/") cur := res for i, p := range parts { + // Set the value for the last part. if i == len(parts)-1 { cur[p] = v break } + // For all but the last part, create a new map value to add a level + // of nesting. if _, ok := cur[p]; !ok { cur[p] = make(map[string]any) } @@ -77,7 +80,7 @@ func (c *constructor) Definitions() any { // for every Go type and referring them using $ref in the corresponding node in // the JSON schema. // -// fns is a list of transformation functions that will be applied to all $defs +// fns is a list of transformation functions that will be applied in order to all $defs // in the schema. func FromType(typ reflect.Type, fns []func(typ reflect.Type, s Schema) Schema) (Schema, error) { c := constructor{ @@ -156,11 +159,6 @@ func (c *constructor) walk(typ reflect.Type) error { } c.seen[typPath] = typ - // Return early directly if it's already been processed. - if _, ok := c.definitions[typPath]; ok { - return nil - } - var s Schema var err error @@ -315,10 +313,6 @@ func (c *constructor) fromTypeMap(typ reflect.Type) (Schema, error) { return Schema{}, fmt.Errorf("expected map, got %s", typ.Kind()) } - if typ.Key().Kind() != reflect.String { - return Schema{}, fmt.Errorf("found map with non-string key: %v", typ.Key()) - } - res := Schema{ Type: ObjectType, } diff --git a/libs/jsonschema/schema.go b/libs/jsonschema/schema.go index 5b58d81d55..10e646165b 100644 --- a/libs/jsonschema/schema.go +++ b/libs/jsonschema/schema.go @@ -41,7 +41,7 @@ type Schema struct { // A boolean type with value false. Setting false here validates that all // properties in the config have been defined in the json schema as properties // - // Its type during runtime will either be Schema or bool + // Its type during runtime will either be *Schema or bool AdditionalProperties any `json:"additionalProperties,omitempty"` // Required properties for the object. Any fields missing the "omitempty" @@ -88,8 +88,6 @@ func (s *Schema) ParseString(v string) (any, error) { type Type string const ( - // Default zero value of a schema. This does not correspond to a type in the - // JSON schema spec and is an internal type defined for convenience. InvalidType Type = "invalid" BooleanType Type = "boolean" StringType Type = "string" From 578019ba22d5aca0af69a8b9f73b8abcc36cef76 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 3 Sep 2024 20:14:02 +0200 Subject: [PATCH 32/62] - --- libs/jsonschema/from_type.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libs/jsonschema/from_type.go b/libs/jsonschema/from_type.go index f916caa0cb..2f5c688b8d 100644 --- a/libs/jsonschema/from_type.go +++ b/libs/jsonschema/from_type.go @@ -44,8 +44,8 @@ type constructor struct { func (c *constructor) Definitions() any { defs := maps.Clone(c.definitions) - // Remove the root type from the definitions. No need to include it in the - // definitions. + // Remove the root type from the definitions. We don't need to include it in + // the definitions because it will be inlined as the root of the generated JSON schema. delete(defs, typePath(c.root)) if len(defs) == 0 { From 176ced190e35baf6bf0ace0f549df5d2ca9e8fe0 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 4 Sep 2024 10:26:47 +0200 Subject: [PATCH 33/62] add tests for bfs assertion --- libs/jsonschema/from_type.go | 4 +++ libs/jsonschema/from_type_test.go | 42 ++++++++++++++++++++++--------- 2 files changed, 34 insertions(+), 12 deletions(-) diff --git a/libs/jsonschema/from_type.go b/libs/jsonschema/from_type.go index 2f5c688b8d..79cd8f1f62 100644 --- a/libs/jsonschema/from_type.go +++ b/libs/jsonschema/from_type.go @@ -216,6 +216,10 @@ func getStructFields(typ reflect.Type) []reflect.StructField { } fieldType := field.Type + + // Embedded types can only be struct{} or pointer to struct{}. Multiple + // levels of pointers are not allowed by the Go compiler. So we only + // dereference pointers once. if fieldType.Kind() == reflect.Pointer { fieldType = fieldType.Elem() } diff --git a/libs/jsonschema/from_type_test.go b/libs/jsonschema/from_type_test.go index ab2f1def8c..7b83a897f4 100644 --- a/libs/jsonschema/from_type_test.go +++ b/libs/jsonschema/from_type_test.go @@ -10,9 +10,10 @@ import ( func TestFromTypeBasic(t *testing.T) { type myStruct struct { - S string `json:"s"` - I *int `json:"i,omitempty"` - V interface{} `json:"v,omitempty"` + S string `json:"s"` + I *int `json:"i,omitempty"` + V interface{} `json:"v,omitempty"` + TriplePointer ***int `json:"triple_pointer,omitempty"` // These fields should be ignored in the resulting schema. NotAnnotated string @@ -84,6 +85,9 @@ func TestFromTypeBasic(t *testing.T) { "v": { Reference: &interfaceRef, }, + "triple_pointer": { + Reference: &intRef, + }, }, AdditionalProperties: false, Required: []string{"s"}, @@ -131,21 +135,35 @@ func TestFromTypeBasic(t *testing.T) { } func TestGetStructFields(t *testing.T) { - type EmbeddedStruct struct { - I int - B bool + type InnerEmbeddedStruct struct { + InnerField float64 + } + + type EmbeddedStructOne struct { + FieldOne int + + *InnerEmbeddedStruct + } + + type EmbeddedStructTwo struct { + FieldTwo bool } type MyStruct struct { - S string - *EmbeddedStruct + *EmbeddedStructOne + EmbeddedStructTwo + + OuterField string } fields := getStructFields(reflect.TypeOf(MyStruct{})) - assert.Len(t, fields, 3) - assert.Equal(t, "S", fields[0].Name) - assert.Equal(t, "I", fields[1].Name) - assert.Equal(t, "B", fields[2].Name) + assert.Len(t, fields, 4) + assert.Equal(t, "OuterField", fields[0].Name) + assert.Equal(t, "FieldOne", fields[1].Name) + + // InnerField occurring after FieldTwo ensures BFS as opposed to DFS traversal. + assert.Equal(t, "FieldTwo", fields[2].Name) + assert.Equal(t, "InnerField", fields[3].Name) } func TestFromTypeNested(t *testing.T) { From 3d5c076a4dd3c32aa66dc6b0789d38cd1cedb39c Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 4 Sep 2024 10:40:04 +0200 Subject: [PATCH 34/62] jsonTags[0] -> fieldName --- libs/jsonschema/from_type.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/libs/jsonschema/from_type.go b/libs/jsonschema/from_type.go index 79cd8f1f62..d89fd5987b 100644 --- a/libs/jsonschema/from_type.go +++ b/libs/jsonschema/from_type.go @@ -255,9 +255,10 @@ func (c *constructor) fromTypeStruct(typ reflect.Type) (Schema, error) { } jsonTags := strings.Split(structField.Tag.Get("json"), ",") + fieldName := jsonTags[0] // Do not include fields in the schema that will not be serialized during // JSON marshalling. - if jsonTags[0] == "" || jsonTags[0] == "-" || !structField.IsExported() { + if fieldName == "" || fieldName == "-" || !structField.IsExported() { continue } @@ -265,7 +266,7 @@ func (c *constructor) fromTypeStruct(typ reflect.Type) (Schema, error) { // required to be present in the API payload. Thus its absence in the // tags list indicates that the field is required. if !slices.Contains(jsonTags, "omitempty") { - res.Required = append(res.Required, jsonTags[0]) + res.Required = append(res.Required, fieldName) } // Walk the fields of the struct. @@ -278,7 +279,7 @@ func (c *constructor) fromTypeStruct(typ reflect.Type) (Schema, error) { // For every property in the struct, add a $ref to the corresponding // $defs block. refPath := path.Join("#/$defs", typPath) - res.Properties[jsonTags[0]] = &Schema{ + res.Properties[fieldName] = &Schema{ Reference: &refPath, } } From be0ad482fff84ab80f5b699b04328a525e82a274 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 4 Sep 2024 10:43:46 +0200 Subject: [PATCH 35/62] str bool --- libs/jsonschema/from_type_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libs/jsonschema/from_type_test.go b/libs/jsonschema/from_type_test.go index 7b83a897f4..4f5c552488 100644 --- a/libs/jsonschema/from_type_test.go +++ b/libs/jsonschema/from_type_test.go @@ -42,14 +42,14 @@ func TestFromTypeBasic(t *testing.T) { }, { name: "string", - typ: reflect.TypeOf(""), + typ: reflect.TypeOf(string("")), expected: Schema{ Type: "string", }, }, { name: "bool", - typ: reflect.TypeOf(true), + typ: reflect.TypeOf(bool(true)), expected: Schema{ Type: "boolean", }, From 8575c47626d1dc0035dbd84d7fab01144a6430cc Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 4 Sep 2024 11:00:53 +0200 Subject: [PATCH 36/62] json pointer comment --- libs/jsonschema/from_type.go | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/libs/jsonschema/from_type.go b/libs/jsonschema/from_type.go index d89fd5987b..a54256bf5e 100644 --- a/libs/jsonschema/from_type.go +++ b/libs/jsonschema/from_type.go @@ -35,12 +35,16 @@ type constructor struct { root reflect.Type } -// The $defs block in a JSON schema cannot contain "/", otherwise it will not be -// correctly parsed by a JSON schema validator (like the Red Hat YAML extension for VSCode). -// So we replace "/" with an additional level of nesting in the output map. +// JSON pointers use "/" as a delimiter to represent nested objects. This means +// we would instead need to use "~1" to represent "/" if we wish to refer to a +// key in a JSON object with a "/" in it. Instead of doing that we replace "/" with an +// additional level of nesting in the output map. Thus the $refs in the generated +// JSON schema can contain "/" without any issues. +// see: https://datatracker.ietf.org/doc/html/rfc6901 // // For example: // {"a/b/c": "value"} is converted to {"a": {"b": {"c": "value"}}} +// the $ref for "value" would be "#/$defs/a/b/c" in the generated JSON schema. func (c *constructor) Definitions() any { defs := maps.Clone(c.definitions) From 46585bbd3fb8e36e23e507fda91a75255ad2d40b Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 4 Sep 2024 11:03:33 +0200 Subject: [PATCH 37/62] type defs as map[string]any --- libs/jsonschema/from_type.go | 2 +- libs/jsonschema/schema.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/libs/jsonschema/from_type.go b/libs/jsonschema/from_type.go index a54256bf5e..21fdb2106b 100644 --- a/libs/jsonschema/from_type.go +++ b/libs/jsonschema/from_type.go @@ -45,7 +45,7 @@ type constructor struct { // For example: // {"a/b/c": "value"} is converted to {"a": {"b": {"c": "value"}}} // the $ref for "value" would be "#/$defs/a/b/c" in the generated JSON schema. -func (c *constructor) Definitions() any { +func (c *constructor) Definitions() map[string]any { defs := maps.Clone(c.definitions) // Remove the root type from the definitions. We don't need to include it in diff --git a/libs/jsonschema/schema.go b/libs/jsonschema/schema.go index 10e646165b..7690ec2f76 100644 --- a/libs/jsonschema/schema.go +++ b/libs/jsonschema/schema.go @@ -15,7 +15,7 @@ import ( type Schema struct { // Definitions that can be reused and referenced throughout the schema. The // syntax for a reference is $ref: #/$defs/ - Definitions any `json:"$defs,omitempty"` + Definitions map[string]any `json:"$defs,omitempty"` // Type of the object Type Type `json:"type,omitempty"` From 6bd7ad0fb2d45f7f36de9bdc9865f1191d8f8119 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 4 Sep 2024 11:15:00 +0200 Subject: [PATCH 38/62] clarify interface comments --- libs/jsonschema/from_type.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/libs/jsonschema/from_type.go b/libs/jsonschema/from_type.go index 21fdb2106b..95914d4fde 100644 --- a/libs/jsonschema/from_type.go +++ b/libs/jsonschema/from_type.go @@ -182,10 +182,9 @@ func (c *constructor) walk(typ reflect.Type) error { case reflect.Float32, reflect.Float64: s = Schema{Type: NumberType} case reflect.Interface: - // Interface or any types are not serialized to JSON by the default JSON - // unmarshaller (json.Unmarshal). They likely thus are parsed by the - // dynamic configuration tree and we should support arbitary values here. - // Eg: variables.default can be anything. + // We cannot determine the schema for fields of interface type just based + // on the type information. Thus we'll set the empty schema here and allow + // arbitrary values. s = Schema{} default: return fmt.Errorf("unsupported type: %s", typ.Kind()) From 66cbb548efac1a684e109a6d4edd47dafe533622 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 4 Sep 2024 11:28:53 +0200 Subject: [PATCH 39/62] return typPath from walk --- libs/jsonschema/from_type.go | 23 +++++++++-------------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/libs/jsonschema/from_type.go b/libs/jsonschema/from_type.go index 95914d4fde..b07fef2f34 100644 --- a/libs/jsonschema/from_type.go +++ b/libs/jsonschema/from_type.go @@ -93,7 +93,7 @@ func FromType(typ reflect.Type, fns []func(typ reflect.Type, s Schema) Schema) ( root: typ, } - err := c.walk(typ) + _, err := c.walk(typ) if err != nil { return Schema{}, err } @@ -149,7 +149,7 @@ func typePath(typ reflect.Type) string { // Walk the Go type, generating $defs for every type encountered, and populating // the corresponding $ref in the JSON schema. -func (c *constructor) walk(typ reflect.Type) error { +func (c *constructor) walk(typ reflect.Type) (string, error) { // Dereference pointers if necessary. for typ.Kind() == reflect.Ptr { typ = typ.Elem() @@ -159,7 +159,7 @@ func (c *constructor) walk(typ reflect.Type) error { // Return early if the type has already been seen, to avoid infinite recursion. if _, ok := c.seen[typPath]; ok { - return nil + return typPath, nil } c.seen[typPath] = typ @@ -187,15 +187,15 @@ func (c *constructor) walk(typ reflect.Type) error { // arbitrary values. s = Schema{} default: - return fmt.Errorf("unsupported type: %s", typ.Kind()) + return "", fmt.Errorf("unsupported type: %s", typ.Kind()) } if err != nil { - return err + return "", err } // Store the computed JSON schema for the type. c.definitions[typPath] = s - return nil + return typPath, nil } // This function returns all member fields of the provided type. @@ -273,8 +273,7 @@ func (c *constructor) fromTypeStruct(typ reflect.Type) (Schema, error) { } // Walk the fields of the struct. - typPath := typePath(structField.Type) - err := c.walk(structField.Type) + typPath, err := c.walk(structField.Type) if err != nil { return Schema{}, err } @@ -299,10 +298,8 @@ func (c *constructor) fromTypeSlice(typ reflect.Type) (Schema, error) { Type: ArrayType, } - typPath := typePath(typ.Elem()) - // Walk the slice element type. - err := c.walk(typ.Elem()) + typPath, err := c.walk(typ.Elem()) if err != nil { return Schema{}, err } @@ -325,10 +322,8 @@ func (c *constructor) fromTypeMap(typ reflect.Type) (Schema, error) { Type: ObjectType, } - typPath := typePath(typ.Elem()) - // Walk the map value type. - err := c.walk(typ.Elem()) + typPath, err := c.walk(typ.Elem()) if err != nil { return Schema{}, err } From ba3f004da18b255c5703d5e186cafc33171566d9 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 4 Sep 2024 11:32:39 +0200 Subject: [PATCH 40/62] schema local to generator --- .codegen.json | 4 ++-- .gitattributes | 2 +- .../internal/schema/generated}/jsonschema.json | 0 3 files changed, 3 insertions(+), 3 deletions(-) rename {cmd/bundle/_generated => bundle/internal/schema/generated}/jsonschema.json (100%) diff --git a/.codegen.json b/.codegen.json index 61a2fe8514..dd76c1241d 100644 --- a/.codegen.json +++ b/.codegen.json @@ -11,10 +11,10 @@ "toolchain": { "required": ["go"], "post_generate": [ - "go run ./bundle/internal/schema/*.go ./cmd/bundle/_generated/jsonschema.json", + "go run ./bundle/internal/schema/*.go ./bundle/internal/schema/generated/jsonschema.json", "echo 'bundle/internal/tf/schema/\\*.go linguist-generated=true' >> ./.gitattributes", "echo 'go.sum linguist-generated=true' >> ./.gitattributes", - "echo 'cmd/bundle/_generated linguist-generated=true' >> ./.gitattributes" + "echo 'bundle/internal/schema/generated/jsonschema.json linguist-generated=true' >> ./.gitattributes" ] } } diff --git a/.gitattributes b/.gitattributes index d77729c10d..766bf2fc85 100755 --- a/.gitattributes +++ b/.gitattributes @@ -120,4 +120,4 @@ cmd/workspace/workspace-conf/workspace-conf.go linguist-generated=true cmd/workspace/workspace/workspace.go linguist-generated=true bundle/internal/tf/schema/\*.go linguist-generated=true go.sum linguist-generated=true -cmd/bundle/_generated linguist-generated=true +bundle/internal/schema/generated/jsonschema.json linguist-generated=true diff --git a/cmd/bundle/_generated/jsonschema.json b/bundle/internal/schema/generated/jsonschema.json similarity index 100% rename from cmd/bundle/_generated/jsonschema.json rename to bundle/internal/schema/generated/jsonschema.json From 446463b9eb24cd64c16d89a5f8ee1df661edb589 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 4 Sep 2024 11:33:28 +0200 Subject: [PATCH 41/62] update schema --- bundle/internal/schema/generated/jsonschema.json | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/bundle/internal/schema/generated/jsonschema.json b/bundle/internal/schema/generated/jsonschema.json index 114a05aaf2..b9edc1e91f 100644 --- a/bundle/internal/schema/generated/jsonschema.json +++ b/bundle/internal/schema/generated/jsonschema.json @@ -758,10 +758,7 @@ "$ref": "#/$defs/github.com/databricks/cli/bundle/config.Lock" } }, - "additionalProperties": false, - "required": [ - "lock" - ] + "additionalProperties": false }, { "type": "string", From bfe9cc4d50b6c5240ddb5f930d176ca34f12998f Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 4 Sep 2024 11:56:52 +0200 Subject: [PATCH 42/62] create generated package --- .codegen.json | 4 ++-- .gitattributes | 2 +- bundle/generated/embed.go | 6 ++++++ bundle/{internal/schema => }/generated/jsonschema.json | 0 cmd/bundle/schema.go | 6 ++---- 5 files changed, 11 insertions(+), 7 deletions(-) create mode 100644 bundle/generated/embed.go rename bundle/{internal/schema => }/generated/jsonschema.json (100%) diff --git a/.codegen.json b/.codegen.json index dd76c1241d..62f75df021 100644 --- a/.codegen.json +++ b/.codegen.json @@ -11,10 +11,10 @@ "toolchain": { "required": ["go"], "post_generate": [ - "go run ./bundle/internal/schema/*.go ./bundle/internal/schema/generated/jsonschema.json", + "go run ./bundle/internal/schema/*.go ./bundle/generated/jsonschema.json", "echo 'bundle/internal/tf/schema/\\*.go linguist-generated=true' >> ./.gitattributes", "echo 'go.sum linguist-generated=true' >> ./.gitattributes", - "echo 'bundle/internal/schema/generated/jsonschema.json linguist-generated=true' >> ./.gitattributes" + "echo 'bundle/generated/jsonschema.json linguist-generated=true' >> ./.gitattributes" ] } } diff --git a/.gitattributes b/.gitattributes index 766bf2fc85..17cef7d263 100755 --- a/.gitattributes +++ b/.gitattributes @@ -120,4 +120,4 @@ cmd/workspace/workspace-conf/workspace-conf.go linguist-generated=true cmd/workspace/workspace/workspace.go linguist-generated=true bundle/internal/tf/schema/\*.go linguist-generated=true go.sum linguist-generated=true -bundle/internal/schema/generated/jsonschema.json linguist-generated=true +bundle/generated/jsonschema.json linguist-generated=true diff --git a/bundle/generated/embed.go b/bundle/generated/embed.go new file mode 100644 index 0000000000..3330b2e172 --- /dev/null +++ b/bundle/generated/embed.go @@ -0,0 +1,6 @@ +package generated + +import _ "embed" + +// go:embed jsonschema.json +var BundleSchema []byte diff --git a/bundle/internal/schema/generated/jsonschema.json b/bundle/generated/jsonschema.json similarity index 100% rename from bundle/internal/schema/generated/jsonschema.json rename to bundle/generated/jsonschema.json diff --git a/cmd/bundle/schema.go b/cmd/bundle/schema.go index d33714c9c2..967250ce34 100644 --- a/cmd/bundle/schema.go +++ b/cmd/bundle/schema.go @@ -3,13 +3,11 @@ package bundle import ( _ "embed" + "github.com/databricks/cli/bundle/generated" "github.com/databricks/cli/cmd/root" "github.com/spf13/cobra" ) -//go:embed _generated/jsonschema.json -var bundleSchemaBytes []byte - func newSchemaCommand() *cobra.Command { cmd := &cobra.Command{ Use: "schema", @@ -18,7 +16,7 @@ func newSchemaCommand() *cobra.Command { } cmd.RunE = func(cmd *cobra.Command, args []string) error { - _, err := cmd.OutOrStdout().Write(bundleSchemaBytes) + _, err := cmd.OutOrStdout().Write(generated.BundleSchema) return err } From a64857ed4c42d251a96288d61ff620f63c79b0c9 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 4 Sep 2024 11:58:02 +0200 Subject: [PATCH 43/62] - --- bundle/generated/embed.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bundle/generated/embed.go b/bundle/generated/embed.go index 3330b2e172..1e674c630c 100644 --- a/bundle/generated/embed.go +++ b/bundle/generated/embed.go @@ -2,5 +2,5 @@ package generated import _ "embed" -// go:embed jsonschema.json +//go:embed jsonschema.json var BundleSchema []byte From d0aa49328114b00855ae7656c0b96c7fadcb8657 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Thu, 5 Sep 2024 14:15:08 +0200 Subject: [PATCH 44/62] - --- bundle/internal/schema/main.go | 1 + 1 file changed, 1 insertion(+) diff --git a/bundle/internal/schema/main.go b/bundle/internal/schema/main.go index 0faa8d09b7..7750c15ee8 100644 --- a/bundle/internal/schema/main.go +++ b/bundle/internal/schema/main.go @@ -38,6 +38,7 @@ func addInterpolationPatterns(typ reflect.Type, s jsonschema.Schema) jsonschema. {Type: jsonschema.StringType, Pattern: interpolationPattern("resources")}, {Type: jsonschema.StringType, Pattern: interpolationPattern("bundle")}, {Type: jsonschema.StringType, Pattern: interpolationPattern("workspace")}, + {Type: jsonschema.StringType, Pattern: interpolationPattern("artifacts")}, {Type: jsonschema.StringType, Pattern: interpolationPattern("var")}, }, } From 4379d7f09bf0090608e7ed44a521cf6012ae004f Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Thu, 5 Sep 2024 14:15:35 +0200 Subject: [PATCH 45/62] allow artifacts referencing --- bundle/generated/jsonschema.json | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/bundle/generated/jsonschema.json b/bundle/generated/jsonschema.json index b9edc1e91f..d791e0799f 100644 --- a/bundle/generated/jsonschema.json +++ b/bundle/generated/jsonschema.json @@ -17,6 +17,10 @@ "type": "string", "pattern": "\\$\\{(workspace(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" }, + { + "type": "string", + "pattern": "\\$\\{(artifacts(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + }, { "type": "string", "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" @@ -40,6 +44,10 @@ "type": "string", "pattern": "\\$\\{(workspace(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" }, + { + "type": "string", + "pattern": "\\$\\{(artifacts(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + }, { "type": "string", "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" @@ -4822,6 +4830,10 @@ "type": "string", "pattern": "\\$\\{(workspace(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" }, + { + "type": "string", + "pattern": "\\$\\{(artifacts(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + }, { "type": "string", "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" @@ -4845,6 +4857,10 @@ "type": "string", "pattern": "\\$\\{(workspace(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" }, + { + "type": "string", + "pattern": "\\$\\{(artifacts(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + }, { "type": "string", "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" From 1dd399f3fe3a71acde7c19f02d016ca2fc7d473c Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Fri, 6 Sep 2024 15:18:31 +0200 Subject: [PATCH 46/62] new line for s --- bundle/internal/schema/main.go | 13 ++++++++----- bundle/{generated => schema}/embed.go | 4 ++-- bundle/{generated => schema}/jsonschema.json | 0 cmd/bundle/schema.go | 4 ++-- 4 files changed, 12 insertions(+), 9 deletions(-) rename bundle/{generated => schema}/embed.go (52%) rename bundle/{generated => schema}/jsonschema.json (100%) diff --git a/bundle/internal/schema/main.go b/bundle/internal/schema/main.go index 7750c15ee8..5a6ef7f248 100644 --- a/bundle/internal/schema/main.go +++ b/bundle/internal/schema/main.go @@ -25,16 +25,19 @@ func addInterpolationPatterns(typ reflect.Type, s jsonschema.Schema) jsonschema. case jsonschema.ArrayType, jsonschema.ObjectType: // arrays and objects can have complex variable values specified. return jsonschema.Schema{ - AnyOf: []jsonschema.Schema{s, { - Type: jsonschema.StringType, - Pattern: interpolationPattern("var"), - }}, + AnyOf: []jsonschema.Schema{ + s, + { + Type: jsonschema.StringType, + Pattern: interpolationPattern("var"), + }}, } case jsonschema.IntegerType, jsonschema.NumberType, jsonschema.BooleanType: // primitives can have variable values, or references like ${bundle.xyz} // or ${workspace.xyz} return jsonschema.Schema{ - AnyOf: []jsonschema.Schema{s, + AnyOf: []jsonschema.Schema{ + s, {Type: jsonschema.StringType, Pattern: interpolationPattern("resources")}, {Type: jsonschema.StringType, Pattern: interpolationPattern("bundle")}, {Type: jsonschema.StringType, Pattern: interpolationPattern("workspace")}, diff --git a/bundle/generated/embed.go b/bundle/schema/embed.go similarity index 52% rename from bundle/generated/embed.go rename to bundle/schema/embed.go index 1e674c630c..68f42a8e5c 100644 --- a/bundle/generated/embed.go +++ b/bundle/schema/embed.go @@ -1,6 +1,6 @@ -package generated +package schema import _ "embed" //go:embed jsonschema.json -var BundleSchema []byte +var Bytes []byte diff --git a/bundle/generated/jsonschema.json b/bundle/schema/jsonschema.json similarity index 100% rename from bundle/generated/jsonschema.json rename to bundle/schema/jsonschema.json diff --git a/cmd/bundle/schema.go b/cmd/bundle/schema.go index 967250ce34..6193f91e34 100644 --- a/cmd/bundle/schema.go +++ b/cmd/bundle/schema.go @@ -3,7 +3,7 @@ package bundle import ( _ "embed" - "github.com/databricks/cli/bundle/generated" + "github.com/databricks/cli/bundle/schema" "github.com/databricks/cli/cmd/root" "github.com/spf13/cobra" ) @@ -16,7 +16,7 @@ func newSchemaCommand() *cobra.Command { } cmd.RunE = func(cmd *cobra.Command, args []string) error { - _, err := cmd.OutOrStdout().Write(generated.BundleSchema) + _, err := cmd.OutOrStdout().Write(schema.Bytes) return err } From 75a571a56d3de5898ef1c7c95eaf52689128d76c Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Fri, 6 Sep 2024 15:22:06 +0200 Subject: [PATCH 47/62] slice for tags --- libs/jsonschema/from_type.go | 32 +++++++++++++++++--------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/libs/jsonschema/from_type.go b/libs/jsonschema/from_type.go index b07fef2f34..07c8716ddb 100644 --- a/libs/jsonschema/from_type.go +++ b/libs/jsonschema/from_type.go @@ -10,17 +10,19 @@ import ( "strings" ) -// Fields tagged "readonly" should not be emitted in the schema as they are -// computed at runtime, and should not be assigned a value by the bundle author. -const readonlyTag = "readonly" - -// Annotation for internal bundle fields that should not be exposed to customers. -// Fields can be tagged as "internal" to remove them from the generated schema. -const internalTag = "internal" - -// Annotation for bundle fields that have been deprecated. -// Fields tagged as "deprecated" are omitted from the generated schema. -const deprecatedTag = "deprecated" +var skipTags = []string{ + // Fields tagged "readonly" should not be emitted in the schema as they are + // computed at runtime, and should not be assigned a value by the bundle author. + "readonly", + + // Annotation for internal bundle fields that should not be exposed to customers. + // Fields can be tagged as "internal" to remove them from the generated schema. + "internal", + + // Annotation for bundle fields that have been deprecated. + // Fields tagged as "deprecated" are omitted from the generated schema. + "deprecated", +} type constructor struct { // Map of typ.PkgPath() + "." + typ.Name() to the schema for that type. @@ -251,10 +253,10 @@ func (c *constructor) fromTypeStruct(typ reflect.Type) (Schema, error) { bundleTags := strings.Split(structField.Tag.Get("bundle"), ",") // Fields marked as "readonly", "internal" or "deprecated" are skipped // while generating the schema - if slices.Contains(bundleTags, readonlyTag) || - slices.Contains(bundleTags, internalTag) || - slices.Contains(bundleTags, deprecatedTag) { - continue + for _, tag := range skipTags { + if slices.Contains(bundleTags, tag) { + continue + } } jsonTags := strings.Split(structField.Tag.Get("json"), ",") From 8cd26318e8b901acb17e11a33f13c116e3dd66e6 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Fri, 6 Sep 2024 15:30:32 +0200 Subject: [PATCH 48/62] fix test --- libs/jsonschema/from_type.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/libs/jsonschema/from_type.go b/libs/jsonschema/from_type.go index 07c8716ddb..0528a925ca 100644 --- a/libs/jsonschema/from_type.go +++ b/libs/jsonschema/from_type.go @@ -81,7 +81,7 @@ func (c *constructor) Definitions() map[string]any { return res } -// FromType converts a reflect.Type to a jsonschema.Schema. Nodes in the final JSON +// FromType converts a [reflect.Type] to a [Schema]. Nodes in the final JSON // schema are guaranteed to be one level deep, which is done using defining $defs // for every Go type and referring them using $ref in the corresponding node in // the JSON schema. @@ -253,11 +253,16 @@ func (c *constructor) fromTypeStruct(typ reflect.Type) (Schema, error) { bundleTags := strings.Split(structField.Tag.Get("bundle"), ",") // Fields marked as "readonly", "internal" or "deprecated" are skipped // while generating the schema + skip := false for _, tag := range skipTags { if slices.Contains(bundleTags, tag) { - continue + skip = true + break } } + if skip { + continue + } jsonTags := strings.Split(structField.Tag.Get("json"), ",") fieldName := jsonTags[0] From c69e6d97df3d3182d6c45a0606feb8658f4a7340 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Fri, 6 Sep 2024 15:32:24 +0200 Subject: [PATCH 49/62] loop over keys --- libs/jsonschema/from_type.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libs/jsonschema/from_type.go b/libs/jsonschema/from_type.go index 0528a925ca..30e70c1009 100644 --- a/libs/jsonschema/from_type.go +++ b/libs/jsonschema/from_type.go @@ -101,8 +101,8 @@ func FromType(typ reflect.Type, fns []func(typ reflect.Type, s Schema) Schema) ( } for _, fn := range fns { - for k, v := range c.definitions { - c.definitions[k] = fn(c.seen[k], v) + for k := range c.definitions { + c.definitions[k] = fn(c.seen[k], c.definitions[k]) } } From 2d62c0ceb7352323fbc6c0d5157d42dc69086b66 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Fri, 6 Sep 2024 16:29:55 +0200 Subject: [PATCH 50/62] Added testS --- bundle/schema/embed_test.go | 71 +++++++++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 bundle/schema/embed_test.go diff --git a/bundle/schema/embed_test.go b/bundle/schema/embed_test.go new file mode 100644 index 0000000000..ee0b5a615e --- /dev/null +++ b/bundle/schema/embed_test.go @@ -0,0 +1,71 @@ +package schema_test + +import ( + "encoding/json" + "testing" + + "github.com/databricks/cli/bundle/schema" + "github.com/databricks/cli/libs/jsonschema" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func walk(defs map[string]any, p ...string) jsonschema.Schema { + v, ok := defs[p[0]] + if !ok { + panic("not found: " + p[0]) + } + + if len(p) == 1 { + b, err := json.Marshal(v) + if err != nil { + panic(err) + } + res := jsonschema.Schema{} + err = json.Unmarshal(b, &res) + if err != nil { + panic(err) + } + return res + } + + return walk(v.(map[string]any), p[1:]...) +} + +func TestJsonSchema(t *testing.T) { + s := jsonschema.Schema{} + err := json.Unmarshal(schema.Bytes, &s) + require.NoError(t, err) + + // Assert job fields have their descriptions loaded. + resourceJob := walk(s.Definitions, "github.com", "databricks", "cli", "bundle", "config", "resources.Job") + fields := []string{"name", "continuous", "deployment", "tasks", "trigger"} + for _, field := range fields { + assert.NotEmpty(t, resourceJob.AnyOf[0].Properties[field].Description) + } + + // Assert descriptions were also loaded for a job task definition. + jobTask := walk(s.Definitions, "github.com", "databricks", "databricks-sdk-go", "service", "jobs.Task") + fields = []string{"notebook_task", "spark_jar_task", "spark_python_task", "spark_submit_task", "description", "depends_on", "environment_key", "for_each_task", "existing_cluster_id"} + for _, field := range fields { + assert.NotEmpty(t, jobTask.AnyOf[0].Properties[field].Description) + } + + // Assert descriptions are loaded for pipelines + pipeline := walk(s.Definitions, "github.com", "databricks", "cli", "bundle", "config", "resources.Pipeline") + fields = []string{"name", "catalog", "clusters", "channel", "continuous", "deployment", "development"} + for _, field := range fields { + assert.NotEmpty(t, pipeline.AnyOf[0].Properties[field].Description) + } + + // Assert enum values are loaded + schedule := walk(s.Definitions, "github.com", "databricks", "databricks-sdk-go", "service", "catalog.MonitorCronSchedule") + assert.Contains(t, schedule.AnyOf[0].Properties["pause_status"].Enum, "PAUSED") + assert.Contains(t, schedule.AnyOf[0].Properties["pause_status"].Enum, "UNPAUSED") + + providers := walk(s.Definitions, "github.com", "databricks", "databricks-sdk-go", "service", "jobs.GitProvider") + assert.Contains(t, providers.Enum, "gitHub") + assert.Contains(t, providers.Enum, "bitbucketCloud") + assert.Contains(t, providers.Enum, "gitHubEnterprise") + assert.Contains(t, providers.Enum, "bitbucketServer") +} From 5f48b58366f7ba49d153374f042314e1392141f9 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 9 Sep 2024 11:18:07 +0200 Subject: [PATCH 51/62] add test for bfs final schema --- libs/jsonschema/from_type.go | 7 ++++++ libs/jsonschema/from_type_test.go | 41 +++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+) diff --git a/libs/jsonschema/from_type.go b/libs/jsonschema/from_type.go index 30e70c1009..7a00124657 100644 --- a/libs/jsonschema/from_type.go +++ b/libs/jsonschema/from_type.go @@ -272,6 +272,13 @@ func (c *constructor) fromTypeStruct(typ reflect.Type) (Schema, error) { continue } + // Skip property if it is already present in the schema. + // This can happen if the same field is defined multiple times across + // a tree of embedded structs. For example see: TestHigherLevelEmbeddedFieldIsInSchema + if _, ok := res.Properties[fieldName]; ok { + continue + } + // "omitempty" tags in the Go SDK structs represent fields that not are // required to be present in the API payload. Thus its absence in the // tags list indicates that the field is required. diff --git a/libs/jsonschema/from_type_test.go b/libs/jsonschema/from_type_test.go index 4f5c552488..174ffad889 100644 --- a/libs/jsonschema/from_type_test.go +++ b/libs/jsonschema/from_type_test.go @@ -6,6 +6,7 @@ import ( "github.com/databricks/cli/libs/jsonschema/test_types" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestFromTypeBasic(t *testing.T) { @@ -166,6 +167,46 @@ func TestGetStructFields(t *testing.T) { assert.Equal(t, "InnerField", fields[3].Name) } +func TestHigherLevelEmbeddedFieldIsInSchema(t *testing.T) { + type Inner struct { + Override string `json:"override,omitempty"` + } + + type EmbeddedOne struct { + Inner + } + + type EmbeddedTwo struct { + Override int `json:"override,omitempty"` + } + + type Outer struct { + EmbeddedOne + EmbeddedTwo + } + + intRef := "#/$defs/int" + expected := Schema{ + Type: "object", + Definitions: map[string]any{ + "int": Schema{ + Type: "integer", + }, + }, + Properties: map[string]*Schema{ + "override": { + Reference: &intRef, + }, + }, + AdditionalProperties: false, + Required: []string{}, + } + + s, err := FromType(reflect.TypeOf(Outer{}), nil) + require.NoError(t, err) + assert.Equal(t, expected, s) +} + func TestFromTypeNested(t *testing.T) { type Inner struct { S string `json:"s"` From ab3dd7ab96225cd7843144fd9c5938d9c272d79b Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 9 Sep 2024 11:21:09 +0200 Subject: [PATCH 52/62] add comment --- libs/jsonschema/from_type.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/libs/jsonschema/from_type.go b/libs/jsonschema/from_type.go index 7a00124657..18a2b3ba5a 100644 --- a/libs/jsonschema/from_type.go +++ b/libs/jsonschema/from_type.go @@ -203,6 +203,10 @@ func (c *constructor) walk(typ reflect.Type) (string, error) { // This function returns all member fields of the provided type. // If the type has embedded (aka anonymous) fields, this function traverses // those in a breadth first manner +// +// BFS is important because we want the a field defined at a higher level embedded +// struct to be given preference over a field with the same name defined at a lower +// level embedded struct. For example see: TestHigherLevelEmbeddedFieldIsInSchema func getStructFields(typ reflect.Type) []reflect.StructField { fields := []reflect.StructField{} bfsQueue := list.New() From 804d379100a72e409517f14bf73454e1411bc726 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 9 Sep 2024 13:57:33 +0200 Subject: [PATCH 53/62] address comments --- .codegen.json | 4 ++-- .gitattributes | 2 +- cmd/bundle/schema.go | 2 -- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/.codegen.json b/.codegen.json index 62f75df021..077d072bba 100644 --- a/.codegen.json +++ b/.codegen.json @@ -11,10 +11,10 @@ "toolchain": { "required": ["go"], "post_generate": [ - "go run ./bundle/internal/schema/*.go ./bundle/generated/jsonschema.json", + "go run ./bundle/internal/schema/*.go ./bundle/schema/jsonschema.json", "echo 'bundle/internal/tf/schema/\\*.go linguist-generated=true' >> ./.gitattributes", "echo 'go.sum linguist-generated=true' >> ./.gitattributes", - "echo 'bundle/generated/jsonschema.json linguist-generated=true' >> ./.gitattributes" + "echo 'bundle/schema/jsonschema.json linguist-generated=true' >> ./.gitattributes" ] } } diff --git a/.gitattributes b/.gitattributes index 17cef7d263..f35c4f81d9 100755 --- a/.gitattributes +++ b/.gitattributes @@ -120,4 +120,4 @@ cmd/workspace/workspace-conf/workspace-conf.go linguist-generated=true cmd/workspace/workspace/workspace.go linguist-generated=true bundle/internal/tf/schema/\*.go linguist-generated=true go.sum linguist-generated=true -bundle/generated/jsonschema.json linguist-generated=true +bundle/schema/jsonschema.json linguist-generated=true diff --git a/cmd/bundle/schema.go b/cmd/bundle/schema.go index 6193f91e34..480618ed79 100644 --- a/cmd/bundle/schema.go +++ b/cmd/bundle/schema.go @@ -1,8 +1,6 @@ package bundle import ( - _ "embed" - "github.com/databricks/cli/bundle/schema" "github.com/databricks/cli/cmd/root" "github.com/spf13/cobra" From 65ed301b06be790bf82ddc921657d128ee07ddfa Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 10 Sep 2024 15:27:39 +0200 Subject: [PATCH 54/62] make interpolation pattern better --- bundle/internal/schema/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bundle/internal/schema/main.go b/bundle/internal/schema/main.go index 5a6ef7f248..3c1fb5da05 100644 --- a/bundle/internal/schema/main.go +++ b/bundle/internal/schema/main.go @@ -13,7 +13,7 @@ import ( ) func interpolationPattern(s string) string { - return fmt.Sprintf(`\$\{(%s(\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\[[0-9]+\])*)*(\[[0-9]+\])*)\}`, s) + return fmt.Sprintf(`\$\{(%s(\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\[[0-9]+\])*)+)\}`, s) } func addInterpolationPatterns(typ reflect.Type, s jsonschema.Schema) jsonschema.Schema { From e1a8e03cea9daa08a1bace737c6c5d7235f15504 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 10 Sep 2024 15:36:52 +0200 Subject: [PATCH 55/62] regenerate schema with interpolation patterns --- bundle/schema/jsonschema.json | 405 +++++++++++++++++----------------- 1 file changed, 206 insertions(+), 199 deletions(-) diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index d791e0799f..4fe978b877 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -7,23 +7,23 @@ }, { "type": "string", - "pattern": "\\$\\{(resources(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(resources(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" }, { "type": "string", - "pattern": "\\$\\{(bundle(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(bundle(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" }, { "type": "string", - "pattern": "\\$\\{(workspace(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(workspace(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" }, { "type": "string", - "pattern": "\\$\\{(artifacts(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(artifacts(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -34,23 +34,23 @@ }, { "type": "string", - "pattern": "\\$\\{(resources(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(resources(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" }, { "type": "string", - "pattern": "\\$\\{(bundle(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(bundle(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" }, { "type": "string", - "pattern": "\\$\\{(workspace(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(workspace(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" }, { "type": "string", - "pattern": "\\$\\{(artifacts(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(artifacts(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -79,7 +79,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -182,7 +182,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -227,7 +227,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -272,7 +272,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -313,7 +313,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -342,7 +342,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -439,7 +439,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -509,7 +509,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -551,7 +551,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -591,7 +591,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -638,7 +638,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -692,7 +692,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -712,7 +712,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -747,7 +747,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -770,7 +770,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -796,7 +796,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -816,7 +816,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -836,7 +836,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -868,7 +868,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -891,7 +891,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -929,7 +929,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -952,7 +952,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -1005,7 +1005,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -1064,7 +1064,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] } @@ -1107,7 +1107,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -1128,7 +1128,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -1146,7 +1146,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -1199,7 +1199,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -1248,7 +1248,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -1273,7 +1273,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -1285,7 +1285,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -1311,7 +1311,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -1332,7 +1332,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -1354,7 +1354,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -1406,7 +1406,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -1444,7 +1444,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -1475,7 +1475,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -1497,7 +1497,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -1615,7 +1615,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -1649,7 +1649,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -1671,7 +1671,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -1692,7 +1692,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -1726,7 +1726,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -1763,7 +1763,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -1793,7 +1793,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -1835,7 +1835,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -1877,7 +1877,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -1898,7 +1898,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -1920,7 +1920,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -1949,7 +1949,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -1974,7 +1974,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -1999,7 +1999,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -2053,7 +2053,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -2074,7 +2074,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -2095,7 +2095,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -2116,7 +2116,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -2154,7 +2154,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -2184,7 +2184,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -2214,7 +2214,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -2259,7 +2259,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -2288,7 +2288,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -2318,7 +2318,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -2357,7 +2357,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -2403,7 +2403,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -2429,7 +2429,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -2454,7 +2454,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -2507,7 +2507,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -2531,7 +2531,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -2553,7 +2553,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -2579,7 +2579,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -2602,7 +2602,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -2633,7 +2633,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -2688,7 +2688,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -2706,7 +2706,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -2739,7 +2739,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -2772,7 +2772,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -2798,7 +2798,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -2823,7 +2823,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -2857,7 +2857,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -2878,7 +2878,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -2945,7 +2945,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -2979,7 +2979,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -3008,7 +3008,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -3026,7 +3026,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -3067,7 +3067,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -3096,7 +3096,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -3129,7 +3129,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -3154,7 +3154,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -3175,7 +3175,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -3197,7 +3197,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -3227,7 +3227,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -3359,7 +3359,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -3384,7 +3384,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -3422,7 +3422,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -3448,7 +3448,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -3481,7 +3481,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -3501,7 +3501,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -3535,7 +3535,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -3557,7 +3557,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -3579,7 +3579,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -3650,7 +3650,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -3675,7 +3675,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -3695,7 +3695,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -3720,7 +3720,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -3742,7 +3742,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -3764,7 +3764,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -3794,7 +3794,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -3824,7 +3824,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -3836,7 +3836,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -3854,7 +3854,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -3876,7 +3876,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -3966,7 +3966,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -4000,7 +4000,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -4025,7 +4025,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -4059,7 +4059,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -4079,7 +4079,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -4113,7 +4113,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -4155,7 +4155,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -4185,7 +4185,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -4208,7 +4208,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -4256,7 +4256,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -4281,7 +4281,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -4311,7 +4311,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -4337,7 +4337,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -4366,7 +4366,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -4396,7 +4396,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -4421,7 +4421,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -4494,7 +4494,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -4523,7 +4523,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -4570,7 +4570,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -4590,7 +4590,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -4627,7 +4627,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -4659,7 +4659,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -4717,7 +4717,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -4734,6 +4734,14 @@ "description": "ARN of the instance profile that the served model will use to access AWS resources.", "$ref": "#/$defs/string" }, + "max_provisioned_throughput": { + "description": "The maximum tokens per second that the endpoint can scale up to.", + "$ref": "#/$defs/int" + }, + "min_provisioned_throughput": { + "description": "The minimum tokens per second that the endpoint can scale down to.", + "$ref": "#/$defs/int" + }, "model_name": { "description": "The name of the model in Databricks Model Registry to be served or if the model resides in Unity Catalog, the full name of model,\nin the form of __catalog_name__.__schema_name__.__model_name__.\n", "$ref": "#/$defs/string" @@ -4775,13 +4783,12 @@ "required": [ "model_name", "model_version", - "scale_to_zero_enabled", - "workload_size" + "scale_to_zero_enabled" ] }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -4805,7 +4812,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] } @@ -4820,23 +4827,23 @@ }, { "type": "string", - "pattern": "\\$\\{(resources(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(resources(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" }, { "type": "string", - "pattern": "\\$\\{(bundle(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(bundle(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" }, { "type": "string", - "pattern": "\\$\\{(workspace(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(workspace(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" }, { "type": "string", - "pattern": "\\$\\{(artifacts(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(artifacts(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -4847,23 +4854,23 @@ }, { "type": "string", - "pattern": "\\$\\{(resources(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(resources(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" }, { "type": "string", - "pattern": "\\$\\{(bundle(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(bundle(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" }, { "type": "string", - "pattern": "\\$\\{(workspace(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(workspace(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" }, { "type": "string", - "pattern": "\\$\\{(artifacts(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(artifacts(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -4884,7 +4891,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -4898,7 +4905,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -4912,7 +4919,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -4926,7 +4933,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -4940,7 +4947,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -4954,7 +4961,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -4968,7 +4975,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -4982,7 +4989,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -4996,7 +5003,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] } @@ -5011,7 +5018,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -5025,7 +5032,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -5039,7 +5046,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] } @@ -5057,7 +5064,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] } @@ -5078,7 +5085,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -5092,7 +5099,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] } @@ -5107,7 +5114,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] } @@ -5125,7 +5132,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -5139,7 +5146,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -5153,7 +5160,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -5167,7 +5174,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -5181,7 +5188,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -5195,7 +5202,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -5209,7 +5216,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -5223,7 +5230,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -5237,7 +5244,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -5251,7 +5258,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -5265,7 +5272,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -5279,7 +5286,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -5293,7 +5300,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -5307,7 +5314,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -5321,7 +5328,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -5335,7 +5342,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -5349,7 +5356,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -5363,7 +5370,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -5377,7 +5384,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -5391,7 +5398,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -5405,7 +5412,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -5419,7 +5426,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -5433,7 +5440,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] }, @@ -5447,7 +5454,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] } @@ -5465,7 +5472,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] } From 48af9d1c570578e9d05ea2b78498a0ccedf43b4c Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 4 Sep 2024 15:44:53 +0200 Subject: [PATCH 56/62] Change variables block to map of any in `Targets` struct --- bundle/config/target.go | 22 ++++++++++++++++++++-- bundle/schema/jsonschema.json | 16 +++++++++++++++- 2 files changed, 35 insertions(+), 3 deletions(-) diff --git a/bundle/config/target.go b/bundle/config/target.go index a2ef4d7356..4b20fac15b 100644 --- a/bundle/config/target.go +++ b/bundle/config/target.go @@ -2,7 +2,6 @@ package config import ( "github.com/databricks/cli/bundle/config/resources" - "github.com/databricks/cli/bundle/config/variable" "github.com/databricks/databricks-sdk-go/service/jobs" ) @@ -38,7 +37,26 @@ type Target struct { // Override default values or lookup name for defined variables // Does not permit defining new variables or redefining existing ones // in the scope of an target - Variables map[string]*variable.Variable `json:"variables,omitempty"` + // + // There are two valid ways to define a variable override in a target: + // 1. Direct value override. We normalize this to the variable.Variable + // struct format when loading the configuration YAML: + // + // variables: + // foo: "value" + // + // 2. Override matching the variable.Variable struct. + // + // variables: + // foo: + // default: "value" + // + // OR + // + // variables: + // foo: + // lookup: "resource_name" + Variables map[string]any `json:"variables,omitempty"` Git Git `json:"git,omitempty"` diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index 4fe978b877..6de293648e 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -995,7 +995,7 @@ "$ref": "#/$defs/github.com/databricks/cli/bundle/config.Sync" }, "variables": { - "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/variable.Variable" + "$ref": "#/$defs/map/interface" }, "workspace": { "$ref": "#/$defs/github.com/databricks/cli/bundle/config.Workspace" @@ -5054,6 +5054,20 @@ } } }, + "interface": { + "anyOf": [ + { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/interface" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, "string": { "anyOf": [ { From 176bb4b382d7da4b26a6287a5edd4e2bd603b1c3 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 4 Sep 2024 15:57:04 +0200 Subject: [PATCH 57/62] fix test --- bundle/config/root_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bundle/config/root_test.go b/bundle/config/root_test.go index c95e6e86cd..b40a5f2b7b 100644 --- a/bundle/config/root_test.go +++ b/bundle/config/root_test.go @@ -139,12 +139,12 @@ func TestRootMergeTargetOverridesWithVariables(t *testing.T) { }, Targets: map[string]*Target{ "development": { - Variables: map[string]*variable.Variable{ - "foo": { + Variables: map[string]any{ + "foo": variable.Variable{ Default: "bar", Description: "wrong", }, - "complex": { + "complex": variable.Variable{ Type: "wrong", Description: "wrong", Default: map[string]interface{}{ From 507b99dca1813989dc24b57357438524258c8af7 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 4 Sep 2024 16:18:34 +0200 Subject: [PATCH 58/62] - --- bundle/config/root_test.go | 6 ++-- bundle/config/target.go | 5 ++- bundle/internal/schema/main.go | 13 +++++++ bundle/schema/jsonschema.json | 64 ++++++++++++++++++++++++++-------- 4 files changed, 69 insertions(+), 19 deletions(-) diff --git a/bundle/config/root_test.go b/bundle/config/root_test.go index b40a5f2b7b..a3934267be 100644 --- a/bundle/config/root_test.go +++ b/bundle/config/root_test.go @@ -139,12 +139,12 @@ func TestRootMergeTargetOverridesWithVariables(t *testing.T) { }, Targets: map[string]*Target{ "development": { - Variables: map[string]any{ - "foo": variable.Variable{ + Variables: map[string]*TargetVariable{ + "foo": { Default: "bar", Description: "wrong", }, - "complex": variable.Variable{ + "complex": { Type: "wrong", Description: "wrong", Default: map[string]interface{}{ diff --git a/bundle/config/target.go b/bundle/config/target.go index 4b20fac15b..718dd3acb7 100644 --- a/bundle/config/target.go +++ b/bundle/config/target.go @@ -2,11 +2,14 @@ package config import ( "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/bundle/config/variable" "github.com/databricks/databricks-sdk-go/service/jobs" ) type Mode string +type TargetVariable variable.Variable + // Target defines overrides for a single target. // This structure is recursively merged into the root configuration. type Target struct { @@ -56,7 +59,7 @@ type Target struct { // variables: // foo: // lookup: "resource_name" - Variables map[string]any `json:"variables,omitempty"` + Variables map[string]*TargetVariable `json:"variables,omitempty"` Git Git `json:"git,omitempty"` diff --git a/bundle/internal/schema/main.go b/bundle/internal/schema/main.go index 3c1fb5da05..51eb1e2668 100644 --- a/bundle/internal/schema/main.go +++ b/bundle/internal/schema/main.go @@ -21,6 +21,19 @@ func addInterpolationPatterns(typ reflect.Type, s jsonschema.Schema) jsonschema. return s } + // The variables block in a target override allows for directly specifying + // the value if it is a primitive type. + if typ == reflect.TypeOf(config.TargetVariable{}) { + return jsonschema.Schema{ + AnyOf: []jsonschema.Schema{s, + {Type: jsonschema.StringType}, + {Type: jsonschema.BooleanType}, + {Type: jsonschema.IntegerType}, + {Type: jsonschema.NumberType}, + }, + } + } + switch s.Type { case jsonschema.ArrayType, jsonschema.ObjectType: // arrays and objects can have complex variable values specified. diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index 6de293648e..95edbae180 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -995,7 +995,7 @@ "$ref": "#/$defs/github.com/databricks/cli/bundle/config.Sync" }, "variables": { - "$ref": "#/$defs/map/interface" + "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config.TargetVariable" }, "workspace": { "$ref": "#/$defs/github.com/databricks/cli/bundle/config.Workspace" @@ -1009,6 +1009,40 @@ } ] }, + "config.TargetVariable": { + "anyOf": [ + { + "type": "object", + "properties": { + "default": { + "$ref": "#/$defs/interface" + }, + "description": { + "$ref": "#/$defs/string" + }, + "lookup": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/variable.Lookup" + }, + "type": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/variable.VariableType" + } + }, + "additionalProperties": false + }, + { + "type": "string" + }, + { + "type": "boolean" + }, + { + "type": "integer" + }, + { + "type": "number" + } + ] + }, "config.Workspace": { "anyOf": [ { @@ -5049,25 +5083,25 @@ "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] + }, + "config.TargetVariable": { + "anyOf": [ + { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config.TargetVariable" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] } } } } }, - "interface": { - "anyOf": [ - { - "type": "object", - "additionalProperties": { - "$ref": "#/$defs/interface" - } - }, - { - "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - } - ] - }, "string": { "anyOf": [ { From f26027ab1cc946445601e07507d36cba409fb77b Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 4 Sep 2024 16:26:20 +0200 Subject: [PATCH 59/62] comment --- bundle/config/target.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/bundle/config/target.go b/bundle/config/target.go index 718dd3acb7..a3cd106b57 100644 --- a/bundle/config/target.go +++ b/bundle/config/target.go @@ -8,6 +8,9 @@ import ( type Mode string +// We alias it here to override the JSON schema associated with a variable value +// in a target override. This is because we allow for directly specifying the value +// in addition to the variable.Variable struct format in a target override. type TargetVariable variable.Variable // Target defines overrides for a single target. From 9689a79f22bc54ae073991620cb598d65986cac5 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 4 Sep 2024 17:42:02 +0200 Subject: [PATCH 60/62] address comments --- bundle/config/root_test.go | 2 +- bundle/config/target.go | 7 +------ bundle/config/variable/variable.go | 5 +++++ bundle/internal/schema/main.go | 2 +- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/bundle/config/root_test.go b/bundle/config/root_test.go index a3934267be..d2c7a9b1fa 100644 --- a/bundle/config/root_test.go +++ b/bundle/config/root_test.go @@ -139,7 +139,7 @@ func TestRootMergeTargetOverridesWithVariables(t *testing.T) { }, Targets: map[string]*Target{ "development": { - Variables: map[string]*TargetVariable{ + Variables: map[string]*variable.TargetVariable{ "foo": { Default: "bar", Description: "wrong", diff --git a/bundle/config/target.go b/bundle/config/target.go index a3cd106b57..fc6ba7b5bb 100644 --- a/bundle/config/target.go +++ b/bundle/config/target.go @@ -8,11 +8,6 @@ import ( type Mode string -// We alias it here to override the JSON schema associated with a variable value -// in a target override. This is because we allow for directly specifying the value -// in addition to the variable.Variable struct format in a target override. -type TargetVariable variable.Variable - // Target defines overrides for a single target. // This structure is recursively merged into the root configuration. type Target struct { @@ -62,7 +57,7 @@ type Target struct { // variables: // foo: // lookup: "resource_name" - Variables map[string]*TargetVariable `json:"variables,omitempty"` + Variables map[string]*variable.TargetVariable `json:"variables,omitempty"` Git Git `json:"git,omitempty"` diff --git a/bundle/config/variable/variable.go b/bundle/config/variable/variable.go index ba94f9c8a5..2362ad10d9 100644 --- a/bundle/config/variable/variable.go +++ b/bundle/config/variable/variable.go @@ -16,6 +16,11 @@ const ( VariableTypeComplex VariableType = "complex" ) +// We alias it here to override the JSON schema associated with a variable value +// in a target override. This is because we allow for directly specifying the value +// in addition to the variable.Variable struct format in a target override. +type TargetVariable Variable + // An input variable for the bundle config type Variable struct { // A type of the variable. This is used to validate the value of the variable diff --git a/bundle/internal/schema/main.go b/bundle/internal/schema/main.go index 51eb1e2668..2e4e67355c 100644 --- a/bundle/internal/schema/main.go +++ b/bundle/internal/schema/main.go @@ -23,7 +23,7 @@ func addInterpolationPatterns(typ reflect.Type, s jsonschema.Schema) jsonschema. // The variables block in a target override allows for directly specifying // the value if it is a primitive type. - if typ == reflect.TypeOf(config.TargetVariable{}) { + if typ == reflect.TypeOf(variable.TargetVariable{}) { return jsonschema.Schema{ AnyOf: []jsonschema.Schema{s, {Type: jsonschema.StringType}, From c365fd424b1aee2304f76a8339cd14807198e3c0 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Thu, 5 Sep 2024 13:37:45 +0200 Subject: [PATCH 61/62] Address comments --- bundle/internal/schema/main.go | 15 +++--- bundle/schema/jsonschema.json | 87 +++++++++++++++------------------- 2 files changed, 47 insertions(+), 55 deletions(-) diff --git a/bundle/internal/schema/main.go b/bundle/internal/schema/main.go index 2e4e67355c..4a2371472f 100644 --- a/bundle/internal/schema/main.go +++ b/bundle/internal/schema/main.go @@ -22,14 +22,17 @@ func addInterpolationPatterns(typ reflect.Type, s jsonschema.Schema) jsonschema. } // The variables block in a target override allows for directly specifying - // the value if it is a primitive type. + // the value of the variable. if typ == reflect.TypeOf(variable.TargetVariable{}) { return jsonschema.Schema{ - AnyOf: []jsonschema.Schema{s, - {Type: jsonschema.StringType}, - {Type: jsonschema.BooleanType}, - {Type: jsonschema.IntegerType}, - {Type: jsonschema.NumberType}, + AnyOf: []jsonschema.Schema{ + // We keep the original schema so that autocomplete suggestions + // continue to work. + s, + // All values are valid for a variable value, be it primitive types + // like string/bool or complex ones like objects/arrays. Thus we override + // the schema to allow all valid JSON values. + {}, }, } } diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index 95edbae180..16adb3af4d 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -642,6 +642,29 @@ } ] }, + "variable.TargetVariable": { + "anyOf": [ + { + "type": "object", + "properties": { + "default": { + "$ref": "#/$defs/interface" + }, + "description": { + "$ref": "#/$defs/string" + }, + "lookup": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/variable.Lookup" + }, + "type": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/variable.VariableType" + } + }, + "additionalProperties": false + }, + {} + ] + }, "variable.Variable": { "type": "object", "properties": { @@ -995,7 +1018,7 @@ "$ref": "#/$defs/github.com/databricks/cli/bundle/config.Sync" }, "variables": { - "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config.TargetVariable" + "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/variable.TargetVariable" }, "workspace": { "$ref": "#/$defs/github.com/databricks/cli/bundle/config.Workspace" @@ -1009,40 +1032,6 @@ } ] }, - "config.TargetVariable": { - "anyOf": [ - { - "type": "object", - "properties": { - "default": { - "$ref": "#/$defs/interface" - }, - "description": { - "$ref": "#/$defs/string" - }, - "lookup": { - "$ref": "#/$defs/github.com/databricks/cli/bundle/config/variable.Lookup" - }, - "type": { - "$ref": "#/$defs/github.com/databricks/cli/bundle/config/variable.VariableType" - } - }, - "additionalProperties": false - }, - { - "type": "string" - }, - { - "type": "boolean" - }, - { - "type": "integer" - }, - { - "type": "number" - } - ] - }, "config.Workspace": { "anyOf": [ { @@ -5027,6 +5016,20 @@ } ] }, + "variable.TargetVariable": { + "anyOf": [ + { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/variable.TargetVariable" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + } + ] + }, "variable.Variable": { "anyOf": [ { @@ -5083,20 +5086,6 @@ "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] - }, - "config.TargetVariable": { - "anyOf": [ - { - "type": "object", - "additionalProperties": { - "$ref": "#/$defs/github.com/databricks/cli/bundle/config.TargetVariable" - } - }, - { - "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" - } - ] } } } From 3b06a94cfc73f2b3d77a451b4c5d6cd30c80f3c1 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 10 Sep 2024 15:39:46 +0200 Subject: [PATCH 62/62] regenerate schema with interpolation patterns --- bundle/schema/jsonschema.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index 16adb3af4d..2db1a5ab4f 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -5026,7 +5026,7 @@ }, { "type": "string", - "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}" + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" } ] },