Added support for complex variables

bundle/config/mutator/resolve_variable_references.go

@@ -17,6 +17,7 @@ type resolveVariableReferences struct {
 	prefixes []string
 	pattern  dyn.Pattern
 	lookupFn func(dyn.Value, dyn.Path) (dyn.Value, error)
+	skipFn   func(dyn.Value) bool
 }
 
 func ResolveVariableReferences(prefixes ...string) bundle.Mutator {
@@ -36,7 +37,11 @@ func ResolveVariableReferencesInComplexVariables() bundle.Mutator {
 		"bundle",
 		"workspace",
 		"variables",
-	}, pattern: dyn.NewPattern(dyn.Key("variables"), dyn.AnyKey(), dyn.Key("value")), lookupFn: lookupForVariables}
+	},
+		pattern:  dyn.NewPattern(dyn.Key("variables"), dyn.AnyKey(), dyn.Key("value")),
+		lookupFn: lookupForComplexVariables,
+		skipFn:   skipResolvingInNonComplexVariables,
+	}
 }
 
 func lookup(v dyn.Value, path dyn.Path) (dyn.Value, error) {
@@ -46,6 +51,34 @@ func lookup(v dyn.Value, path dyn.Path) (dyn.Value, error) {
 	return dyn.GetByPath(v, path)
 }
 
+func lookupForComplexVariables(v dyn.Value, path dyn.Path) (dyn.Value, error) {
+	if path[0].Key() != "variables" {
+		return lookup(v, path)
+	}
+
+	varV, err := dyn.GetByPath(v, path[:len(path)-1])
+	if err != nil {
+		return dyn.InvalidValue, err
+	}
+
+	var vv variable.Variable
+	err = convert.ToTyped(&vv, varV)
+	if err != nil {
+		return dyn.InvalidValue, err
+	}
+
+	if vv.Type == variable.VariableTypeComplex {
+		return dyn.InvalidValue, fmt.Errorf("complex variables cannot contain references to another complex variables")
+	}
+
+	return lookup(v, path)
+}
+
+func skipResolvingInNonComplexVariables(v dyn.Value) bool {
+	_, ok := v.AsMap()
+	return !ok
+}
+
 func lookupForVariables(v dyn.Value, path dyn.Path) (dyn.Value, error) {
 	if path[0].Key() != "variables" {
 		return lookup(v, path)
@@ -108,17 +141,27 @@ func (m *resolveVariableReferences) Apply(ctx context.Context, b *bundle.Bundle)
 			// Resolve variable references in all values.
 			return dynvar.Resolve(v, func(path dyn.Path) (dyn.Value, error) {
 				// Rewrite the shorthand path ${var.foo} into ${variables.foo.value}.
-				if path.HasPrefix(varPath) && len(path) == 2 {
-					path = dyn.NewPath(
+				if path.HasPrefix(varPath) {
+					newPath := dyn.NewPath(
 						dyn.Key("variables"),
 						path[1],
 						dyn.Key("value"),
 					)
+
+					if len(path) > 2 {
+						newPath = newPath.Append(path[2:]...)
+					}
+
+					path = newPath
 				}
 
 				// Perform resolution only if the path starts with one of the specified prefixes.
 				for _, prefix := range prefixes {
 					if path.HasPrefix(prefix) {
+						// Skip resolution if there is a skip function and it returns true.
+						if m.skipFn != nil && m.skipFn(v) {
+							return dyn.InvalidValue, dynvar.ErrSkipResolution
+						}
 						return m.lookupFn(normalized, path)
 					}
 				}

bundle/config/mutator/resolve_variable_references_test.go

@@ -191,3 +191,118 @@ func TestResolveVariableReferencesForPrimitiveNonStringFields(t *testing.T) {
 	assert.Equal(t, 2, b.Config.Resources.Jobs["job1"].JobSettings.Tasks[0].NewCluster.Autoscale.MaxWorkers)
 	assert.Equal(t, 0.5, b.Config.Resources.Jobs["job1"].JobSettings.Tasks[0].NewCluster.AzureAttributes.SpotBidMaxPrice)
 }
+
+func TestResolveComplexVariable(t *testing.T) {
+	b := &bundle.Bundle{
+		Config: config.Root{
+			Bundle: config.Bundle{
+				Name: "example",
+			},
+			Variables: map[string]*variable.Variable{
+				"cluster": {
+					Value: map[string]any{
+						"node_type_id": "Standard_DS3_v2",
+						"num_workers":  2,
+					},
+				},
+			},
+
+			Resources: config.Resources{
+				Jobs: map[string]*resources.Job{
+					"job1": {
+						JobSettings: &jobs.JobSettings{
+							JobClusters: []jobs.JobCluster{
+								{
+									NewCluster: compute.ClusterSpec{
+										NodeTypeId: "random",
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+
+	ctx := context.Background()
+
+	// Assign the variables to the dynamic configuration.
+	diags := bundle.ApplyFunc(ctx, b, func(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
+		err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) {
+			var p dyn.Path
+			var err error
+
+			// Set the notification settings.
+			p = dyn.MustPathFromString("resources.jobs.job1.job_clusters[0]")
+			v, err = dyn.SetByPath(v, p.Append(dyn.Key("new_cluster")), dyn.V("${var.cluster}"))
+			require.NoError(t, err)
+
+			return v, nil
+		})
+		return diag.FromErr(err)
+	})
+	require.NoError(t, diags.Error())
+
+	diags = bundle.Apply(ctx, b, ResolveVariableReferences("bundle", "workspace", "variables"))
+	require.NoError(t, diags.Error())
+	require.Equal(t, "Standard_DS3_v2", b.Config.Resources.Jobs["job1"].JobSettings.JobClusters[0].NewCluster.NodeTypeId)
+	require.Equal(t, 2, b.Config.Resources.Jobs["job1"].JobSettings.JobClusters[0].NewCluster.NumWorkers)
+}
+
+func TestResolveComplexVariableReferencesToFields(t *testing.T) {
+	b := &bundle.Bundle{
+		Config: config.Root{
+			Bundle: config.Bundle{
+				Name: "example",
+			},
+			Variables: map[string]*variable.Variable{
+				"cluster": {
+					Value: map[string]any{
+						"node_type_id": "Standard_DS3_v2",
+						"num_workers":  2,
+					},
+				},
+			},
+
+			Resources: config.Resources{
+				Jobs: map[string]*resources.Job{
+					"job1": {
+						JobSettings: &jobs.JobSettings{
+							JobClusters: []jobs.JobCluster{
+								{
+									NewCluster: compute.ClusterSpec{
+										NodeTypeId: "random",
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+
+	ctx := context.Background()
+
+	// Assign the variables to the dynamic configuration.
+	diags := bundle.ApplyFunc(ctx, b, func(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
+		err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) {
+			var p dyn.Path
+			var err error
+
+			// Set the notification settings.
+			p = dyn.MustPathFromString("resources.jobs.job1.job_clusters[0].new_cluster")
+			v, err = dyn.SetByPath(v, p.Append(dyn.Key("node_type_id")), dyn.V("${var.cluster.node_type_id}"))
+			require.NoError(t, err)
+
+			return v, nil
+		})
+		return diag.FromErr(err)
+	})
+	require.NoError(t, diags.Error())
+
+	diags = bundle.Apply(ctx, b, ResolveVariableReferences("bundle", "workspace", "variables"))
+	require.NoError(t, diags.Error())
+	require.Equal(t, "Standard_DS3_v2", b.Config.Resources.Jobs["job1"].JobSettings.JobClusters[0].NewCluster.NodeTypeId)
+}

bundle/tests/complex_variables_test.go

@@ -38,6 +38,8 @@ func TestComplexVariables(t *testing.T) {
 	require.Contains(t, b.Config.Resources.Jobs["my_job"].Tasks[0].Libraries, compute.Library{
 		Whl: "/path/to/whl",
 	})
+
+	require.Equal(t, "task with spark version 13.2.x-scala2.11 and jar /path/to/jar", b.Config.Resources.Jobs["my_job"].Tasks[0].TaskKey)
 }
 
 func TestComplexVariablesOverride(t *testing.T) {

bundle/tests/variables/complex/databricks.yml

@@ -10,7 +10,8 @@ resources:
       tasks:
       - task_key: test
         job_cluster_key: key
-        libraries: ${var.libraries}
+        libraries: ${variables.libraries.value}
+        task_key: "task with spark version ${var.cluster.spark_version} and jar ${var.libraries[0].jar}"
 
 variables:
   node_type:

libs/dyn/convert/from_typed.go

@@ -37,10 +37,6 @@ func FromTyped(src any, ref dyn.Value) (dyn.Value, error) {
 // Private implementation of FromTyped that allows for additional options not exposed
 // in the public API.
 func fromTyped(src any, ref dyn.Value, options ...fromTypedOptions) (dyn.Value, error) {
-	if src == nil {
-		return dyn.NilValue, nil
-	}
-
 	srcv := reflect.ValueOf(src)
 
 	// Dereference pointer if necessary

libs/dyn/dynvar/ref.go

@@ -6,7 +6,7 @@ import (
 	"github.com/databricks/cli/libs/dyn"
 )
 
-const VariableRegex = `\$\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\}`
+const VariableRegex = `\$\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\[[0-9]+\])?)*(\[[0-9]+\])?)\}`
 
 var re = regexp.MustCompile(VariableRegex)
 

libs/dyn/dynvar/resolve_test.go

@@ -247,3 +247,49 @@ func TestResolveWithInterpolateAliasedRef(t *testing.T) {
 	assert.Equal(t, "a", getByPath(t, out, "b").MustString())
 	assert.Equal(t, "a", getByPath(t, out, "c").MustString())
 }
+
+func TestResolveIndexedRefs(t *testing.T) {
+	in := dyn.V(map[string]dyn.Value{
+		"slice": dyn.V([]dyn.Value{dyn.V("a"), dyn.V("b")}),
+		"a":     dyn.V("a: ${slice[0]}"),
+	})
+
+	out, err := dynvar.Resolve(in, dynvar.DefaultLookup(in))
+	require.NoError(t, err)
+
+	assert.Equal(t, "a: a", getByPath(t, out, "a").MustString())
+}
+
+func TestResolveIndexedRefsFromMap(t *testing.T) {
+	in := dyn.V(map[string]dyn.Value{
+		"map": dyn.V(
+			map[string]dyn.Value{
+				"slice": dyn.V([]dyn.Value{dyn.V("a")}),
+			}),
+		"a": dyn.V("a: ${map.slice[0]}"),
+	})
+
+	out, err := dynvar.Resolve(in, dynvar.DefaultLookup(in))
+	require.NoError(t, err)
+
+	assert.Equal(t, "a: a", getByPath(t, out, "a").MustString())
+}
+
+func TestResolveMapFieldFromIndexedRefs(t *testing.T) {
+	in := dyn.V(map[string]dyn.Value{
+		"map": dyn.V(
+			map[string]dyn.Value{
+				"slice": dyn.V([]dyn.Value{
+					dyn.V(map[string]dyn.Value{
+						"value": dyn.V("a"),
+					}),
+				}),
+			}),
+		"a": dyn.V("a: ${map.slice[0].value}"),
+	})
+
+	out, err := dynvar.Resolve(in, dynvar.DefaultLookup(in))
+	require.NoError(t, err)
+
+	assert.Equal(t, "a: a", getByPath(t, out, "a").MustString())
+}

libs/dyn/path.go

@@ -18,11 +18,11 @@ func (c pathComponent) Index() int {
 	return c.index
 }
 
-func (c pathComponent) isKey() bool {
+func (c pathComponent) IsKey() bool {
 	return c.key != ""
 }
 
-func (c pathComponent) isIndex() bool {
+func (c pathComponent) IsIndex() bool {
 	return c.key == ""
 }
 

libs/dyn/visit.go

@@ -68,7 +68,7 @@ func (component pathComponent) visit(v Value, prefix Path, suffix Pattern, opts
 	path := append(prefix, component)
 
 	switch {
-	case component.isKey():
+	case component.IsKey():
 		// Expect a map to be set if this is a key.
 		m, ok := v.AsMap()
 		if !ok {
@@ -101,7 +101,7 @@ func (component pathComponent) visit(v Value, prefix Path, suffix Pattern, opts
 			l: v.l,
 		}, nil
 
-	case component.isIndex():
+	case component.IsIndex():
 		// Expect a sequence to be set if this is an index.
 		s, ok := v.AsSequence()
 		if !ok {

libs/dyn/visit_set.go

@@ -32,7 +32,7 @@ func SetByPath(v Value, p Path, nv Value) (Value, error) {
 			path := append(prefix, component)
 
 			switch {
-			case component.isKey():
+			case component.IsKey():
 				// Expect a map to be set if this is a key.
 				m, ok := v.AsMap()
 				if !ok {
@@ -48,7 +48,7 @@ func SetByPath(v Value, p Path, nv Value) (Value, error) {
 					l: v.l,
 				}, nil
 
-			case component.isIndex():
+			case component.IsIndex():
 				// Expect a sequence to be set if this is an index.
 				s, ok := v.AsSequence()
 				if !ok {