From f8b286b9454666db03bb02dac4f36fe7b9ffc75a Mon Sep 17 00:00:00 2001
From: Andrew Nester <andrew.nester@databricks.com>
Date: Mon, 3 Jun 2024 18:45:19 +0200
Subject: [PATCH 01/14] Change variable struct to use any type instead of
 string

---
 .../resolve_resource_references_test.go       | 21 ++++++------
 .../resolve_variable_references_test.go       |  6 +---
 bundle/config/mutator/set_variables.go        |  4 +--
 bundle/config/mutator/set_variables_test.go   | 32 +++++++++----------
 bundle/config/root_test.go                    |  8 ++---
 bundle/config/variable/variable.go            | 12 ++++---
 bundle/tests/complex_variables_test.go        | 13 ++++++++
 bundle/tests/variables/complex/databricks.yml | 23 +++++++++++++
 bundle/tests/variables_test.go                |  4 +--
 libs/dyn/convert/from_typed.go                | 12 ++++++-
 libs/dyn/convert/from_typed_test.go           | 32 +++++++++++++++++++
 libs/dyn/convert/normalize.go                 |  7 ++++
 libs/dyn/convert/normalize_test.go            | 16 ++++++++++
 libs/dyn/convert/to_typed.go                  |  7 ++++
 libs/dyn/convert/to_typed_test.go             | 22 +++++++++++++
 15 files changed, 172 insertions(+), 47 deletions(-)
 create mode 100644 bundle/tests/complex_variables_test.go
 create mode 100644 bundle/tests/variables/complex/databricks.yml

diff --git a/bundle/config/mutator/resolve_resource_references_test.go b/bundle/config/mutator/resolve_resource_references_test.go
index 60636bcc68..31d17a6c81 100644
--- a/bundle/config/mutator/resolve_resource_references_test.go
+++ b/bundle/config/mutator/resolve_resource_references_test.go
@@ -34,7 +34,7 @@ func TestResolveClusterReference(t *testing.T) {
 					},
 				},
 				"some-variable": {
-					Value: &justString,
+					Value: justString,
 				},
 			},
 		},
@@ -52,8 +52,8 @@ func TestResolveClusterReference(t *testing.T) {
 
 	diags := bundle.Apply(context.Background(), b, ResolveResourceReferences())
 	require.NoError(t, diags.Error())
-	require.Equal(t, "1234-5678-abcd", *b.Config.Variables["my-cluster-id-1"].Value)
-	require.Equal(t, "9876-5432-xywz", *b.Config.Variables["my-cluster-id-2"].Value)
+	require.Equal(t, "1234-5678-abcd", b.Config.Variables["my-cluster-id-1"].Value)
+	require.Equal(t, "9876-5432-xywz", b.Config.Variables["my-cluster-id-2"].Value)
 }
 
 func TestResolveNonExistentClusterReference(t *testing.T) {
@@ -68,7 +68,7 @@ func TestResolveNonExistentClusterReference(t *testing.T) {
 					},
 				},
 				"some-variable": {
-					Value: &justString,
+					Value: justString,
 				},
 			},
 		},
@@ -104,7 +104,7 @@ func TestNoLookupIfVariableIsSet(t *testing.T) {
 
 	diags := bundle.Apply(context.Background(), b, ResolveResourceReferences())
 	require.NoError(t, diags.Error())
-	require.Equal(t, "random value", *b.Config.Variables["my-cluster-id"].Value)
+	require.Equal(t, "random value", b.Config.Variables["my-cluster-id"].Value)
 }
 
 func TestResolveServicePrincipal(t *testing.T) {
@@ -131,14 +131,11 @@ func TestResolveServicePrincipal(t *testing.T) {
 
 	diags := bundle.Apply(context.Background(), b, ResolveResourceReferences())
 	require.NoError(t, diags.Error())
-	require.Equal(t, "app-1234", *b.Config.Variables["my-sp"].Value)
+	require.Equal(t, "app-1234", b.Config.Variables["my-sp"].Value)
 }
 
 func TestResolveVariableReferencesInVariableLookups(t *testing.T) {
-	s := func(s string) *string {
-		return &s
-	}
-
+	s := "bar"
 	b := &bundle.Bundle{
 		Config: config.Root{
 			Bundle: config.Bundle{
@@ -146,7 +143,7 @@ func TestResolveVariableReferencesInVariableLookups(t *testing.T) {
 			},
 			Variables: map[string]*variable.Variable{
 				"foo": {
-					Value: s("bar"),
+					Value: s,
 				},
 				"lookup": {
 					Lookup: &variable.Lookup{
@@ -167,7 +164,7 @@ func TestResolveVariableReferencesInVariableLookups(t *testing.T) {
 	diags := bundle.Apply(context.Background(), b, bundle.Seq(ResolveVariableReferencesInLookup(), ResolveResourceReferences()))
 	require.NoError(t, diags.Error())
 	require.Equal(t, "cluster-bar-dev", b.Config.Variables["lookup"].Lookup.Cluster)
-	require.Equal(t, "1234-5678-abcd", *b.Config.Variables["lookup"].Value)
+	require.Equal(t, "1234-5678-abcd", b.Config.Variables["lookup"].Value)
 }
 
 func TestResolveLookupVariableReferencesInVariableLookups(t *testing.T) {
diff --git a/bundle/config/mutator/resolve_variable_references_test.go b/bundle/config/mutator/resolve_variable_references_test.go
index 651ea3d2ce..bc0cdcea0e 100644
--- a/bundle/config/mutator/resolve_variable_references_test.go
+++ b/bundle/config/mutator/resolve_variable_references_test.go
@@ -43,10 +43,6 @@ func TestResolveVariableReferences(t *testing.T) {
 }
 
 func TestResolveVariableReferencesToBundleVariables(t *testing.T) {
-	s := func(s string) *string {
-		return &s
-	}
-
 	b := &bundle.Bundle{
 		Config: config.Root{
 			Bundle: config.Bundle{
@@ -57,7 +53,7 @@ func TestResolveVariableReferencesToBundleVariables(t *testing.T) {
 			},
 			Variables: map[string]*variable.Variable{
 				"foo": {
-					Value: s("bar"),
+					Value: "bar",
 				},
 			},
 		},
diff --git a/bundle/config/mutator/set_variables.go b/bundle/config/mutator/set_variables.go
index eae1fe2ab7..48d4f6c0f6 100644
--- a/bundle/config/mutator/set_variables.go
+++ b/bundle/config/mutator/set_variables.go
@@ -39,9 +39,9 @@ func setVariable(ctx context.Context, v *variable.Variable, name string) diag.Di
 
 	// case: Set the variable to its default value
 	if v.HasDefault() {
-		err := v.Set(*v.Default)
+		err := v.Set(v.Default)
 		if err != nil {
-			return diag.Errorf(`failed to assign default value from config "%s" to variable %s with error: %v`, *v.Default, name, err)
+			return diag.Errorf(`failed to assign default value from config "%s" to variable %s with error: %v`, v.Default, name, err)
 		}
 		return nil
 	}
diff --git a/bundle/config/mutator/set_variables_test.go b/bundle/config/mutator/set_variables_test.go
index ae4f798969..41942e47bc 100644
--- a/bundle/config/mutator/set_variables_test.go
+++ b/bundle/config/mutator/set_variables_test.go
@@ -15,7 +15,7 @@ func TestSetVariableFromProcessEnvVar(t *testing.T) {
 	defaultVal := "default"
 	variable := variable.Variable{
 		Description: "a test variable",
-		Default:     &defaultVal,
+		Default:     defaultVal,
 	}
 
 	// set value for variable as an environment variable
@@ -23,19 +23,19 @@ func TestSetVariableFromProcessEnvVar(t *testing.T) {
 
 	diags := setVariable(context.Background(), &variable, "foo")
 	require.NoError(t, diags.Error())
-	assert.Equal(t, *variable.Value, "process-env")
+	assert.Equal(t, variable.Value, "process-env")
 }
 
 func TestSetVariableUsingDefaultValue(t *testing.T) {
 	defaultVal := "default"
 	variable := variable.Variable{
 		Description: "a test variable",
-		Default:     &defaultVal,
+		Default:     defaultVal,
 	}
 
 	diags := setVariable(context.Background(), &variable, "foo")
 	require.NoError(t, diags.Error())
-	assert.Equal(t, *variable.Value, "default")
+	assert.Equal(t, variable.Value, "default")
 }
 
 func TestSetVariableWhenAlreadyAValueIsAssigned(t *testing.T) {
@@ -43,15 +43,15 @@ func TestSetVariableWhenAlreadyAValueIsAssigned(t *testing.T) {
 	val := "assigned-value"
 	variable := variable.Variable{
 		Description: "a test variable",
-		Default:     &defaultVal,
-		Value:       &val,
+		Default:     defaultVal,
+		Value:       val,
 	}
 
 	// since a value is already assigned to the variable, it would not be overridden
 	// by the default value
 	diags := setVariable(context.Background(), &variable, "foo")
 	require.NoError(t, diags.Error())
-	assert.Equal(t, *variable.Value, "assigned-value")
+	assert.Equal(t, variable.Value, "assigned-value")
 }
 
 func TestSetVariableEnvVarValueDoesNotOverridePresetValue(t *testing.T) {
@@ -59,8 +59,8 @@ func TestSetVariableEnvVarValueDoesNotOverridePresetValue(t *testing.T) {
 	val := "assigned-value"
 	variable := variable.Variable{
 		Description: "a test variable",
-		Default:     &defaultVal,
-		Value:       &val,
+		Default:     defaultVal,
+		Value:       val,
 	}
 
 	// set value for variable as an environment variable
@@ -70,7 +70,7 @@ func TestSetVariableEnvVarValueDoesNotOverridePresetValue(t *testing.T) {
 	// by the value from environment
 	diags := setVariable(context.Background(), &variable, "foo")
 	require.NoError(t, diags.Error())
-	assert.Equal(t, *variable.Value, "assigned-value")
+	assert.Equal(t, variable.Value, "assigned-value")
 }
 
 func TestSetVariablesErrorsIfAValueCouldNotBeResolved(t *testing.T) {
@@ -92,15 +92,15 @@ func TestSetVariablesMutator(t *testing.T) {
 			Variables: map[string]*variable.Variable{
 				"a": {
 					Description: "resolved to default value",
-					Default:     &defaultValForA,
+					Default:     defaultValForA,
 				},
 				"b": {
 					Description: "resolved from environment vairables",
-					Default:     &defaultValForB,
+					Default:     defaultValForB,
 				},
 				"c": {
 					Description: "has already been assigned a value",
-					Value:       &valForC,
+					Value:       valForC,
 				},
 			},
 		},
@@ -110,7 +110,7 @@ func TestSetVariablesMutator(t *testing.T) {
 
 	diags := bundle.Apply(context.Background(), b, SetVariables())
 	require.NoError(t, diags.Error())
-	assert.Equal(t, "default-a", *b.Config.Variables["a"].Value)
-	assert.Equal(t, "env-var-b", *b.Config.Variables["b"].Value)
-	assert.Equal(t, "assigned-val-c", *b.Config.Variables["c"].Value)
+	assert.Equal(t, "default-a", b.Config.Variables["a"].Value)
+	assert.Equal(t, "env-var-b", b.Config.Variables["b"].Value)
+	assert.Equal(t, "assigned-val-c", b.Config.Variables["c"].Value)
 }
diff --git a/bundle/config/root_test.go b/bundle/config/root_test.go
index b567688480..07f1feece0 100644
--- a/bundle/config/root_test.go
+++ b/bundle/config/root_test.go
@@ -51,7 +51,7 @@ func TestInitializeVariables(t *testing.T) {
 	root := &Root{
 		Variables: map[string]*variable.Variable{
 			"foo": {
-				Default:     &fooDefault,
+				Default:     fooDefault,
 				Description: "an optional variable since default is defined",
 			},
 			"bar": {
@@ -62,8 +62,8 @@ func TestInitializeVariables(t *testing.T) {
 
 	err := root.InitializeVariables([]string{"foo=123", "bar=456"})
 	assert.NoError(t, err)
-	assert.Equal(t, "123", *(root.Variables["foo"].Value))
-	assert.Equal(t, "456", *(root.Variables["bar"].Value))
+	assert.Equal(t, "123", (root.Variables["foo"].Value))
+	assert.Equal(t, "456", (root.Variables["bar"].Value))
 }
 
 func TestInitializeVariablesWithAnEqualSignInValue(t *testing.T) {
@@ -77,7 +77,7 @@ func TestInitializeVariablesWithAnEqualSignInValue(t *testing.T) {
 
 	err := root.InitializeVariables([]string{"foo=123=567"})
 	assert.NoError(t, err)
-	assert.Equal(t, "123=567", *(root.Variables["foo"].Value))
+	assert.Equal(t, "123=567", (root.Variables["foo"].Value))
 }
 
 func TestInitializeVariablesInvalidFormat(t *testing.T) {
diff --git a/bundle/config/variable/variable.go b/bundle/config/variable/variable.go
index 5e700a9b0c..7e563ff088 100644
--- a/bundle/config/variable/variable.go
+++ b/bundle/config/variable/variable.go
@@ -4,10 +4,12 @@ import (
 	"fmt"
 )
 
+type VariableValue = any
+
 // An input variable for the bundle config
 type Variable struct {
 	// A default value which then makes the variable optional
-	Default *string `json:"default,omitempty"`
+	Default VariableValue `json:"default,omitempty"`
 
 	// Documentation for this input variable
 	Description string `json:"description,omitempty"`
@@ -21,7 +23,7 @@ type Variable struct {
 	// 4. Default value defined in variable definition
 	// 5. Throw error, since if no default value is defined, then the variable
 	//    is required
-	Value *string `json:"value,omitempty" bundle:"readonly"`
+	Value VariableValue `json:"value,omitempty" bundle:"readonly"`
 
 	// The value of this field will be used to lookup the resource by name
 	// And assign the value of the variable to ID of the resource found.
@@ -39,10 +41,10 @@ func (v *Variable) HasValue() bool {
 	return v.Value != nil
 }
 
-func (v *Variable) Set(val string) error {
+func (v *Variable) Set(val VariableValue) error {
 	if v.HasValue() {
-		return fmt.Errorf("variable has already been assigned value: %s", *v.Value)
+		return fmt.Errorf("variable has already been assigned value: %s", v.Value)
 	}
-	v.Value = &val
+	v.Value = val
 	return nil
 }
diff --git a/bundle/tests/complex_variables_test.go b/bundle/tests/complex_variables_test.go
new file mode 100644
index 0000000000..b9721280af
--- /dev/null
+++ b/bundle/tests/complex_variables_test.go
@@ -0,0 +1,13 @@
+package config_tests
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestComplexVariables(t *testing.T) {
+	t.Skip("Skipping until complex variables are implemented")
+	_, diags := loadTargetWithDiags("variables/complex", "default")
+	require.Empty(t, diags)
+}
diff --git a/bundle/tests/variables/complex/databricks.yml b/bundle/tests/variables/complex/databricks.yml
new file mode 100644
index 0000000000..120144886b
--- /dev/null
+++ b/bundle/tests/variables/complex/databricks.yml
@@ -0,0 +1,23 @@
+bundle:
+  name: complex-variables
+
+resources:
+  jobs:
+    my_job:
+      job_clusters:
+        - job_cluster_key: key
+          new_cluster: ${var.cluster}
+      tasks:
+      - task_key: test
+        job_cluster_key: key
+
+variables:
+  cluster:
+    description: "A cluster definition"
+    default:
+      spark_version: "13.2.x-scala2.11"
+      node_type_id: "Standard_DS3_v2"
+      num_workers: 2
+      spark_conf:
+        spark.speculation: true
+        spark.databricks.delta.retentionDurationCheck.enabled: false
diff --git a/bundle/tests/variables_test.go b/bundle/tests/variables_test.go
index f518026846..b812e4e942 100644
--- a/bundle/tests/variables_test.go
+++ b/bundle/tests/variables_test.go
@@ -106,8 +106,8 @@ func TestVariablesWithoutDefinition(t *testing.T) {
 	require.NoError(t, diags.Error())
 	require.True(t, b.Config.Variables["a"].HasValue())
 	require.True(t, b.Config.Variables["b"].HasValue())
-	assert.Equal(t, "foo", *b.Config.Variables["a"].Value)
-	assert.Equal(t, "bar", *b.Config.Variables["b"].Value)
+	assert.Equal(t, "foo", b.Config.Variables["a"].Value)
+	assert.Equal(t, "bar", b.Config.Variables["b"].Value)
 }
 
 func TestVariablesWithTargetLookupOverrides(t *testing.T) {
diff --git a/libs/dyn/convert/from_typed.go b/libs/dyn/convert/from_typed.go
index ae491d8ab7..ec4d12a270 100644
--- a/libs/dyn/convert/from_typed.go
+++ b/libs/dyn/convert/from_typed.go
@@ -99,8 +99,18 @@ func fromTypedStruct(src reflect.Value, ref dyn.Value) (dyn.Value, error) {
 			refv = dyn.NilValue
 		}
 
+		var options []fromTypedOptions
+		if v.Kind() == reflect.Interface {
+			options = append(options, includeZeroValuedScalars)
+		}
+
 		// Convert the field taking into account the reference value (may be equal to config.NilValue).
-		nv, err := fromTyped(v.Interface(), refv)
+		vint := v.Interface()
+		if vint == nil {
+			continue
+		}
+
+		nv, err := fromTyped(vint, refv, options...)
 		if err != nil {
 			return dyn.InvalidValue, err
 		}
diff --git a/libs/dyn/convert/from_typed_test.go b/libs/dyn/convert/from_typed_test.go
index f75470f420..dbb76bc66a 100644
--- a/libs/dyn/convert/from_typed_test.go
+++ b/libs/dyn/convert/from_typed_test.go
@@ -619,3 +619,35 @@ func TestFromTypedFloatTypeError(t *testing.T) {
 	_, err := FromTyped(src, ref)
 	require.Error(t, err)
 }
+
+func TestFromTypedAny(t *testing.T) {
+	type Tmp struct {
+		Foo any `json:"foo"`
+		Bar any `json:"bar"`
+		Foz any `json:"foz"`
+		Baz any `json:"baz"`
+	}
+
+	src := Tmp{
+		Foo: "foo",
+		Bar: false,
+		Foz: 0,
+		Baz: map[string]any{
+			"foo": "foo",
+			"bar": 1234,
+		},
+	}
+
+	ref := dyn.NilValue
+	nv, err := FromTyped(src, ref)
+	require.NoError(t, err)
+	assert.Equal(t, dyn.V(map[string]dyn.Value{
+		"foo": dyn.V("foo"),
+		"bar": dyn.V(false),
+		"foz": dyn.V(int64(0)),
+		"baz": dyn.V(map[string]dyn.Value{
+			"foo": dyn.V("foo"),
+			"bar": dyn.V(int64(1234)),
+		}),
+	}), nv)
+}
diff --git a/libs/dyn/convert/normalize.go b/libs/dyn/convert/normalize.go
index 35d4d82106..1c31e56d56 100644
--- a/libs/dyn/convert/normalize.go
+++ b/libs/dyn/convert/normalize.go
@@ -56,6 +56,8 @@ func (n normalizeOptions) normalizeType(typ reflect.Type, src dyn.Value, seen []
 		return n.normalizeInt(typ, src, path)
 	case reflect.Float32, reflect.Float64:
 		return n.normalizeFloat(typ, src, path)
+	case reflect.Interface:
+		return n.normalizeInterface(typ, src, path)
 	}
 
 	return dyn.InvalidValue, diag.Errorf("unsupported type: %s", typ.Kind())
@@ -371,3 +373,8 @@ func (n normalizeOptions) normalizeFloat(typ reflect.Type, src dyn.Value, path d
 
 	return dyn.NewValue(out, src.Location()), diags
 }
+
+func (n normalizeOptions) normalizeInterface(typ reflect.Type, src dyn.Value, path dyn.Path) (dyn.Value, diag.Diagnostics) {
+	out := src.AsAny()
+	return dyn.NewValue(out, src.Location()), nil
+}
diff --git a/libs/dyn/convert/normalize_test.go b/libs/dyn/convert/normalize_test.go
index 843b4ea591..cb5ad19db9 100644
--- a/libs/dyn/convert/normalize_test.go
+++ b/libs/dyn/convert/normalize_test.go
@@ -725,3 +725,19 @@ func TestNormalizeAnchors(t *testing.T) {
 		"foo": "bar",
 	}, vout.AsAny())
 }
+
+func TestNormalizeBoolToAny(t *testing.T) {
+	var typ any
+	vin := dyn.NewValue(false, dyn.Location{File: "file", Line: 1, Column: 1})
+	vout, err := Normalize(&typ, vin)
+	assert.Len(t, err, 0)
+	assert.Equal(t, dyn.NewValue(false, dyn.Location{File: "file", Line: 1, Column: 1}), vout)
+}
+
+func TestNormalizeIntToAny(t *testing.T) {
+	var typ any
+	vin := dyn.NewValue(10, dyn.Location{File: "file", Line: 1, Column: 1})
+	vout, err := Normalize(&typ, vin)
+	assert.Len(t, err, 0)
+	assert.Equal(t, dyn.NewValue(10, dyn.Location{File: "file", Line: 1, Column: 1}), vout)
+}
diff --git a/libs/dyn/convert/to_typed.go b/libs/dyn/convert/to_typed.go
index f10853a2e4..71cc3cf7cb 100644
--- a/libs/dyn/convert/to_typed.go
+++ b/libs/dyn/convert/to_typed.go
@@ -46,6 +46,8 @@ func ToTyped(dst any, src dyn.Value) error {
 		return toTypedInt(dstv, src)
 	case reflect.Float32, reflect.Float64:
 		return toTypedFloat(dstv, src)
+	case reflect.Interface:
+		return toTypedInterface(dstv, src)
 	}
 
 	return fmt.Errorf("unsupported type: %s", dstv.Kind())
@@ -260,3 +262,8 @@ func toTypedFloat(dst reflect.Value, src dyn.Value) error {
 		msg:   fmt.Sprintf("expected a float, found a %s", src.Kind()),
 	}
 }
+
+func toTypedInterface(dst reflect.Value, src dyn.Value) error {
+	dst.Set(reflect.ValueOf(src.AsAny()))
+	return nil
+}
diff --git a/libs/dyn/convert/to_typed_test.go b/libs/dyn/convert/to_typed_test.go
index 56d98a3cf4..6aecec3701 100644
--- a/libs/dyn/convert/to_typed_test.go
+++ b/libs/dyn/convert/to_typed_test.go
@@ -511,3 +511,25 @@ func TestToTypedWithAliasKeyType(t *testing.T) {
 	assert.Equal(t, "bar", out["foo"])
 	assert.Equal(t, "baz", out["bar"])
 }
+
+func TestToTypedAnyWithBool(t *testing.T) {
+	var out interface{}
+	err := ToTyped(&out, dyn.V(false))
+	require.NoError(t, err)
+	assert.Equal(t, false, out)
+
+	err = ToTyped(&out, dyn.V(true))
+	require.NoError(t, err)
+	assert.Equal(t, true, out)
+}
+
+func TestToTypedAnyWithMap(t *testing.T) {
+	var out interface{}
+	v := dyn.V(map[string]dyn.Value{
+		"foo": dyn.V("bar"),
+		"bar": dyn.V("baz"),
+	})
+	err := ToTyped(&out, v)
+	require.NoError(t, err)
+	assert.Equal(t, map[string]interface{}{"foo": "bar", "bar": "baz"}, out)
+}

From f8b7d64cef7ff0445df9f38f54434d4c8f6a7d54 Mon Sep 17 00:00:00 2001
From: Andrew Nester <andrew.nester@databricks.com>
Date: Tue, 4 Jun 2024 12:02:58 +0200
Subject: [PATCH 02/14] addressed feedback

---
 bundle/config/variable/variable.go     |  3 +++
 bundle/tests/complex_variables_test.go |  1 -
 libs/dyn/convert/from_typed.go         | 11 +++++------
 libs/dyn/convert/normalize.go          |  3 +--
 libs/dyn/convert/to_typed_test.go      |  6 +++---
 5 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/bundle/config/variable/variable.go b/bundle/config/variable/variable.go
index 7e563ff088..d27e369c49 100644
--- a/bundle/config/variable/variable.go
+++ b/bundle/config/variable/variable.go
@@ -4,6 +4,9 @@ import (
 	"fmt"
 )
 
+// We are using `any` because since introduction of complex variables,
+// variables can be of any type.
+// Type alias is used to make it easier to understand the code.
 type VariableValue = any
 
 // An input variable for the bundle config
diff --git a/bundle/tests/complex_variables_test.go b/bundle/tests/complex_variables_test.go
index b9721280af..afab04876d 100644
--- a/bundle/tests/complex_variables_test.go
+++ b/bundle/tests/complex_variables_test.go
@@ -7,7 +7,6 @@ import (
 )
 
 func TestComplexVariables(t *testing.T) {
-	t.Skip("Skipping until complex variables are implemented")
 	_, diags := loadTargetWithDiags("variables/complex", "default")
 	require.Empty(t, diags)
 }
diff --git a/libs/dyn/convert/from_typed.go b/libs/dyn/convert/from_typed.go
index ec4d12a270..8532a71663 100644
--- a/libs/dyn/convert/from_typed.go
+++ b/libs/dyn/convert/from_typed.go
@@ -39,6 +39,10 @@ func FromTyped(src any, ref dyn.Value) (dyn.Value, error) {
 // Private implementation of FromTyped that allows for additional options not exposed
 // in the public API.
 func fromTyped(src any, ref dyn.Value, options ...fromTypedOptions) (dyn.Value, error) {
+	if src == nil {
+		return dyn.NilValue, nil
+	}
+
 	srcv := reflect.ValueOf(src)
 
 	// Dereference pointer if necessary
@@ -105,12 +109,7 @@ func fromTypedStruct(src reflect.Value, ref dyn.Value) (dyn.Value, error) {
 		}
 
 		// Convert the field taking into account the reference value (may be equal to config.NilValue).
-		vint := v.Interface()
-		if vint == nil {
-			continue
-		}
-
-		nv, err := fromTyped(vint, refv, options...)
+		nv, err := fromTyped(v.Interface(), refv, options...)
 		if err != nil {
 			return dyn.InvalidValue, err
 		}
diff --git a/libs/dyn/convert/normalize.go b/libs/dyn/convert/normalize.go
index 1c31e56d56..d09979cffd 100644
--- a/libs/dyn/convert/normalize.go
+++ b/libs/dyn/convert/normalize.go
@@ -375,6 +375,5 @@ func (n normalizeOptions) normalizeFloat(typ reflect.Type, src dyn.Value, path d
 }
 
 func (n normalizeOptions) normalizeInterface(typ reflect.Type, src dyn.Value, path dyn.Path) (dyn.Value, diag.Diagnostics) {
-	out := src.AsAny()
-	return dyn.NewValue(out, src.Location()), nil
+	return src, nil
 }
diff --git a/libs/dyn/convert/to_typed_test.go b/libs/dyn/convert/to_typed_test.go
index 6aecec3701..5e37f28631 100644
--- a/libs/dyn/convert/to_typed_test.go
+++ b/libs/dyn/convert/to_typed_test.go
@@ -513,7 +513,7 @@ func TestToTypedWithAliasKeyType(t *testing.T) {
 }
 
 func TestToTypedAnyWithBool(t *testing.T) {
-	var out interface{}
+	var out any
 	err := ToTyped(&out, dyn.V(false))
 	require.NoError(t, err)
 	assert.Equal(t, false, out)
@@ -524,12 +524,12 @@ func TestToTypedAnyWithBool(t *testing.T) {
 }
 
 func TestToTypedAnyWithMap(t *testing.T) {
-	var out interface{}
+	var out any
 	v := dyn.V(map[string]dyn.Value{
 		"foo": dyn.V("bar"),
 		"bar": dyn.V("baz"),
 	})
 	err := ToTyped(&out, v)
 	require.NoError(t, err)
-	assert.Equal(t, map[string]interface{}{"foo": "bar", "bar": "baz"}, out)
+	assert.Equal(t, map[string]any{"foo": "bar", "bar": "baz"}, out)
 }

From 26d1d5a69a6ca4c2f260c22480fd9fa16b4743a1 Mon Sep 17 00:00:00 2001
From: Andrew Nester <andrew.nester@databricks.com>
Date: Tue, 4 Jun 2024 12:36:29 +0200
Subject: [PATCH 03/14] added support for complex variables

---
 bundle/tests/complex_variables_test.go | 17 ++++++++++++-
 libs/dyn/convert/from_typed.go         | 15 ++++++++++++
 libs/dyn/convert/normalize.go          | 33 +++++++++++++++++++++-----
 libs/dyn/convert/to_typed.go           | 18 ++++++++++++++
 4 files changed, 76 insertions(+), 7 deletions(-)

diff --git a/bundle/tests/complex_variables_test.go b/bundle/tests/complex_variables_test.go
index afab04876d..c532cc290f 100644
--- a/bundle/tests/complex_variables_test.go
+++ b/bundle/tests/complex_variables_test.go
@@ -1,12 +1,27 @@
 package config_tests
 
 import (
+	"context"
 	"testing"
 
+	"github.com/databricks/cli/bundle"
+	"github.com/databricks/cli/bundle/config/mutator"
 	"github.com/stretchr/testify/require"
 )
 
 func TestComplexVariables(t *testing.T) {
-	_, diags := loadTargetWithDiags("variables/complex", "default")
+	b, diags := loadTargetWithDiags("variables/complex", "default")
 	require.Empty(t, diags)
+
+	diags = bundle.Apply(context.Background(), b, bundle.Seq(
+		mutator.SetVariables(),
+		mutator.ResolveVariableReferences(
+			"variables",
+		),
+	))
+	require.NoError(t, diags.Error())
+
+	require.Equal(t, "13.2.x-scala2.11", b.Config.Resources.Jobs["my_job"].JobClusters[0].NewCluster.SparkVersion)
+	require.Equal(t, 2, b.Config.Resources.Jobs["my_job"].JobClusters[0].NewCluster.NumWorkers)
+	require.Equal(t, "true", b.Config.Resources.Jobs["my_job"].JobClusters[0].NewCluster.SparkConf["spark.speculation"])
 }
diff --git a/libs/dyn/convert/from_typed.go b/libs/dyn/convert/from_typed.go
index 8532a71663..f56ee5ab5d 100644
--- a/libs/dyn/convert/from_typed.go
+++ b/libs/dyn/convert/from_typed.go
@@ -84,6 +84,11 @@ func fromTyped(src any, ref dyn.Value, options ...fromTypedOptions) (dyn.Value,
 func fromTypedStruct(src reflect.Value, ref dyn.Value) (dyn.Value, error) {
 	// Check that the reference value is compatible or nil.
 	switch ref.Kind() {
+	case dyn.KindString:
+		// Ignore pure variable references (e.g. ${var.foo}).
+		if dynvar.IsPureVariableReference(ref.MustString()) {
+			return ref, nil
+		}
 	case dyn.KindMap, dyn.KindNil:
 	default:
 		return dyn.InvalidValue, fmt.Errorf("unhandled type: %s", ref.Kind())
@@ -125,6 +130,11 @@ func fromTypedStruct(src reflect.Value, ref dyn.Value) (dyn.Value, error) {
 func fromTypedMap(src reflect.Value, ref dyn.Value) (dyn.Value, error) {
 	// Check that the reference value is compatible or nil.
 	switch ref.Kind() {
+	case dyn.KindString:
+		// Ignore pure variable references (e.g. ${var.foo}).
+		if dynvar.IsPureVariableReference(ref.MustString()) {
+			return ref, nil
+		}
 	case dyn.KindMap, dyn.KindNil:
 	default:
 		return dyn.InvalidValue, fmt.Errorf("unhandled type: %s", ref.Kind())
@@ -168,6 +178,11 @@ func fromTypedMap(src reflect.Value, ref dyn.Value) (dyn.Value, error) {
 func fromTypedSlice(src reflect.Value, ref dyn.Value) (dyn.Value, error) {
 	// Check that the reference value is compatible or nil.
 	switch ref.Kind() {
+	case dyn.KindString:
+		// Ignore pure variable references (e.g. ${var.foo}).
+		if dynvar.IsPureVariableReference(ref.MustString()) {
+			return ref, nil
+		}
 	case dyn.KindSequence, dyn.KindNil:
 	default:
 		return dyn.InvalidValue, fmt.Errorf("unhandled type: %s", ref.Kind())
diff --git a/libs/dyn/convert/normalize.go b/libs/dyn/convert/normalize.go
index d09979cffd..2eedcac563 100644
--- a/libs/dyn/convert/normalize.go
+++ b/libs/dyn/convert/normalize.go
@@ -168,9 +168,16 @@ func (n normalizeOptions) normalizeStruct(typ reflect.Type, src dyn.Value, seen
 		return dyn.NewValue(out, src.Location()), diags
 	case dyn.KindNil:
 		return src, diags
-	}
 
-	return dyn.InvalidValue, diags.Append(typeMismatch(dyn.KindMap, src, path))
+	default:
+		// Return verbatim if it's a pure variable reference.
+		if dynvar.IsPureVariableReference(src.MustString()) {
+			return src, nil
+		}
+
+		// Cannot interpret as a slice.
+		return dyn.InvalidValue, diags.Append(typeMismatch(dyn.KindMap, src, path))
+	}
 }
 
 func (n normalizeOptions) normalizeMap(typ reflect.Type, src dyn.Value, seen []reflect.Type, path dyn.Path) (dyn.Value, diag.Diagnostics) {
@@ -199,9 +206,16 @@ func (n normalizeOptions) normalizeMap(typ reflect.Type, src dyn.Value, seen []r
 		return dyn.NewValue(out, src.Location()), diags
 	case dyn.KindNil:
 		return src, diags
-	}
 
-	return dyn.InvalidValue, diags.Append(typeMismatch(dyn.KindMap, src, path))
+	default:
+		// Return verbatim if it's a pure variable reference.
+		if dynvar.IsPureVariableReference(src.MustString()) {
+			return src, nil
+		}
+
+		// Cannot interpret as a slice.
+		return dyn.InvalidValue, diags.Append(typeMismatch(dyn.KindMap, src, path))
+	}
 }
 
 func (n normalizeOptions) normalizeSlice(typ reflect.Type, src dyn.Value, seen []reflect.Type, path dyn.Path) (dyn.Value, diag.Diagnostics) {
@@ -227,9 +241,16 @@ func (n normalizeOptions) normalizeSlice(typ reflect.Type, src dyn.Value, seen [
 		return dyn.NewValue(out, src.Location()), diags
 	case dyn.KindNil:
 		return src, diags
-	}
 
-	return dyn.InvalidValue, diags.Append(typeMismatch(dyn.KindSequence, src, path))
+	default:
+		// Return verbatim if it's a pure variable reference.
+		if dynvar.IsPureVariableReference(src.MustString()) {
+			return src, nil
+		}
+
+		// Cannot interpret as a slice.
+		return dyn.InvalidValue, diags.Append(typeMismatch(dyn.KindSequence, src, path))
+	}
 }
 
 func (n normalizeOptions) normalizeString(typ reflect.Type, src dyn.Value, path dyn.Path) (dyn.Value, diag.Diagnostics) {
diff --git a/libs/dyn/convert/to_typed.go b/libs/dyn/convert/to_typed.go
index 71cc3cf7cb..91d6445a12 100644
--- a/libs/dyn/convert/to_typed.go
+++ b/libs/dyn/convert/to_typed.go
@@ -103,6 +103,12 @@ func toTypedStruct(dst reflect.Value, src dyn.Value) error {
 	case dyn.KindNil:
 		dst.SetZero()
 		return nil
+	case dyn.KindString:
+		// Ignore pure variable references (e.g. ${var.foo}).
+		if dynvar.IsPureVariableReference(src.MustString()) {
+			dst.SetZero()
+			return nil
+		}
 	}
 
 	return TypeError{
@@ -134,6 +140,12 @@ func toTypedMap(dst reflect.Value, src dyn.Value) error {
 	case dyn.KindNil:
 		dst.SetZero()
 		return nil
+	case dyn.KindString:
+		// Ignore pure variable references (e.g. ${var.foo}).
+		if dynvar.IsPureVariableReference(src.MustString()) {
+			dst.SetZero()
+			return nil
+		}
 	}
 
 	return TypeError{
@@ -159,6 +171,12 @@ func toTypedSlice(dst reflect.Value, src dyn.Value) error {
 	case dyn.KindNil:
 		dst.SetZero()
 		return nil
+	case dyn.KindString:
+		// Ignore pure variable references (e.g. ${var.foo}).
+		if dynvar.IsPureVariableReference(src.MustString()) {
+			dst.SetZero()
+			return nil
+		}
 	}
 
 	return TypeError{

From 4503657aafe094e9dc2228140c0b442a2732695d Mon Sep 17 00:00:00 2001
From: Andrew Nester <andrew.nester@databricks.com>
Date: Tue, 4 Jun 2024 13:54:05 +0200
Subject: [PATCH 04/14] support overrides and do not allow setting via vars or
 envs

---
 bundle/config/mutator/set_variables.go        |  4 ++++
 bundle/config/root.go                         | 22 ++++++++++++++++-
 bundle/config/variable/variable.go            | 24 +++++++++++++++++++
 bundle/tests/complex_variables_test.go        | 17 +++++++++++++
 bundle/tests/variables/complex/databricks.yml | 13 ++++++++++
 5 files changed, 79 insertions(+), 1 deletion(-)

diff --git a/bundle/config/mutator/set_variables.go b/bundle/config/mutator/set_variables.go
index 48d4f6c0f6..ca6adb1e80 100644
--- a/bundle/config/mutator/set_variables.go
+++ b/bundle/config/mutator/set_variables.go
@@ -30,6 +30,10 @@ func setVariable(ctx context.Context, v *variable.Variable, name string) diag.Di
 	// case: read and set variable value from process environment
 	envVarName := bundleVarPrefix + name
 	if val, ok := env.Lookup(ctx, envVarName); ok {
+		if v.IsComplex() {
+			return diag.Errorf(`setting via environment variables (%s) is not supported for complex variable %s`, envVarName, name)
+		}
+
 		err := v.Set(val)
 		if err != nil {
 			return diag.Errorf(`failed to assign value "%s" to variable %s from environment variable %s with error: %v`, val, name, envVarName, err)
diff --git a/bundle/config/root.go b/bundle/config/root.go
index 88197c2b87..5a4e7053d8 100644
--- a/bundle/config/root.go
+++ b/bundle/config/root.go
@@ -263,6 +263,11 @@ func (r *Root) InitializeVariables(vars []string) error {
 		if _, ok := r.Variables[name]; !ok {
 			return fmt.Errorf("variable %s has not been defined", name)
 		}
+
+		if r.Variables[name].IsComplex() {
+			return fmt.Errorf("setting variables of complex type via --var flag is not supported: %s", name)
+		}
+
 		err := r.Variables[name].Set(val)
 		if err != nil {
 			return fmt.Errorf("failed to assign %s to %s: %s", val, name, err)
@@ -415,7 +420,7 @@ func rewriteShorthands(v dyn.Value) (dyn.Value, error) {
 		}
 
 		// For each variable, normalize its contents if it is a single string.
-		return dyn.Map(target, "variables", dyn.Foreach(func(_ dyn.Path, variable dyn.Value) (dyn.Value, error) {
+		return dyn.Map(target, "variables", dyn.Foreach(func(p dyn.Path, variable dyn.Value) (dyn.Value, error) {
 			switch variable.Kind() {
 
 			case dyn.KindString, dyn.KindBool, dyn.KindFloat, dyn.KindInt:
@@ -426,6 +431,21 @@ func rewriteShorthands(v dyn.Value) (dyn.Value, error) {
 					"default": variable,
 				}, variable.Location()), nil
 
+			case dyn.KindMap, dyn.KindSequence:
+				// Check if the original definition of variable has a type field.
+				typeV, err := dyn.GetByPath(v, p.Append(dyn.Key("type")))
+				if err != nil {
+					return variable, nil
+				}
+
+				if typeV.MustString() == "complex" {
+					return dyn.NewValue(map[string]dyn.Value{
+						"default": variable,
+					}, variable.Location()), nil
+				}
+
+				return variable, nil
+
 			default:
 				return variable, nil
 			}
diff --git a/bundle/config/variable/variable.go b/bundle/config/variable/variable.go
index d27e369c49..ba94f9c8a5 100644
--- a/bundle/config/variable/variable.go
+++ b/bundle/config/variable/variable.go
@@ -2,6 +2,7 @@ package variable
 
 import (
 	"fmt"
+	"reflect"
 )
 
 // We are using `any` because since introduction of complex variables,
@@ -9,8 +10,17 @@ import (
 // Type alias is used to make it easier to understand the code.
 type VariableValue = any
 
+type VariableType string
+
+const (
+	VariableTypeComplex VariableType = "complex"
+)
+
 // An input variable for the bundle config
 type Variable struct {
+	// A type of the variable. This is used to validate the value of the variable
+	Type VariableType `json:"type,omitempty"`
+
 	// A default value which then makes the variable optional
 	Default VariableValue `json:"default,omitempty"`
 
@@ -48,6 +58,20 @@ func (v *Variable) Set(val VariableValue) error {
 	if v.HasValue() {
 		return fmt.Errorf("variable has already been assigned value: %s", v.Value)
 	}
+
+	rv := reflect.ValueOf(val)
+	switch rv.Kind() {
+	case reflect.Struct, reflect.Array, reflect.Slice, reflect.Map:
+		if v.Type != VariableTypeComplex {
+			return fmt.Errorf("variable type is not complex")
+		}
+	}
+
 	v.Value = val
+
 	return nil
 }
+
+func (v *Variable) IsComplex() bool {
+	return v.Type == VariableTypeComplex
+}
diff --git a/bundle/tests/complex_variables_test.go b/bundle/tests/complex_variables_test.go
index c532cc290f..36b2374c74 100644
--- a/bundle/tests/complex_variables_test.go
+++ b/bundle/tests/complex_variables_test.go
@@ -25,3 +25,20 @@ func TestComplexVariables(t *testing.T) {
 	require.Equal(t, 2, b.Config.Resources.Jobs["my_job"].JobClusters[0].NewCluster.NumWorkers)
 	require.Equal(t, "true", b.Config.Resources.Jobs["my_job"].JobClusters[0].NewCluster.SparkConf["spark.speculation"])
 }
+
+func TestComplexVariablesOverride(t *testing.T) {
+	b, diags := loadTargetWithDiags("variables/complex", "dev")
+	require.Empty(t, diags)
+
+	diags = bundle.Apply(context.Background(), b, bundle.Seq(
+		mutator.SetVariables(),
+		mutator.ResolveVariableReferences(
+			"variables",
+		),
+	))
+	require.NoError(t, diags.Error())
+
+	require.Equal(t, "14.2.x-scala2.11", b.Config.Resources.Jobs["my_job"].JobClusters[0].NewCluster.SparkVersion)
+	require.Equal(t, 4, b.Config.Resources.Jobs["my_job"].JobClusters[0].NewCluster.NumWorkers)
+	require.Equal(t, "false", b.Config.Resources.Jobs["my_job"].JobClusters[0].NewCluster.SparkConf["spark.speculation"])
+}
diff --git a/bundle/tests/variables/complex/databricks.yml b/bundle/tests/variables/complex/databricks.yml
index 120144886b..db13807348 100644
--- a/bundle/tests/variables/complex/databricks.yml
+++ b/bundle/tests/variables/complex/databricks.yml
@@ -13,6 +13,7 @@ resources:
 
 variables:
   cluster:
+    type: complex
     description: "A cluster definition"
     default:
       spark_version: "13.2.x-scala2.11"
@@ -21,3 +22,15 @@ variables:
       spark_conf:
         spark.speculation: true
         spark.databricks.delta.retentionDurationCheck.enabled: false
+
+targets:
+  default:
+  dev:
+    variables:
+      cluster:
+        spark_version: "14.2.x-scala2.11"
+        node_type_id: "Standard_DS3_v2"
+        num_workers: 4
+        spark_conf:
+          spark.speculation: false
+          spark.databricks.delta.retentionDurationCheck.enabled: false

From aff20cfba05d9929123ca8f25549f8271422c90a Mon Sep 17 00:00:00 2001
From: Andrew Nester <andrew.nester@databricks.com>
Date: Mon, 17 Jun 2024 11:50:09 +0200
Subject: [PATCH 05/14] added tests and fixes

---
 libs/dyn/convert/from_typed_test.go |   2 +
 libs/dyn/convert/normalize.go       |  24 +++---
 libs/dyn/convert/normalize_test.go  | 124 ++++++++++++++++++++++++++++
 3 files changed, 138 insertions(+), 12 deletions(-)

diff --git a/libs/dyn/convert/from_typed_test.go b/libs/dyn/convert/from_typed_test.go
index dbb76bc66a..dd1c237cfb 100644
--- a/libs/dyn/convert/from_typed_test.go
+++ b/libs/dyn/convert/from_typed_test.go
@@ -635,6 +635,7 @@ func TestFromTypedAny(t *testing.T) {
 		Baz: map[string]any{
 			"foo": "foo",
 			"bar": 1234,
+			"qux": 0,
 		},
 	}
 
@@ -648,6 +649,7 @@ func TestFromTypedAny(t *testing.T) {
 		"baz": dyn.V(map[string]dyn.Value{
 			"foo": dyn.V("foo"),
 			"bar": dyn.V(int64(1234)),
+			"qux": dyn.V(int64(0)),
 		}),
 	}), nv)
 }
diff --git a/libs/dyn/convert/normalize.go b/libs/dyn/convert/normalize.go
index 2eedcac563..128bedf954 100644
--- a/libs/dyn/convert/normalize.go
+++ b/libs/dyn/convert/normalize.go
@@ -169,15 +169,15 @@ func (n normalizeOptions) normalizeStruct(typ reflect.Type, src dyn.Value, seen
 	case dyn.KindNil:
 		return src, diags
 
-	default:
+	case dyn.KindString:
 		// Return verbatim if it's a pure variable reference.
 		if dynvar.IsPureVariableReference(src.MustString()) {
 			return src, nil
 		}
-
-		// Cannot interpret as a slice.
-		return dyn.InvalidValue, diags.Append(typeMismatch(dyn.KindMap, src, path))
 	}
+
+	// Cannot interpret as a strcut.
+	return dyn.InvalidValue, diags.Append(typeMismatch(dyn.KindMap, src, path))
 }
 
 func (n normalizeOptions) normalizeMap(typ reflect.Type, src dyn.Value, seen []reflect.Type, path dyn.Path) (dyn.Value, diag.Diagnostics) {
@@ -207,15 +207,15 @@ func (n normalizeOptions) normalizeMap(typ reflect.Type, src dyn.Value, seen []r
 	case dyn.KindNil:
 		return src, diags
 
-	default:
+	case dyn.KindString:
 		// Return verbatim if it's a pure variable reference.
 		if dynvar.IsPureVariableReference(src.MustString()) {
 			return src, nil
 		}
-
-		// Cannot interpret as a slice.
-		return dyn.InvalidValue, diags.Append(typeMismatch(dyn.KindMap, src, path))
 	}
+
+	// Cannot interpret as a map.
+	return dyn.InvalidValue, diags.Append(typeMismatch(dyn.KindMap, src, path))
 }
 
 func (n normalizeOptions) normalizeSlice(typ reflect.Type, src dyn.Value, seen []reflect.Type, path dyn.Path) (dyn.Value, diag.Diagnostics) {
@@ -242,15 +242,15 @@ func (n normalizeOptions) normalizeSlice(typ reflect.Type, src dyn.Value, seen [
 	case dyn.KindNil:
 		return src, diags
 
-	default:
+	case dyn.KindString:
 		// Return verbatim if it's a pure variable reference.
 		if dynvar.IsPureVariableReference(src.MustString()) {
 			return src, nil
 		}
-
-		// Cannot interpret as a slice.
-		return dyn.InvalidValue, diags.Append(typeMismatch(dyn.KindSequence, src, path))
 	}
+
+	// Cannot interpret as a slice.
+	return dyn.InvalidValue, diags.Append(typeMismatch(dyn.KindSequence, src, path))
 }
 
 func (n normalizeOptions) normalizeString(typ reflect.Type, src dyn.Value, path dyn.Path) (dyn.Value, diag.Diagnostics) {
diff --git a/libs/dyn/convert/normalize_test.go b/libs/dyn/convert/normalize_test.go
index cb5ad19db9..299ffcabdd 100644
--- a/libs/dyn/convert/normalize_test.go
+++ b/libs/dyn/convert/normalize_test.go
@@ -223,6 +223,52 @@ func TestNormalizeStructIncludeMissingFieldsOnRecursiveType(t *testing.T) {
 	}), vout)
 }
 
+func TestNormalizeStructVariableReference(t *testing.T) {
+	type Tmp struct {
+		Foo string `json:"foo"`
+	}
+
+	var typ Tmp
+	vin := dyn.NewValue("${var.foo}", dyn.Location{File: "file", Line: 1, Column: 1})
+	vout, err := Normalize(typ, vin)
+	assert.Empty(t, err)
+	assert.Equal(t, vin, vout)
+}
+
+func TestNormalizeStructRandomStringError(t *testing.T) {
+	type Tmp struct {
+		Foo string `json:"foo"`
+	}
+
+	var typ Tmp
+	vin := dyn.NewValue("var foo", dyn.Location{File: "file", Line: 1, Column: 1})
+	_, err := Normalize(typ, vin)
+	assert.Len(t, err, 1)
+	assert.Equal(t, diag.Diagnostic{
+		Severity: diag.Warning,
+		Summary:  `expected map, found string`,
+		Location: vin.Location(),
+		Path:     dyn.EmptyPath,
+	}, err[0])
+}
+
+func TestNormalizeStructIntError(t *testing.T) {
+	type Tmp struct {
+		Foo string `json:"foo"`
+	}
+
+	var typ Tmp
+	vin := dyn.NewValue(1, dyn.Location{File: "file", Line: 1, Column: 1})
+	_, err := Normalize(typ, vin)
+	assert.Len(t, err, 1)
+	assert.Equal(t, diag.Diagnostic{
+		Severity: diag.Warning,
+		Summary:  `expected map, found int`,
+		Location: vin.Location(),
+		Path:     dyn.EmptyPath,
+	}, err[0])
+}
+
 func TestNormalizeMap(t *testing.T) {
 	var typ map[string]string
 	vin := dyn.V(map[string]dyn.Value{
@@ -312,6 +358,40 @@ func TestNormalizeMapNestedError(t *testing.T) {
 	)
 }
 
+func TestNormalizeMapVariableReference(t *testing.T) {
+	var typ map[string]string
+	vin := dyn.NewValue("${var.foo}", dyn.Location{File: "file", Line: 1, Column: 1})
+	vout, err := Normalize(typ, vin)
+	assert.Empty(t, err)
+	assert.Equal(t, vin, vout)
+}
+
+func TestNormalizeMapRandomStringError(t *testing.T) {
+	var typ map[string]string
+	vin := dyn.NewValue("var foo", dyn.Location{File: "file", Line: 1, Column: 1})
+	_, err := Normalize(typ, vin)
+	assert.Len(t, err, 1)
+	assert.Equal(t, diag.Diagnostic{
+		Severity: diag.Warning,
+		Summary:  `expected map, found string`,
+		Location: vin.Location(),
+		Path:     dyn.EmptyPath,
+	}, err[0])
+}
+
+func TestNormalizeMapIntError(t *testing.T) {
+	var typ map[string]string
+	vin := dyn.NewValue(1, dyn.Location{File: "file", Line: 1, Column: 1})
+	_, err := Normalize(typ, vin)
+	assert.Len(t, err, 1)
+	assert.Equal(t, diag.Diagnostic{
+		Severity: diag.Warning,
+		Summary:  `expected map, found int`,
+		Location: vin.Location(),
+		Path:     dyn.EmptyPath,
+	}, err[0])
+}
+
 func TestNormalizeSlice(t *testing.T) {
 	var typ []string
 	vin := dyn.V([]dyn.Value{
@@ -400,6 +480,40 @@ func TestNormalizeSliceNestedError(t *testing.T) {
 	)
 }
 
+func TestNormalizeSliceVariableReference(t *testing.T) {
+	var typ []string
+	vin := dyn.NewValue("${var.foo}", dyn.Location{File: "file", Line: 1, Column: 1})
+	vout, err := Normalize(typ, vin)
+	assert.Empty(t, err)
+	assert.Equal(t, vin, vout)
+}
+
+func TestNormalizeSliceRandomStringError(t *testing.T) {
+	var typ []string
+	vin := dyn.NewValue("var foo", dyn.Location{File: "file", Line: 1, Column: 1})
+	_, err := Normalize(typ, vin)
+	assert.Len(t, err, 1)
+	assert.Equal(t, diag.Diagnostic{
+		Severity: diag.Warning,
+		Summary:  `expected sequence, found string`,
+		Location: vin.Location(),
+		Path:     dyn.EmptyPath,
+	}, err[0])
+}
+
+func TestNormalizeSliceIntError(t *testing.T) {
+	var typ []string
+	vin := dyn.NewValue(1, dyn.Location{File: "file", Line: 1, Column: 1})
+	_, err := Normalize(typ, vin)
+	assert.Len(t, err, 1)
+	assert.Equal(t, diag.Diagnostic{
+		Severity: diag.Warning,
+		Summary:  `expected sequence, found int`,
+		Location: vin.Location(),
+		Path:     dyn.EmptyPath,
+	}, err[0])
+}
+
 func TestNormalizeString(t *testing.T) {
 	var typ string
 	vin := dyn.V("string")
@@ -741,3 +855,13 @@ func TestNormalizeIntToAny(t *testing.T) {
 	assert.Len(t, err, 0)
 	assert.Equal(t, dyn.NewValue(10, dyn.Location{File: "file", Line: 1, Column: 1}), vout)
 }
+
+func TestNormalizeSliceToAny(t *testing.T) {
+	var typ any
+	v1 := dyn.NewValue(1, dyn.Location{File: "file", Line: 1, Column: 1})
+	v2 := dyn.NewValue(2, dyn.Location{File: "file", Line: 1, Column: 1})
+	vin := dyn.NewValue([]dyn.Value{v1, v2}, dyn.Location{File: "file", Line: 1, Column: 1})
+	vout, err := Normalize(&typ, vin)
+	assert.Len(t, err, 0)
+	assert.Equal(t, dyn.NewValue([]dyn.Value{v1, v2}, dyn.Location{File: "file", Line: 1, Column: 1}), vout)
+}

From f5d71271e044072fd544faca7a6f6d90018654e7 Mon Sep 17 00:00:00 2001
From: Andrew Nester <andrew.nester@databricks.com>
Date: Tue, 18 Jun 2024 13:37:39 +0200
Subject: [PATCH 06/14] Update libs/dyn/convert/normalize.go

Co-authored-by: shreyas-goenka <88374338+shreyas-goenka@users.noreply.github.com>
---
 libs/dyn/convert/normalize.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libs/dyn/convert/normalize.go b/libs/dyn/convert/normalize.go
index 128bedf954..ad82e20ef5 100644
--- a/libs/dyn/convert/normalize.go
+++ b/libs/dyn/convert/normalize.go
@@ -176,7 +176,7 @@ func (n normalizeOptions) normalizeStruct(typ reflect.Type, src dyn.Value, seen
 		}
 	}
 
-	// Cannot interpret as a strcut.
+	// Cannot interpret as a struct.
 	return dyn.InvalidValue, diags.Append(typeMismatch(dyn.KindMap, src, path))
 }
 

From f7eba5fad4e2dff8171963176a1ff7900d983b07 Mon Sep 17 00:00:00 2001
From: Andrew Nester <andrew.nester@databricks.com>
Date: Wed, 19 Jun 2024 15:47:44 +0200
Subject: [PATCH 07/14] fixes

---
 .../mutator/resolve_resource_references_test.go      | 11 ++---------
 bundle/tests/complex_variables_test.go               | 12 ++++++++++++
 bundle/tests/variables/complex/databricks.yml        |  9 +++++++++
 bundle/tests/variables_test.go                       |  6 +++---
 libs/dyn/convert/from_typed_test.go                  |  2 ++
 5 files changed, 28 insertions(+), 12 deletions(-)

diff --git a/bundle/config/mutator/resolve_resource_references_test.go b/bundle/config/mutator/resolve_resource_references_test.go
index 6590b39d7f..86a03b23e4 100644
--- a/bundle/config/mutator/resolve_resource_references_test.go
+++ b/bundle/config/mutator/resolve_resource_references_test.go
@@ -194,22 +194,15 @@ func TestResolveLookupVariableReferencesInVariableLookups(t *testing.T) {
 }
 
 func TestNoResolveLookupIfVariableSetWithEnvVariable(t *testing.T) {
-	s := func(s string) *string {
-		return &s
-	}
-
 	b := &bundle.Bundle{
 		Config: config.Root{
 			Bundle: config.Bundle{
 				Target: "dev",
 			},
 			Variables: map[string]*variable.Variable{
-				"foo": {
-					Value: s("bar"),
-				},
 				"lookup": {
 					Lookup: &variable.Lookup{
-						Cluster: "cluster-${var.foo}-${bundle.target}",
+						Cluster: "cluster-${bundle.target}",
 					},
 				},
 			},
@@ -224,5 +217,5 @@ func TestNoResolveLookupIfVariableSetWithEnvVariable(t *testing.T) {
 
 	diags := bundle.Apply(ctx, b, bundle.Seq(SetVariables(), ResolveVariableReferencesInLookup(), ResolveResourceReferences()))
 	require.NoError(t, diags.Error())
-	require.Equal(t, "1234-5678-abcd", *b.Config.Variables["lookup"].Value)
+	require.Equal(t, "1234-5678-abcd", b.Config.Variables["lookup"].Value)
 }
diff --git a/bundle/tests/complex_variables_test.go b/bundle/tests/complex_variables_test.go
index 36b2374c74..e6e78a8b41 100644
--- a/bundle/tests/complex_variables_test.go
+++ b/bundle/tests/complex_variables_test.go
@@ -6,6 +6,7 @@ import (
 
 	"github.com/databricks/cli/bundle"
 	"github.com/databricks/cli/bundle/config/mutator"
+	"github.com/databricks/databricks-sdk-go/service/compute"
 	"github.com/stretchr/testify/require"
 )
 
@@ -24,6 +25,17 @@ func TestComplexVariables(t *testing.T) {
 	require.Equal(t, "13.2.x-scala2.11", b.Config.Resources.Jobs["my_job"].JobClusters[0].NewCluster.SparkVersion)
 	require.Equal(t, 2, b.Config.Resources.Jobs["my_job"].JobClusters[0].NewCluster.NumWorkers)
 	require.Equal(t, "true", b.Config.Resources.Jobs["my_job"].JobClusters[0].NewCluster.SparkConf["spark.speculation"])
+
+	require.Equal(t, 3, len(b.Config.Resources.Jobs["my_job"].Tasks[0].Libraries))
+	require.Contains(t, b.Config.Resources.Jobs["my_job"].Tasks[0].Libraries, compute.Library{
+		Jar: "/path/to/jar",
+	})
+	require.Contains(t, b.Config.Resources.Jobs["my_job"].Tasks[0].Libraries, compute.Library{
+		Egg: "/path/to/egg",
+	})
+	require.Contains(t, b.Config.Resources.Jobs["my_job"].Tasks[0].Libraries, compute.Library{
+		Whl: "/path/to/whl",
+	})
 }
 
 func TestComplexVariablesOverride(t *testing.T) {
diff --git a/bundle/tests/variables/complex/databricks.yml b/bundle/tests/variables/complex/databricks.yml
index db13807348..ecc4ecd94d 100644
--- a/bundle/tests/variables/complex/databricks.yml
+++ b/bundle/tests/variables/complex/databricks.yml
@@ -10,6 +10,7 @@ resources:
       tasks:
       - task_key: test
         job_cluster_key: key
+        libraries: ${var.libraries}
 
 variables:
   cluster:
@@ -22,6 +23,14 @@ variables:
       spark_conf:
         spark.speculation: true
         spark.databricks.delta.retentionDurationCheck.enabled: false
+  libraries:
+    type: complex
+    description: "A libraries definition"
+    default:
+      - jar: "/path/to/jar"
+      - egg: "/path/to/egg"
+      - whl: "/path/to/whl"
+
 
 targets:
   default:
diff --git a/bundle/tests/variables_test.go b/bundle/tests/variables_test.go
index c46ddcd681..7cf0f72f03 100644
--- a/bundle/tests/variables_test.go
+++ b/bundle/tests/variables_test.go
@@ -140,9 +140,9 @@ func TestVariablesWithTargetLookupOverrides(t *testing.T) {
 	))
 
 	require.NoError(t, diags.Error())
-	assert.Equal(t, "4321", *b.Config.Variables["d"].Value)
-	assert.Equal(t, "1234", *b.Config.Variables["e"].Value)
-	assert.Equal(t, "9876", *b.Config.Variables["f"].Value)
+	assert.Equal(t, "4321", b.Config.Variables["d"].Value)
+	assert.Equal(t, "1234", b.Config.Variables["e"].Value)
+	assert.Equal(t, "9876", b.Config.Variables["f"].Value)
 }
 
 func TestVariableTargetOverrides(t *testing.T) {
diff --git a/libs/dyn/convert/from_typed_test.go b/libs/dyn/convert/from_typed_test.go
index dd1c237cfb..066c86ef7d 100644
--- a/libs/dyn/convert/from_typed_test.go
+++ b/libs/dyn/convert/from_typed_test.go
@@ -636,6 +636,7 @@ func TestFromTypedAny(t *testing.T) {
 			"foo": "foo",
 			"bar": 1234,
 			"qux": 0,
+			"nil": nil,
 		},
 	}
 
@@ -650,6 +651,7 @@ func TestFromTypedAny(t *testing.T) {
 			"foo": dyn.V("foo"),
 			"bar": dyn.V(int64(1234)),
 			"qux": dyn.V(int64(0)),
+			"nil": dyn.V(nil),
 		}),
 	}), nv)
 }

From ddd14eb4595fa1aa51e164bc84aaf9160185e909 Mon Sep 17 00:00:00 2001
From: Andrew Nester <andrew.nester@databricks.com>
Date: Mon, 24 Jun 2024 12:29:49 +0200
Subject: [PATCH 08/14] allow using variables inside complex variables

---
 bundle/config/mutator/resolve_variable_references.go | 8 ++++++++
 bundle/phases/initialize.go                          | 6 ++++--
 bundle/tests/complex_variables_test.go               | 4 ++++
 bundle/tests/variables/complex/databricks.yml        | 7 +++++--
 libs/dyn/convert/from_typed.go                       | 2 +-
 5 files changed, 22 insertions(+), 5 deletions(-)

diff --git a/bundle/config/mutator/resolve_variable_references.go b/bundle/config/mutator/resolve_variable_references.go
index f7fce6c821..218d5a65c6 100644
--- a/bundle/config/mutator/resolve_variable_references.go
+++ b/bundle/config/mutator/resolve_variable_references.go
@@ -31,6 +31,14 @@ func ResolveVariableReferencesInLookup() bundle.Mutator {
 	}, pattern: dyn.NewPattern(dyn.Key("variables"), dyn.AnyKey(), dyn.Key("lookup")), lookupFn: lookupForVariables}
 }
 
+func ResolveVariableReferencesInComplexVariables() bundle.Mutator {
+	return &resolveVariableReferences{prefixes: []string{
+		"bundle",
+		"workspace",
+		"variables",
+	}, pattern: dyn.NewPattern(dyn.Key("variables"), dyn.AnyKey(), dyn.Key("value")), lookupFn: lookupForVariables}
+}
+
 func lookup(v dyn.Value, path dyn.Path) (dyn.Value, error) {
 	// Future opportunity: if we lookup this path in both the given root
 	// and the synthesized root, we know if it was explicitly set or implied to be empty.
diff --git a/bundle/phases/initialize.go b/bundle/phases/initialize.go
index d96ee0ebf9..e24ce74e6b 100644
--- a/bundle/phases/initialize.go
+++ b/bundle/phases/initialize.go
@@ -29,10 +29,12 @@ func Initialize() bundle.Mutator {
 			mutator.ExpandWorkspaceRoot(),
 			mutator.DefineDefaultWorkspacePaths(),
 			mutator.SetVariables(),
-			// Intentionally placed before ResolveVariableReferencesInLookup, ResolveResourceReferences
-			// and ResolveVariableReferences. See what is expected in PythonMutatorPhaseInit doc
+			// Intentionally placed before ResolveVariableReferencesInLookup, ResolveResourceReferences,
+			// ResolveVariableReferencesInComplexVariables and ResolveVariableReferences.
+			// See what is expected in PythonMutatorPhaseInit doc
 			pythonmutator.PythonMutator(pythonmutator.PythonMutatorPhaseInit),
 			mutator.ResolveVariableReferencesInLookup(),
+			mutator.ResolveVariableReferencesInComplexVariables(),
 			mutator.ResolveResourceReferences(),
 			mutator.ResolveVariableReferences(
 				"bundle",
diff --git a/bundle/tests/complex_variables_test.go b/bundle/tests/complex_variables_test.go
index e6e78a8b41..57724d3752 100644
--- a/bundle/tests/complex_variables_test.go
+++ b/bundle/tests/complex_variables_test.go
@@ -16,6 +16,7 @@ func TestComplexVariables(t *testing.T) {
 
 	diags = bundle.Apply(context.Background(), b, bundle.Seq(
 		mutator.SetVariables(),
+		mutator.ResolveVariableReferencesInComplexVariables(),
 		mutator.ResolveVariableReferences(
 			"variables",
 		),
@@ -23,6 +24,7 @@ func TestComplexVariables(t *testing.T) {
 	require.NoError(t, diags.Error())
 
 	require.Equal(t, "13.2.x-scala2.11", b.Config.Resources.Jobs["my_job"].JobClusters[0].NewCluster.SparkVersion)
+	require.Equal(t, "Standard_DS3_v2", b.Config.Resources.Jobs["my_job"].JobClusters[0].NewCluster.NodeTypeId)
 	require.Equal(t, 2, b.Config.Resources.Jobs["my_job"].JobClusters[0].NewCluster.NumWorkers)
 	require.Equal(t, "true", b.Config.Resources.Jobs["my_job"].JobClusters[0].NewCluster.SparkConf["spark.speculation"])
 
@@ -44,6 +46,7 @@ func TestComplexVariablesOverride(t *testing.T) {
 
 	diags = bundle.Apply(context.Background(), b, bundle.Seq(
 		mutator.SetVariables(),
+		mutator.ResolveVariableReferencesInComplexVariables(),
 		mutator.ResolveVariableReferences(
 			"variables",
 		),
@@ -51,6 +54,7 @@ func TestComplexVariablesOverride(t *testing.T) {
 	require.NoError(t, diags.Error())
 
 	require.Equal(t, "14.2.x-scala2.11", b.Config.Resources.Jobs["my_job"].JobClusters[0].NewCluster.SparkVersion)
+	require.Equal(t, "Standard_DS3_v3", b.Config.Resources.Jobs["my_job"].JobClusters[0].NewCluster.NodeTypeId)
 	require.Equal(t, 4, b.Config.Resources.Jobs["my_job"].JobClusters[0].NewCluster.NumWorkers)
 	require.Equal(t, "false", b.Config.Resources.Jobs["my_job"].JobClusters[0].NewCluster.SparkConf["spark.speculation"])
 }
diff --git a/bundle/tests/variables/complex/databricks.yml b/bundle/tests/variables/complex/databricks.yml
index ecc4ecd94d..1a3bdb5c5d 100644
--- a/bundle/tests/variables/complex/databricks.yml
+++ b/bundle/tests/variables/complex/databricks.yml
@@ -13,12 +13,14 @@ resources:
         libraries: ${var.libraries}
 
 variables:
+  node_type:
+    default: "Standard_DS3_v2"
   cluster:
     type: complex
     description: "A cluster definition"
     default:
       spark_version: "13.2.x-scala2.11"
-      node_type_id: "Standard_DS3_v2"
+      node_type_id: ${var.node_type}
       num_workers: 2
       spark_conf:
         spark.speculation: true
@@ -36,9 +38,10 @@ targets:
   default:
   dev:
     variables:
+      node_type: "Standard_DS3_v3"
       cluster:
         spark_version: "14.2.x-scala2.11"
-        node_type_id: "Standard_DS3_v2"
+        node_type_id: ${var.node_type}
         num_workers: 4
         spark_conf:
           spark.speculation: false
diff --git a/libs/dyn/convert/from_typed.go b/libs/dyn/convert/from_typed.go
index b561bc0b04..b772571f96 100644
--- a/libs/dyn/convert/from_typed.go
+++ b/libs/dyn/convert/from_typed.go
@@ -111,7 +111,7 @@ func fromTypedStruct(src reflect.Value, ref dyn.Value, options ...fromTypedOptio
 
 		var options []fromTypedOptions
 		if v.Kind() == reflect.Interface {
-			options = append(options, includeZeroValuedScalars)
+			options = append(options, includeZeroValues)
 		}
 
 		// Convert the field taking into account the reference value (may be equal to config.NilValue).

From 0c4bf7193716ee954d7d1c693694504e6092ba17 Mon Sep 17 00:00:00 2001
From: Andrew Nester <andrew.nester@databricks.com>
Date: Mon, 24 Jun 2024 15:01:41 +0200
Subject: [PATCH 09/14] addressed feedback

---
 .../mutator/resolve_variable_references.go    |  49 +++++++-
 .../resolve_variable_references_test.go       | 115 ++++++++++++++++++
 bundle/tests/complex_variables_test.go        |   2 +
 bundle/tests/variables/complex/databricks.yml |   3 +-
 libs/dyn/convert/from_typed.go                |   4 -
 libs/dyn/dynvar/ref.go                        |   2 +-
 libs/dyn/dynvar/resolve_test.go               |  46 +++++++
 libs/dyn/path.go                              |   4 +-
 libs/dyn/visit.go                             |   4 +-
 libs/dyn/visit_set.go                         |   4 +-
 10 files changed, 218 insertions(+), 15 deletions(-)

diff --git a/bundle/config/mutator/resolve_variable_references.go b/bundle/config/mutator/resolve_variable_references.go
index 218d5a65c6..cddc85cba8 100644
--- a/bundle/config/mutator/resolve_variable_references.go
+++ b/bundle/config/mutator/resolve_variable_references.go
@@ -17,6 +17,7 @@ type resolveVariableReferences struct {
 	prefixes []string
 	pattern  dyn.Pattern
 	lookupFn func(dyn.Value, dyn.Path) (dyn.Value, error)
+	skipFn   func(dyn.Value) bool
 }
 
 func ResolveVariableReferences(prefixes ...string) bundle.Mutator {
@@ -36,7 +37,11 @@ func ResolveVariableReferencesInComplexVariables() bundle.Mutator {
 		"bundle",
 		"workspace",
 		"variables",
-	}, pattern: dyn.NewPattern(dyn.Key("variables"), dyn.AnyKey(), dyn.Key("value")), lookupFn: lookupForVariables}
+	},
+		pattern:  dyn.NewPattern(dyn.Key("variables"), dyn.AnyKey(), dyn.Key("value")),
+		lookupFn: lookupForComplexVariables,
+		skipFn:   skipResolvingInNonComplexVariables,
+	}
 }
 
 func lookup(v dyn.Value, path dyn.Path) (dyn.Value, error) {
@@ -46,6 +51,34 @@ func lookup(v dyn.Value, path dyn.Path) (dyn.Value, error) {
 	return dyn.GetByPath(v, path)
 }
 
+func lookupForComplexVariables(v dyn.Value, path dyn.Path) (dyn.Value, error) {
+	if path[0].Key() != "variables" {
+		return lookup(v, path)
+	}
+
+	varV, err := dyn.GetByPath(v, path[:len(path)-1])
+	if err != nil {
+		return dyn.InvalidValue, err
+	}
+
+	var vv variable.Variable
+	err = convert.ToTyped(&vv, varV)
+	if err != nil {
+		return dyn.InvalidValue, err
+	}
+
+	if vv.Type == variable.VariableTypeComplex {
+		return dyn.InvalidValue, fmt.Errorf("complex variables cannot contain references to another complex variables")
+	}
+
+	return lookup(v, path)
+}
+
+func skipResolvingInNonComplexVariables(v dyn.Value) bool {
+	_, ok := v.AsMap()
+	return !ok
+}
+
 func lookupForVariables(v dyn.Value, path dyn.Path) (dyn.Value, error) {
 	if path[0].Key() != "variables" {
 		return lookup(v, path)
@@ -108,17 +141,27 @@ func (m *resolveVariableReferences) Apply(ctx context.Context, b *bundle.Bundle)
 			// Resolve variable references in all values.
 			return dynvar.Resolve(v, func(path dyn.Path) (dyn.Value, error) {
 				// Rewrite the shorthand path ${var.foo} into ${variables.foo.value}.
-				if path.HasPrefix(varPath) && len(path) == 2 {
-					path = dyn.NewPath(
+				if path.HasPrefix(varPath) {
+					newPath := dyn.NewPath(
 						dyn.Key("variables"),
 						path[1],
 						dyn.Key("value"),
 					)
+
+					if len(path) > 2 {
+						newPath = newPath.Append(path[2:]...)
+					}
+
+					path = newPath
 				}
 
 				// Perform resolution only if the path starts with one of the specified prefixes.
 				for _, prefix := range prefixes {
 					if path.HasPrefix(prefix) {
+						// Skip resolution if there is a skip function and it returns true.
+						if m.skipFn != nil && m.skipFn(v) {
+							return dyn.InvalidValue, dynvar.ErrSkipResolution
+						}
 						return m.lookupFn(normalized, path)
 					}
 				}
diff --git a/bundle/config/mutator/resolve_variable_references_test.go b/bundle/config/mutator/resolve_variable_references_test.go
index bc0cdcea0e..1fea3e5590 100644
--- a/bundle/config/mutator/resolve_variable_references_test.go
+++ b/bundle/config/mutator/resolve_variable_references_test.go
@@ -191,3 +191,118 @@ func TestResolveVariableReferencesForPrimitiveNonStringFields(t *testing.T) {
 	assert.Equal(t, 2, b.Config.Resources.Jobs["job1"].JobSettings.Tasks[0].NewCluster.Autoscale.MaxWorkers)
 	assert.Equal(t, 0.5, b.Config.Resources.Jobs["job1"].JobSettings.Tasks[0].NewCluster.AzureAttributes.SpotBidMaxPrice)
 }
+
+func TestResolveComplexVariable(t *testing.T) {
+	b := &bundle.Bundle{
+		Config: config.Root{
+			Bundle: config.Bundle{
+				Name: "example",
+			},
+			Variables: map[string]*variable.Variable{
+				"cluster": {
+					Value: map[string]any{
+						"node_type_id": "Standard_DS3_v2",
+						"num_workers":  2,
+					},
+				},
+			},
+
+			Resources: config.Resources{
+				Jobs: map[string]*resources.Job{
+					"job1": {
+						JobSettings: &jobs.JobSettings{
+							JobClusters: []jobs.JobCluster{
+								{
+									NewCluster: compute.ClusterSpec{
+										NodeTypeId: "random",
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+
+	ctx := context.Background()
+
+	// Assign the variables to the dynamic configuration.
+	diags := bundle.ApplyFunc(ctx, b, func(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
+		err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) {
+			var p dyn.Path
+			var err error
+
+			// Set the notification settings.
+			p = dyn.MustPathFromString("resources.jobs.job1.job_clusters[0]")
+			v, err = dyn.SetByPath(v, p.Append(dyn.Key("new_cluster")), dyn.V("${var.cluster}"))
+			require.NoError(t, err)
+
+			return v, nil
+		})
+		return diag.FromErr(err)
+	})
+	require.NoError(t, diags.Error())
+
+	diags = bundle.Apply(ctx, b, ResolveVariableReferences("bundle", "workspace", "variables"))
+	require.NoError(t, diags.Error())
+	require.Equal(t, "Standard_DS3_v2", b.Config.Resources.Jobs["job1"].JobSettings.JobClusters[0].NewCluster.NodeTypeId)
+	require.Equal(t, 2, b.Config.Resources.Jobs["job1"].JobSettings.JobClusters[0].NewCluster.NumWorkers)
+}
+
+func TestResolveComplexVariableReferencesToFields(t *testing.T) {
+	b := &bundle.Bundle{
+		Config: config.Root{
+			Bundle: config.Bundle{
+				Name: "example",
+			},
+			Variables: map[string]*variable.Variable{
+				"cluster": {
+					Value: map[string]any{
+						"node_type_id": "Standard_DS3_v2",
+						"num_workers":  2,
+					},
+				},
+			},
+
+			Resources: config.Resources{
+				Jobs: map[string]*resources.Job{
+					"job1": {
+						JobSettings: &jobs.JobSettings{
+							JobClusters: []jobs.JobCluster{
+								{
+									NewCluster: compute.ClusterSpec{
+										NodeTypeId: "random",
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+
+	ctx := context.Background()
+
+	// Assign the variables to the dynamic configuration.
+	diags := bundle.ApplyFunc(ctx, b, func(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
+		err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) {
+			var p dyn.Path
+			var err error
+
+			// Set the notification settings.
+			p = dyn.MustPathFromString("resources.jobs.job1.job_clusters[0].new_cluster")
+			v, err = dyn.SetByPath(v, p.Append(dyn.Key("node_type_id")), dyn.V("${var.cluster.node_type_id}"))
+			require.NoError(t, err)
+
+			return v, nil
+		})
+		return diag.FromErr(err)
+	})
+	require.NoError(t, diags.Error())
+
+	diags = bundle.Apply(ctx, b, ResolveVariableReferences("bundle", "workspace", "variables"))
+	require.NoError(t, diags.Error())
+	require.Equal(t, "Standard_DS3_v2", b.Config.Resources.Jobs["job1"].JobSettings.JobClusters[0].NewCluster.NodeTypeId)
+}
diff --git a/bundle/tests/complex_variables_test.go b/bundle/tests/complex_variables_test.go
index 57724d3752..ffe80e418d 100644
--- a/bundle/tests/complex_variables_test.go
+++ b/bundle/tests/complex_variables_test.go
@@ -38,6 +38,8 @@ func TestComplexVariables(t *testing.T) {
 	require.Contains(t, b.Config.Resources.Jobs["my_job"].Tasks[0].Libraries, compute.Library{
 		Whl: "/path/to/whl",
 	})
+
+	require.Equal(t, "task with spark version 13.2.x-scala2.11 and jar /path/to/jar", b.Config.Resources.Jobs["my_job"].Tasks[0].TaskKey)
 }
 
 func TestComplexVariablesOverride(t *testing.T) {
diff --git a/bundle/tests/variables/complex/databricks.yml b/bundle/tests/variables/complex/databricks.yml
index 1a3bdb5c5d..f7535ad4be 100644
--- a/bundle/tests/variables/complex/databricks.yml
+++ b/bundle/tests/variables/complex/databricks.yml
@@ -10,7 +10,8 @@ resources:
       tasks:
       - task_key: test
         job_cluster_key: key
-        libraries: ${var.libraries}
+        libraries: ${variables.libraries.value}
+        task_key: "task with spark version ${var.cluster.spark_version} and jar ${var.libraries[0].jar}"
 
 variables:
   node_type:
diff --git a/libs/dyn/convert/from_typed.go b/libs/dyn/convert/from_typed.go
index b772571f96..15c5b7978b 100644
--- a/libs/dyn/convert/from_typed.go
+++ b/libs/dyn/convert/from_typed.go
@@ -37,10 +37,6 @@ func FromTyped(src any, ref dyn.Value) (dyn.Value, error) {
 // Private implementation of FromTyped that allows for additional options not exposed
 // in the public API.
 func fromTyped(src any, ref dyn.Value, options ...fromTypedOptions) (dyn.Value, error) {
-	if src == nil {
-		return dyn.NilValue, nil
-	}
-
 	srcv := reflect.ValueOf(src)
 
 	// Dereference pointer if necessary
diff --git a/libs/dyn/dynvar/ref.go b/libs/dyn/dynvar/ref.go
index e6340269f7..e9c5969885 100644
--- a/libs/dyn/dynvar/ref.go
+++ b/libs/dyn/dynvar/ref.go
@@ -6,7 +6,7 @@ import (
 	"github.com/databricks/cli/libs/dyn"
 )
 
-const VariableRegex = `\$\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\}`
+const VariableRegex = `\$\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\[[0-9]+\])?)*(\[[0-9]+\])?)\}`
 
 var re = regexp.MustCompile(VariableRegex)
 
diff --git a/libs/dyn/dynvar/resolve_test.go b/libs/dyn/dynvar/resolve_test.go
index bbecbb7760..b27fdccb74 100644
--- a/libs/dyn/dynvar/resolve_test.go
+++ b/libs/dyn/dynvar/resolve_test.go
@@ -247,3 +247,49 @@ func TestResolveWithInterpolateAliasedRef(t *testing.T) {
 	assert.Equal(t, "a", getByPath(t, out, "b").MustString())
 	assert.Equal(t, "a", getByPath(t, out, "c").MustString())
 }
+
+func TestResolveIndexedRefs(t *testing.T) {
+	in := dyn.V(map[string]dyn.Value{
+		"slice": dyn.V([]dyn.Value{dyn.V("a"), dyn.V("b")}),
+		"a":     dyn.V("a: ${slice[0]}"),
+	})
+
+	out, err := dynvar.Resolve(in, dynvar.DefaultLookup(in))
+	require.NoError(t, err)
+
+	assert.Equal(t, "a: a", getByPath(t, out, "a").MustString())
+}
+
+func TestResolveIndexedRefsFromMap(t *testing.T) {
+	in := dyn.V(map[string]dyn.Value{
+		"map": dyn.V(
+			map[string]dyn.Value{
+				"slice": dyn.V([]dyn.Value{dyn.V("a")}),
+			}),
+		"a": dyn.V("a: ${map.slice[0]}"),
+	})
+
+	out, err := dynvar.Resolve(in, dynvar.DefaultLookup(in))
+	require.NoError(t, err)
+
+	assert.Equal(t, "a: a", getByPath(t, out, "a").MustString())
+}
+
+func TestResolveMapFieldFromIndexedRefs(t *testing.T) {
+	in := dyn.V(map[string]dyn.Value{
+		"map": dyn.V(
+			map[string]dyn.Value{
+				"slice": dyn.V([]dyn.Value{
+					dyn.V(map[string]dyn.Value{
+						"value": dyn.V("a"),
+					}),
+				}),
+			}),
+		"a": dyn.V("a: ${map.slice[0].value}"),
+	})
+
+	out, err := dynvar.Resolve(in, dynvar.DefaultLookup(in))
+	require.NoError(t, err)
+
+	assert.Equal(t, "a: a", getByPath(t, out, "a").MustString())
+}
diff --git a/libs/dyn/path.go b/libs/dyn/path.go
index 76377e2dce..1d0d4afabd 100644
--- a/libs/dyn/path.go
+++ b/libs/dyn/path.go
@@ -18,11 +18,11 @@ func (c pathComponent) Index() int {
 	return c.index
 }
 
-func (c pathComponent) isKey() bool {
+func (c pathComponent) IsKey() bool {
 	return c.key != ""
 }
 
-func (c pathComponent) isIndex() bool {
+func (c pathComponent) IsIndex() bool {
 	return c.key == ""
 }
 
diff --git a/libs/dyn/visit.go b/libs/dyn/visit.go
index 3fe3561943..d0f2bfee5e 100644
--- a/libs/dyn/visit.go
+++ b/libs/dyn/visit.go
@@ -68,7 +68,7 @@ func (component pathComponent) visit(v Value, prefix Path, suffix Pattern, opts
 	path := append(prefix, component)
 
 	switch {
-	case component.isKey():
+	case component.IsKey():
 		// Expect a map to be set if this is a key.
 		m, ok := v.AsMap()
 		if !ok {
@@ -101,7 +101,7 @@ func (component pathComponent) visit(v Value, prefix Path, suffix Pattern, opts
 			l: v.l,
 		}, nil
 
-	case component.isIndex():
+	case component.IsIndex():
 		// Expect a sequence to be set if this is an index.
 		s, ok := v.AsSequence()
 		if !ok {
diff --git a/libs/dyn/visit_set.go b/libs/dyn/visit_set.go
index b086fb8a91..cf3b3b3127 100644
--- a/libs/dyn/visit_set.go
+++ b/libs/dyn/visit_set.go
@@ -32,7 +32,7 @@ func SetByPath(v Value, p Path, nv Value) (Value, error) {
 			path := append(prefix, component)
 
 			switch {
-			case component.isKey():
+			case component.IsKey():
 				// Expect a map to be set if this is a key.
 				m, ok := v.AsMap()
 				if !ok {
@@ -48,7 +48,7 @@ func SetByPath(v Value, p Path, nv Value) (Value, error) {
 					l: v.l,
 				}, nil
 
-			case component.isIndex():
+			case component.IsIndex():
 				// Expect a sequence to be set if this is an index.
 				s, ok := v.AsSequence()
 				if !ok {

From 44b17cd4ab856df4568447fc0006e0d45f0904b3 Mon Sep 17 00:00:00 2001
From: Andrew Nester <andrew.nester@databricks.com>
Date: Mon, 24 Jun 2024 15:23:02 +0200
Subject: [PATCH 10/14] fix test

---
 bundle/schema/schema_test.go | 96 ++++++++++++++++++------------------
 1 file changed, 48 insertions(+), 48 deletions(-)

diff --git a/bundle/schema/schema_test.go b/bundle/schema/schema_test.go
index ea4fd10201..73ace2170e 100644
--- a/bundle/schema/schema_test.go
+++ b/bundle/schema/schema_test.go
@@ -20,7 +20,7 @@ func TestIntSchema(t *testing.T) {
 				},
 				{
 					"type": "string",
-					"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}"
+					"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])?)*(\\[[0-9]+\\])?)\\}"
 				}
 			]
 		}`
@@ -47,7 +47,7 @@ func TestBooleanSchema(t *testing.T) {
 				},
 				{
 					"type": "string",
-					"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}"
+					"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])?)*(\\[[0-9]+\\])?)\\}"
 				}
 			]
 		}`
@@ -123,7 +123,7 @@ func TestStructOfPrimitivesSchema(t *testing.T) {
 						},
 						{
 							"type": "string",
-							"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}"
+							"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])?)*(\\[[0-9]+\\])?)\\}"
 						}
 					]
 				},
@@ -134,7 +134,7 @@ func TestStructOfPrimitivesSchema(t *testing.T) {
 						},
 						{
 							"type": "string",
-							"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}"
+							"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])?)*(\\[[0-9]+\\])?)\\}"
 						}
 					]
 				},
@@ -145,7 +145,7 @@ func TestStructOfPrimitivesSchema(t *testing.T) {
 						},
 						{
 							"type": "string",
-							"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}"
+							"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])?)*(\\[[0-9]+\\])?)\\}"
 						}
 					]
 				},
@@ -156,7 +156,7 @@ func TestStructOfPrimitivesSchema(t *testing.T) {
 						},
 						{
 							"type": "string",
-							"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}"
+							"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])?)*(\\[[0-9]+\\])?)\\}"
 						}
 					]
 				},
@@ -167,7 +167,7 @@ func TestStructOfPrimitivesSchema(t *testing.T) {
 						},
 						{
 							"type": "string",
-							"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}"
+							"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])?)*(\\[[0-9]+\\])?)\\}"
 						}
 					]
 				},
@@ -178,7 +178,7 @@ func TestStructOfPrimitivesSchema(t *testing.T) {
 						},
 						{
 							"type": "string",
-							"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}"
+							"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])?)*(\\[[0-9]+\\])?)\\}"
 						}
 					]
 				},
@@ -189,7 +189,7 @@ func TestStructOfPrimitivesSchema(t *testing.T) {
 						},
 						{
 							"type": "string",
-							"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}"
+							"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])?)*(\\[[0-9]+\\])?)\\}"
 						}
 					]
 				},
@@ -200,7 +200,7 @@ func TestStructOfPrimitivesSchema(t *testing.T) {
 						},
 						{
 							"type": "string",
-							"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}"
+							"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])?)*(\\[[0-9]+\\])?)\\}"
 						}
 					]
 				},
@@ -214,7 +214,7 @@ func TestStructOfPrimitivesSchema(t *testing.T) {
 						},
 						{
 							"type": "string",
-							"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}"
+							"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])?)*(\\[[0-9]+\\])?)\\}"
 						}
 					]
 				},
@@ -225,7 +225,7 @@ func TestStructOfPrimitivesSchema(t *testing.T) {
 						},
 						{
 							"type": "string",
-							"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}"
+							"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])?)*(\\[[0-9]+\\])?)\\}"
 						}
 					]
 				},
@@ -236,7 +236,7 @@ func TestStructOfPrimitivesSchema(t *testing.T) {
 						},
 						{
 							"type": "string",
-							"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}"
+							"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])?)*(\\[[0-9]+\\])?)\\}"
 						}
 					]
 				},
@@ -247,7 +247,7 @@ func TestStructOfPrimitivesSchema(t *testing.T) {
 						},
 						{
 							"type": "string",
-							"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}"
+							"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])?)*(\\[[0-9]+\\])?)\\}"
 						}
 					]
 				},
@@ -258,7 +258,7 @@ func TestStructOfPrimitivesSchema(t *testing.T) {
 						},
 						{
 							"type": "string",
-							"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}"
+							"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])?)*(\\[[0-9]+\\])?)\\}"
 						}
 					]
 				}
@@ -326,7 +326,7 @@ func TestStructOfStructsSchema(t *testing.T) {
 										},
 										{
 											"type": "string",
-											"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}"
+											"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])?)*(\\[[0-9]+\\])?)\\}"
 										}
 									]
 								},
@@ -391,7 +391,7 @@ func TestStructOfMapsSchema(t *testing.T) {
 									},
 									{
 										"type": "string",
-										"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}"
+										"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])?)*(\\[[0-9]+\\])?)\\}"
 									}
 								]
 							}
@@ -481,7 +481,7 @@ func TestMapOfPrimitivesSchema(t *testing.T) {
 					},
 					{
 						"type": "string",
-						"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}"
+						"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])?)*(\\[[0-9]+\\])?)\\}"
 					}
 				]
 			}
@@ -518,7 +518,7 @@ func TestMapOfStructSchema(t *testing.T) {
 							},
 							{
 								"type": "string",
-								"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}"
+								"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])?)*(\\[[0-9]+\\])?)\\}"
 							}
 						]
 					}
@@ -556,7 +556,7 @@ func TestMapOfMapSchema(t *testing.T) {
 						},
 						{
 							"type": "string",
-							"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}"
+							"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])?)*(\\[[0-9]+\\])?)\\}"
 						}
 					]
 				}
@@ -661,7 +661,7 @@ func TestSliceOfMapSchema(t *testing.T) {
 						},
 						{
 							"type": "string",
-							"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}"
+							"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])?)*(\\[[0-9]+\\])?)\\}"
 						}
 					]
 				}
@@ -699,7 +699,7 @@ func TestSliceOfStructSchema(t *testing.T) {
 							},
 							{
 								"type": "string",
-								"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}"
+								"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])?)*(\\[[0-9]+\\])?)\\}"
 							}
 						]
 					}
@@ -757,7 +757,7 @@ func TestEmbeddedStructSchema(t *testing.T) {
 						},
 						{
 							"type": "string",
-							"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}"
+							"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])?)*(\\[[0-9]+\\])?)\\}"
 						}
 					]
 				},
@@ -797,7 +797,7 @@ func TestEmbeddedStructSchema(t *testing.T) {
 											},
 											{
 												"type": "string",
-												"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}"
+												"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])?)*(\\[[0-9]+\\])?)\\}"
 											}
 										]
 									},
@@ -892,7 +892,7 @@ func TestNonAnnotatedFieldsAreSkipped(t *testing.T) {
 						},
 						{
 							"type": "string",
-							"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}"
+							"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])?)*(\\[[0-9]+\\])?)\\}"
 						}
 					]
 				}
@@ -934,7 +934,7 @@ func TestDashFieldsAreSkipped(t *testing.T) {
 						},
 						{
 							"type": "string",
-							"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}"
+							"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])?)*(\\[[0-9]+\\])?)\\}"
 						}
 					]
 				}
@@ -987,7 +987,7 @@ func TestPointerInStructSchema(t *testing.T) {
 								},
 								{
 									"type": "string",
-									"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}"
+									"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])?)*(\\[[0-9]+\\])?)\\}"
 								}
 							]
 						}
@@ -1004,7 +1004,7 @@ func TestPointerInStructSchema(t *testing.T) {
 						},
 						{
 							"type": "string",
-							"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}"
+							"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])?)*(\\[[0-9]+\\])?)\\}"
 						}
 					]
 				},
@@ -1018,7 +1018,7 @@ func TestPointerInStructSchema(t *testing.T) {
 								},
 								{
 									"type": "string",
-									"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}"
+									"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])?)*(\\[[0-9]+\\])?)\\}"
 								}
 							]
 						}
@@ -1035,7 +1035,7 @@ func TestPointerInStructSchema(t *testing.T) {
 						},
 						{
 							"type": "string",
-							"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}"
+							"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])?)*(\\[[0-9]+\\])?)\\}"
 						}
 					]
 				},
@@ -1106,7 +1106,7 @@ func TestGenericSchema(t *testing.T) {
 								},
 								{
 									"type": "string",
-									"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}"
+									"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])?)*(\\[[0-9]+\\])?)\\}"
 								}
 							]
 						},
@@ -1129,7 +1129,7 @@ func TestGenericSchema(t *testing.T) {
 								},
 								{
 									"type": "string",
-									"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}"
+									"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])?)*(\\[[0-9]+\\])?)\\}"
 								}
 							]
 						},
@@ -1157,7 +1157,7 @@ func TestGenericSchema(t *testing.T) {
 											},
 											{
 												"type": "string",
-												"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}"
+												"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])?)*(\\[[0-9]+\\])?)\\}"
 											}
 										]
 									},
@@ -1180,7 +1180,7 @@ func TestGenericSchema(t *testing.T) {
 											},
 											{
 												"type": "string",
-												"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}"
+												"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])?)*(\\[[0-9]+\\])?)\\}"
 											}
 										]
 									},
@@ -1210,7 +1210,7 @@ func TestGenericSchema(t *testing.T) {
 															},
 															{
 																"type": "string",
-																"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}"
+																"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])?)*(\\[[0-9]+\\])?)\\}"
 															}
 														]
 													},
@@ -1236,7 +1236,7 @@ func TestGenericSchema(t *testing.T) {
 															},
 															{
 																"type": "string",
-																"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}"
+																"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])?)*(\\[[0-9]+\\])?)\\}"
 															}
 														]
 													},
@@ -1322,7 +1322,7 @@ func TestFieldsWithoutOmitEmptyAreRequired(t *testing.T) {
 						},
 						{
 							"type": "string",
-							"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}"
+							"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])?)*(\\[[0-9]+\\])?)\\}"
 						}
 					]
 				},
@@ -1333,7 +1333,7 @@ func TestFieldsWithoutOmitEmptyAreRequired(t *testing.T) {
 						},
 						{
 							"type": "string",
-							"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}"
+							"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])?)*(\\[[0-9]+\\])?)\\}"
 						}
 					]
 				},
@@ -1347,7 +1347,7 @@ func TestFieldsWithoutOmitEmptyAreRequired(t *testing.T) {
 								},
 								{
 									"type": "string",
-									"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}"
+									"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])?)*(\\[[0-9]+\\])?)\\}"
 								}
 							]
 						},
@@ -1429,7 +1429,7 @@ func TestDocIngestionForObject(t *testing.T) {
 								},
 								{
 									"type": "string",
-									"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}"
+									"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])?)*(\\[[0-9]+\\])?)\\}"
 								}
 							]
 						}
@@ -1512,7 +1512,7 @@ func TestDocIngestionForSlice(t *testing.T) {
 									},
 									{
 										"type": "string",
-										"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}"
+										"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])?)*(\\[[0-9]+\\])?)\\}"
 									}
 								]
 							},
@@ -1524,7 +1524,7 @@ func TestDocIngestionForSlice(t *testing.T) {
 									},
 									{
 										"type": "string",
-										"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}"
+										"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])?)*(\\[[0-9]+\\])?)\\}"
 									}
 								]
 							}
@@ -1611,7 +1611,7 @@ func TestDocIngestionForMap(t *testing.T) {
 									},
 									{
 										"type": "string",
-										"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}"
+										"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])?)*(\\[[0-9]+\\])?)\\}"
 									}
 								]
 							},
@@ -1623,7 +1623,7 @@ func TestDocIngestionForMap(t *testing.T) {
 									},
 									{
 										"type": "string",
-										"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}"
+										"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])?)*(\\[[0-9]+\\])?)\\}"
 									}
 								]
 							}
@@ -1683,7 +1683,7 @@ func TestDocIngestionForTopLevelPrimitive(t *testing.T) {
 						},
 						{
 							"type": "string",
-							"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}"
+							"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])?)*(\\[[0-9]+\\])?)\\}"
 						}
 					]
 				}
@@ -1761,7 +1761,7 @@ func TestInterfaceGeneratesEmptySchema(t *testing.T) {
 						},
 						{
 							"type": "string",
-							"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}"
+							"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])?)*(\\[[0-9]+\\])?)\\}"
 						}
 					]
 				},
@@ -1810,7 +1810,7 @@ func TestBundleReadOnlytag(t *testing.T) {
 						},
 						{
 							"type": "string",
-							"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}"
+							"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])?)*(\\[[0-9]+\\])?)\\}"
 						}
 					]
 				},
@@ -1870,7 +1870,7 @@ func TestBundleInternalTag(t *testing.T) {
 						},
 						{
 							"type": "string",
-							"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}"
+							"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])?)*(\\[[0-9]+\\])?)\\}"
 						}
 					]
 				},

From 95c0032ef728f60361043fbbe93bc294f5995677 Mon Sep 17 00:00:00 2001
From: Andrew Nester <andrew.nester@databricks.com>
Date: Mon, 24 Jun 2024 17:11:32 +0200
Subject: [PATCH 11/14] allow nested indexed refs

---
 libs/dyn/dynvar/ref.go          |  2 +-
 libs/dyn/dynvar/resolve_test.go | 14 ++++++++++++++
 libs/dyn/path.go                |  4 ++--
 libs/dyn/visit.go               |  4 ++--
 libs/dyn/visit_set.go           |  4 ++--
 5 files changed, 21 insertions(+), 7 deletions(-)

diff --git a/libs/dyn/dynvar/ref.go b/libs/dyn/dynvar/ref.go
index e9c5969885..bf160fa85b 100644
--- a/libs/dyn/dynvar/ref.go
+++ b/libs/dyn/dynvar/ref.go
@@ -6,7 +6,7 @@ import (
 	"github.com/databricks/cli/libs/dyn"
 )
 
-const VariableRegex = `\$\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\[[0-9]+\])?)*(\[[0-9]+\])?)\}`
+const VariableRegex = `\$\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\[[0-9]+\])*)*(\[[0-9]+\])*)\}`
 
 var re = regexp.MustCompile(VariableRegex)
 
diff --git a/libs/dyn/dynvar/resolve_test.go b/libs/dyn/dynvar/resolve_test.go
index b27fdccb74..498322a425 100644
--- a/libs/dyn/dynvar/resolve_test.go
+++ b/libs/dyn/dynvar/resolve_test.go
@@ -293,3 +293,17 @@ func TestResolveMapFieldFromIndexedRefs(t *testing.T) {
 
 	assert.Equal(t, "a: a", getByPath(t, out, "a").MustString())
 }
+
+func TestResolveNestedIndexedRefs(t *testing.T) {
+	in := dyn.V(map[string]dyn.Value{
+		"slice": dyn.V([]dyn.Value{
+			dyn.V([]dyn.Value{dyn.V("a")}),
+		}),
+		"a": dyn.V("a: ${slice[0][0]}"),
+	})
+
+	out, err := dynvar.Resolve(in, dynvar.DefaultLookup(in))
+	require.NoError(t, err)
+
+	assert.Equal(t, "a: a", getByPath(t, out, "a").MustString())
+}
diff --git a/libs/dyn/path.go b/libs/dyn/path.go
index 1d0d4afabd..76377e2dce 100644
--- a/libs/dyn/path.go
+++ b/libs/dyn/path.go
@@ -18,11 +18,11 @@ func (c pathComponent) Index() int {
 	return c.index
 }
 
-func (c pathComponent) IsKey() bool {
+func (c pathComponent) isKey() bool {
 	return c.key != ""
 }
 
-func (c pathComponent) IsIndex() bool {
+func (c pathComponent) isIndex() bool {
 	return c.key == ""
 }
 
diff --git a/libs/dyn/visit.go b/libs/dyn/visit.go
index d0f2bfee5e..3fe3561943 100644
--- a/libs/dyn/visit.go
+++ b/libs/dyn/visit.go
@@ -68,7 +68,7 @@ func (component pathComponent) visit(v Value, prefix Path, suffix Pattern, opts
 	path := append(prefix, component)
 
 	switch {
-	case component.IsKey():
+	case component.isKey():
 		// Expect a map to be set if this is a key.
 		m, ok := v.AsMap()
 		if !ok {
@@ -101,7 +101,7 @@ func (component pathComponent) visit(v Value, prefix Path, suffix Pattern, opts
 			l: v.l,
 		}, nil
 
-	case component.IsIndex():
+	case component.isIndex():
 		// Expect a sequence to be set if this is an index.
 		s, ok := v.AsSequence()
 		if !ok {
diff --git a/libs/dyn/visit_set.go b/libs/dyn/visit_set.go
index cf3b3b3127..b086fb8a91 100644
--- a/libs/dyn/visit_set.go
+++ b/libs/dyn/visit_set.go
@@ -32,7 +32,7 @@ func SetByPath(v Value, p Path, nv Value) (Value, error) {
 			path := append(prefix, component)
 
 			switch {
-			case component.IsKey():
+			case component.isKey():
 				// Expect a map to be set if this is a key.
 				m, ok := v.AsMap()
 				if !ok {
@@ -48,7 +48,7 @@ func SetByPath(v Value, p Path, nv Value) (Value, error) {
 					l: v.l,
 				}, nil
 
-			case component.IsIndex():
+			case component.isIndex():
 				// Expect a sequence to be set if this is an index.
 				s, ok := v.AsSequence()
 				if !ok {

From 59dba117771c16db1169a510af26eece6e1245aa Mon Sep 17 00:00:00 2001
From: Andrew Nester <andrew.nester@databricks.com>
Date: Tue, 25 Jun 2024 10:58:49 +0200
Subject: [PATCH 12/14] fixed test

---
 bundle/schema/schema_test.go | 96 ++++++++++++++++++------------------
 1 file changed, 48 insertions(+), 48 deletions(-)

diff --git a/bundle/schema/schema_test.go b/bundle/schema/schema_test.go
index 73ace2170e..6d9df0cc7c 100644
--- a/bundle/schema/schema_test.go
+++ b/bundle/schema/schema_test.go
@@ -20,7 +20,7 @@ func TestIntSchema(t *testing.T) {
 				},
 				{
 					"type": "string",
-					"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])?)*(\\[[0-9]+\\])?)\\}"
+					"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}"
 				}
 			]
 		}`
@@ -47,7 +47,7 @@ func TestBooleanSchema(t *testing.T) {
 				},
 				{
 					"type": "string",
-					"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])?)*(\\[[0-9]+\\])?)\\}"
+					"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}"
 				}
 			]
 		}`
@@ -123,7 +123,7 @@ func TestStructOfPrimitivesSchema(t *testing.T) {
 						},
 						{
 							"type": "string",
-							"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])?)*(\\[[0-9]+\\])?)\\}"
+							"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}"
 						}
 					]
 				},
@@ -134,7 +134,7 @@ func TestStructOfPrimitivesSchema(t *testing.T) {
 						},
 						{
 							"type": "string",
-							"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])?)*(\\[[0-9]+\\])?)\\}"
+							"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}"
 						}
 					]
 				},
@@ -145,7 +145,7 @@ func TestStructOfPrimitivesSchema(t *testing.T) {
 						},
 						{
 							"type": "string",
-							"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])?)*(\\[[0-9]+\\])?)\\}"
+							"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}"
 						}
 					]
 				},
@@ -156,7 +156,7 @@ func TestStructOfPrimitivesSchema(t *testing.T) {
 						},
 						{
 							"type": "string",
-							"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])?)*(\\[[0-9]+\\])?)\\}"
+							"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}"
 						}
 					]
 				},
@@ -167,7 +167,7 @@ func TestStructOfPrimitivesSchema(t *testing.T) {
 						},
 						{
 							"type": "string",
-							"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])?)*(\\[[0-9]+\\])?)\\}"
+							"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}"
 						}
 					]
 				},
@@ -178,7 +178,7 @@ func TestStructOfPrimitivesSchema(t *testing.T) {
 						},
 						{
 							"type": "string",
-							"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])?)*(\\[[0-9]+\\])?)\\}"
+							"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}"
 						}
 					]
 				},
@@ -189,7 +189,7 @@ func TestStructOfPrimitivesSchema(t *testing.T) {
 						},
 						{
 							"type": "string",
-							"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])?)*(\\[[0-9]+\\])?)\\}"
+							"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}"
 						}
 					]
 				},
@@ -200,7 +200,7 @@ func TestStructOfPrimitivesSchema(t *testing.T) {
 						},
 						{
 							"type": "string",
-							"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])?)*(\\[[0-9]+\\])?)\\}"
+							"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}"
 						}
 					]
 				},
@@ -214,7 +214,7 @@ func TestStructOfPrimitivesSchema(t *testing.T) {
 						},
 						{
 							"type": "string",
-							"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])?)*(\\[[0-9]+\\])?)\\}"
+							"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}"
 						}
 					]
 				},
@@ -225,7 +225,7 @@ func TestStructOfPrimitivesSchema(t *testing.T) {
 						},
 						{
 							"type": "string",
-							"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])?)*(\\[[0-9]+\\])?)\\}"
+							"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}"
 						}
 					]
 				},
@@ -236,7 +236,7 @@ func TestStructOfPrimitivesSchema(t *testing.T) {
 						},
 						{
 							"type": "string",
-							"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])?)*(\\[[0-9]+\\])?)\\}"
+							"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}"
 						}
 					]
 				},
@@ -247,7 +247,7 @@ func TestStructOfPrimitivesSchema(t *testing.T) {
 						},
 						{
 							"type": "string",
-							"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])?)*(\\[[0-9]+\\])?)\\}"
+							"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}"
 						}
 					]
 				},
@@ -258,7 +258,7 @@ func TestStructOfPrimitivesSchema(t *testing.T) {
 						},
 						{
 							"type": "string",
-							"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])?)*(\\[[0-9]+\\])?)\\}"
+							"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}"
 						}
 					]
 				}
@@ -326,7 +326,7 @@ func TestStructOfStructsSchema(t *testing.T) {
 										},
 										{
 											"type": "string",
-											"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])?)*(\\[[0-9]+\\])?)\\}"
+											"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}"
 										}
 									]
 								},
@@ -391,7 +391,7 @@ func TestStructOfMapsSchema(t *testing.T) {
 									},
 									{
 										"type": "string",
-										"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])?)*(\\[[0-9]+\\])?)\\}"
+										"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}"
 									}
 								]
 							}
@@ -481,7 +481,7 @@ func TestMapOfPrimitivesSchema(t *testing.T) {
 					},
 					{
 						"type": "string",
-						"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])?)*(\\[[0-9]+\\])?)\\}"
+						"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}"
 					}
 				]
 			}
@@ -518,7 +518,7 @@ func TestMapOfStructSchema(t *testing.T) {
 							},
 							{
 								"type": "string",
-								"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])?)*(\\[[0-9]+\\])?)\\}"
+								"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}"
 							}
 						]
 					}
@@ -556,7 +556,7 @@ func TestMapOfMapSchema(t *testing.T) {
 						},
 						{
 							"type": "string",
-							"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])?)*(\\[[0-9]+\\])?)\\}"
+							"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}"
 						}
 					]
 				}
@@ -661,7 +661,7 @@ func TestSliceOfMapSchema(t *testing.T) {
 						},
 						{
 							"type": "string",
-							"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])?)*(\\[[0-9]+\\])?)\\}"
+							"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}"
 						}
 					]
 				}
@@ -699,7 +699,7 @@ func TestSliceOfStructSchema(t *testing.T) {
 							},
 							{
 								"type": "string",
-								"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])?)*(\\[[0-9]+\\])?)\\}"
+								"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}"
 							}
 						]
 					}
@@ -757,7 +757,7 @@ func TestEmbeddedStructSchema(t *testing.T) {
 						},
 						{
 							"type": "string",
-							"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])?)*(\\[[0-9]+\\])?)\\}"
+							"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}"
 						}
 					]
 				},
@@ -797,7 +797,7 @@ func TestEmbeddedStructSchema(t *testing.T) {
 											},
 											{
 												"type": "string",
-												"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])?)*(\\[[0-9]+\\])?)\\}"
+												"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}"
 											}
 										]
 									},
@@ -892,7 +892,7 @@ func TestNonAnnotatedFieldsAreSkipped(t *testing.T) {
 						},
 						{
 							"type": "string",
-							"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])?)*(\\[[0-9]+\\])?)\\}"
+							"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}"
 						}
 					]
 				}
@@ -934,7 +934,7 @@ func TestDashFieldsAreSkipped(t *testing.T) {
 						},
 						{
 							"type": "string",
-							"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])?)*(\\[[0-9]+\\])?)\\}"
+							"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}"
 						}
 					]
 				}
@@ -987,7 +987,7 @@ func TestPointerInStructSchema(t *testing.T) {
 								},
 								{
 									"type": "string",
-									"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])?)*(\\[[0-9]+\\])?)\\}"
+									"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}"
 								}
 							]
 						}
@@ -1004,7 +1004,7 @@ func TestPointerInStructSchema(t *testing.T) {
 						},
 						{
 							"type": "string",
-							"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])?)*(\\[[0-9]+\\])?)\\}"
+							"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}"
 						}
 					]
 				},
@@ -1018,7 +1018,7 @@ func TestPointerInStructSchema(t *testing.T) {
 								},
 								{
 									"type": "string",
-									"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])?)*(\\[[0-9]+\\])?)\\}"
+									"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}"
 								}
 							]
 						}
@@ -1035,7 +1035,7 @@ func TestPointerInStructSchema(t *testing.T) {
 						},
 						{
 							"type": "string",
-							"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])?)*(\\[[0-9]+\\])?)\\}"
+							"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}"
 						}
 					]
 				},
@@ -1106,7 +1106,7 @@ func TestGenericSchema(t *testing.T) {
 								},
 								{
 									"type": "string",
-									"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])?)*(\\[[0-9]+\\])?)\\}"
+									"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}"
 								}
 							]
 						},
@@ -1129,7 +1129,7 @@ func TestGenericSchema(t *testing.T) {
 								},
 								{
 									"type": "string",
-									"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])?)*(\\[[0-9]+\\])?)\\}"
+									"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}"
 								}
 							]
 						},
@@ -1157,7 +1157,7 @@ func TestGenericSchema(t *testing.T) {
 											},
 											{
 												"type": "string",
-												"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])?)*(\\[[0-9]+\\])?)\\}"
+												"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}"
 											}
 										]
 									},
@@ -1180,7 +1180,7 @@ func TestGenericSchema(t *testing.T) {
 											},
 											{
 												"type": "string",
-												"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])?)*(\\[[0-9]+\\])?)\\}"
+												"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}"
 											}
 										]
 									},
@@ -1210,7 +1210,7 @@ func TestGenericSchema(t *testing.T) {
 															},
 															{
 																"type": "string",
-																"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])?)*(\\[[0-9]+\\])?)\\}"
+																"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}"
 															}
 														]
 													},
@@ -1236,7 +1236,7 @@ func TestGenericSchema(t *testing.T) {
 															},
 															{
 																"type": "string",
-																"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])?)*(\\[[0-9]+\\])?)\\}"
+																"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}"
 															}
 														]
 													},
@@ -1322,7 +1322,7 @@ func TestFieldsWithoutOmitEmptyAreRequired(t *testing.T) {
 						},
 						{
 							"type": "string",
-							"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])?)*(\\[[0-9]+\\])?)\\}"
+							"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}"
 						}
 					]
 				},
@@ -1333,7 +1333,7 @@ func TestFieldsWithoutOmitEmptyAreRequired(t *testing.T) {
 						},
 						{
 							"type": "string",
-							"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])?)*(\\[[0-9]+\\])?)\\}"
+							"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}"
 						}
 					]
 				},
@@ -1347,7 +1347,7 @@ func TestFieldsWithoutOmitEmptyAreRequired(t *testing.T) {
 								},
 								{
 									"type": "string",
-									"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])?)*(\\[[0-9]+\\])?)\\}"
+									"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}"
 								}
 							]
 						},
@@ -1429,7 +1429,7 @@ func TestDocIngestionForObject(t *testing.T) {
 								},
 								{
 									"type": "string",
-									"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])?)*(\\[[0-9]+\\])?)\\}"
+									"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}"
 								}
 							]
 						}
@@ -1512,7 +1512,7 @@ func TestDocIngestionForSlice(t *testing.T) {
 									},
 									{
 										"type": "string",
-										"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])?)*(\\[[0-9]+\\])?)\\}"
+										"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}"
 									}
 								]
 							},
@@ -1524,7 +1524,7 @@ func TestDocIngestionForSlice(t *testing.T) {
 									},
 									{
 										"type": "string",
-										"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])?)*(\\[[0-9]+\\])?)\\}"
+										"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}"
 									}
 								]
 							}
@@ -1611,7 +1611,7 @@ func TestDocIngestionForMap(t *testing.T) {
 									},
 									{
 										"type": "string",
-										"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])?)*(\\[[0-9]+\\])?)\\}"
+										"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}"
 									}
 								]
 							},
@@ -1623,7 +1623,7 @@ func TestDocIngestionForMap(t *testing.T) {
 									},
 									{
 										"type": "string",
-										"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])?)*(\\[[0-9]+\\])?)\\}"
+										"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}"
 									}
 								]
 							}
@@ -1683,7 +1683,7 @@ func TestDocIngestionForTopLevelPrimitive(t *testing.T) {
 						},
 						{
 							"type": "string",
-							"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])?)*(\\[[0-9]+\\])?)\\}"
+							"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}"
 						}
 					]
 				}
@@ -1761,7 +1761,7 @@ func TestInterfaceGeneratesEmptySchema(t *testing.T) {
 						},
 						{
 							"type": "string",
-							"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])?)*(\\[[0-9]+\\])?)\\}"
+							"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}"
 						}
 					]
 				},
@@ -1810,7 +1810,7 @@ func TestBundleReadOnlytag(t *testing.T) {
 						},
 						{
 							"type": "string",
-							"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])?)*(\\[[0-9]+\\])?)\\}"
+							"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}"
 						}
 					]
 				},
@@ -1870,7 +1870,7 @@ func TestBundleInternalTag(t *testing.T) {
 						},
 						{
 							"type": "string",
-							"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])?)*(\\[[0-9]+\\])?)\\}"
+							"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)*(\\[[0-9]+\\])*)\\}"
 						}
 					]
 				},

From 327d05e4288543d6f98b27bbaece5ee4dedca2c8 Mon Sep 17 00:00:00 2001
From: Andrew Nester <andrew.nester@databricks.com>
Date: Tue, 25 Jun 2024 14:01:05 +0200
Subject: [PATCH 13/14] added test for referencing complex in complex error

---
 .../resolve_variable_references_test.go       | 68 ++++++++++++++++++-
 bundle/phases/initialize.go                   |  2 +-
 2 files changed, 67 insertions(+), 3 deletions(-)

diff --git a/bundle/config/mutator/resolve_variable_references_test.go b/bundle/config/mutator/resolve_variable_references_test.go
index 1fea3e5590..2b88a2495d 100644
--- a/bundle/config/mutator/resolve_variable_references_test.go
+++ b/bundle/config/mutator/resolve_variable_references_test.go
@@ -204,6 +204,7 @@ func TestResolveComplexVariable(t *testing.T) {
 						"node_type_id": "Standard_DS3_v2",
 						"num_workers":  2,
 					},
+					Type: variable.VariableTypeComplex,
 				},
 			},
 
@@ -233,7 +234,6 @@ func TestResolveComplexVariable(t *testing.T) {
 			var p dyn.Path
 			var err error
 
-			// Set the notification settings.
 			p = dyn.MustPathFromString("resources.jobs.job1.job_clusters[0]")
 			v, err = dyn.SetByPath(v, p.Append(dyn.Key("new_cluster")), dyn.V("${var.cluster}"))
 			require.NoError(t, err)
@@ -262,6 +262,7 @@ func TestResolveComplexVariableReferencesToFields(t *testing.T) {
 						"node_type_id": "Standard_DS3_v2",
 						"num_workers":  2,
 					},
+					Type: variable.VariableTypeComplex,
 				},
 			},
 
@@ -291,7 +292,6 @@ func TestResolveComplexVariableReferencesToFields(t *testing.T) {
 			var p dyn.Path
 			var err error
 
-			// Set the notification settings.
 			p = dyn.MustPathFromString("resources.jobs.job1.job_clusters[0].new_cluster")
 			v, err = dyn.SetByPath(v, p.Append(dyn.Key("node_type_id")), dyn.V("${var.cluster.node_type_id}"))
 			require.NoError(t, err)
@@ -306,3 +306,67 @@ func TestResolveComplexVariableReferencesToFields(t *testing.T) {
 	require.NoError(t, diags.Error())
 	require.Equal(t, "Standard_DS3_v2", b.Config.Resources.Jobs["job1"].JobSettings.JobClusters[0].NewCluster.NodeTypeId)
 }
+
+func TestResolveComplexVariableReferencesWithComplexVariablesError(t *testing.T) {
+	b := &bundle.Bundle{
+		Config: config.Root{
+			Bundle: config.Bundle{
+				Name: "example",
+			},
+			Variables: map[string]*variable.Variable{
+				"cluster": {
+					Value: map[string]any{
+						"node_type_id": "Standard_DS3_v2",
+						"num_workers":  2,
+						"spark_conf":   "${var.spark_conf}",
+					},
+					Type: variable.VariableTypeComplex,
+				},
+				"spark_conf": {
+					Value: map[string]any{
+						"spark.executor.memory": "4g",
+						"spark.executor.cores":  "2",
+					},
+					Type: variable.VariableTypeComplex,
+				},
+			},
+
+			Resources: config.Resources{
+				Jobs: map[string]*resources.Job{
+					"job1": {
+						JobSettings: &jobs.JobSettings{
+							JobClusters: []jobs.JobCluster{
+								{
+									NewCluster: compute.ClusterSpec{
+										NodeTypeId: "random",
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+
+	ctx := context.Background()
+
+	// Assign the variables to the dynamic configuration.
+	diags := bundle.ApplyFunc(ctx, b, func(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
+		err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) {
+			var p dyn.Path
+			var err error
+
+			p = dyn.MustPathFromString("resources.jobs.job1.job_clusters[0]")
+			v, err = dyn.SetByPath(v, p.Append(dyn.Key("new_cluster")), dyn.V("${var.cluster}"))
+			require.NoError(t, err)
+
+			return v, nil
+		})
+		return diag.FromErr(err)
+	})
+	require.NoError(t, diags.Error())
+
+	diags = bundle.Apply(ctx, b, bundle.Seq(ResolveVariableReferencesInComplexVariables(), ResolveVariableReferences("bundle", "workspace", "variables")))
+	require.ErrorContains(t, diags.Error(), "complex variables cannot contain references to another complex variables")
+}
diff --git a/bundle/phases/initialize.go b/bundle/phases/initialize.go
index e24ce74e6b..79fca9df64 100644
--- a/bundle/phases/initialize.go
+++ b/bundle/phases/initialize.go
@@ -34,8 +34,8 @@ func Initialize() bundle.Mutator {
 			// See what is expected in PythonMutatorPhaseInit doc
 			pythonmutator.PythonMutator(pythonmutator.PythonMutatorPhaseInit),
 			mutator.ResolveVariableReferencesInLookup(),
-			mutator.ResolveVariableReferencesInComplexVariables(),
 			mutator.ResolveResourceReferences(),
+			mutator.ResolveVariableReferencesInComplexVariables(),
 			mutator.ResolveVariableReferences(
 				"bundle",
 				"workspace",

From 2e3c97b846c8a0373f180fe4b3a7bc49385fa84c Mon Sep 17 00:00:00 2001
From: Andrew Nester <andrew.nester@databricks.com>
Date: Wed, 26 Jun 2024 11:36:01 +0200
Subject: [PATCH 14/14] added missing test

---
 bundle/config/mutator/set_variables_test.go | 15 +++++++++++++++
 bundle/config/root_test.go                  | 13 +++++++++++++
 2 files changed, 28 insertions(+)

diff --git a/bundle/config/mutator/set_variables_test.go b/bundle/config/mutator/set_variables_test.go
index 41942e47bc..65dedee978 100644
--- a/bundle/config/mutator/set_variables_test.go
+++ b/bundle/config/mutator/set_variables_test.go
@@ -114,3 +114,18 @@ func TestSetVariablesMutator(t *testing.T) {
 	assert.Equal(t, "env-var-b", b.Config.Variables["b"].Value)
 	assert.Equal(t, "assigned-val-c", b.Config.Variables["c"].Value)
 }
+
+func TestSetComplexVariablesViaEnvVariablesIsNotAllowed(t *testing.T) {
+	defaultVal := "default"
+	variable := variable.Variable{
+		Description: "a test variable",
+		Default:     defaultVal,
+		Type:        variable.VariableTypeComplex,
+	}
+
+	// set value for variable as an environment variable
+	t.Setenv("BUNDLE_VAR_foo", "process-env")
+
+	diags := setVariable(context.Background(), &variable, "foo")
+	assert.ErrorContains(t, diags.Error(), "setting via environment variables (BUNDLE_VAR_foo) is not supported for complex variable foo")
+}
diff --git a/bundle/config/root_test.go b/bundle/config/root_test.go
index 07f1feece0..27cc3d22bc 100644
--- a/bundle/config/root_test.go
+++ b/bundle/config/root_test.go
@@ -119,3 +119,16 @@ func TestRootMergeTargetOverridesWithMode(t *testing.T) {
 	require.NoError(t, root.MergeTargetOverrides("development"))
 	assert.Equal(t, Development, root.Bundle.Mode)
 }
+
+func TestInitializeComplexVariablesViaFlagIsNotAllowed(t *testing.T) {
+	root := &Root{
+		Variables: map[string]*variable.Variable{
+			"foo": {
+				Type: variable.VariableTypeComplex,
+			},
+		},
+	}
+
+	err := root.InitializeVariables([]string{"foo=123"})
+	assert.ErrorContains(t, err, "setting variables of complex type via --var flag is not supported: foo")
+}