fix

bundle/config/root.go

@@ -408,13 +408,21 @@ func (r *Root) MergeTargetOverrides(name string) error {
 
 var variableKeywords = []string{"default", "lookup"}
 
-// isFullVariableDef checks if the given value is a variable definition.
-// A variable definition is a map with one of the following
-// keys: "default", "value", "lookup".
-func isFullVariableDef(v dyn.Value) bool {
+// isFullVariableOverrideDef checks if the given value is a full syntax varaible override.
+// A full syntax variable override is a map with only one of the following
+// keys: "default", "lookup".
+func isFullVariableOverrideDef(v dyn.Value) bool {
+	mv, ok := v.AsMap()
+	if !ok {
+		return false
+	}
+
+	if mv.Len() != 1 {
+		return false
+	}
+
 	for _, keyword := range variableKeywords {
-		vv, err := dyn.Get(v, keyword)
-		if err == nil && vv.Kind() != dyn.KindInvalid {
+		if _, ok := mv.GetByString(keyword); ok {
 			return true
 		}
 	}
@@ -449,6 +457,11 @@ func rewriteShorthands(v dyn.Value) (dyn.Value, error) {
 				}, variable.Locations()), nil
 
 			case dyn.KindMap, dyn.KindSequence:
+				// If it's a full variable definition, leave it as is.
+				if isFullVariableOverrideDef(variable) {
+					return variable, nil
+				}
+
 				// Check if the original definition of variable has a type field.
 				// If it has a type field, it means the shorthand is a value of a complex type.
 				// Type might not be found if the variable overriden in a separate file
@@ -463,11 +476,6 @@ func rewriteShorthands(v dyn.Value) (dyn.Value, error) {
 					}
 				}
 
-				// If it's a full variable definition, leave it as is.
-				if isFullVariableDef(variable) {
-					return variable, nil
-				}
-
 				// If it's a shorthand, rewrite it to a full variable definition.
 				return dyn.NewValue(map[string]dyn.Value{
 					"default": variable,

bundle/tests/complex_variables_test.go

@@ -81,10 +81,10 @@ func TestComplexVariablesOverrideWithMultipleFiles(t *testing.T) {
 		),
 	))
 	require.NoError(t, diags.Error())
-	for i, cluster := range b.Config.Resources.Jobs["my_job"].JobClusters {
-		require.Equalf(t, "14.2.x-scala2.11", cluster.NewCluster.SparkVersion, "cluster: %v", i)
-		require.Equalf(t, "Standard_DS3_v2", cluster.NewCluster.NodeTypeId, "cluster: %v", i)
-		require.Equalf(t, 4, cluster.NewCluster.NumWorkers, "cluster: %v", i)
-		require.Equalf(t, "false", cluster.NewCluster.SparkConf["spark.speculation"], "cluster: %v", i)
+	for _, cluster := range b.Config.Resources.Jobs["my_job"].JobClusters {
+		require.Equalf(t, "14.2.x-scala2.11", cluster.NewCluster.SparkVersion, "cluster: %v", cluster.JobClusterKey)
+		require.Equalf(t, "Standard_DS3_v2", cluster.NewCluster.NodeTypeId, "cluster: %v", cluster.JobClusterKey)
+		require.Equalf(t, 4, cluster.NewCluster.NumWorkers, "cluster: %v", cluster.JobClusterKey)
+		require.Equalf(t, "false", cluster.NewCluster.SparkConf["spark.speculation"], "cluster: %v", cluster.JobClusterKey)
 	}
 }

bundle/tests/variables/complex_multiple_files/databricks.yml

@@ -9,14 +9,44 @@ resources:
           new_cluster: ${var.cluster1}
         - job_cluster_key: key2
           new_cluster: ${var.cluster2}
-
+        - job_cluster_key: key3
+          new_cluster: ${var.cluster3}
+        - job_cluster_key: key4
+          new_cluster: ${var.cluster4}
 variables:
   cluster1:
     type: complex
     description: "A cluster definition"
   cluster2:
     type: complex
     description: "A cluster definition"
+  cluster3:
+    type: complex
+    description: "A cluster definition"
+  cluster4:
+    type: complex
+    description: "A cluster definition"
 
 include:
   - ./variables/*.yml
+
+
+targets:
+  default:
+  dev:
+    variables:
+      cluster3:
+        spark_version: "14.2.x-scala2.11"
+        node_type_id: "Standard_DS3_v2"
+        num_workers: 4
+        spark_conf:
+          spark.speculation: false
+          spark.databricks.delta.retentionDurationCheck.enabled: false
+      cluster4:
+        default:
+          spark_version: "14.2.x-scala2.11"
+          node_type_id: "Standard_DS3_v2"
+          num_workers: 4
+          spark_conf:
+            spark.speculation: false
+            spark.databricks.delta.retentionDurationCheck.enabled: false