feat: allow user to specify/override outputs from the setup stage (#1741

)

cli/bptest/cmd.go

@@ -14,6 +14,7 @@ import (
 var flags struct {
 	testDir   string
 	testStage string
+	setupVars map[string]string
 }
 
 func init() {
@@ -25,6 +26,7 @@ func init() {
 
 	Cmd.PersistentFlags().StringVar(&flags.testDir, "test-dir", "", "Path to directory containing integration tests (default is computed by scanning current working directory)")
 	runCmd.Flags().StringVar(&flags.testStage, "stage", "", "Test stage to execute (default is running all stages in order - init, apply, verify, teardown)")
+	runCmd.Flags().StringToStringVar(&flags.setupVars, "setup-var", map[string]string{}, "Specify outputs from the setup phase (useful with --stage=verify)")
 }
 
 var Cmd = &cobra.Command{
@@ -90,7 +92,7 @@ var runCmd = &cobra.Command{
 		if err != nil {
 			return err
 		}
-		testCmd, err := getTestCmd(intTestDir, testStage, args[0], relTestPkg)
+		testCmd, err := getTestCmd(intTestDir, testStage, args[0], relTestPkg, flags.setupVars)
 		if err != nil {
 			return err
 		}

cli/bptest/run.go

@@ -28,6 +28,8 @@ const (
 	// startBufSize is the initial of the buffer token
 	maxScanTokenSize = 10 * 1024 * 1024
 	startBufSize     = 4096
+	// This must be kept in sync with what github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/tft parses.
+	setupEnvVarPrefix = "CFT_SETUP_"
 )
 
 var allTestArgs = []string{"-p", "1", "-count", "1", "-timeout", "0"}
@@ -99,13 +101,18 @@ func streamExec(cmd *exec.Cmd) error {
 }
 
 // getTestCmd returns a prepared cmd for running the specified tests(s)
-func getTestCmd(intTestDir string, testStage string, testName string, relTestPkg string) (*exec.Cmd, error) {
+func getTestCmd(intTestDir string, testStage string, testName string, relTestPkg string, setupVars map[string]string) (*exec.Cmd, error) {
+
 	// pass all current env vars to test command
 	env := os.Environ()
 	// set test stage env var if specified
 	if testStage != "" {
 		env = append(env, fmt.Sprintf("%s=%s", testStageEnvVarKey, testStage))
 	}
+	// Load the env with any setup-vars specified
+	for k, v := range setupVars {
+		env = append(env, fmt.Sprintf("%s%s=%s", setupEnvVarPrefix, k, v))
+	}
 
 	// determine binary and args used for test execution
 	testArgs := append([]string{relTestPkg}, allTestArgs...)

cli/bptest/run_test.go

@@ -68,7 +68,9 @@ func TestGetTestCmd(t *testing.T) {
 		testStage  string
 		testName   string
 		relTestPkg string
+		setupVars  map[string]string
 		wantArgs   []string
+		wantEnv    []string
 		errMsg     string
 	}{
 		{
@@ -87,6 +89,15 @@ func TestGetTestCmd(t *testing.T) {
 			testName:  "TestFoo",
 			testStage: "init",
 			wantArgs:  []string{"./...", "-run", "TestFoo", "-p", "1", "-count", "1", "-timeout", "0"},
+			wantEnv:   []string{"RUN_STAGE=init"},
+		},
+		{
+			name:      "setup vars",
+			testName:  "TestFoo",
+			testStage: "verify",
+			setupVars: map[string]string{"my-key": "my-value"},
+			wantArgs:  []string{"./...", "-run", "TestFoo", "-p", "1", "-count", "1", "-timeout", "0"},
+			wantEnv:   []string{"RUN_STAGE=verify", "CFT_SETUP_my-key=my-value"},
 		},
 	}
 	for _, tt := range tests {
@@ -98,7 +109,7 @@ func TestGetTestCmd(t *testing.T) {
 			if tt.relTestPkg == "" {
 				tt.relTestPkg = "./..."
 			}
-			gotCmd, err := getTestCmd(tt.intTestDir, tt.testStage, tt.testName, tt.relTestPkg)
+			gotCmd, err := getTestCmd(tt.intTestDir, tt.testStage, tt.testName, tt.relTestPkg, tt.setupVars)
 			if tt.errMsg != "" {
 				assert.NotNil(err)
 				assert.Contains(err.Error(), tt.errMsg)
@@ -109,6 +120,7 @@ func TestGetTestCmd(t *testing.T) {
 					assert.Contains(gotCmd.Env, fmt.Sprintf("RUN_STAGE=%s", tt.testStage))
 				}
 			}
+			assert.Subset(gotCmd.Env, tt.wantEnv)
 		})
 	}
 }

infra/blueprint-test/pkg/tft/terraform.go

@@ -70,6 +70,7 @@ type TFBlueprintTest struct {
 	apply                         func(*assert.Assertions) // apply function
 	verify                        func(*assert.Assertions) // verify function
 	teardown                      func(*assert.Assertions) // teardown function
+	setupOutputOverrides          map[string]interface{}   // override outputs from the Setup phase
 }
 
 type tftOption func(*TFBlueprintTest)
@@ -149,6 +150,13 @@ func WithLogger(logger *logger.Logger) tftOption {
 	}
 }
 
+// WithSetupOutputs overrides output values from the setup stage
+func WithSetupOutputs(vars map[string]interface{}) tftOption {
+	return func(f *TFBlueprintTest) {
+		f.setupOutputOverrides = vars
+	}
+}
+
 // NewTFBlueprintTest sets defaults, validates and returns a TFBlueprintTest.
 func NewTFBlueprintTest(t testing.TB, opts ...tftOption) *TFBlueprintTest {
 	tft := &TFBlueprintTest{
@@ -216,6 +224,14 @@ func NewTFBlueprintTest(t testing.TB, opts ...tftOption) *TFBlueprintTest {
 			tft.logger.Logf(tft.t, "Skipping credential activation %s output from setup", setupKeyOutputName)
 		}
 	}
+	// Load env vars to supplement/override setup
+	tft.logger.Logf(tft.t, "Loading setup from environment")
+	if tft.setupOutputOverrides == nil {
+		tft.setupOutputOverrides = make(map[string]interface{})
+	}
+	for k, v := range extractFromEnv("CFT_SETUP_") {
+		tft.setupOutputOverrides[k] = v
+	}
 
 	tft.logger.Logf(tft.t, "Running tests TF configs in %s", tft.tfDir)
 	return tft
@@ -282,6 +298,13 @@ func (b *TFBlueprintTest) GetStringOutput(name string) string {
 // GetTFSetupOutputListVal returns TF output from setup for a given key as list.
 // It fails test if given key does not output a list type.
 func (b *TFBlueprintTest) GetTFSetupOutputListVal(key string) []string {
+	if v, ok := b.setupOutputOverrides[key]; ok {
+		if listval, ok := v.([]string); ok {
+			return listval
+		} else {
+			b.t.Fatalf("Setup Override %s is not a list value", key)
+		}
+	}
 	if b.setupDir == "" {
 		b.t.Fatal("Setup path not set")
 	}
@@ -291,6 +314,9 @@ func (b *TFBlueprintTest) GetTFSetupOutputListVal(key string) []string {
 // GetTFSetupStringOutput returns TF setup output for a given key as string.
 // It fails test if given key does not output a primitive or if setupDir is not configured.
 func (b *TFBlueprintTest) GetTFSetupStringOutput(key string) string {
+	if v, ok := b.setupOutputOverrides[key]; ok {
+		return v.(string)
+	}
 	if b.setupDir == "" {
 		b.t.Fatal("Setup path not set")
 	}
@@ -304,6 +330,24 @@ func loadTFEnvVar(m map[string]string, new map[string]string) {
 	}
 }
 
+// extractFromEnv parses environment variables with the given prefix, and returns a key-value map.
+// e.g. CFT_SETUP_key=value returns map[string]string{"key": "value"}
+func extractFromEnv(prefix string) map[string]interface{} {
+	r := make(map[string]interface{})
+	for _, s := range os.Environ() {
+		k, v, ok := strings.Cut(s, "=")
+		if !ok {
+			// skip malformed entries in os.Environ
+			continue
+		}
+		// For env vars with the prefix, extract the key and value
+		if setupvar, ok := strings.CutPrefix(k, prefix); ok {
+			r[setupvar] = v
+		}
+	}
+	return r
+}
+
 // ShouldSkip checks if a test should be skipped
 func (b *TFBlueprintTest) ShouldSkip() bool {
 	return b.BlueprintTestConfig.Spec.Skip

infra/blueprint-test/pkg/tft/terraform_test.go

@@ -75,20 +75,28 @@ output "simple_map" {
 	}
 }
 
-func getTFOutputMap(t *testing.T, tf string) map[string]interface{} {
+// newTestDir creates a new directory suitable for use as TFDir
+func newTestDir(t *testing.T, pattern string, input string) string {
 	t.Helper()
 	assert := assert.New(t)
 
 	// setup tf file
-	tfDir, err := os.MkdirTemp("", "")
+	tfDir, err := os.MkdirTemp("", pattern)
 	assert.NoError(err)
-	defer os.RemoveAll(tfDir)
 	tfFilePath := path.Join(tfDir, "test.tf")
-	err = os.WriteFile(tfFilePath, []byte(tf), 0644)
+	err = os.WriteFile(tfFilePath, []byte(input), 0644)
 	assert.NoError(err)
+	return tfDir
+}
+
+func getTFOutputMap(t *testing.T, tf string) map[string]interface{} {
+	t.Helper()
+
+	tfDir := newTestDir(t, "", tf)
+	defer os.RemoveAll(tfDir)
 
 	// apply tf and get outputs
-	tOpts := &terraform.Options{TerraformDir: path.Dir(tfFilePath), Logger: logger.Discard}
+	tOpts := &terraform.Options{TerraformDir: tfDir, Logger: logger.Discard}
 	terraform.Init(t, tOpts)
 	terraform.Apply(t, tOpts)
 	return terraform.OutputAll(t, tOpts)
@@ -121,3 +129,146 @@ func TestGetKVFromOutputString(t *testing.T) {
 		})
 	}
 }
+
+func TestSetupOverrideString(t *testing.T) {
+	tests := []struct {
+		name      string
+		tfOutputs string
+		overrides map[string]interface{}
+		want      map[string]string
+	}{
+		{name: "no overrides",
+			tfOutputs: `
+			output "simple_string" {
+			  value = "foo"
+			}
+
+			output "simple_num" {
+			  value = 1
+			}
+
+			output "simple_bool" {
+			  value = true
+			}
+			`,
+			overrides: map[string]interface{}{},
+			want: map[string]string{
+				"simple_string": "foo",
+				"simple_num":    "1",
+				"simple_bool":   "true",
+			},
+		},
+		{name: "all overrides",
+			tfOutputs: `
+			output "simple_string" {
+			  value = "foo"
+			}
+
+			output "simple_num" {
+			  value = 1
+			}
+
+			output "simple_bool" {
+			  value = true
+			}
+			`,
+			overrides: map[string]interface{}{
+				"simple_string": "bar",
+				"simple_num":    "2",
+				"simple_bool":   "false",
+			},
+			want: map[string]string{
+				"simple_string": "bar",
+				"simple_num":    "2",
+				"simple_bool":   "false",
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			emptyDir := newTestDir(t, "empty*", "")
+			setupDir := newTestDir(t, "setup-*", tt.tfOutputs)
+			defer os.RemoveAll(emptyDir)
+			defer os.RemoveAll(setupDir)
+			b := NewTFBlueprintTest(&testingiface.RuntimeT{},
+				WithSetupOutputs(tt.overrides),
+				WithTFDir(emptyDir),
+				WithSetupPath(setupDir))
+			// create outputs from setup
+			_, err := terraform.ApplyE(t, &terraform.Options{TerraformDir: setupDir})
+			if err != nil {
+				t.Fatalf("Failed to apply setup: %v", err)
+			}
+			for k, want := range tt.want {
+				if b.GetTFSetupStringOutput(k) != want {
+					t.Errorf("unexpected string output for %s: want %s got %s", k, want, b.GetStringOutput(k))
+				}
+			}
+		})
+	}
+}
+func TestSetupOverrideList(t *testing.T) {
+	tests := []struct {
+		name      string
+		tfOutputs string
+		overrides map[string]interface{}
+		want      map[string][]string
+	}{
+		{name: "no overrides",
+			tfOutputs: `
+				output "simple_list" {
+					value = ["foo","bar"]
+				}
+			`,
+			overrides: map[string]interface{}{},
+			want: map[string][]string{
+				"simple_list": {"foo", "bar"},
+			},
+		},
+		{name: "all overrides",
+			tfOutputs: `
+				output "simple_list" {
+					value = ["foo","bar"]
+				}
+			`,
+			overrides: map[string]interface{}{
+				"simple_list": []string{"apple", "orange"},
+			},
+			want: map[string][]string{
+				"simple_list": {"apple", "orange"},
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			emptyDir := newTestDir(t, "empty*", "")
+			setupDir := newTestDir(t, "setup-*", tt.tfOutputs)
+			defer os.RemoveAll(emptyDir)
+			defer os.RemoveAll(setupDir)
+			b := NewTFBlueprintTest(&testingiface.RuntimeT{},
+				WithSetupOutputs(tt.overrides),
+				WithTFDir(emptyDir),
+				WithSetupPath(setupDir))
+			// create outputs from setup
+			_, err := terraform.ApplyE(t, &terraform.Options{TerraformDir: setupDir})
+			if err != nil {
+				t.Fatalf("Failed to apply setup: %v", err)
+			}
+			for k, want := range tt.want {
+				got := b.GetTFSetupOutputListVal(k)
+				assert.ElementsMatchf(t, got, want, "list mismatch: want %s got %s", want)
+			}
+		})
+	}
+
+}
+
+func TestSetupOverrideFromEnv(t *testing.T) {
+	t.Setenv("CFT_SETUP_my-key", "my-value")
+	emptyDir := newTestDir(t, "empty*", "")
+	defer os.RemoveAll(emptyDir)
+	b := NewTFBlueprintTest(&testingiface.RuntimeT{},
+		WithTFDir(emptyDir))
+	got := b.GetTFSetupStringOutput("my-key")
+	assert.Equal(t, got, "my-value")
+}