feat(tft): add filemutex for tft plugin cache (#2140)

infra/blueprint-test/examples/simple_krm_blueprint/pod.yaml

@@ -12,7 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-
 apiVersion: v1
 kind: Pod
 metadata:

infra/blueprint-test/go.mod

@@ -4,6 +4,7 @@ go 1.21
 
 require (
 	github.com/GoogleContainerTools/kpt-functions-sdk/go/api v0.0.0-20230427202446-3255accc518d
+	github.com/alexflint/go-filemutex v1.3.0
 	github.com/gruntwork-io/terratest v0.46.11
 	github.com/hashicorp/terraform-config-inspect v0.0.0-20231204233900-a34142ec2a72
 	github.com/mitchellh/go-testing-interface v1.14.2-0.20210821155943-2d9075ca8770
@@ -87,7 +88,7 @@ require (
 	golang.org/x/net v0.17.0 // indirect
 	golang.org/x/oauth2 v0.12.0 // indirect
 	golang.org/x/sync v0.4.0 // indirect
-	golang.org/x/sys v0.15.0 // indirect
+	golang.org/x/sys v0.16.0 // indirect
 	golang.org/x/term v0.15.0 // indirect
 	golang.org/x/text v0.14.0 // indirect
 	golang.org/x/time v0.3.0 // indirect

infra/blueprint-test/go.sum

@@ -192,6 +192,8 @@ github.com/GoogleContainerTools/kpt-functions-sdk/go/api v0.0.0-20230427202446-3
 github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
 github.com/agext/levenshtein v1.2.3 h1:YB2fHEn0UJagG8T1rrWknE3ZQzWM06O8AMAatNn7lmo=
 github.com/agext/levenshtein v1.2.3/go.mod h1:JEDfjyjHDjOF/1e4FlBE/PkbqA9OfWu2ki2W0IB5558=
+github.com/alexflint/go-filemutex v1.3.0 h1:LgE+nTUWnQCyRKbpoceKZsPQbs84LivvgwUymZXdOcM=
+github.com/alexflint/go-filemutex v1.3.0/go.mod h1:U0+VA/i30mGBlLCrFPGtTe9y6wGQfNAWPBTekHQ+c8A=
 github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
 github.com/apparentlymart/go-textseg/v15 v15.0.0 h1:uYvfpb3DyLSCGWnctWKGj857c6ew1u1fNQOlOtuGxQY=
 github.com/apparentlymart/go-textseg/v15 v15.0.0/go.mod h1:K8XmNZdhEBkdlyDdvbmmsvpAG721bKi0joRfFdHIWJ4=
@@ -734,8 +736,8 @@ golang.org/x/sys v0.0.0-20220624220833-87e55d714810/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc=
-golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU=
+golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=

infra/blueprint-test/pkg/tft/terraform.go

@@ -30,14 +30,18 @@ import (
 	"github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/discovery"
 	"github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/gcloud"
 	"github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/utils"
+	"github.com/alexflint/go-filemutex"
 	"github.com/gruntwork-io/terratest/modules/logger"
 	"github.com/gruntwork-io/terratest/modules/terraform"
 	"github.com/hashicorp/terraform-config-inspect/tfconfig"
 	"github.com/mitchellh/go-testing-interface"
 	"github.com/stretchr/testify/assert"
 )
 
-const setupKeyOutputName = "sa_key"
+const (
+	setupKeyOutputName = "sa_key"
+	tftCacheMutexFilename  = "/tmp/bpt-tft-cache.lock"
+)
 
 var (
 	CommonRetryableErrors = map[string]string{
@@ -73,6 +77,7 @@ type TFBlueprintTest struct {
 	verify                        func(*assert.Assertions) // verify function
 	teardown                      func(*assert.Assertions) // teardown function
 	setupOutputOverrides          map[string]interface{}   // override outputs from the Setup phase
+	tftCacheMutex                 *filemutex.FileMutex     // Mutex to protect Terraform plugin cache
 }
 
 type tftOption func(*TFBlueprintTest)
@@ -167,11 +172,17 @@ func WithSetupOutputs(vars map[string]interface{}) tftOption {
 
 // NewTFBlueprintTest sets defaults, validates and returns a TFBlueprintTest.
 func NewTFBlueprintTest(t testing.TB, opts ...tftOption) *TFBlueprintTest {
+	var err error
 	tft := &TFBlueprintTest{
 		name:      fmt.Sprintf("%s TF Blueprint", t.Name()),
 		tfEnvVars: make(map[string]string),
 		t:         t,
 	}
+	// initiate tft cache file mutex
+	tft.tftCacheMutex, err = filemutex.New(tftCacheMutexFilename)
+	if err != nil {
+		t.Fatalf("tft lock file <%s> could not created: %v", tftCacheMutexFilename, err)
+	}
 	// default TF blueprint methods
 	tft.init = tft.DefaultInit
 	tft.apply = tft.DefaultApply
@@ -204,7 +215,6 @@ func NewTFBlueprintTest(t testing.TB, opts ...tftOption) *TFBlueprintTest {
 	}
 
 	// discover test config
-	var err error
 	tft.BlueprintTestConfig, err = discovery.GetTestConfig(path.Join(tft.tfDir, discovery.DefaultTestConfigFilename))
 	if err != nil {
 		t.Fatal(err)
@@ -481,21 +491,57 @@ func (b *TFBlueprintTest) DefaultApply(assert *assert.Assertions) {
 
 // Init runs the default or custom init function for the blueprint.
 func (b *TFBlueprintTest) Init(assert *assert.Assertions) {
+	// allow only single write as Terraform plugin cache isn't concurrent safe
+	if err := b.tftCacheMutex.Lock(); err != nil {
+		b.t.Fatalf("Could not acquire lock: %v", err)
+	}
+	defer func() {
+		if err := b.tftCacheMutex.Unlock(); err != nil {
+			b.t.Fatalf("Could not release lock: %v", err)
+		}
+	}()
 	b.init(assert)
 }
 
 // Apply runs the default or custom apply function for the blueprint.
 func (b *TFBlueprintTest) Apply(assert *assert.Assertions) {
+	// allow only parallel reads as Terraform plugin cache isn't concurrent safe
+	if err := b.tftCacheMutex.RLock(); err != nil {
+		b.t.Fatalf("Could not acquire read lock: %v", err)
+	}
+	defer func() {
+		if err := b.tftCacheMutex.RUnlock(); err != nil {
+			b.t.Fatalf("Could not release read lock: %v", err)
+		}
+	}()
 	b.apply(assert)
 }
 
 // Verify runs the default or custom verify function for the blueprint.
 func (b *TFBlueprintTest) Verify(assert *assert.Assertions) {
+	// allow only parallel reads as Terraform plugin cache isn't concurrent safe
+	if err := b.tftCacheMutex.RLock(); err != nil {
+		b.t.Fatalf("Could not acquire read lock: %v", err)
+	}
+	defer func() {
+		if err := b.tftCacheMutex.RUnlock(); err != nil {
+			b.t.Fatalf("Could not release read lock: %v", err)
+		}
+	}()
 	b.verify(assert)
 }
 
 // Teardown runs the default or custom teardown function for the blueprint.
 func (b *TFBlueprintTest) Teardown(assert *assert.Assertions) {
+	// allow only parallel reads as Terraform plugin cache isn't concurrent safe
+	if err := b.tftCacheMutex.RLock(); err != nil {
+		b.t.Fatalf("Could not acquire read lock:%v", err)
+	}
+	defer func() {
+		if err := b.tftCacheMutex.RUnlock(); err != nil {
+			b.t.Fatalf("Could not release read lock: %v", err)
+		}
+	}()
 	b.teardown(assert)
 }
 

infra/blueprint-test/test/krm_simple_blueprint_test.go

@@ -1,6 +1,7 @@
 package test
 
 import (
+	"strings"
 	"testing"
 
 	"github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/gcloud"
@@ -26,7 +27,8 @@ func TestKRMSimpleBlueprint(t *testing.T) {
 			k8sOpts := k8s.KubectlOptions{}
 			op, err := k8s.RunKubectlAndGetOutputE(t, &k8sOpts, "get", "pod", "simple-krm-blueprint", "--no-headers", "-o", "custom-columns=:metadata.name")
 			assert.NoError(err)
-			assert.Equal("simple-krm-blueprint", op)
+			result := strings.Split(op, "\n")
+			assert.Equal("simple-krm-blueprint", result[len(result)-1])
 		})
 	networkBlueprint.Test()
 }

infra/blueprint-test/test/setup/main.tf

@@ -1,5 +1,5 @@
 /**
- * Copyright 2021-2023 Google LLC
+ * Copyright 2021-2024 Google LLC
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -64,6 +64,6 @@ resource "google_service_account_key" "key" {
 
 module "kubernetes-engine_example_simple_autopilot_public" {
   source  = "terraform-google-modules/kubernetes-engine/google//examples/simple_autopilot_public"
-  version                     = "~> 26.0"
+  version                     = "~> 30.0"
   project_id                  = module.project.project_id
 }