fix: change tofu unpacking to default to terraform registry (#1001)

* fix: default to registry.terraform.io as the default unpacking location when reading a brokerpak and creating the plugins folder, we were defaulting the domain of the providers to registry.opentofu.org instead of registry.terraform.io. This was not consistent with our default for the download location, which is registry.terraform.io. It now defaults to terraform registry in both places. This means brokerpaks which don't indicate any domain in their provider definitions in the manifest.yml file, will need to fully qualify the provider in the required_providers block to avoid errors, as OpenTofu will default to the opentofu registry. [#187379198](https://www.pivotaltracker.com/story/show/187379198) * fix: workaround to prevent tofu from renaming providers on show when fixing [this issue in tofu](opentofu/opentofu#1139) some logic was introduced to change the hostname of the providers to [#187199940](https://www.pivotaltracker.com/story/show/187199940) point to the opentofu registry (in-memory) when executing a show, if the terraform code didn't specify a hostname in the provider's block. However this functionality seems to be buggy and it is renaming providers regardless, which breaks the tf_attribute_skip functionality. A way to disable this rename is to set the OPENTOFU_STATEFILE_PROVIDER_ADDRESS_TRANSLATION environment variable to `0`. This PR implements that workaround for the broker. [#187379198](https://www.pivotaltracker.com/story/show/187379198) Co-authored-by: Felisia Martini <[email protected]> * chore: tidy up go.mod [#187379198](https://www.pivotaltracker.com/story/show/187379198) Co-authored-by: Marcela Campo <[email protected]> * chore: add reference to opentofu issue [#187379198](https://www.pivotaltracker.com/story/show/187379198) Co-authored-by: Felisia Martini <[email protected]> --------- Co-authored-by: Felisia Martini <[email protected]>
cloudfoundry · Apr 9, 2024 · 9848ca2 · 9848ca2
1 parent 53b4080
commit 9848ca2
Show file tree

Hide file tree

Showing 38 changed files with 249 additions and 39 deletions.
diff --git a/go.sum b/go.sum
@@ -718,8 +718,6 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug
 golang.org/x/net v0.0.0-20220909164309-bea034e7d591/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
 golang.org/x/net v0.0.0-20221014081412-f15817d10f9b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
 golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
-golang.org/x/net v0.22.0 h1:9sGLhx7iRIHEiX0oAJ3MRZMUCElJgy7Br1nO+AMN3Tc=
-golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
 golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs=
 golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=

diff --git a/integrationtest/fixtures/binding-cleanup/fake-bad-bind.tf b/integrationtest/fixtures/binding-cleanup/fake-bad-bind.tf
@@ -1,4 +1,12 @@
 // Fails as required "length" parameter is missing
+terraform {
+  required_providers {
+    random = {
+      source  = "registry.terraform.io/hashicorp/random"
+    }
+  }
+}
+
 resource "random_string" "random" {}
 
 output bind_output { value = random_string.random.result }
diff --git a/integrationtest/fixtures/binding-cleanup/fake-good-bind.tf b/integrationtest/fixtures/binding-cleanup/fake-good-bind.tf
@@ -1,3 +1,11 @@
+terraform {
+  required_providers {
+    random = {
+      source  = "registry.terraform.io/hashicorp/random"
+    }
+  }
+}
+
 resource "random_string" "random" {
   length = 10
 }

diff --git a/integrationtest/fixtures/binding-cleanup/fake-provision.tf b/integrationtest/fixtures/binding-cleanup/fake-provision.tf
@@ -1,3 +1,11 @@
+terraform {
+  required_providers {
+    random = {
+      source  = "registry.terraform.io/hashicorp/random"
+    }
+  }
+}
+
 resource "random_uuid" "random" {}
 
 output provision_output { value = random_uuid.random.result }
diff --git a/integrationtest/fixtures/golden-path/fake-uuid-bind.tf b/integrationtest/fixtures/golden-path/fake-uuid-bind.tf
@@ -1,3 +1,11 @@
+terraform {
+  required_providers {
+    random = {
+      source  = "registry.terraform.io/hashicorp/random"
+    }
+  }
+}
+
 resource "random_uuid" "random" {}
 
 output bind_output { value = random_uuid.random.result }
diff --git a/integrationtest/fixtures/golden-path/fake-uuid-provision.tf b/integrationtest/fixtures/golden-path/fake-uuid-provision.tf
@@ -1,3 +1,11 @@
+terraform {
+  required_providers {
+    random = {
+      source  = "registry.terraform.io/hashicorp/random"
+    }
+  }
+}
+
 resource "random_uuid" "random" {}
 
 output provision_output { value = random_uuid.random.result }
diff --git a/integrationtest/fixtures/maintenance-info/fake-provision.tf b/integrationtest/fixtures/maintenance-info/fake-provision.tf
@@ -1,7 +1,7 @@
 terraform {
   required_providers {
     random = {
-      source = "hashicorp/random"
+      source = "registry.terraform.io/hashicorp/random"
       version = "3.5.1"
     }
   }

diff --git a/integrationtest/fixtures/osbapi/fake-provision.tf b/integrationtest/fixtures/osbapi/fake-provision.tf
@@ -1,4 +1,10 @@
-provider "random" { }
+terraform {
+  required_providers {
+    random = {
+      source  = "registry.terraform.io/hashicorp/random"
+    }
+  }
+}
 
 resource "random_integer" "priority" {
   min = 1

diff --git a/integrationtest/fixtures/prevent-destroy-during-update/fake-provision.tf b/integrationtest/fixtures/prevent-destroy-during-update/fake-provision.tf
@@ -1,4 +1,10 @@
-provider "random" { }
+terraform {
+  required_providers {
+    random = {
+      source  = "registry.terraform.io/hashicorp/random"
+    }
+  }
+}
 
 variable length { type = number }
 

diff --git a/integrationtest/fixtures/provision-cleanup/fake-string-provision.tf b/integrationtest/fixtures/provision-cleanup/fake-string-provision.tf
@@ -1,4 +1,12 @@
 // Fails as required "length" parameter is missing
+terraform {
+  required_providers {
+    random = {
+      source  = "registry.terraform.io/hashicorp/random"
+    }
+  }
+}
+
 resource "random_string" "random" {}
 
 output provision_output { value = random_string.random.result }
diff --git a/integrationtest/fixtures/subsume/fake-string-service.yml b/integrationtest/fixtures/subsume/fake-string-service.yml
@@ -24,6 +24,7 @@ provision:
   import_parameters_to_delete: [ "random_string.random.id", "random_string.random.result" ]
   template_refs:
     main: fake-string-provision.tf
+    versions: versions.tf
   outputs:
     - field_name: provision_output
       type: string

diff --git a/integrationtest/fixtures/subsume/fake-uuid-service.yml b/integrationtest/fixtures/subsume/fake-uuid-service.yml
@@ -21,6 +21,7 @@ provision:
   import_parameters_to_delete: [ "random_uuid.random.id" , "random_uuid.random.result" ]
   template_refs:
     main: fake-uuid-provision.tf
+    versions: versions.tf
   outputs:
     - field_name: provision_output
       type: string

diff --git a/integrationtest/fixtures/subsume/versions.tf b/integrationtest/fixtures/subsume/versions.tf
@@ -0,0 +1,7 @@
+terraform {
+  required_providers {
+    random = {
+      source  = "registry.terraform.io/hashicorp/random"
+    }
+  }
+}
diff --git a/integrationtest/fixtures/termination-recovery/fake-uuid-bind.tf b/integrationtest/fixtures/termination-recovery/fake-uuid-bind.tf
@@ -1,3 +1,11 @@
+terraform {
+  required_providers {
+    random = {
+      source  = "registry.terraform.io/hashicorp/random"
+    }
+  }
+}
+
 resource "random_uuid" "random" {}
 
 output bind_output { value = random_uuid.random.result }
diff --git a/integrationtest/fixtures/termination-recovery/fake-uuid-provision.tf b/integrationtest/fixtures/termination-recovery/fake-uuid-provision.tf
@@ -1,3 +1,11 @@
+terraform {
+  required_providers {
+    random = {
+      source  = "registry.terraform.io/hashicorp/random"
+    }
+  }
+}
+
 resource "random_uuid" "random" {}
 
 output provision_output { value = random_uuid.random.result }
diff --git a/integrationtest/fixtures/terraform-block-action-before-upgrade-updated/fake-provision.tf b/integrationtest/fixtures/terraform-block-action-before-upgrade-updated/fake-provision.tf
@@ -1,7 +1,7 @@
 terraform {
   required_providers {
     random = {
-      source = "hashicorp/random"
+      source = "registry.terraform.io/hashicorp/random"
       version = "3.5.1"
     }
   }

diff --git a/integrationtest/fixtures/terraform-block-action-before-upgrade/fake-provision.tf b/integrationtest/fixtures/terraform-block-action-before-upgrade/fake-provision.tf
@@ -1,3 +1,11 @@
+terraform {
+  required_providers {
+    random = {
+      source  = "registry.terraform.io/hashicorp/random"
+    }
+  }
+}
+
 provider "random" { }
 
 resource "random_integer" "priority" {

diff --git a/integrationtest/fixtures/terraform-module-upgrade-updated/fake-provision.tf b/integrationtest/fixtures/terraform-module-upgrade-updated/fake-provision.tf
@@ -1,7 +1,7 @@
 terraform {
   required_providers {
     random = {
-      source  = "hashicorp/random"
+      source  = "registry.terraform.io/hashicorp/random"
       version = "3.5.1"
     }
   }

diff --git a/integrationtest/fixtures/terraform-module-upgrade/fake-provision.tf b/integrationtest/fixtures/terraform-module-upgrade/fake-provision.tf
@@ -1,4 +1,10 @@
-provider "random" { }
+terraform {
+  required_providers {
+    random = {
+      source  = "registry.terraform.io/hashicorp/random"
+    }
+  }
+}
 
 resource "random_password" "password" {
   length = 5

diff --git a/integrationtest/fixtures/terraform-rename-provider/fake-provision.tf b/integrationtest/fixtures/terraform-rename-provider/fake-provision.tf
@@ -1,7 +1,7 @@
 terraform {
   required_providers {
     random = {
-      source = "ContentSquare/random"
+      source = "registry.terraform.io/ContentSquare/random"
       version = "3.1.0"
     }
   }

diff --git a/integrationtest/fixtures/terraform-upgrade-updated/fake-bind.tf b/integrationtest/fixtures/terraform-upgrade-updated/fake-bind.tf
@@ -1,7 +1,7 @@
 terraform {
   required_providers {
     random = {
-      source  = "hashicorp/random"
+      source  = "registry.terraform.io/hashicorp/random"
       version = "3.5.1"
     }
   }

diff --git a/integrationtest/fixtures/terraform-upgrade-updated/fake-provision.tf b/integrationtest/fixtures/terraform-upgrade-updated/fake-provision.tf
@@ -1,7 +1,7 @@
 terraform {
   required_providers {
     random = {
-      source  = "hashicorp/random"
+      source  = "registry.terraform.io/hashicorp/random"
       version = "3.5.1"
     }
   }

diff --git a/integrationtest/fixtures/terraform-upgrade/fake-bind.tf b/integrationtest/fixtures/terraform-upgrade/fake-bind.tf
@@ -1,4 +1,10 @@
-provider "random" { }
+terraform {
+  required_providers {
+    random = {
+      source  = "registry.terraform.io/hashicorp/random"
+    }
+  }
+}
 
 resource "random_integer" "priority" {
   min = 1

diff --git a/integrationtest/fixtures/terraform-upgrade/fake-provision.tf b/integrationtest/fixtures/terraform-upgrade/fake-provision.tf
@@ -1,4 +1,10 @@
-provider "random" { }
+terraform {
+  required_providers {
+    random = {
+      source  = "registry.terraform.io/hashicorp/random"
+    }
+  }
+}
 
 resource "random_integer" "priority" {
   min = 1

diff --git a/integrationtest/fixtures/tf_attribute_skip/fake-provision.tf b/integrationtest/fixtures/tf_attribute_skip/fake-provision.tf
@@ -1,15 +1,5 @@
 ## new versions of terraform fail with a different error to the one we want to trigger if this file is empty.
 ## adding some dummy terraform to test errors when a particular field is not present.
-
-terraform {
-  required_providers {
-    random = {
-      source  = "hashicorp/random"
-      version = "3.5.1"
-    }
-  }
-}
-
 resource "random_integer" "priority" {
   min = 3
   max = 4

diff --git a/integrationtest/fixtures/tf_attribute_skip/fake-service.yml b/integrationtest/fixtures/tf_attribute_skip/fake-service.yml
@@ -20,6 +20,7 @@ plans:
 provision:
   template_refs:
     main: fake-provision.tf
+    versions: versions.tf
   user_inputs:
     - field_name: skip
       type: boolean

diff --git a/integrationtest/fixtures/tf_attribute_skip/versions.tf b/integrationtest/fixtures/tf_attribute_skip/versions.tf
@@ -0,0 +1,8 @@
+terraform {
+  required_providers {
+    random = {
+      source  = "registry.terraform.io/hashicorp/random"
+      version = "3.5.1"
+    }
+  }
+}
diff --git a/integrationtest/subsume_test.go b/integrationtest/subsume_test.go
@@ -31,7 +31,8 @@ var _ = Describe("Subsume", func() {
 	It("can subsume a resource", func() {
 		const serviceOfferingGUID = "547cad88-fa93-11eb-9f44-97feefe52547"
 		const servicePlanGUID = "59624c68-fa93-11eb-9081-e79b0e1ab5ae"
-		broker.Provision(serviceOfferingGUID, servicePlanGUID, testdrive.WithProvisionParams(`{"value":"a97fd57a-fa94-11eb-8256-930255607a99"}`))
+		_, err := broker.Provision(serviceOfferingGUID, servicePlanGUID, testdrive.WithProvisionParams(`{"value":"a97fd57a-fa94-11eb-8256-930255607a99"}`))
+		Expect(err).NotTo(HaveOccurred())
 	})
 
 	It("cancels a subsume operation when a resource would be deleted", func() {

diff --git a/internal/brokerpak/manifest/parser_test.go b/internal/brokerpak/manifest/parser_test.go
@@ -320,9 +320,9 @@ var _ = Describe("Parser", func() {
 			Expect(err).NotTo(HaveOccurred())
 			Expect(m.TerraformProviders[1].Provider.String()).To(Equal(expected))
 		},
-		Entry("empty 'provider' field", "", "registry.opentofu.org/hashicorp/foo"),
-		Entry("just type", "lala", "registry.opentofu.org/hashicorp/lala"),
-		Entry("type and namespace", "mycorp/lala", "registry.opentofu.org/mycorp/lala"),
+		Entry("empty 'provider' field", "", "registry.terraform.io/hashicorp/foo"),
+		Entry("just type", "lala", "registry.terraform.io/hashicorp/lala"),
+		Entry("type and namespace", "mycorp/lala", "registry.terraform.io/mycorp/lala"),
 		Entry("fully qualified", "mything.io/mycorp/lala", "mything.io/mycorp/lala"),
 	)
 

diff --git a/internal/brokerpak/reader/reader.go b/internal/brokerpak/reader/reader.go
@@ -19,6 +19,7 @@ import (
 	"archive/zip"
 	"fmt"
 	"io"
+	"log"
 	"os"
 	"path"
 	"path/filepath"
@@ -268,6 +269,11 @@ func (pak *BrokerPakReader) readBytes(name string) ([]byte, error) {
 
 func providerInstallPath(destination string, tfProvider manifest.TerraformProvider) string {
 	plat := platform.CurrentPlatform()
+	log.Println("ProviderInstallPath:", filepath.Join(
+		destination,
+		tfProvider.Provider.String(),
+		tfProvider.Version.String(),
+		fmt.Sprintf("%s_%s", plat.Os, plat.Arch)))
 	return filepath.Join(
 		destination,
 		tfProvider.Provider.String(),

diff --git a/internal/brokerpak/reader/reader_test.go b/internal/brokerpak/reader/reader_test.go
@@ -35,6 +35,7 @@ var _ = Describe("reader", func() {
 				withTerraform(binaryV160),
 				withProvider("", "terraform-provider-google-beta", "1.19.0", "x4"),
 				withProvider("other-namespace/google", "terraform-provider-google", "1.19.0", "x5"),
+				withProvider("custom.registry.org/other-namespace/custom", "terraform-provider-custom", "1.19.0", "x5"),
 			)
 
 			pakReader, err := reader.OpenBrokerPak(pk)
@@ -44,11 +45,15 @@ var _ = Describe("reader", func() {
 			Expect(pakReader.ExtractPlatformBins(binOutput)).NotTo(HaveOccurred())
 
 			plat := fmt.Sprintf("%s_%s", runtime.GOOS, runtime.GOARCH)
-			hashicorpBinOutput := filepath.Join(binOutput, "registry.opentofu.org", "hashicorp")
+			hashicorpBinOutput := filepath.Join(binOutput, "registry.terraform.io", "hashicorp")
 			Expect(filepath.Join(hashicorpBinOutput, "google-beta", "1.19.0", plat, "terraform-provider-google-beta_v1.19.0_x4")).To(BeAnExistingFile())
 
-			otherNamespaceBinOutput := filepath.Join(binOutput, "registry.opentofu.org", "other-namespace")
+			otherNamespaceBinOutput := filepath.Join(binOutput, "registry.terraform.io", "other-namespace")
 			Expect(filepath.Join(otherNamespaceBinOutput, "google", "1.19.0", plat, "terraform-provider-google_v1.19.0_x5")).To(BeAnExistingFile())
+
+			customDomainBinOutput := filepath.Join(binOutput, "custom.registry.org", "other-namespace")
+			Expect(filepath.Join(customDomainBinOutput, "custom", "1.19.0", plat, "terraform-provider-custom_v1.19.0_x5")).To(BeAnExistingFile())
+
 		})
 
 		Context("single version of tofu", func() {

diff --git a/internal/tfproviderfqn/tfproviderfqn.go b/internal/tfproviderfqn/tfproviderfqn.go
@@ -5,7 +5,7 @@ import "fmt"
 
 const (
 	prefix           = "terraform-provider-"
-	defaultRegistry  = "registry.opentofu.org"
+	defaultRegistry  = "registry.terraform.io"
 	defaultNamespace = "hashicorp"
 )