Merge pull request #7435 from patrickdillon/412-aws-shared-vpc

CORS-2792: AWS Shared VPC Backport [release-4.12]

data/data/aws/cluster/main.tf

@@ -70,9 +70,11 @@ module "dns" {
   cluster_id               = var.cluster_id
   tags                     = local.tags
   internal_zone            = var.aws_internal_zone
+  internal_zone_role       = var.aws_internal_zone_role
   vpc_id                   = module.vpc.vpc_id
   region                   = var.aws_region
   publish_strategy         = var.aws_publish_strategy
+  custom_endpoints         = var.custom_endpoints
 }
 
 module "vpc" {

data/data/aws/cluster/route53/base.tf

@@ -9,6 +9,27 @@ locals {
   use_alias = ! local.use_cname
 }
 
+provider "aws" {
+  alias = "private_hosted_zone"
+
+  assume_role {
+    role_arn = var.internal_zone_role
+  }
+
+  region = var.region
+
+  skip_region_validation = true
+
+  endpoints {
+    ec2     = lookup(var.custom_endpoints, "ec2", null)
+    elb     = lookup(var.custom_endpoints, "elasticloadbalancing", null)
+    iam     = lookup(var.custom_endpoints, "iam", null)
+    route53 = lookup(var.custom_endpoints, "route53", null)
+    s3      = lookup(var.custom_endpoints, "s3", null)
+    sts     = lookup(var.custom_endpoints, "sts", null)
+  }
+}
+
 data "aws_route53_zone" "public" {
   count = local.public_endpoints ? 1 : 0
 
@@ -18,6 +39,8 @@ data "aws_route53_zone" "public" {
 }
 
 data "aws_route53_zone" "int" {
+  provider = aws.private_hosted_zone
+
   zone_id = var.internal_zone == null ? aws_route53_zone.new_int[0].id : var.internal_zone
 }
 
@@ -54,7 +77,8 @@ resource "aws_route53_record" "api_external_alias" {
 }
 
 resource "aws_route53_record" "api_internal_alias" {
-  count = local.use_alias ? 1 : 0
+  provider = aws.private_hosted_zone
+  count    = local.use_alias ? 1 : 0
 
   zone_id = data.aws_route53_zone.int.zone_id
   name    = "api-int.${var.cluster_domain}"
@@ -68,7 +92,8 @@ resource "aws_route53_record" "api_internal_alias" {
 }
 
 resource "aws_route53_record" "api_external_internal_zone_alias" {
-  count = local.use_alias ? 1 : 0
+  provider = aws.private_hosted_zone
+  count    = local.use_alias ? 1 : 0
 
   zone_id = data.aws_route53_zone.int.zone_id
   name    = "api.${var.cluster_domain}"
@@ -93,7 +118,8 @@ resource "aws_route53_record" "api_external_cname" {
 }
 
 resource "aws_route53_record" "api_internal_cname" {
-  count = local.use_cname ? 1 : 0
+  provider = aws.private_hosted_zone
+  count    = local.use_cname ? 1 : 0
 
   zone_id = data.aws_route53_zone.int.zone_id
   name    = "api-int.${var.cluster_domain}"
@@ -104,7 +130,8 @@ resource "aws_route53_record" "api_internal_cname" {
 }
 
 resource "aws_route53_record" "api_external_internal_zone_cname" {
-  count = local.use_cname ? 1 : 0
+  provider = aws.private_hosted_zone
+  count    = local.use_cname ? 1 : 0
 
   zone_id = data.aws_route53_zone.int.zone_id
   name    = "api.${var.cluster_domain}"

data/data/aws/cluster/route53/variables.tf

@@ -28,6 +28,12 @@ variable "internal_zone" {
   description = "An existing hosted zone (zone ID) to use for the internal API."
 }
 
+variable "internal_zone_role" {
+  type        = string
+  default     = null
+  description = "(optional) A role to assume when using an existing hosted zone from another account."
+}
+
 variable "api_external_lb_dns_name" {
   description = "External API's LB DNS name"
   type        = string
@@ -63,3 +69,16 @@ variable "region" {
   type = string
   description = "The target AWS region for the cluster."
 }
+
+variable "custom_endpoints" {
+  type = map(string)
+
+  description = <<EOF
+(optional) Custom AWS endpoints to override existing services.
+Check - https://www.terraform.io/docs/providers/aws/guides/custom-service-endpoints.html
+
+Example: `{ "key" = "value", "foo" = "bar" }`
+EOF
+
+  default = {}
+}

data/data/aws/variables-aws.tf

@@ -144,6 +144,13 @@ variable "aws_internal_zone" {
   description = "(optional) An existing hosted zone (zone ID) to use for the internal API."
 }
 
+variable "aws_internal_zone_role" {
+  type        = string
+  default     = null
+  description = "(optional) A role to assume when using an existing hosted zone from another account."
+}
+
+
 variable "aws_publish_strategy" {
   type        = string
   description = "The cluster publishing strategy, either Internal or External"

data/data/install.openshift.io_installconfigs.yaml

@@ -1746,6 +1746,14 @@ spec:
                       the subnets. Leave the hosted zone unset to have the installer
                       create the hosted zone on your behalf.
                     type: string
+                  hostedZoneRole:
+                    description: HostedZoneRole is the ARN of an IAM role to be assumed
+                      when performing operations on the provided HostedZone. HostedZoneRole
+                      can be used in a shared VPC scenario when the private hosted
+                      zone belongs to a different account than the rest of the cluster
+                      resources. If HostedZoneRole is set, HostedZone must also be
+                      set.
+                    type: string
                   lbType:
                     description: "LBType is an optional field to specify a load balancer
                       type. \n When this field is specified, the default ingresscontroller

go.mod

@@ -45,7 +45,7 @@ require (
 	github.com/metal3-io/baremetal-operator/apis v0.0.0
 	github.com/metal3-io/baremetal-operator/pkg/hardwareutils v0.0.0
 	github.com/nutanix-cloud-native/prism-go-client v0.2.1-0.20220804130801-c8a253627c64
-	github.com/openshift/api v3.9.1-0.20191111211345-a27ff30ebf09+incompatible
+	github.com/openshift/api v0.0.0-20230817133225-564be9ddb58e
 	github.com/openshift/assisted-image-service v0.0.0-20220506122314-2f689a1084b8
 	github.com/openshift/assisted-service v0.0.0-20220928142635-a40422bdea61
 	github.com/openshift/assisted-service/api v0.0.0
@@ -246,7 +246,7 @@ replace sigs.k8s.io/controller-tools => sigs.k8s.io/controller-tools v0.3.1-0.20
 
 // Override the OpenShift API version in hive
 
-replace github.com/openshift/api => github.com/openshift/api v0.0.0-20221004120407-c46852673d03
+replace github.com/openshift/api => github.com/openshift/api v0.0.0-20230817133225-564be9ddb58e
 
 replace github.com/terraform-providers/terraform-provider-nutanix => github.com/nutanix/terraform-provider-nutanix v1.5.0
 

go.sum

@@ -1159,8 +1159,8 @@ github.com/opencontainers/runc v1.0.0-rc90/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2r
 github.com/opencontainers/runc v1.1.2 h1:2VSZwLx5k/BfsBxMMipG/LYUnmqOD/BPkIVgQUcTlLw=
 github.com/opencontainers/runtime-spec v0.1.2-0.20190507144316-5b71a03e2700/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
 github.com/opencontainers/selinux v1.5.2/go.mod h1:yTcKuYAh6R95iDpefGLQaPaRwJFwyzAJufJyiTt7s0g=
-github.com/openshift/api v0.0.0-20221004120407-c46852673d03 h1:e8oBXPs3eCJWIeEVw29l+GLvxdo3BPhdpkUd/FzPjeg=
-github.com/openshift/api v0.0.0-20221004120407-c46852673d03/go.mod h1:JRz+ZvTqu9u7t6suhhPTacbFl5K65Y6rJbNM7HjWA3g=
+github.com/openshift/api v0.0.0-20230817133225-564be9ddb58e h1:Hbw58VzpO9SktwYwXhiiubgvGmNTNeK6mxGtjPQ0uy4=
+github.com/openshift/api v0.0.0-20230817133225-564be9ddb58e/go.mod h1:aQ6LDasvHMvHZXqLHnX2GRmnfTWCF/iIwz8EMTTIE9A=
 github.com/openshift/assisted-image-service v0.0.0-20220506122314-2f689a1084b8 h1:oZ3VAWiM8tPRBM+vYI4GBmlrqyoqizcgZ7pBy5EX2K8=
 github.com/openshift/assisted-image-service v0.0.0-20220506122314-2f689a1084b8/go.mod h1:bH4+AsmPy8mQQvtgedBm2Crs93TDWeXEMlIPrlEMpjA=
 github.com/openshift/assisted-service v0.0.0-20220928142635-a40422bdea61 h1:HtUYJBAdRgfVYide0bq3GsT/4n5uPWteA1rIZVXEL7k=
@@ -1178,7 +1178,6 @@ github.com/openshift/baremetal-operator/pkg/hardwareutils v0.0.0-20220128094204-
 github.com/openshift/build-machinery-go v0.0.0-20200211121458-5e3d6e570160/go.mod h1:1CkcsT3aVebzRBzVTSbiKSkJMsC/CASqxesfqEMfJEc=
 github.com/openshift/build-machinery-go v0.0.0-20200819073603-48aa266c95f7/go.mod h1:b1BuldmJlbA/xYtdZvKi+7j5YGB44qJUJDZ9zwiNCfE=
 github.com/openshift/build-machinery-go v0.0.0-20210712174854-1bb7fd1518d3/go.mod h1:b1BuldmJlbA/xYtdZvKi+7j5YGB44qJUJDZ9zwiNCfE=
-github.com/openshift/build-machinery-go v0.0.0-20220913142420-e25cf57ea46d/go.mod h1:b1BuldmJlbA/xYtdZvKi+7j5YGB44qJUJDZ9zwiNCfE=
 github.com/openshift/client-go v0.0.0-20200827190008-3062137373b5/go.mod h1:5rGmrkQ8DJEUXA+AR3rEjfH+HFyg4/apY9iCQFgvPfE=
 github.com/openshift/client-go v0.0.0-20210730113412-1811c1b3fc0e/go.mod h1:P1pjphFOgm/nYjmtouHGaSLGtdP25dQICJnYtcYhfEs=
 github.com/openshift/client-go v0.0.0-20211025111749-96ca2abfc56c/go.mod h1:xigLF97kzy1PZuDsC0Lfu6GlzChRt62+2Ts/nG3sPHY=

pkg/asset/cluster/aws/aws.go

@@ -11,6 +11,7 @@ import (
 	"github.com/pkg/errors"
 
 	"github.com/openshift/installer/pkg/asset/installconfig"
+	awsic "github.com/openshift/installer/pkg/asset/installconfig/aws"
 	"github.com/openshift/installer/pkg/types"
 	awstypes "github.com/openshift/installer/pkg/types/aws"
 )
@@ -26,6 +27,7 @@ func Metadata(clusterID, infraID string, config *types.InstallConfig) *awstypes.
 		}},
 		ServiceEndpoints: config.AWS.ServiceEndpoints,
 		ClusterDomain:    config.ClusterDomain(),
+		HostedZoneRole:   config.AWS.HostedZoneRole,
 	}
 }
 
@@ -79,7 +81,8 @@ func tagSharedVPCResources(ctx context.Context, clusterID string, installConfig
 	}
 
 	if zone := installConfig.Config.AWS.HostedZone; zone != "" {
-		route53Client := route53.New(session)
+		r53cfg := awsic.GetR53ClientCfg(session, installConfig.Config.AWS.HostedZoneRole)
+		route53Client := route53.New(session, r53cfg)
 		if _, err := route53Client.ChangeTagsForResourceWithContext(ctx, &route53.ChangeTagsForResourceInput{
 			ResourceType: aws.String("hostedzone"),
 			ResourceId:   aws.String(zone),

pkg/asset/cluster/tfvars.go

@@ -281,6 +281,7 @@ func (t *TerraformVariables) Generate(parents asset.Parents) error {
 			PrivateSubnets:        privateSubnets,
 			PublicSubnets:         publicSubnets,
 			InternalZone:          installConfig.Config.AWS.HostedZone,
+			InternalZoneRole:      installConfig.Config.AWS.HostedZoneRole,
 			Services:              installConfig.Config.AWS.ServiceEndpoints,
 			Publish:               installConfig.Config.Publish,
 			MasterConfigs:         masterConfigs,

pkg/asset/installconfig/aws/permissions.go

@@ -32,6 +32,12 @@ const (
 	// PermissionDeleteSharedInstanceRole is a set of permissions required when the installer destroys resources from a
 	// cluster with user-supplied IAM roles for instances.
 	PermissionDeleteSharedInstanceRole PermissionGroup = "delete-shared-instance-role"
+
+	// PermissionCreateHostedZone is a set of permissions required when the installer creates a route53 hosted zone.
+	PermissionCreateHostedZone PermissionGroup = "create-hosted-zone"
+
+	// PermissionDeleteHostedZone is a set of permissions required when the installer destroys a route53 hosted zone.
+	PermissionDeleteHostedZone PermissionGroup = "delete-hosted-zone"
 )
 
 var permissions = map[PermissionGroup][]string{
@@ -131,8 +137,6 @@ var permissions = map[PermissionGroup][]string{
 		// Route53 related perms
 		"route53:ChangeResourceRecordSets",
 		"route53:ChangeTagsForResource",
-		"route53:CreateHostedZone",
-		"route53:DeleteHostedZone",
 		"route53:GetChange",
 		"route53:GetHostedZone",
 		"route53:ListHostedZones",
@@ -233,6 +237,12 @@ var permissions = map[PermissionGroup][]string{
 	PermissionDeleteSharedInstanceRole: {
 		"iam:UntagRole",
 	},
+	PermissionCreateHostedZone: {
+		"route53:CreateHostedZone",
+	},
+	PermissionDeleteHostedZone: {
+		"route53:DeleteHostedZone",
+	},
 }
 
 // ValidateCreds will try to create an AWS session, and also verify that the current credentials

pkg/asset/installconfig/aws/route53.go

@@ -3,6 +3,7 @@ package aws
 import (
 	"fmt"
 	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/credentials/stscreds"
 	awss "github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/route53"
 	"github.com/openshift/installer/pkg/types"
@@ -15,10 +16,10 @@ import (
 
 // API represents the calls made to the API.
 type API interface {
-	GetHostedZone(hostedZone string) (*route53.GetHostedZoneOutput, error)
-	ValidateZoneRecords(zone *route53.HostedZone, zoneName string, zonePath *field.Path, ic *types.InstallConfig) field.ErrorList
+	GetHostedZone(hostedZone string, cfg *aws.Config) (*route53.GetHostedZoneOutput, error)
+	ValidateZoneRecords(zone *route53.HostedZone, zoneName string, zonePath *field.Path, ic *types.InstallConfig, cfg *aws.Config) field.ErrorList
 	GetBaseDomain(baseDomainName string) (*route53.HostedZone, error)
-	GetSubDomainDNSRecords(hostedZone *route53.HostedZone, ic *types.InstallConfig) ([]string, error)
+	GetSubDomainDNSRecords(hostedZone *route53.HostedZone, ic *types.InstallConfig, cfg *aws.Config) ([]string, error)
 }
 
 // Client makes calls to the AWS Route53 API.
@@ -35,9 +36,9 @@ func NewClient(ssn *awss.Session) *Client {
 }
 
 // GetHostedZone attempts to get the hosted zone from the AWS Route53 instance
-func (c *Client) GetHostedZone(hostedZone string) (*route53.GetHostedZoneOutput, error) {
+func (c *Client) GetHostedZone(hostedZone string, cfg *aws.Config) (*route53.GetHostedZoneOutput, error) {
 	// build a new Route53 instance from the same session that made it here
-	r53 := route53.New(c.ssn)
+	r53 := route53.New(c.ssn, cfg)
 
 	// validate that the hosted zone exists
 	hostedZoneOutput, err := r53.GetHostedZone(&route53.GetHostedZoneInput{Id: aws.String(hostedZone)})
@@ -48,10 +49,10 @@ func (c *Client) GetHostedZone(hostedZone string) (*route53.GetHostedZoneOutput,
 }
 
 // ValidateZoneRecords Attempts to validate each of the candidate HostedZones against the Config
-func (c *Client) ValidateZoneRecords(zone *route53.HostedZone, zoneName string, zonePath *field.Path, ic *types.InstallConfig) field.ErrorList {
+func (c *Client) ValidateZoneRecords(zone *route53.HostedZone, zoneName string, zonePath *field.Path, ic *types.InstallConfig, cfg *aws.Config) field.ErrorList {
 	allErrs := field.ErrorList{}
 
-	problematicRecords, err := c.GetSubDomainDNSRecords(zone, ic)
+	problematicRecords, err := c.GetSubDomainDNSRecords(zone, ic, cfg)
 	if err != nil {
 		allErrs = append(allErrs, field.InternalError(zonePath,
 			errors.Wrapf(err, "could not list record sets for domain %q", zoneName)))
@@ -70,15 +71,15 @@ func (c *Client) ValidateZoneRecords(zone *route53.HostedZone, zoneName string,
 
 // GetSubDomainDNSRecords Validates the hostedZone against the cluster domain, and ensures that the
 // cluster domain does not have a current record set for the hostedZone
-func (c *Client) GetSubDomainDNSRecords(hostedZone *route53.HostedZone, ic *types.InstallConfig) ([]string, error) {
+func (c *Client) GetSubDomainDNSRecords(hostedZone *route53.HostedZone, ic *types.InstallConfig, cfg *aws.Config) ([]string, error) {
 	dottedClusterDomain := ic.ClusterDomain() + "."
 
 	// validate that the domain of the hosted zone is the cluster domain or a parent of the cluster domain
 	if !isHostedZoneDomainParentOfClusterDomain(hostedZone, dottedClusterDomain) {
 		return nil, errors.Errorf("hosted zone domain %q is not a parent of the cluster domain %q", *hostedZone.Name, dottedClusterDomain)
 	}
 
-	r53 := route53.New(c.ssn)
+	r53 := route53.New(c.ssn, cfg)
 
 	var problematicRecords []string
 	// validate that the hosted zone does not already have any record sets for the cluster domain
@@ -132,3 +133,14 @@ func (c *Client) GetBaseDomain(baseDomainName string) (*route53.HostedZone, erro
 	}
 	return baseDomainZone, nil
 }
+
+// GetR53ClientCfg creates a config for the route53 client by determining
+// whether it is needed to obtain STS assume role credentials.
+func GetR53ClientCfg(sess *awss.Session, roleARN string) *aws.Config {
+	if roleARN == "" {
+		return nil
+	}
+
+	creds := stscreds.NewCredentials(sess, roleARN)
+	return &aws.Config{Credentials: creds}
+}