Backport of Allow connections through Terminating Gateways from peere…

…d clusters NET-3463 into release/1.16.x (#19092) Allow connections through Terminating Gateways from peered clusters NET-3463 (#18959) * Add InboundPeerTrustBundle maps to Terminating Gateway * Add notify and cancelation of watch for inbound peer trust bundles * Pass peer trust bundles to the RBAC creation function * Regenerate Golden Files * add changelog, also adds another spot that needed peeredTrustBundles * Add basic test for terminating gateway with peer trust bundle * Add intention to cluster peered golden test * rerun codegen * update changelog * really update the changelog --------- Co-authored-by: Thomas Eckert <[email protected]> Co-authored-by: Melisa Griffin <[email protected]>
hashicorp · Oct 5, 2023 · a16914f · a16914f
1 parent 19bb5db
commit a16914f
Show file tree

Hide file tree

Showing 7 changed files with 384 additions and 17 deletions.
diff --git a/.changelog/18959.txt b/.changelog/18959.txt
@@ -0,0 +1,3 @@
+```release-note:bug
+gateways: Fix a bug where a service in a peered datacenter could not access an external node service through a terminating gateway
+```
diff --git a/agent/proxycfg/proxycfg.deepcopy.go b/agent/proxycfg/proxycfg.deepcopy.go
@@ -822,5 +822,27 @@ func (o *configSnapshotTerminatingGateway) DeepCopy() *configSnapshotTerminating
 			cp.HostnameServices[k2] = cp_HostnameServices_v2
 		}
 	}
+	if o.WatchedInboundPeerTrustBundles != nil {
+		cp.WatchedInboundPeerTrustBundles = make(map[structs.ServiceName]context.CancelFunc, len(o.WatchedInboundPeerTrustBundles))
+		for k2, v2 := range o.WatchedInboundPeerTrustBundles {
+			cp.WatchedInboundPeerTrustBundles[k2] = v2
+		}
+	}
+	if o.InboundPeerTrustBundles != nil {
+		cp.InboundPeerTrustBundles = make(map[structs.ServiceName][]*pbpeering.PeeringTrustBundle, len(o.InboundPeerTrustBundles))
+		for k2, v2 := range o.InboundPeerTrustBundles {
+			var cp_InboundPeerTrustBundles_v2 []*pbpeering.PeeringTrustBundle
+			if v2 != nil {
+				cp_InboundPeerTrustBundles_v2 = make([]*pbpeering.PeeringTrustBundle, len(v2))
+				copy(cp_InboundPeerTrustBundles_v2, v2)
+				for i3 := range v2 {
+					if v2[i3] != nil {
+						cp_InboundPeerTrustBundles_v2[i3] = v2[i3].DeepCopy()
+					}
+				}
+			}
+			cp.InboundPeerTrustBundles[k2] = cp_InboundPeerTrustBundles_v2
+		}
+	}
 	return &cp
 }
diff --git a/agent/proxycfg/snapshot.go b/agent/proxycfg/snapshot.go
@@ -282,6 +282,15 @@ type configSnapshotTerminatingGateway struct {
 	// HostnameServices is a map of service name to service instances with a hostname as the address.
 	// If hostnames are configured they must be provided to Envoy via CDS not EDS.
 	HostnameServices map[structs.ServiceName]structs.CheckServiceNodes
+
+	// WatchedInboundPeerTrustBundles is a map of service name to a cancel function. This cancel
+	// function is tied to the watch of the inbound peer trust bundles for the gateway.
+	WatchedInboundPeerTrustBundles map[structs.ServiceName]context.CancelFunc
+
+	// InboundPeerTrustBundles is a map of service name to a list of peering trust bundles.
+	// These bundles are used to configure RBAC policies for inbound filter chains on the gateway
+	// from services that are in a cluster-peered datacenter.
+	InboundPeerTrustBundles map[structs.ServiceName][]*pbpeering.PeeringTrustBundle
 }
 
 // ValidServices returns the list of service keys that have enough data to be emitted.

diff --git a/agent/proxycfg/terminating_gateway.go b/agent/proxycfg/terminating_gateway.go
@@ -8,8 +8,10 @@ import (
 	"fmt"
 	"strings"
 
+	cachetype "github.com/hashicorp/consul/agent/cache-types"
 	"github.com/hashicorp/consul/agent/leafcert"
 	"github.com/hashicorp/consul/agent/structs"
+	"github.com/hashicorp/consul/proto/private/pbpeering"
 )
 
 type handlerTerminatingGateway struct {
@@ -68,6 +70,8 @@ func (s *handlerTerminatingGateway) initialize(ctx context.Context) (ConfigSnaps
 	snap.TerminatingGateway.GatewayServices = make(map[structs.ServiceName]structs.GatewayService)
 	snap.TerminatingGateway.DestinationServices = make(map[structs.ServiceName]structs.GatewayService)
 	snap.TerminatingGateway.HostnameServices = make(map[structs.ServiceName]structs.CheckServiceNodes)
+	snap.TerminatingGateway.WatchedInboundPeerTrustBundles = make(map[structs.ServiceName]context.CancelFunc)
+	snap.TerminatingGateway.InboundPeerTrustBundles = make(map[structs.ServiceName][]*pbpeering.PeeringTrustBundle)
 	return snap, nil
 }
 
@@ -168,6 +172,29 @@ func (s *handlerTerminatingGateway) handleUpdate(ctx context.Context, u UpdateEv
 				snap.TerminatingGateway.WatchedIntentions[svc.Service] = cancel
 			}
 
+			if _, ok := snap.TerminatingGateway.WatchedInboundPeerTrustBundles[svc.Service]; !ok {
+				ctx, cancel := context.WithCancel(ctx)
+
+				err := s.dataSources.TrustBundleList.Notify(ctx, &cachetype.TrustBundleListRequest{
+					Request: &pbpeering.TrustBundleListByServiceRequest{
+						ServiceName: svc.Service.Name,
+						Namespace:   svc.Service.EnterpriseMeta.NamespaceOrDefault(),
+						Partition:   svc.Service.EnterpriseMeta.PartitionOrDefault(),
+					},
+					QueryOptions: structs.QueryOptions{Token: s.token},
+				}, peerTrustBundleIDPrefix+svc.Service.String(), s.ch)
+
+				if err != nil {
+					logger.Error("failed to register watch for peer trust bundles",
+						"service", svc.Service.String(),
+						"error", err,
+					)
+					cancel()
+					return err
+				}
+				snap.TerminatingGateway.WatchedInboundPeerTrustBundles[svc.Service] = cancel
+			}
+
 			// Watch leaf certificate for the service
 			// This cert is used to terminate mTLS connections on the service's behalf
 			if _, ok := snap.TerminatingGateway.WatchedLeaves[svc.Service]; !ok {
@@ -299,6 +326,16 @@ func (s *handlerTerminatingGateway) handleUpdate(ctx context.Context, u UpdateEv
 			}
 		}
 
+		// Cancel watches for peered trust bundle that were not in the update
+		for sn, cancelFn := range snap.TerminatingGateway.WatchedInboundPeerTrustBundles {
+			if _, ok := svcMap[sn]; !ok {
+				logger.Debug("canceling watch for peered trust bundle", "service", sn.String())
+				delete(snap.TerminatingGateway.WatchedInboundPeerTrustBundles, sn)
+				delete(snap.TerminatingGateway.InboundPeerTrustBundles, sn)
+				cancelFn()
+			}
+		}
+
 		// Cancel intention watches for services that were not in the update
 		for sn, cancelFn := range snap.TerminatingGateway.WatchedIntentions {
 			if _, ok := svcMap[sn]; !ok {
@@ -374,6 +411,16 @@ func (s *handlerTerminatingGateway) handleUpdate(ctx context.Context, u UpdateEv
 		sn := structs.ServiceNameFromString(strings.TrimPrefix(u.CorrelationID, serviceIntentionsIDPrefix))
 		snap.TerminatingGateway.Intentions[sn] = resp
 
+	case strings.HasPrefix(u.CorrelationID, peerTrustBundleIDPrefix):
+		resp, ok := u.Result.(*pbpeering.TrustBundleListByServiceResponse)
+		if !ok {
+			return fmt.Errorf("invalid type for response: %T", u.Result)
+		}
+		if len(resp.Bundles) > 0 {
+			sn := structs.ServiceNameFromString(strings.TrimPrefix(u.CorrelationID, peerTrustBundleIDPrefix))
+			snap.TerminatingGateway.InboundPeerTrustBundles[sn] = resp.Bundles
+		}
+
 	default:
 		// do nothing
 	}

diff --git a/agent/xds/listeners.go b/agent/xds/listeners.go
@@ -1638,6 +1638,7 @@ func (s *ResourceGenerator) makeTerminatingGatewayListener(
 
 		intentions := cfgSnap.TerminatingGateway.Intentions[svc]
 		svcConfig := cfgSnap.TerminatingGateway.ServiceConfigs[svc]
+		peerTrustBundles := cfgSnap.TerminatingGateway.InboundPeerTrustBundles[svc]
 
 		cfg, err := ParseProxyConfig(svcConfig.ProxyConfig)
 		if err != nil {
@@ -1651,10 +1652,11 @@ func (s *ResourceGenerator) makeTerminatingGatewayListener(
 		}
 
 		opts := terminatingGatewayFilterChainOpts{
-			cluster:    clusterName,
-			service:    svc,
-			intentions: intentions,
-			protocol:   cfg.Protocol,
+			cluster:          clusterName,
+			service:          svc,
+			intentions:       intentions,
+			protocol:         cfg.Protocol,
+			peerTrustBundles: peerTrustBundles,
 		}
 
 		clusterChain, err := s.makeFilterChainTerminatingGateway(cfgSnap, opts)
@@ -1682,6 +1684,7 @@ func (s *ResourceGenerator) makeTerminatingGatewayListener(
 	for _, svc := range cfgSnap.TerminatingGateway.ValidDestinations() {
 		intentions := cfgSnap.TerminatingGateway.Intentions[svc]
 		svcConfig := cfgSnap.TerminatingGateway.ServiceConfigs[svc]
+		peerTrustBundles := cfgSnap.TerminatingGateway.InboundPeerTrustBundles[svc]
 
 		cfg, err := ParseProxyConfig(svcConfig.ProxyConfig)
 		if err != nil {
@@ -1698,10 +1701,11 @@ func (s *ResourceGenerator) makeTerminatingGatewayListener(
 		dest = &svcConfig.Destination
 
 		opts := terminatingGatewayFilterChainOpts{
-			service:    svc,
-			intentions: intentions,
-			protocol:   cfg.Protocol,
-			port:       dest.Port,
+			service:          svc,
+			intentions:       intentions,
+			protocol:         cfg.Protocol,
+			port:             dest.Port,
+			peerTrustBundles: peerTrustBundles,
 		}
 		for _, address := range dest.Addresses {
 			clusterName := clusterNameForDestination(cfgSnap, svc.Name, address, svc.NamespaceOrDefault(), svc.PartitionOrDefault())
@@ -1758,12 +1762,13 @@ func (s *ResourceGenerator) makeTerminatingGatewayListener(
 }
 
 type terminatingGatewayFilterChainOpts struct {
-	cluster    string
-	service    structs.ServiceName
-	intentions structs.SimplifiedIntentions
-	protocol   string
-	address    string // only valid for destination listeners
-	port       int    // only valid for destination listeners
+	cluster          string
+	service          structs.ServiceName
+	intentions       structs.SimplifiedIntentions
+	protocol         string
+	address          string // only valid for destination listeners
+	port             int    // only valid for destination listeners
+	peerTrustBundles []*pbpeering.PeeringTrustBundle
 }
 
 func (s *ResourceGenerator) makeFilterChainTerminatingGateway(cfgSnap *proxycfg.ConfigSnapshot, tgtwyOpts terminatingGatewayFilterChainOpts) (*envoy_listener_v3.FilterChain, error) {
@@ -1799,7 +1804,7 @@ func (s *ResourceGenerator) makeFilterChainTerminatingGateway(cfgSnap *proxycfg.
 				datacenter:  cfgSnap.Datacenter,
 				partition:   cfgSnap.ProxyID.PartitionOrDefault(),
 			},
-			nil, // TODO(peering): verify intentions w peers don't apply to terminatingGateway
+			tgtwyOpts.peerTrustBundles,
 		)
 		if err != nil {
 			return nil, err
@@ -1845,7 +1850,7 @@ func (s *ResourceGenerator) makeFilterChainTerminatingGateway(cfgSnap *proxycfg.
 				datacenter:  cfgSnap.Datacenter,
 				partition:   cfgSnap.ProxyID.PartitionOrDefault(),
 			},
-			nil, // TODO(peering): verify intentions w peers don't apply to terminatingGateway
+			tgtwyOpts.peerTrustBundles,
 			cfgSnap.JWTProviders,
 		)
 		if err != nil {
@@ -2115,7 +2120,6 @@ func (s *ResourceGenerator) makeMeshGatewayPeerFilterChain(
 		// RDS, Envoy's Route Discovery Service, is only used for HTTP services.
 		useRDS = useHTTPFilter
 	)
-
 	if useHTTPFilter && cfgSnap.MeshGateway.Leaf == nil {
 		return nil, nil // ignore; not ready
 	}

diff --git a/agent/xds/listeners_test.go b/agent/xds/listeners_test.go
@@ -20,6 +20,7 @@ import (
 	"github.com/hashicorp/consul/agent/proxycfg"
 	"github.com/hashicorp/consul/agent/structs"
 	"github.com/hashicorp/consul/envoyextensions/xdscommon"
+	"github.com/hashicorp/consul/proto/private/pbpeering"
 	"github.com/hashicorp/consul/sdk/testutil"
 	"github.com/hashicorp/consul/types"
 )
@@ -983,6 +984,41 @@ func TestListenersFromSnapshot(t *testing.T) {
 				})
 			},
 		},
+		{
+			name: "terminating-gateway-with-peer-trust-bundle",
+			create: func(t testinf.T) *proxycfg.ConfigSnapshot {
+				roots, _ := proxycfg.TestCerts(t)
+				return proxycfg.TestConfigSnapshotTerminatingGateway(t, true, nil, []proxycfg.UpdateEvent{
+					{
+						CorrelationID: "peer-trust-bundle:web",
+						Result: &pbpeering.TrustBundleListByServiceResponse{
+							Bundles: []*pbpeering.PeeringTrustBundle{
+								{
+									TrustDomain: "foo.bar.gov",
+									PeerName:    "dc1",
+									Partition:   "default",
+									RootPEMs: []string{
+										roots.Roots[0].RootCert,
+									},
+									ExportedPartition: "dc1",
+									CreateIndex:       0,
+									ModifyIndex:       0,
+								},
+							},
+						},
+					},
+					{
+						CorrelationID: "service-intentions:web",
+						Result: structs.SimplifiedIntentions{
+							{
+								SourceName:      "*",
+								DestinationName: "web",
+							},
+						},
+					},
+				})
+			},
+		},
 		{
 			name: "ingress-with-tls-listener",
 			create: func(t testinf.T) *proxycfg.ConfigSnapshot {