diff --git a/xds/googledirectpath/googlec2p.go b/xds/googledirectpath/googlec2p.go index 936bf2da3274..fab8097e41b7 100644 --- a/xds/googledirectpath/googlec2p.go +++ b/xds/googledirectpath/googlec2p.go @@ -30,6 +30,7 @@ import ( "fmt" "math/rand" "net/url" + "sync" "time" "google.golang.org/grpc/grpclog" @@ -46,7 +47,7 @@ const ( c2pScheme = "google-c2p" c2pAuthority = "traffic-director-c2p.xds.googleapis.com" - tdURL = "dns:///directpath-pa.googleapis.com" + defaultUniverseDomain = "googleapis.com" zoneURL = "http://metadata.google.internal/computeMetadata/v1/instance/zone" ipv6URL = "http://metadata.google.internal/computeMetadata/v1/instance/network-interfaces/0/ipv6s" ipv6CapableMetadataName = "TRAFFICDIRECTOR_DIRECTPATH_C2P_IPV6_CAPABLE" @@ -56,17 +57,66 @@ const ( dnsName, xdsName = "dns", "xds" ) -// For overriding in unittests. var ( + logger = internalgrpclog.NewPrefixLogger(grpclog.Component("directpath"), logPrefix) + universeDomainMu sync.Mutex + universeDomain = "" + // For overriding in unittests. onGCE = googlecloud.OnGCE randInt = rand.Int - logger = internalgrpclog.NewPrefixLogger(grpclog.Component("directpath"), logPrefix) ) func init() { resolver.Register(c2pResolverBuilder{}) } +// SetUniverseDomain informs the gRPC library of the universe domain +// in which the process is running (for example, "googleapis.com"). +// It is the caller's responsibility to ensure that the domain is correct. +// +// This setting is used by the "google-c2p" resolver (the resolver used +// for URIs with the "google-c2p" scheme) to configure its dependencies. +// +// If a gRPC channel is created with the "google-c2p" URI scheme and this +// function has NOT been called, then gRPC configures the universe domain as +// "googleapis.com". +// +// Returns nil if either: +// +// a) The universe domain has not yet been configured. +// b) The universe domain has been configured and matches the provided value. +// +// Otherwise, returns an error. +func SetUniverseDomain(domain string) error { + universeDomainMu.Lock() + defer universeDomainMu.Unlock() + if domain == "" { + return fmt.Errorf("universe domain cannot be empty") + } + if universeDomain == "" { + universeDomain = domain + return nil + } + if universeDomain != domain { + return fmt.Errorf("universe domain cannot be set to %s, already set to different value: %s", domain, universeDomain) + } + return nil +} + +func getXdsServerURI() string { + universeDomainMu.Lock() + defer universeDomainMu.Unlock() + if universeDomain == "" { + universeDomain = defaultUniverseDomain + } + // Put env var override logic after default value logic so + // that tests still run the default value logic. + if envconfig.C2PResolverTestOnlyTrafficDirectorURI != "" { + return envconfig.C2PResolverTestOnlyTrafficDirectorURI + } + return fmt.Sprintf("dns:///directpath-pa.%s", universeDomain) +} + type c2pResolverBuilder struct{} func (c2pResolverBuilder) Build(t resolver.Target, cc resolver.ClientConn, opts resolver.BuildOptions) (resolver.Resolver, error) { @@ -90,11 +140,7 @@ func (c2pResolverBuilder) Build(t resolver.Target, cc resolver.ClientConn, opts go func() { zoneCh <- getZone(httpReqTimeout) }() go func() { ipv6CapableCh <- getIPv6Capable(httpReqTimeout) }() - xdsServerURI := envconfig.C2PResolverTestOnlyTrafficDirectorURI - if xdsServerURI == "" { - xdsServerURI = tdURL - } - + xdsServerURI := getXdsServerURI() nodeCfg := newNodeConfig(<-zoneCh, <-ipv6CapableCh) xdsServerCfg := newXdsServerConfig(xdsServerURI) authoritiesCfg := newAuthoritiesConfig(xdsServerCfg) diff --git a/xds/googledirectpath/googlec2p_test.go b/xds/googledirectpath/googlec2p_test.go index afa4ea0c7f55..4b101b308014 100644 --- a/xds/googledirectpath/googlec2p_test.go +++ b/xds/googledirectpath/googlec2p_test.go @@ -82,6 +82,21 @@ func simulateRunningOnGCE(t *testing.T, gce bool) { t.Cleanup(func() { onGCE = oldOnGCE }) } +// ensure universeDomain is set to the expected default, +// and clean it up again after the test. +func useCleanUniverseDomain(t *testing.T) { + universeDomainMu.Lock() + defer universeDomainMu.Unlock() + if universeDomain != "" { + t.Fatalf("universe domain unexpectedly initialized: %v", universeDomain) + } + t.Cleanup(func() { + universeDomainMu.Lock() + universeDomain = "" + universeDomainMu.Unlock() + }) +} + // Tests the scenario where the bootstrap env vars are set and we're running on // GCE. The test builds a google-c2p resolver and verifies that an xDS resolver // is built and that we don't fallback to DNS (because federation is enabled by @@ -89,6 +104,7 @@ func simulateRunningOnGCE(t *testing.T, gce bool) { func (s) TestBuildWithBootstrapEnvSet(t *testing.T) { replaceResolvers(t) simulateRunningOnGCE(t, true) + useCleanUniverseDomain(t) builder := resolver.Get(c2pScheme) for i, envP := range []*string{&envconfig.XDSBootstrapFileName, &envconfig.XDSBootstrapFileContent} { @@ -118,6 +134,7 @@ func (s) TestBuildWithBootstrapEnvSet(t *testing.T) { func (s) TestBuildNotOnGCE(t *testing.T) { replaceResolvers(t) simulateRunningOnGCE(t, false) + useCleanUniverseDomain(t) builder := resolver.Get(c2pScheme) // Build the google-c2p resolver. @@ -152,6 +169,7 @@ func bootstrapConfig(t *testing.T, opts bootstrap.ConfigOptionsForTesting) *boot func (s) TestBuildXDS(t *testing.T) { replaceResolvers(t) simulateRunningOnGCE(t, true) + useCleanUniverseDomain(t) builder := resolver.Get(c2pScheme) // Override the zone returned by the metadata server. @@ -295,6 +313,7 @@ func (s) TestBuildXDS(t *testing.T) { // google-c2p scheme with a non-empty authority and verifies that it fails with // an expected error. func (s) TestBuildFailsWhenCalledWithAuthority(t *testing.T) { + useCleanUniverseDomain(t) uri := "google-c2p://an-authority/resource" cc, err := grpc.Dial(uri, grpc.WithTransportCredentials(insecure.NewCredentials())) defer func() { @@ -307,3 +326,188 @@ func (s) TestBuildFailsWhenCalledWithAuthority(t *testing.T) { t.Fatalf("grpc.Dial(%s) returned error: %v, want: %v", uri, err, wantErr) } } + +func (s) TestSetUniverseDomainNonDefault(t *testing.T) { + replaceResolvers(t) + simulateRunningOnGCE(t, true) + useCleanUniverseDomain(t) + builder := resolver.Get(c2pScheme) + + // Override the zone returned by the metadata server. + oldGetZone := getZone + getZone = func(time.Duration) string { return "test-zone" } + defer func() { getZone = oldGetZone }() + + // Override IPv6 capability returned by the metadata server. + oldGetIPv6Capability := getIPv6Capable + getIPv6Capable = func(time.Duration) bool { return false } + defer func() { getIPv6Capable = oldGetIPv6Capability }() + + // Override the random func used in the node ID. + origRandInd := randInt + randInt = func() int { return 666 } + defer func() { randInt = origRandInd }() + + // Set the universe domain + testUniverseDomain := "test-universe-domain.test" + if err := SetUniverseDomain(testUniverseDomain); err != nil { + t.Fatalf("SetUniverseDomain(%s) failed: %v", testUniverseDomain, err) + } + + // Now set universe domain to something different, it should fail + domain := "test-universe-domain-2.test" + err := SetUniverseDomain(domain) + wantErr := "already set" + if err == nil || !strings.Contains(err.Error(), wantErr) { + t.Fatalf("googlec2p.SetUniverseDomain(%s) returned error: %v, want: %v", domain, err, wantErr) + } + + // Now explicitly set universe domain to the default, it should also fail + domain = "googleapis.com" + err = SetUniverseDomain(domain) + wantErr = "already set" + if err == nil || !strings.Contains(err.Error(), wantErr) { + t.Fatalf("googlec2p.SetUniverseDomain(%s) returned error: %v, want: %v", domain, err, wantErr) + } + + // Now set universe domain to the original value, it should work + if err := SetUniverseDomain(testUniverseDomain); err != nil { + t.Fatalf("googlec2p.SetUniverseDomain(%s) failed: %v", testUniverseDomain, err) + } + + // Build the google-c2p resolver. + r, err := builder.Build(resolver.Target{}, nil, resolver.BuildOptions{}) + if err != nil { + t.Fatalf("failed to build resolver: %v", err) + } + defer r.Close() + + // Build should return xDS, not DNS. + if r != testXDSResolver { + t.Fatalf("Build() returned %#v, want xds resolver", r) + } + + gotConfig, err := bootstrap.GetConfiguration() + if err != nil { + t.Fatalf("Failed to get bootstrap config: %v", err) + } + + // Check that we use directpath-pa.test-universe-domain.test in the + // bootstrap config. + wantBootstrapConfig := bootstrapConfig(t, bootstrap.ConfigOptionsForTesting{ + Servers: []byte(`[{ + "server_uri": "dns:///directpath-pa.test-universe-domain.test", + "channel_creds": [{"type": "google_default"}], + "server_features": ["ignore_resource_deletion"] + }]`), + Authorities: map[string]json.RawMessage{ + "traffic-director-c2p.xds.googleapis.com": []byte(`{ + "xds_servers": [ + { + "server_uri": "dns:///directpath-pa.test-universe-domain.test", + "channel_creds": [{"type": "google_default"}], + "server_features": ["ignore_resource_deletion"] + } + ] + }`), + }, + Node: []byte(`{ + "id": "C2P-666", + "locality": {"zone": "test-zone"} + }`), + }) + if diff := cmp.Diff(wantBootstrapConfig, gotConfig); diff != "" { + t.Fatalf("Unexpected diff in bootstrap config (-want +got):\n%s", diff) + } +} + +func (s) TestDefaultUniverseDomain(t *testing.T) { + replaceResolvers(t) + simulateRunningOnGCE(t, true) + useCleanUniverseDomain(t) + builder := resolver.Get(c2pScheme) + + // Override the zone returned by the metadata server. + oldGetZone := getZone + getZone = func(time.Duration) string { return "test-zone" } + defer func() { getZone = oldGetZone }() + + // Override IPv6 capability returned by the metadata server. + oldGetIPv6Capability := getIPv6Capable + getIPv6Capable = func(time.Duration) bool { return false } + defer func() { getIPv6Capable = oldGetIPv6Capability }() + + // Override the random func used in the node ID. + origRandInd := randInt + randInt = func() int { return 666 } + defer func() { randInt = origRandInd }() + + // Build the google-c2p resolver. + r, err := builder.Build(resolver.Target{}, nil, resolver.BuildOptions{}) + if err != nil { + t.Fatalf("failed to build resolver: %v", err) + } + defer r.Close() + + // Build should return xDS, not DNS. + if r != testXDSResolver { + t.Fatalf("Build() returned %#v, want xds resolver", r) + } + + gotConfig, err := bootstrap.GetConfiguration() + if err != nil { + t.Fatalf("Failed to get bootstrap config: %v", err) + } + + // Check that we use directpath-pa.googleapis.com in the bootstrap config + wantBootstrapConfig := bootstrapConfig(t, bootstrap.ConfigOptionsForTesting{ + Servers: []byte(`[{ + "server_uri": "dns:///directpath-pa.googleapis.com", + "channel_creds": [{"type": "google_default"}], + "server_features": ["ignore_resource_deletion"] + }]`), + Authorities: map[string]json.RawMessage{ + "traffic-director-c2p.xds.googleapis.com": []byte(`{ + "xds_servers": [ + { + "server_uri": "dns:///directpath-pa.googleapis.com", + "channel_creds": [{"type": "google_default"}], + "server_features": ["ignore_resource_deletion"] + } + ] + }`), + }, + Node: []byte(`{ + "id": "C2P-666", + "locality": {"zone": "test-zone"} + }`), + }) + if diff := cmp.Diff(wantBootstrapConfig, gotConfig); diff != "" { + t.Fatalf("Unexpected diff in bootstrap config (-want +got):\n%s", diff) + } + + // Now set universe domain to something different than the default, it should fail + domain := "test-universe-domain.test" + err = SetUniverseDomain(domain) + wantErr := "already set" + if err == nil || !strings.Contains(err.Error(), wantErr) { + t.Fatalf("googlec2p.SetUniverseDomain(%s) returned error: %v, want: %v", domain, err, wantErr) + } + + // Now explicitly set universe domain to the default, it should work + domain = "googleapis.com" + if err := SetUniverseDomain(domain); err != nil { + t.Fatalf("googlec2p.SetUniverseDomain(%s) failed: %v", domain, err) + } +} + +func (s) TestSetUniverseDomainEmptyString(t *testing.T) { + replaceResolvers(t) + simulateRunningOnGCE(t, true) + useCleanUniverseDomain(t) + wantErr := "cannot be empty" + err := SetUniverseDomain("") + if err == nil || !strings.Contains(err.Error(), wantErr) { + t.Fatalf("googlec2p.SetUniverseDomain(\"\") returned error: %v, want: %v", err, wantErr) + } +}