Skip to content

Commit

Permalink
resolver/google-c2p: introduce SetUniverseDomain API (#7719)
Browse files Browse the repository at this point in the history
  • Loading branch information
@apolcyn
apolcyn authored Oct 21, 2024
1 parent 98959d9 commit 14e2a20
Show file tree
Hide file tree
Showing 2 changed files with 258 additions and 8 deletions.
62 changes: 54 additions & 8 deletions xds/googledirectpath/googlec2p.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ import (
"fmt"
"math/rand"
"net/url"
"sync"
"time"

"google.golang.org/grpc/grpclog"
Expand All @@ -46,7 +47,7 @@ const (
c2pScheme = "google-c2p"
c2pAuthority = "traffic-director-c2p.xds.googleapis.com"

tdURL = "dns:///directpath-pa.googleapis.com"
defaultUniverseDomain = "googleapis.com"
zoneURL = "http://metadata.google.internal/computeMetadata/v1/instance/zone"
ipv6URL = "http://metadata.google.internal/computeMetadata/v1/instance/network-interfaces/0/ipv6s"
ipv6CapableMetadataName = "TRAFFICDIRECTOR_DIRECTPATH_C2P_IPV6_CAPABLE"
Expand All @@ -56,17 +57,66 @@ const (
dnsName, xdsName = "dns", "xds"
)

// For overriding in unittests.
var (
logger = internalgrpclog.NewPrefixLogger(grpclog.Component("directpath"), logPrefix)
universeDomainMu sync.Mutex
universeDomain = ""
// For overriding in unittests.
onGCE = googlecloud.OnGCE
randInt = rand.Int
logger = internalgrpclog.NewPrefixLogger(grpclog.Component("directpath"), logPrefix)
)

func init() {
resolver.Register(c2pResolverBuilder{})
}

// SetUniverseDomain informs the gRPC library of the universe domain
// in which the process is running (for example, "googleapis.com").
// It is the caller's responsibility to ensure that the domain is correct.
//
// This setting is used by the "google-c2p" resolver (the resolver used
// for URIs with the "google-c2p" scheme) to configure its dependencies.
//
// If a gRPC channel is created with the "google-c2p" URI scheme and this
// function has NOT been called, then gRPC configures the universe domain as
// "googleapis.com".
//
// Returns nil if either:
//
// a) The universe domain has not yet been configured.
// b) The universe domain has been configured and matches the provided value.
//
// Otherwise, returns an error.
func SetUniverseDomain(domain string) error {
universeDomainMu.Lock()
defer universeDomainMu.Unlock()
if domain == "" {
return fmt.Errorf("universe domain cannot be empty")
}
if universeDomain == "" {
universeDomain = domain
return nil
}
if universeDomain != domain {
return fmt.Errorf("universe domain cannot be set to %s, already set to different value: %s", domain, universeDomain)
}
return nil
}

func getXdsServerURI() string {
universeDomainMu.Lock()
defer universeDomainMu.Unlock()
if universeDomain == "" {
universeDomain = defaultUniverseDomain
}
// Put env var override logic after default value logic so
// that tests still run the default value logic.
if envconfig.C2PResolverTestOnlyTrafficDirectorURI != "" {
return envconfig.C2PResolverTestOnlyTrafficDirectorURI
}
return fmt.Sprintf("dns:///directpath-pa.%s", universeDomain)
}

type c2pResolverBuilder struct{}

func (c2pResolverBuilder) Build(t resolver.Target, cc resolver.ClientConn, opts resolver.BuildOptions) (resolver.Resolver, error) {
Expand All @@ -90,11 +140,7 @@ func (c2pResolverBuilder) Build(t resolver.Target, cc resolver.ClientConn, opts
go func() { zoneCh <- getZone(httpReqTimeout) }()
go func() { ipv6CapableCh <- getIPv6Capable(httpReqTimeout) }()

xdsServerURI := envconfig.C2PResolverTestOnlyTrafficDirectorURI
if xdsServerURI == "" {
xdsServerURI = tdURL
}

xdsServerURI := getXdsServerURI()
nodeCfg := newNodeConfig(<-zoneCh, <-ipv6CapableCh)
xdsServerCfg := newXdsServerConfig(xdsServerURI)
authoritiesCfg := newAuthoritiesConfig(xdsServerCfg)
Expand Down
204 changes: 204 additions & 0 deletions xds/googledirectpath/googlec2p_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -82,13 +82,29 @@ func simulateRunningOnGCE(t *testing.T, gce bool) {
t.Cleanup(func() { onGCE = oldOnGCE })
}

// ensure universeDomain is set to the expected default,
// and clean it up again after the test.
func useCleanUniverseDomain(t *testing.T) {
universeDomainMu.Lock()
defer universeDomainMu.Unlock()
if universeDomain != "" {
t.Fatalf("universe domain unexpectedly initialized: %v", universeDomain)
}
t.Cleanup(func() {
universeDomainMu.Lock()
universeDomain = ""
universeDomainMu.Unlock()
})
}

// Tests the scenario where the bootstrap env vars are set and we're running on
// GCE. The test builds a google-c2p resolver and verifies that an xDS resolver
// is built and that we don't fallback to DNS (because federation is enabled by
// default).
func (s) TestBuildWithBootstrapEnvSet(t *testing.T) {
replaceResolvers(t)
simulateRunningOnGCE(t, true)
useCleanUniverseDomain(t)

builder := resolver.Get(c2pScheme)
for i, envP := range []*string{&envconfig.XDSBootstrapFileName, &envconfig.XDSBootstrapFileContent} {
Expand Down Expand Up @@ -118,6 +134,7 @@ func (s) TestBuildWithBootstrapEnvSet(t *testing.T) {
func (s) TestBuildNotOnGCE(t *testing.T) {
replaceResolvers(t)
simulateRunningOnGCE(t, false)
useCleanUniverseDomain(t)
builder := resolver.Get(c2pScheme)

// Build the google-c2p resolver.
Expand Down Expand Up @@ -152,6 +169,7 @@ func bootstrapConfig(t *testing.T, opts bootstrap.ConfigOptionsForTesting) *boot
func (s) TestBuildXDS(t *testing.T) {
replaceResolvers(t)
simulateRunningOnGCE(t, true)
useCleanUniverseDomain(t)
builder := resolver.Get(c2pScheme)

// Override the zone returned by the metadata server.
Expand Down Expand Up @@ -295,6 +313,7 @@ func (s) TestBuildXDS(t *testing.T) {
// google-c2p scheme with a non-empty authority and verifies that it fails with
// an expected error.
func (s) TestBuildFailsWhenCalledWithAuthority(t *testing.T) {
useCleanUniverseDomain(t)
uri := "google-c2p://an-authority/resource"
cc, err := grpc.Dial(uri, grpc.WithTransportCredentials(insecure.NewCredentials()))
defer func() {
Expand All @@ -307,3 +326,188 @@ func (s) TestBuildFailsWhenCalledWithAuthority(t *testing.T) {
t.Fatalf("grpc.Dial(%s) returned error: %v, want: %v", uri, err, wantErr)
}
}

func (s) TestSetUniverseDomainNonDefault(t *testing.T) {
replaceResolvers(t)
simulateRunningOnGCE(t, true)
useCleanUniverseDomain(t)
builder := resolver.Get(c2pScheme)

// Override the zone returned by the metadata server.
oldGetZone := getZone
getZone = func(time.Duration) string { return "test-zone" }
defer func() { getZone = oldGetZone }()

// Override IPv6 capability returned by the metadata server.
oldGetIPv6Capability := getIPv6Capable
getIPv6Capable = func(time.Duration) bool { return false }
defer func() { getIPv6Capable = oldGetIPv6Capability }()

// Override the random func used in the node ID.
origRandInd := randInt
randInt = func() int { return 666 }
defer func() { randInt = origRandInd }()

// Set the universe domain
testUniverseDomain := "test-universe-domain.test"
if err := SetUniverseDomain(testUniverseDomain); err != nil {
t.Fatalf("SetUniverseDomain(%s) failed: %v", testUniverseDomain, err)
}

// Now set universe domain to something different, it should fail
domain := "test-universe-domain-2.test"
err := SetUniverseDomain(domain)
wantErr := "already set"
if err == nil || !strings.Contains(err.Error(), wantErr) {
t.Fatalf("googlec2p.SetUniverseDomain(%s) returned error: %v, want: %v", domain, err, wantErr)
}

// Now explicitly set universe domain to the default, it should also fail
domain = "googleapis.com"
err = SetUniverseDomain(domain)
wantErr = "already set"
if err == nil || !strings.Contains(err.Error(), wantErr) {
t.Fatalf("googlec2p.SetUniverseDomain(%s) returned error: %v, want: %v", domain, err, wantErr)
}

// Now set universe domain to the original value, it should work
if err := SetUniverseDomain(testUniverseDomain); err != nil {
t.Fatalf("googlec2p.SetUniverseDomain(%s) failed: %v", testUniverseDomain, err)
}

// Build the google-c2p resolver.
r, err := builder.Build(resolver.Target{}, nil, resolver.BuildOptions{})
if err != nil {
t.Fatalf("failed to build resolver: %v", err)
}
defer r.Close()

// Build should return xDS, not DNS.
if r != testXDSResolver {
t.Fatalf("Build() returned %#v, want xds resolver", r)
}

gotConfig, err := bootstrap.GetConfiguration()
if err != nil {
t.Fatalf("Failed to get bootstrap config: %v", err)
}

// Check that we use directpath-pa.test-universe-domain.test in the
// bootstrap config.
wantBootstrapConfig := bootstrapConfig(t, bootstrap.ConfigOptionsForTesting{
Servers: []byte(`[{
"server_uri": "dns:///directpath-pa.test-universe-domain.test",
"channel_creds": [{"type": "google_default"}],
"server_features": ["ignore_resource_deletion"]
}]`),
Authorities: map[string]json.RawMessage{
"traffic-director-c2p.xds.googleapis.com": []byte(`{
"xds_servers": [
{
"server_uri": "dns:///directpath-pa.test-universe-domain.test",
"channel_creds": [{"type": "google_default"}],
"server_features": ["ignore_resource_deletion"]
}
]
}`),
},
Node: []byte(`{
"id": "C2P-666",
"locality": {"zone": "test-zone"}
}`),
})
if diff := cmp.Diff(wantBootstrapConfig, gotConfig); diff != "" {
t.Fatalf("Unexpected diff in bootstrap config (-want +got):\n%s", diff)
}
}

func (s) TestDefaultUniverseDomain(t *testing.T) {
replaceResolvers(t)
simulateRunningOnGCE(t, true)
useCleanUniverseDomain(t)
builder := resolver.Get(c2pScheme)

// Override the zone returned by the metadata server.
oldGetZone := getZone
getZone = func(time.Duration) string { return "test-zone" }
defer func() { getZone = oldGetZone }()

// Override IPv6 capability returned by the metadata server.
oldGetIPv6Capability := getIPv6Capable
getIPv6Capable = func(time.Duration) bool { return false }
defer func() { getIPv6Capable = oldGetIPv6Capability }()

// Override the random func used in the node ID.
origRandInd := randInt
randInt = func() int { return 666 }
defer func() { randInt = origRandInd }()

// Build the google-c2p resolver.
r, err := builder.Build(resolver.Target{}, nil, resolver.BuildOptions{})
if err != nil {
t.Fatalf("failed to build resolver: %v", err)
}
defer r.Close()

// Build should return xDS, not DNS.
if r != testXDSResolver {
t.Fatalf("Build() returned %#v, want xds resolver", r)
}

gotConfig, err := bootstrap.GetConfiguration()
if err != nil {
t.Fatalf("Failed to get bootstrap config: %v", err)
}

// Check that we use directpath-pa.googleapis.com in the bootstrap config
wantBootstrapConfig := bootstrapConfig(t, bootstrap.ConfigOptionsForTesting{
Servers: []byte(`[{
"server_uri": "dns:///directpath-pa.googleapis.com",
"channel_creds": [{"type": "google_default"}],
"server_features": ["ignore_resource_deletion"]
}]`),
Authorities: map[string]json.RawMessage{
"traffic-director-c2p.xds.googleapis.com": []byte(`{
"xds_servers": [
{
"server_uri": "dns:///directpath-pa.googleapis.com",
"channel_creds": [{"type": "google_default"}],
"server_features": ["ignore_resource_deletion"]
}
]
}`),
},
Node: []byte(`{
"id": "C2P-666",
"locality": {"zone": "test-zone"}
}`),
})
if diff := cmp.Diff(wantBootstrapConfig, gotConfig); diff != "" {
t.Fatalf("Unexpected diff in bootstrap config (-want +got):\n%s", diff)
}

// Now set universe domain to something different than the default, it should fail
domain := "test-universe-domain.test"
err = SetUniverseDomain(domain)
wantErr := "already set"
if err == nil || !strings.Contains(err.Error(), wantErr) {
t.Fatalf("googlec2p.SetUniverseDomain(%s) returned error: %v, want: %v", domain, err, wantErr)
}

// Now explicitly set universe domain to the default, it should work
domain = "googleapis.com"
if err := SetUniverseDomain(domain); err != nil {
t.Fatalf("googlec2p.SetUniverseDomain(%s) failed: %v", domain, err)
}
}

func (s) TestSetUniverseDomainEmptyString(t *testing.T) {
replaceResolvers(t)
simulateRunningOnGCE(t, true)
useCleanUniverseDomain(t)
wantErr := "cannot be empty"
err := SetUniverseDomain("")
if err == nil || !strings.Contains(err.Error(), wantErr) {
t.Fatalf("googlec2p.SetUniverseDomain(\"\") returned error: %v, want: %v", err, wantErr)
}
}

0 comments on commit 14e2a20

Please sign in to comment.