Skip to content

Commit

Permalink
tests: run each mock test in a different project
Browse files Browse the repository at this point in the history 
This also allows for context isolation and (in theory) parallel execution.
  • Loading branch information
@justinsb
justinsb committed Jan 20, 2024
1 parent 59c7576 commit 0c1c13f
Show file tree
Hide file tree
Showing 4 changed files with 98 additions and 87 deletions.
50 changes: 46 additions & 4 deletions config/tests/samples/create/harness.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ import (
"time"

exportparameters "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/cli/cmd/export/parameters"
"google.golang.org/api/cloudresourcemanager/v1"
cloudresourcemanagerv1 "google.golang.org/api/cloudresourcemanager/v1"
"google.golang.org/api/option"

Expand All @@ -37,6 +38,7 @@ import (
"github.com/GoogleCloudPlatform/k8s-config-connector/pkg/logging"
"github.com/GoogleCloudPlatform/k8s-config-connector/pkg/test"
testenvironment "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/test/environment"
testgcp "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/test/gcp"
testwebhook "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/test/webhook"
cnrmwebhook "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/webhook"
"golang.org/x/oauth2"
Expand All @@ -63,6 +65,8 @@ type Harness struct {
*testing.T
Ctx context.Context

Project testgcp.GCPProject

client client.Client
restConfig *rest.Config

Expand Down Expand Up @@ -203,12 +207,14 @@ func NewHarness(t *testing.T, ctx context.Context) *Harness {

roundTripper := http.RoundTripper(mockCloud)

h.Ctx = context.WithValue(h.Ctx, httpRoundTripperKey, roundTripper)
ctx = context.WithValue(ctx, httpRoundTripperKey, roundTripper)
h.Ctx = ctx

kccConfig.HTTPClient = &http.Client{Transport: roundTripper}

// Also hook the oauth2 library
h.Ctx = context.WithValue(h.Ctx, oauth2.HTTPClient, kccConfig.HTTPClient)
ctx = context.WithValue(ctx, oauth2.HTTPClient, kccConfig.HTTPClient)
h.Ctx = ctx

h.gcpAccessToken = "dummytoken"
kccConfig.GCPAccessToken = h.gcpAccessToken
Expand All @@ -218,6 +224,42 @@ func NewHarness(t *testing.T, ctx context.Context) *Harness {
t.Fatalf("E2E_GCP_TARGET=%q not supported", targetGCP)
}

if os.Getenv("E2E_GCP_TARGET") == "mock" {
// Some fixed-value fake org-ids for testing.
// We used fixed values so that the output is predictable (for golden testing)
testgcp.TestFolderID.Set("123451001")
testgcp.TestFolder2ID.Set("123451002")
testgcp.TestOrgID.Set("123450001")
testgcp.TestBillingAccountID.Set("123456-777777-000001")
testgcp.IAMIntegrationTestsOrganizationID.Set("123450002")
testgcp.IAMIntegrationTestsBillingAccountID.Set("123456-777777-000002")
testgcp.TestAttachedClusterName.Set("xks-cluster")

crm := h.getCloudResourceManagerClient(kccConfig.HTTPClient)
req := &cloudresourcemanager.Project{
ProjectId: "mock-project",
}
op, err := crm.Projects.Create(req).Context(ctx).Do()
if err != nil {
t.Fatalf("error creating project: %v", err)
}
if !op.Done {
t.Fatalf("expected mock create project operation to be done immediately")
}
found, err := crm.Projects.Get(req.ProjectId).Context(ctx).Do()
if err != nil {
t.Fatalf("error reading created project: %v", err)
}
project := testgcp.GCPProject{
ProjectID: found.ProjectId,
ProjectNumber: found.ProjectNumber,
}
testgcp.TestKCCAttachedClusterProject.Set("mock-project")
h.Project = project
} else {
h.Project = testgcp.GetDefaultProject(t)
}

// Log DCL requests
if artifacts := os.Getenv("ARTIFACTS"); artifacts != "" {
outputDir := filepath.Join(artifacts, "http-logs")
Expand Down Expand Up @@ -308,8 +350,8 @@ func (h *Harness) ExportParams() exportparameters.Parameters {
return exportParams
}

func (h *Harness) GetCloudResourceManagerClient() *cloudresourcemanagerv1.Service {
s, err := cloudresourcemanagerv1.NewService(h.Ctx, option.WithHTTPClient(h.kccConfig.HTTPClient))
func (h *Harness) getCloudResourceManagerClient(httpClient *http.Client) *cloudresourcemanagerv1.Service {
s, err := cloudresourcemanagerv1.NewService(h.Ctx, option.WithHTTPClient(httpClient))
if err != nil {
h.Fatalf("error building cloudresourcemanagerv1 client: %v", err)
}
Expand Down
21 changes: 8 additions & 13 deletions config/tests/samples/create/samples.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ import (
"github.com/ghodss/yaml"
"k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/apimachinery/pkg/util/wait"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/log"
Expand All @@ -64,8 +65,8 @@ func networksInSampleCount(sample Sample) int {
return count
}

func SetupNamespacesAndApplyDefaults(t *Harness, samples []Sample, project testgcp.GCPProject) {
namespaceNames := getNamespaces(samples)
func SetupNamespacesAndApplyDefaults(t *Harness, resources []*unstructured.Unstructured, project testgcp.GCPProject) {
namespaceNames := getNamespaces(resources)
setupNamespaces(t, namespaceNames, project)
}

Expand All @@ -75,18 +76,12 @@ func setupNamespaces(t *Harness, namespaces []string, project testgcp.GCPProject
}
}

func getNamespaces(samples []Sample) []string {
namespaces := make(map[string]bool)
for _, sample := range samples {
for _, unstruct := range sample.Resources {
namespaces[unstruct.GetNamespace()] = true
}
}
results := make([]string, 0, len(namespaces))
for k := range namespaces {
results = append(results, k)
func getNamespaces(resources []*unstructured.Unstructured) []string {
namespaces := sets.NewString()
for _, unstruct := range resources {
namespaces.Insert(unstruct.GetNamespace())
}
return results
return namespaces.List()
}

type CreateDeleteTestOptions struct {
Expand Down
2 changes: 1 addition & 1 deletion config/tests/samples/create/samples_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -257,7 +257,7 @@ func TestAll(t *testing.T) {
ctx := context.TODO()

h := NewHarnessWithManager(t, ctx, mgr)
SetupNamespacesAndApplyDefaults(h, []Sample{s}, project)
SetupNamespacesAndApplyDefaults(h, s.Resources, project)

networkCount := int64(networksInSampleCount(s))
if networkCount > 0 {
Expand Down
112 changes: 43 additions & 69 deletions tests/e2e/unified_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,6 @@ import (
"github.com/GoogleCloudPlatform/k8s-config-connector/pkg/test/resourcefixture"
testvariable "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/test/resourcefixture/variable"
testyaml "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/test/yaml"
"google.golang.org/api/cloudresourcemanager/v1"

"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime/schema"
Expand All @@ -46,57 +45,25 @@ func TestAllInSeries(t *testing.T) {
cancel()
})

testHarness := create.NewHarness(t, ctx)

var project testgcp.GCPProject
if os.Getenv("E2E_GCP_TARGET") == "mock" {
// Some fixed-value fake org-ids for testing.
// We used fixed values so that the output is predictable (for golden testing)
testgcp.TestFolderID.Set("123451001")
testgcp.TestFolder2ID.Set("123451002")
testgcp.TestOrgID.Set("123450001")
testgcp.TestBillingAccountID.Set("123456-777777-000001")
testgcp.IAMIntegrationTestsOrganizationID.Set("123450002")
testgcp.IAMIntegrationTestsBillingAccountID.Set("123456-777777-000002")
testgcp.TestAttachedClusterName.Set("xks-cluster")

crm := testHarness.GetCloudResourceManagerClient()
req := &cloudresourcemanager.Project{
ProjectId: "mock-project",
}
op, err := crm.Projects.Create(req).Context(testHarness.Ctx).Do()
if err != nil {
testHarness.Fatalf("error creating project: %v", err)
}
if !op.Done {
testHarness.Fatalf("expected mock create project operation to be done immediately")
}
found, err := crm.Projects.Get(req.ProjectId).Context(testHarness.Ctx).Do()
if err != nil {
testHarness.Fatalf("error reading created project: %v", err)
}
project = testgcp.GCPProject{
ProjectID: found.ProjectId,
ProjectNumber: found.ProjectNumber,
}
testgcp.TestKCCAttachedClusterProject.Set("mock-project")
} else {
project = testgcp.GetDefaultProject(t)
}

t.Run("samples", func(t *testing.T) {
samples := create.LoadAllSamples(t, project)
samples := create.ListAllSamples(t)

for _, s := range samples {
s := s
for _, sampleKey := range samples {
sampleKey := sampleKey
// TODO(b/259496928): Randomize the resource names for parallel execution when/if needed.

t.Run(s.Name, func(t *testing.T) {
create.MaybeSkip(t, s.Name, s.Resources)
t.Run(sampleKey.Name, func(t *testing.T) {
// Quickly load the sample with a dummy project, just to see if we should skip it
{
dummySample := create.LoadSample(t, sampleKey, testgcp.GCPProject{ProjectID: "test-skip", ProjectNumber: 123456789})
create.MaybeSkip(t, sampleKey.Name, dummySample.Resources)
}

h := testHarness.ForSubtest(t)
h := create.NewHarness(t, ctx)
project := h.Project
s := create.LoadSample(t, sampleKey, project)

create.SetupNamespacesAndApplyDefaults(h, []create.Sample{s}, project)
create.SetupNamespacesAndApplyDefaults(h, s.Resources, project)

// Hack: set project-id because mockkubeapiserver does not support webhooks
for _, u := range s.Resources {
Expand All @@ -119,37 +86,44 @@ func TestAllInSeries(t *testing.T) {
fixture := fixture
// TODO(b/259496928): Randomize the resource names for parallel execution when/if needed.

testID := testvariable.NewUniqueId()
t.Run(fixture.Name, func(t *testing.T) {
uniqueID := testvariable.NewUniqueId()

loadFixture := func(project testgcp.GCPProject) (*unstructured.Unstructured, create.CreateDeleteTestOptions) {
primaryResource := bytesToUnstructured(t, fixture.Create, uniqueID, project)

s := create.Sample{
Name: fixture.Name,
}
opt := create.CreateDeleteTestOptions{CleanupResources: true}
opt.Create = append(opt.Create, primaryResource)

createResource := bytesToUnstructured(t, fixture.Create, testID, project)
s.Resources = append(s.Resources, createResource)
if fixture.Dependencies != nil {
dependencyYamls := testyaml.SplitYAML(t, fixture.Dependencies)
for _, dependBytes := range dependencyYamls {
depUnstruct := bytesToUnstructured(t, dependBytes, uniqueID, project)
opt.Create = append(opt.Create, depUnstruct)
}
}

exportResources := []*unstructured.Unstructured{createResource}
if fixture.Update != nil {
u := bytesToUnstructured(t, fixture.Update, uniqueID, project)
opt.Updates = append(opt.Updates, u)
}
return primaryResource, opt
}

if fixture.Dependencies != nil {
dependencyYamls := testyaml.SplitYAML(t, fixture.Dependencies)
for _, dependBytes := range dependencyYamls {
depUnstruct := bytesToUnstructured(t, dependBytes, testID, project)
s.Resources = append(s.Resources, depUnstruct)
// Quickly load the fixture with a dummy project, just to see if we should skip it
{
_, opt := loadFixture(testgcp.GCPProject{ProjectID: "test-skip", ProjectNumber: 123456789})
create.MaybeSkip(t, fixture.Name, opt.Create)
}
}

opt := create.CreateDeleteTestOptions{Create: s.Resources, CleanupResources: true}
if fixture.Update != nil {
u := bytesToUnstructured(t, fixture.Update, testID, project)
opt.Updates = append(opt.Updates, u)
}
h := create.NewHarness(t, ctx)
project := h.Project

t.Run(s.Name, func(t *testing.T) {
create.MaybeSkip(t, s.Name, s.Resources)
primaryResource, opt := loadFixture(project)

h := testHarness.ForSubtest(t)
exportResources := []*unstructured.Unstructured{primaryResource}

create.SetupNamespacesAndApplyDefaults(h, []create.Sample{s}, project)
create.SetupNamespacesAndApplyDefaults(h, opt.Create, project)

opt.CleanupResources = false // We delete explicitly below
create.RunCreateDeleteTest(h, opt)
Expand Down Expand Up @@ -186,7 +160,7 @@ func TestAllInSeries(t *testing.T) {
h.CompareGoldenFile(expectedPath, string(output), h.IgnoreComments, h.ReplaceString(project.ProjectID, "example-project-id"))
}

create.DeleteResources(h, s.Resources)
create.DeleteResources(h, opt.Create)
})
}
})
Expand Down

0 comments on commit 0c1c13f

Please sign in to comment.