Skip to content

Commit

Permalink
run fixtures tests in parallel
Browse files Browse the repository at this point in the history
  • Loading branch information
@gemmahou
gemmahou committed Aug 15, 2024
1 parent 6485c02 commit d8b9bc6
Show file tree
Hide file tree
Showing 2 changed files with 117 additions and 78 deletions.
2 changes: 1 addition & 1 deletion scripts/github-actions/tests-e2e-fixtures
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,4 +35,4 @@ GOLDEN_REQUEST_CHECKS=1 \
E2E_KUBE_TARGET=envtest \
E2E_GCP_TARGET=mock \
KCC_USE_DIRECT_RECONCILERS="SQLInstance" \
go test -test.count=1 -timeout 3600s -v ./tests/e2e -run TestAllInSeries/fixtures
go test -test.count=1 -timeout 3600s -v ./tests/e2e -run TestAllInParallel/fixtures
193 changes: 116 additions & 77 deletions tests/e2e/unified_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,7 @@ import (
_ "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/controller/direct/register"
)

// TestAllInSeries Run both samples and fixtures test in series
func TestAllInSeries(t *testing.T) {
if os.Getenv("RUN_E2E") == "" {
t.Skip("RUN_E2E not set; skipping")
Expand All @@ -63,54 +64,22 @@ func TestAllInSeries(t *testing.T) {
cancel()
})

subtestTimeout := time.Hour
if targetGCP := os.Getenv("E2E_GCP_TARGET"); targetGCP == "mock" {
// We allow a total of 3 minutes: 2 for the test itself (for deep object chains with retries),
// and 1 minute to shutdown envtest / allow kube-apiserver requests to time-out.
subtestTimeout = 3 * time.Minute
}

t.Run("samples", func(t *testing.T) {
samples := create.ListAllSamples(t)

for _, sampleKey := range samples {
sampleKey := sampleKey
// TODO(b/259496928): Randomize the resource names for parallel execution when/if needed.

t.Run(sampleKey.Name, func(t *testing.T) {
ctx := addTestTimeout(ctx, t, subtestTimeout)

// Quickly load the sample with a dummy project, just to see if we should skip it
{
dummySample := create.LoadSample(t, sampleKey, testgcp.GCPProject{ProjectID: "test-skip", ProjectNumber: 123456789})
create.MaybeSkip(t, sampleKey.Name, dummySample.Resources)
if s := os.Getenv("ONLY_TEST_APIGROUP"); s != "" {
t.Skipf("skipping test because cannot determine group for samples, with ONLY_TEST_APIGROUP=%s", s)
}
}

h := create.NewHarness(ctx, t)
project := h.Project
s := create.LoadSample(t, sampleKey, project)

create.SetupNamespacesAndApplyDefaults(h, s.Resources, project)

// Hack: set project-id because mockkubeapiserver does not support webhooks
for _, u := range s.Resources {
annotations := u.GetAnnotations()
if annotations == nil {
annotations = make(map[string]string)
}
annotations["cnrm.cloud.google.com/project-id"] = project.ProjectID
u.SetAnnotations(annotations)
}

create.RunCreateDeleteTest(h, create.CreateDeleteTestOptions{Create: s.Resources, CleanupResources: true})
})
runSamplesTests(ctx, t, sampleKey)
}
})

testFixturesInSeries(ctx, t, false, cancel)
t.Run("fixtures", func(t *testing.T) {
fixtures := resourcefixture.Load(t)
for _, fixture := range fixtures {
runFixturesTests(ctx, t, fixture, false)
}
})
}

// TestPauseInSeries is a basic smoke test to prove that if CC pauses actuation of resources
Expand All @@ -123,64 +92,134 @@ func TestPauseInSeries(t *testing.T) {
cancel()
})

testFixturesInSeries(ctx, t, true, cancel)
t.Run("fixtures", func(t *testing.T) {
fixtures := resourcefixture.Load(t)
for _, fixture := range fixtures {
runFixturesTests(ctx, t, fixture, true)
}
})
}

func testFixturesInSeries(ctx context.Context, t *testing.T, testPause bool, cancel context.CancelFunc) {
t.Helper()
// TestAllInParallel An option to run samples and fixtures test in parallel by services
func TestAllInParallel(t *testing.T) {
if os.Getenv("RUN_E2E") == "" {
t.Skip("RUN_E2E not set; skipping")
}

ctx := context.Background()
ctx, cancel := context.WithCancel(ctx)
t.Cleanup(func() {
// Do a cleanup while we can still handle the error.
t.Logf("shutting down manager")
cancel()
})

t.Run("samples", func(t *testing.T) {
// TODO(yuhou): Get service name from samples test
t.Errorf("Run samples tests in parallel is unsupported")
})

t.Run("fixtures", func(t *testing.T) {
fixtures := resourcefixture.Load(t)
groups := sets.NewString()
for _, fixture := range fixtures {
groups.Insert(fixture.GVK.Group)
}

for _, group := range groups.List() {
group := group
groupTestName := strings.TrimSuffix(group, ".cnrm.cloud.google.com")
t.Run(groupTestName, func(t *testing.T) {
t.Parallel()
for _, fixture := range fixtures {
if fixture.GVK.Group != group {
continue
}
runFixturesTests(ctx, t, fixture, false)
}
})
}
})
}

func runSamplesTests(ctx context.Context, t *testing.T, sampleKey create.SampleKey) {
subtestTimeout := time.Hour
if targetGCP := os.Getenv("E2E_GCP_TARGET"); targetGCP == "mock" {
// We allow a total of 3 minutes: 2 for the test itself (for deep object chains with retries),
// and 1 minute to shutdown envtest / allow kube-apiserver requests to time-out.
subtestTimeout = 3 * time.Minute
}
if os.Getenv("RUN_E2E") == "" {
t.Skip("RUN_E2E not set; skipping")
}
if testPause && os.Getenv("GOLDEN_REQUEST_CHECKS") == "" {
t.Skip("GOLDEN_REQUEST_CHECKS not set; skipping as this test relies on the golden files.")
}

t.Run("fixtures", func(t *testing.T) {
fixtures := resourcefixture.Load(t)
for _, fixture := range fixtures {
fixture := fixture
// TODO(b/259496928): Randomize the resource names for parallel execution when/if needed.
t.Run(sampleKey.Name, func(t *testing.T) {
ctx = addTestTimeout(ctx, t, subtestTimeout)
// Quickly load the sample with a dummy project, just to see if we should skip it
{
dummySample := create.LoadSample(t, sampleKey, testgcp.GCPProject{ProjectID: "test-skip", ProjectNumber: 123456789})
create.MaybeSkip(t, sampleKey.Name, dummySample.Resources)
if s := os.Getenv("ONLY_TEST_APIGROUP"); s != "" {
t.Skipf("skipping test because cannot determine group for samples, with ONLY_TEST_APIGROUP=%s", s)
}
}

t.Run(fixture.Name, func(t *testing.T) {
ctx := addTestTimeout(ctx, t, subtestTimeout)
h := create.NewHarness(ctx, t)
project := h.Project
s := create.LoadSample(t, sampleKey, project)

loadFixture := func(project testgcp.GCPProject, uniqueID string) (*unstructured.Unstructured, create.CreateDeleteTestOptions) {
primaryResource := bytesToUnstructured(t, fixture.Create, uniqueID, project)
create.SetupNamespacesAndApplyDefaults(h, s.Resources, project)

opt := create.CreateDeleteTestOptions{CleanupResources: true}
// Hack: set project-id because mockkubeapiserver does not support webhooks
for _, u := range s.Resources {
annotations := u.GetAnnotations()
if annotations == nil {
annotations = make(map[string]string)
}
annotations["cnrm.cloud.google.com/project-id"] = project.ProjectID
u.SetAnnotations(annotations)
}

if fixture.Dependencies != nil {
dependencyYamls := testyaml.SplitYAML(t, fixture.Dependencies)
for _, dependBytes := range dependencyYamls {
depUnstruct := bytesToUnstructured(t, dependBytes, uniqueID, project)
opt.Create = append(opt.Create, depUnstruct)
}
}
create.RunCreateDeleteTest(h, create.CreateDeleteTestOptions{Create: s.Resources, CleanupResources: true})
})
}

opt.Create = append(opt.Create, primaryResource)
func runFixturesTests(ctx context.Context, t *testing.T, fixture resourcefixture.ResourceFixture, testPause bool) {
subtestTimeout := time.Hour
if targetGCP := os.Getenv("E2E_GCP_TARGET"); targetGCP == "mock" {
// We allow a total of 3 minutes: 2 for the test itself (for deep object chains with retries),
// and 1 minute to shutdown envtest / allow kube-apiserver requests to time-out.
subtestTimeout = 3 * time.Minute
}

if fixture.Update != nil {
u := bytesToUnstructured(t, fixture.Update, uniqueID, project)
opt.Updates = append(opt.Updates, u)
}
return primaryResource, opt
t.Helper()
if testPause && os.Getenv("GOLDEN_REQUEST_CHECKS") == "" {
t.Skip("GOLDEN_REQUEST_CHECKS not set; skipping as this test relies on the golden files.")
}

t.Run(fixture.Name, func(t *testing.T) {
ctx = addTestTimeout(ctx, t, subtestTimeout)
loadFixture := func(project testgcp.GCPProject, uniqueID string) (*unstructured.Unstructured, create.CreateDeleteTestOptions) {
primaryResource := bytesToUnstructured(t, fixture.Create, uniqueID, project)

opt := create.CreateDeleteTestOptions{CleanupResources: true}

if fixture.Dependencies != nil {
dependencyYamls := testyaml.SplitYAML(t, fixture.Dependencies)
for _, dependBytes := range dependencyYamls {
depUnstruct := bytesToUnstructured(t, dependBytes, uniqueID, project)
opt.Create = append(opt.Create, depUnstruct)
}
}

runScenario(ctx, t, testPause, fixture, loadFixture)
})
opt.Create = append(opt.Create, primaryResource)

if fixture.Update != nil {
u := bytesToUnstructured(t, fixture.Update, uniqueID, project)
opt.Updates = append(opt.Updates, u)
}
return primaryResource, opt
}
})

// Do a cleanup while we can still handle the error.
t.Logf("shutting down manager")
cancel()
runScenario(ctx, t, testPause, fixture, loadFixture)
})
}

func runScenario(ctx context.Context, t *testing.T, testPause bool, fixture resourcefixture.ResourceFixture, loadFixture func(project testgcp.GCPProject, uniqueID string) (*unstructured.Unstructured, create.CreateDeleteTestOptions)) {
Expand Down

0 comments on commit d8b9bc6

Please sign in to comment.