From d66185276cfa7062cd8af0e42f30102e0f235d38 Mon Sep 17 00:00:00 2001 From: Tony Redondo Date: Tue, 8 Oct 2024 18:01:46 +0200 Subject: [PATCH] internal/civisibility: refactor the test suite for civisibility gotesting instrumentation for multiple testing scenarios. --- .../gotesting/testcontroller_test.go | 451 ++++++++++++++++++ .../integrations/gotesting/testing_test.go | 174 ------- 2 files changed, 451 insertions(+), 174 deletions(-) create mode 100644 internal/civisibility/integrations/gotesting/testcontroller_test.go diff --git a/internal/civisibility/integrations/gotesting/testcontroller_test.go b/internal/civisibility/integrations/gotesting/testcontroller_test.go new file mode 100644 index 0000000000..48ff206d20 --- /dev/null +++ b/internal/civisibility/integrations/gotesting/testcontroller_test.go @@ -0,0 +1,451 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2024 Datadog, Inc. + +package gotesting + +import ( + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "os" + "os/exec" + "testing" + + "gopkg.in/DataDog/dd-trace-go.v1/ddtrace/ext" + "gopkg.in/DataDog/dd-trace-go.v1/ddtrace/mocktracer" + "gopkg.in/DataDog/dd-trace-go.v1/internal" + "gopkg.in/DataDog/dd-trace-go.v1/internal/civisibility/constants" + "gopkg.in/DataDog/dd-trace-go.v1/internal/civisibility/integrations" + "gopkg.in/DataDog/dd-trace-go.v1/internal/civisibility/utils/net" + "gopkg.in/DataDog/dd-trace-go.v1/internal/log" +) + +var currentM *testing.M +var mTracer mocktracer.Tracer + +// TestMain is the entry point for testing and runs before any test. +func TestMain(m *testing.M) { + log.SetLevel(log.LevelDebug) + + // We need to spawn separated test process for each scenario + scenarios := []string{"TestFlakyTestRetries", "TestEarlyFlakeDetection", "TestFlakyTestRetriesAndEarlyFlakeDetection"} + + if internal.BoolEnv(scenarios[0], false) { + fmt.Printf("Scenario %s started.\n", scenarios[0]) + runFlakyTestRetriesTests(m) + } else if internal.BoolEnv(scenarios[1], false) { + fmt.Printf("Scenario %s started.\n", scenarios[1]) + runEarlyFlakyTestDetectionTests(m) + } else if internal.BoolEnv(scenarios[2], false) { + fmt.Printf("Scenario %s started.\n", scenarios[2]) + runFlakyTestRetriesWithEarlyFlakyTestDetectionTests(m) + } else { + fmt.Println("Starting tests...") + for _, v := range scenarios { + cmd := exec.Command(os.Args[0], os.Args[1:]...) + cmd.Stdout, cmd.Stderr = os.Stdout, os.Stderr + cmd.Env = append(cmd.Env, os.Environ()...) + cmd.Env = append(cmd.Env, fmt.Sprintf("%s=true", v)) + fmt.Printf("Running scenario: %s:\n", v) + err := cmd.Run() + fmt.Printf("Done.\n\n") + if err != nil { + if exiterr, ok := err.(*exec.ExitError); ok { + fmt.Printf("Scenario %s failed with exit code: %d\n", v, exiterr.ExitCode()) + os.Exit(exiterr.ExitCode()) + } else { + fmt.Printf("cmd.Run: %v\n", err) + os.Exit(1) + } + break + } + } + } + + os.Exit(0) +} + +func runFlakyTestRetriesTests(m *testing.M) { + // mock the settings api to enable automatic test retries + server := setUpHttpServer(true, false, nil) + defer server.Close() + + // set a custom retry count + os.Setenv(constants.CIVisibilityFlakyRetryCountEnvironmentVariable, "10") + + // initialize the mock tracer for doing assertions on the finished spans + currentM = m + mTracer = integrations.InitializeCIVisibilityMock() + + // execute the tests, because we are expecting some tests to fail and check the assertion later + // we don't store the exit code from the test runner + exitCode := RunM(m) + if exitCode != 1 { + panic("expected the exit code to be 1. We have a failing test on purpose.") + } + + // get all finished spans + finishedSpans := mTracer.FinishedSpans() + + // 1 session span + // 1 module span + // 2 suite span (testing_test.go and reflections_test.go) + // 5 tests from reflections_test.go + // 1 TestMyTest01 + // 1 TestMyTest02 + 2 subtests + // 1 Test_Foo + 3 subtests + // 1 TestWithExternalCalls + 2 subtests + // 1 TestSkip + // 1 TestRetryWithPanic + 3 retry tests from testing_test.go + // 1 TestRetryWithFail + 3 retry tests from testing_test.go + // 1 TestRetryAlwaysFail + 10 retry tests from testing_test.go + // 1 TestNormalPassingAfterRetryAlwaysFail + // 1 TestEarlyFlakeDetection + // 2 normal spans from testing_test.go + checkSpansByResourceName(finishedSpans, "gopkg.in/DataDog/dd-trace-go.v1/internal/civisibility/integrations/gotesting", 1) + checkSpansByResourceName(finishedSpans, "reflections_test.go", 1) + checkSpansByResourceName(finishedSpans, "testing_test.go", 1) + checkSpansByResourceName(finishedSpans, "testing_test.go.TestMyTest01", 1) + checkSpansByResourceName(finishedSpans, "testing_test.go.TestMyTest02", 1) + checkSpansByResourceName(finishedSpans, "testing_test.go.TestMyTest02/sub01", 1) + checkSpansByResourceName(finishedSpans, "testing_test.go.TestMyTest02/sub01/sub03", 1) + checkSpansByResourceName(finishedSpans, "testing_test.go.Test_Foo", 1) + checkSpansByResourceName(finishedSpans, "testing_test.go.Test_Foo/yellow_should_return_color", 1) + checkSpansByResourceName(finishedSpans, "testing_test.go.Test_Foo/banana_should_return_fruit", 1) + checkSpansByResourceName(finishedSpans, "testing_test.go.Test_Foo/duck_should_return_animal", 1) + checkSpansByResourceName(finishedSpans, "testing_test.go.TestWithExternalCalls", 1) + checkSpansByResourceName(finishedSpans, "testing_test.go.TestWithExternalCalls/default", 1) + checkSpansByResourceName(finishedSpans, "testing_test.go.TestWithExternalCalls/custom-name", 1) + checkSpansByResourceName(finishedSpans, "testing_test.go.TestSkip", 1) + checkSpansByResourceName(finishedSpans, "testing_test.go.TestRetryWithPanic", 4) + checkSpansByResourceName(finishedSpans, "testing_test.go.TestRetryWithFail", 4) + checkSpansByResourceName(finishedSpans, "testing_test.go.TestRetryAlwaysFail", 11) + checkSpansByResourceName(finishedSpans, "testing_test.go.TestNormalPassingAfterRetryAlwaysFail", 1) + checkSpansByResourceName(finishedSpans, "testing_test.go.TestEarlyFlakeDetection", 1) + + checkSpansByType(finishedSpans, + 44, + 1, + 1, + 2, + 38, + 2) + + os.Exit(0) +} + +func runEarlyFlakyTestDetectionTests(m *testing.M) { + // mock the settings api to enable automatic test retries + server := setUpHttpServer(false, true, &net.EfdResponseData{ + Tests: net.EfdResponseDataModules{ + "gopkg.in/DataDog/dd-trace-go.v1/internal/civisibility/integrations/gotesting": net.EfdResponseDataSuites{ + "reflections_test.go": []string{ + "TestGetFieldPointerFrom", + "TestGetInternalTestArray", + "TestGetInternalBenchmarkArray", + "TestCommonPrivateFields_AddLevel", + "TestGetBenchmarkPrivateFields", + }, + }, + }, + }) + defer server.Close() + + // initialize the mock tracer for doing assertions on the finished spans + currentM = m + mTracer = integrations.InitializeCIVisibilityMock() + + // execute the tests, because we are expecting some tests to fail and check the assertion later + // we don't store the exit code from the test runner + exitCode := RunM(m) + if exitCode != 1 { + panic("expected the exit code to be 1. We have a failing test on purpose.") + } + + // get all finished spans + finishedSpans := mTracer.FinishedSpans() + + // 1 session span + // 1 module span + // 2 suite span (testing_test.go and reflections_test.go) + // 5 tests from reflections_test.go + // 11 TestMyTest01 + // 11 TestMyTest02 + 22 subtests + // 11 Test_Foo + 33 subtests + // 11 TestWithExternalCalls + 22 subtests + // 11 TestSkip + // 11 TestRetryWithPanic + // 11 TestRetryWithFail + // 11 TestRetryAlwaysFail + // 11 TestNormalPassingAfterRetryAlwaysFail + // 11 TestEarlyFlakeDetection + // 22 normal spans from testing_test.go + checkSpansByResourceName(finishedSpans, "gopkg.in/DataDog/dd-trace-go.v1/internal/civisibility/integrations/gotesting", 1) + checkSpansByResourceName(finishedSpans, "reflections_test.go", 1) + checkSpansByResourceName(finishedSpans, "testing_test.go", 1) + checkSpansByResourceName(finishedSpans, "testing_test.go.TestMyTest01", 11) + checkSpansByResourceName(finishedSpans, "testing_test.go.TestMyTest02", 11) + // checkSpansByResourceName(finishedSpans, "testing_test.go.TestMyTest02/sub01", 11) + // checkSpansByResourceName(finishedSpans, "testing_test.go.TestMyTest02/sub01/sub03", 11) + checkSpansByResourceName(finishedSpans, "testing_test.go.Test_Foo", 11) + // checkSpansByResourceName(finishedSpans, "testing_test.go.Test_Foo/yellow_should_return_color", 11) + // checkSpansByResourceName(finishedSpans, "testing_test.go.Test_Foo/banana_should_return_fruit", 11) + // checkSpansByResourceName(finishedSpans, "testing_test.go.Test_Foo/duck_should_return_animal", 11) + checkSpansByResourceName(finishedSpans, "testing_test.go.TestWithExternalCalls", 11) + // checkSpansByResourceName(finishedSpans, "testing_test.go.TestWithExternalCalls/default", 11) + // checkSpansByResourceName(finishedSpans, "testing_test.go.TestWithExternalCalls/custom-name", 11) + checkSpansByResourceName(finishedSpans, "testing_test.go.TestSkip", 11) + checkSpansByResourceName(finishedSpans, "testing_test.go.TestRetryWithPanic", 11) + checkSpansByResourceName(finishedSpans, "testing_test.go.TestRetryWithFail", 11) + checkSpansByResourceName(finishedSpans, "testing_test.go.TestRetryAlwaysFail", 11) + checkSpansByResourceName(finishedSpans, "testing_test.go.TestNormalPassingAfterRetryAlwaysFail", 11) + checkSpansByResourceName(finishedSpans, "testing_test.go.TestEarlyFlakeDetection", 11) + + checkSpansByType(finishedSpans, + 218, + 1, + 1, + 2, + 192, + 22) + + os.Exit(0) +} + +func runFlakyTestRetriesWithEarlyFlakyTestDetectionTests(m *testing.M) { + // mock the settings api to enable automatic test retries + server := setUpHttpServer(true, true, &net.EfdResponseData{ + Tests: net.EfdResponseDataModules{ + "gopkg.in/DataDog/dd-trace-go.v1/internal/civisibility/integrations/gotesting": net.EfdResponseDataSuites{ + "reflections_test.go": []string{ + "TestGetFieldPointerFrom", + "TestGetInternalTestArray", + "TestGetInternalBenchmarkArray", + "TestCommonPrivateFields_AddLevel", + "TestGetBenchmarkPrivateFields", + }, + "testing_test.go": []string{ + "TestMyTest01", + "TestMyTest02", + "Test_Foo", + "TestWithExternalCalls", + "TestSkip", + "TestRetryWithPanic", + "TestRetryWithFail", + "TestRetryAlwaysFail", + "TestNormalPassingAfterRetryAlwaysFail", + }, + }, + }, + }) + defer server.Close() + + // set a custom retry count + os.Setenv(constants.CIVisibilityFlakyRetryCountEnvironmentVariable, "10") + + // initialize the mock tracer for doing assertions on the finished spans + currentM = m + mTracer = integrations.InitializeCIVisibilityMock() + + // execute the tests, because we are expecting some tests to fail and check the assertion later + // we don't store the exit code from the test runner + exitCode := RunM(m) + if exitCode != 1 { + panic("expected the exit code to be 1. We have a failing test on purpose.") + } + + // get all finished spans + finishedSpans := mTracer.FinishedSpans() + + // 1 session span + // 1 module span + // 2 suite span (testing_test.go and reflections_test.go) + // 5 tests from reflections_test.go + // 1 TestMyTest01 + // 1 TestMyTest02 + 2 subtests + // 1 Test_Foo + 3 subtests + // 1 TestWithExternalCalls + 2 subtests + // 1 TestSkip + // 1 TestRetryWithPanic + 3 retry tests from testing_test.go + // 1 TestRetryWithFail + 3 retry tests from testing_test.go + // 1 TestRetryAlwaysFail + 10 retry tests from testing_test.go + // 1 TestNormalPassingAfterRetryAlwaysFail + // 11 TestEarlyFlakeDetection + 10 retries + // 2 normal spans from testing_test.go + checkSpansByResourceName(finishedSpans, "gopkg.in/DataDog/dd-trace-go.v1/internal/civisibility/integrations/gotesting", 1) + checkSpansByResourceName(finishedSpans, "reflections_test.go", 1) + checkSpansByResourceName(finishedSpans, "testing_test.go", 1) + checkSpansByResourceName(finishedSpans, "testing_test.go.TestMyTest01", 1) + checkSpansByResourceName(finishedSpans, "testing_test.go.TestMyTest02", 1) + checkSpansByResourceName(finishedSpans, "testing_test.go.TestMyTest02/sub01", 1) + checkSpansByResourceName(finishedSpans, "testing_test.go.TestMyTest02/sub01/sub03", 1) + checkSpansByResourceName(finishedSpans, "testing_test.go.Test_Foo", 1) + checkSpansByResourceName(finishedSpans, "testing_test.go.Test_Foo/yellow_should_return_color", 1) + checkSpansByResourceName(finishedSpans, "testing_test.go.Test_Foo/banana_should_return_fruit", 1) + checkSpansByResourceName(finishedSpans, "testing_test.go.Test_Foo/duck_should_return_animal", 1) + checkSpansByResourceName(finishedSpans, "testing_test.go.TestWithExternalCalls", 1) + checkSpansByResourceName(finishedSpans, "testing_test.go.TestWithExternalCalls/default", 1) + checkSpansByResourceName(finishedSpans, "testing_test.go.TestWithExternalCalls/custom-name", 1) + checkSpansByResourceName(finishedSpans, "testing_test.go.TestSkip", 1) + checkSpansByResourceName(finishedSpans, "testing_test.go.TestRetryWithPanic", 4) + checkSpansByResourceName(finishedSpans, "testing_test.go.TestRetryWithFail", 4) + checkSpansByResourceName(finishedSpans, "testing_test.go.TestRetryAlwaysFail", 11) + checkSpansByResourceName(finishedSpans, "testing_test.go.TestNormalPassingAfterRetryAlwaysFail", 1) + checkSpansByResourceName(finishedSpans, "testing_test.go.TestEarlyFlakeDetection", 21) + + checkSpansByType(finishedSpans, + 64, + 1, + 1, + 2, + 58, + 2) + + os.Exit(0) +} + +func checkSpansByType(finishedSpans []mocktracer.Span, + totalFinishedSpansCount int, sessionSpansCount int, moduleSpansCount int, + suiteSpansCount int, testSpansCount int, normalSpansCount int) { + calculatedFinishedSpans := len(finishedSpans) + fmt.Printf("Number of spans received: %d\n", calculatedFinishedSpans) + if calculatedFinishedSpans < totalFinishedSpansCount { + panic(fmt.Sprintf("expected at least %d finished spans, got %d", totalFinishedSpansCount, calculatedFinishedSpans)) + } + + sessionSpans := getSpansWithType(finishedSpans, constants.SpanTypeTestSession) + calculatedSessionSpans := len(sessionSpans) + fmt.Printf("Number of sessions received: %d\n", calculatedSessionSpans) + showResourcesNameFromSpans(sessionSpans) + if calculatedSessionSpans != sessionSpansCount { + panic(fmt.Sprintf("expected exactly %d session span, got %d", sessionSpansCount, calculatedSessionSpans)) + } + + moduleSpans := getSpansWithType(finishedSpans, constants.SpanTypeTestModule) + calculatedModuleSpans := len(moduleSpans) + fmt.Printf("Number of modules received: %d\n", calculatedModuleSpans) + showResourcesNameFromSpans(moduleSpans) + if calculatedModuleSpans != moduleSpansCount { + panic(fmt.Sprintf("expected exactly %d module span, got %d", moduleSpansCount, calculatedModuleSpans)) + } + + suiteSpans := getSpansWithType(finishedSpans, constants.SpanTypeTestSuite) + calculatedSuiteSpans := len(suiteSpans) + fmt.Printf("Number of suites received: %d\n", calculatedSuiteSpans) + showResourcesNameFromSpans(suiteSpans) + if calculatedSuiteSpans != suiteSpansCount { + panic(fmt.Sprintf("expected exactly %d suite spans, got %d", suiteSpansCount, calculatedSuiteSpans)) + } + + testSpans := getSpansWithType(finishedSpans, constants.SpanTypeTest) + calculatedTestSpans := len(testSpans) + fmt.Printf("Number of tests received: %d\n", calculatedTestSpans) + showResourcesNameFromSpans(testSpans) + if calculatedTestSpans != testSpansCount { + panic(fmt.Sprintf("expected exactly %d test spans, got %d", testSpansCount, calculatedTestSpans)) + } + + normalSpans := getSpansWithType(finishedSpans, ext.SpanTypeHTTP) + calculatedNormalSpans := len(normalSpans) + fmt.Printf("Number of http spans received: %d\n", calculatedNormalSpans) + showResourcesNameFromSpans(normalSpans) + if calculatedNormalSpans != normalSpansCount { + panic(fmt.Sprintf("expected exactly %d normal spans, got %d", normalSpansCount, calculatedNormalSpans)) + } +} + +func checkSpansByResourceName(finishedSpans []mocktracer.Span, resourceName string, count int) { + numOfSpans := len(getSpansWithResourceName(finishedSpans, resourceName)) + if numOfSpans != count { + panic(fmt.Sprintf("expected exactly %d spans with resource name: %s, got %d", count, resourceName, numOfSpans)) + } +} + +func setUpHttpServer(flakyRetriesEnabled bool, earlyFlakyDetectionEnabled bool, earlyFlakyDetectionData *net.EfdResponseData) *httptest.Server { + // mock the settings api to enable automatic test retries + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Printf("MockApi received request: %s\n", r.URL.Path) + + // Settings request + if r.URL.Path == "/api/v2/libraries/tests/services/setting" { + w.Header().Set("Content-Type", "application/json") + response := struct { + Data struct { + ID string `json:"id"` + Type string `json:"type"` + Attributes net.SettingsResponseData `json:"attributes"` + } `json:"data,omitempty"` + }{} + + // let's enable flaky test retries + response.Data.Attributes = net.SettingsResponseData{ + FlakyTestRetriesEnabled: flakyRetriesEnabled, + } + response.Data.Attributes.EarlyFlakeDetection.Enabled = earlyFlakyDetectionEnabled + response.Data.Attributes.EarlyFlakeDetection.SlowTestRetries.FiveS = 10 + response.Data.Attributes.EarlyFlakeDetection.SlowTestRetries.TenS = 5 + response.Data.Attributes.EarlyFlakeDetection.SlowTestRetries.ThirtyS = 3 + response.Data.Attributes.EarlyFlakeDetection.SlowTestRetries.FiveM = 2 + + fmt.Printf("MockApi sending response: %v\n", response) + json.NewEncoder(w).Encode(&response) + } + + if earlyFlakyDetectionEnabled && r.URL.Path == "/api/v2/ci/libraries/tests" { + w.Header().Set("Content-Type", "application/json") + response := struct { + Data struct { + ID string `json:"id"` + Type string `json:"type"` + Attributes net.EfdResponseData `json:"attributes"` + } `json:"data,omitempty"` + }{} + + if earlyFlakyDetectionData != nil { + response.Data.Attributes = *earlyFlakyDetectionData + } + + fmt.Printf("MockApi sending response: %v\n", response) + json.NewEncoder(w).Encode(&response) + } + })) + + // set the custom agentless url and the flaky retry count env-var + fmt.Printf("Using mockapi at: %s\n", server.URL) + os.Setenv(constants.CIVisibilityAgentlessEnabledEnvironmentVariable, "1") + os.Setenv(constants.CIVisibilityAgentlessURLEnvironmentVariable, server.URL) + os.Setenv(constants.APIKeyEnvironmentVariable, "12345") + + return server +} + +func getSpansWithType(spans []mocktracer.Span, spanType string) []mocktracer.Span { + var result []mocktracer.Span + for _, span := range spans { + if span.Tag(ext.SpanType) == spanType { + result = append(result, span) + } + } + + return result +} + +func getSpansWithResourceName(spans []mocktracer.Span, resourceName string) []mocktracer.Span { + var result []mocktracer.Span + for _, span := range spans { + if span.Tag(ext.ResourceName) == resourceName { + result = append(result, span) + } + } + + return result +} + +func showResourcesNameFromSpans(spans []mocktracer.Span) { + for i, span := range spans { + fmt.Printf(" [%d] = %v\n", i, span.Tag(ext.ResourceName)) + } +} diff --git a/internal/civisibility/integrations/gotesting/testing_test.go b/internal/civisibility/integrations/gotesting/testing_test.go index 1f9fa12e5a..64b89b8029 100644 --- a/internal/civisibility/integrations/gotesting/testing_test.go +++ b/internal/civisibility/integrations/gotesting/testing_test.go @@ -6,14 +6,11 @@ package gotesting import ( - "encoding/json" "fmt" "net/http" "net/http/httptest" - "os" "runtime" "slices" - "strconv" "testing" ddhttp "gopkg.in/DataDog/dd-trace-go.v1/contrib/net/http" @@ -21,164 +18,10 @@ import ( "gopkg.in/DataDog/dd-trace-go.v1/ddtrace/mocktracer" ddtracer "gopkg.in/DataDog/dd-trace-go.v1/ddtrace/tracer" "gopkg.in/DataDog/dd-trace-go.v1/internal/civisibility/constants" - "gopkg.in/DataDog/dd-trace-go.v1/internal/civisibility/integrations" - "gopkg.in/DataDog/dd-trace-go.v1/internal/civisibility/utils/net" - "gopkg.in/DataDog/dd-trace-go.v1/internal/log" "github.com/stretchr/testify/assert" ) -var currentM *testing.M -var mTracer mocktracer.Tracer - -// TestMain is the entry point for testing and runs before any test. -func TestMain(m *testing.M) { - - log.SetLevel(log.LevelDebug) - - // mock the settings api to enable automatic test retries - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - fmt.Printf("MockApi received request: %s\n", r.URL.Path) - - // Settings request - if r.URL.Path == "/api/v2/libraries/tests/services/setting" { - w.Header().Set("Content-Type", "application/json") - response := struct { - Data struct { - ID string `json:"id"` - Type string `json:"type"` - Attributes net.SettingsResponseData `json:"attributes"` - } `json:"data,omitempty"` - }{} - - // let's enable flaky test retries - response.Data.Attributes = net.SettingsResponseData{ - FlakyTestRetriesEnabled: true, - } - response.Data.Attributes.EarlyFlakeDetection.Enabled = false - response.Data.Attributes.EarlyFlakeDetection.SlowTestRetries.FiveS = 10 - - fmt.Printf("MockApi sending response: %v\n", response) - json.NewEncoder(w).Encode(&response) - } - - if r.URL.Path == "/api/v2/ci/libraries/tests" { - w.Header().Set("Content-Type", "application/json") - response := struct { - Data struct { - ID string `json:"id"` - Type string `json:"type"` - Attributes net.EfdResponseData `json:"attributes"` - } `json:"data,omitempty"` - }{} - - response.Data.Attributes = net.EfdResponseData{ - Tests: net.EfdResponseDataModules{ - "gopkg.in/DataDog/dd-trace-go.v1/internal/civisibility/integrations/gotesting": net.EfdResponseDataSuites{ - "reflections_test.go": []string{ - "TestGetFieldPointerFrom", - "TestGetInternalTestArray", - "TestGetInternalBenchmarkArray", - "TestCommonPrivateFields_AddLevel", - "TestGetBenchmarkPrivateFields", - }, - "testing_test.go": []string{ - "TestMyTest01", - "TestMyTest02", - "Test_Foo", - "TestWithExternalCalls", - "TestSkip", - "TestRetryWithPanic", - "TestRetryWithFail", - "TestRetryAlwaysFail", - "TestNormalPassingAfterRetryAlwaysFail", - }, - }, - }, - } - - fmt.Printf("MockApi sending response: %v\n", response) - json.NewEncoder(w).Encode(&response) - } - })) - defer server.Close() - - // set the custom agentless url and the flaky retry count env-var - fmt.Printf("Using mockapi at: %s\n", server.URL) - os.Setenv(constants.CIVisibilityAgentlessEnabledEnvironmentVariable, "1") - os.Setenv(constants.CIVisibilityAgentlessURLEnvironmentVariable, server.URL) - os.Setenv(constants.APIKeyEnvironmentVariable, "12345") - os.Setenv(constants.CIVisibilityFlakyRetryCountEnvironmentVariable, "10") - - // initialize the mock tracer for doing assertions on the finished spans - currentM = m - mTracer = integrations.InitializeCIVisibilityMock() - - // execute the tests, because we are expecting some tests to fail and check the assertion later - // we don't store the exit code from the test runner - exitCode := RunM(m) - if exitCode != 1 { - panic("expected the exit code to be 1. We have a failing test on purpose.") - } - - // get all finished spans - finishedSpans := mTracer.FinishedSpans() - - // 1 session span - // 1 module span - // 2 suite span (testing_test.go and reflections_test.go) - // 6 tests spans from testing_test.go - // 7 sub stest spans from testing_test.go - // 1 TestRetryWithPanic + 3 retry tests from testing_test.go - // 1 TestRetryWithFail + 3 retry tests from testing_test.go - // 1 TestRetryAlwaysFail + 10 retry tests from testing_test.go - // 1 TestEarlyFlakeDetection - // 2 normal spans from testing_test.go - // 5 tests from reflections_test.go - // 2 benchmark spans (optional - require the -bench option) - fmt.Printf("Number of spans received: %d\n", len(finishedSpans)) - if len(finishedSpans) < 38 { - panic("expected at least 38 finished spans, got " + strconv.Itoa(len(finishedSpans))) - } - - sessionSpans := getSpansWithType(finishedSpans, constants.SpanTypeTestSession) - fmt.Printf("Number of sessions received: %d\n", len(sessionSpans)) - showResourcesNameFromSpans(sessionSpans) - if len(sessionSpans) != 1 { - panic("expected exactly 1 session span, got " + strconv.Itoa(len(sessionSpans))) - } - - moduleSpans := getSpansWithType(finishedSpans, constants.SpanTypeTestModule) - fmt.Printf("Number of modules received: %d\n", len(moduleSpans)) - showResourcesNameFromSpans(moduleSpans) - if len(moduleSpans) != 1 { - panic("expected exactly 1 module span, got " + strconv.Itoa(len(moduleSpans))) - } - - suiteSpans := getSpansWithType(finishedSpans, constants.SpanTypeTestSuite) - fmt.Printf("Number of suites received: %d\n", len(suiteSpans)) - showResourcesNameFromSpans(suiteSpans) - if len(suiteSpans) != 2 { - panic("expected exactly 2 suite spans, got " + strconv.Itoa(len(suiteSpans))) - } - - testSpans := getSpansWithType(finishedSpans, constants.SpanTypeTest) - fmt.Printf("Number of tests received: %d\n", len(testSpans)) - showResourcesNameFromSpans(testSpans) - if len(testSpans) != 38 { - panic("expected exactly 38 test spans, got " + strconv.Itoa(len(testSpans))) - } - - httpSpans := getSpansWithType(finishedSpans, ext.SpanTypeHTTP) - fmt.Printf("Number of http spans received: %d\n", len(httpSpans)) - showResourcesNameFromSpans(httpSpans) - if len(httpSpans) != 2 { - panic("expected exactly 2 normal spans, got " + strconv.Itoa(len(httpSpans))) - } - - os.Exit(0) -} - // TestMyTest01 demonstrates instrumentation of InternalTests func TestMyTest01(t *testing.T) { assertTest(t) @@ -505,20 +348,3 @@ func assertCommon(assert *assert.Assertions, span mocktracer.Span) { } assert.Contains(spanTags, constants.CIWorkspacePath) } - -func getSpansWithType(spans []mocktracer.Span, spanType string) []mocktracer.Span { - var result []mocktracer.Span - for _, span := range spans { - if span.Tag(ext.SpanType) == spanType { - result = append(result, span) - } - } - - return result -} - -func showResourcesNameFromSpans(spans []mocktracer.Span) { - for i, span := range spans { - fmt.Printf(" [%d] = %v\n", i, span.Tag(ext.ResourceName)) - } -}