From e2b6417065823e2dd8d39199c765fd9f54619bc6 Mon Sep 17 00:00:00 2001 From: Soumyajit Das Date: Thu, 9 Jan 2025 19:22:40 +0530 Subject: [PATCH] feat: [CI-15495]: Send telemetry data via step response --- api/api.go | 44 +++++++++++++++++++++ pipeline/runtime/run.go | 15 +++---- pipeline/runtime/runtest.go | 31 ++++++++------- pipeline/runtime/runtest_test.go | 6 ++- pipeline/runtime/runtestsV2.go | 40 ++++++++++--------- pipeline/runtime/runtestsV2_test.go | 28 +++++++------ pipeline/runtime/step_executor.go | 32 ++++++++------- pipeline/runtime/step_executor_stateless.go | 6 +-- ti/instrumentation/common/helper.go | 12 ++++++ ti/instrumentation/instrumentation.go | 4 +- ti/report/parser/junit/junit.go | 1 + ti/report/report.go | 14 ++++++- ti/savings/cache/gradle/helper.go | 19 +++++++++ ti/savings/savings.go | 9 ++++- 14 files changed, 185 insertions(+), 76 deletions(-) diff --git a/api/api.go b/api/api.go index db39c802..b8d2c7d4 100644 --- a/api/api.go +++ b/api/api.go @@ -105,6 +105,48 @@ type ( Files []*spec.File `json:"files,omitempty"` StepStatus StepStatusConfig `json:"step_status,omitempty"` } + + TelemetryData struct { + BuildIntelligenceMetaData BuildIntelligenceMetaData `json:"build_intelligence_data,omitempty"` + TestIntelligenceMetaData TestIntelligenceMetaData `json:"test_intelligence_data,omitempty"` + CacheIntelligenceMetaData CacheIntelligenceMetaData `json:"cache_intelligence_data,omitempty"` + DlcMetadata DlcMetadata `json:"dlc_metadata,omitempty"` + Errors []string `json:"errors,omitempty"` + } + + BuildIntelligenceMetaData struct { + BuildTasks int `json:"build_tasks,omitempty"` + TasksRestored int `json:"tasks_restored,omitempty"` + StepType string `json:"step_type,omitempty"` + BuildTool string `json:"build_tool,omitempty"` + Language string `json:"language,omitempty"` + Errors []string `json:"errors,omitempty"` + } + + TestIntelligenceMetaData struct { + TotalTests int `json:"total_tests,omitempty"` + TotalTestClasses int `json:"total_test_classes,omitempty"` + TotalSelectedTests int `json:"total_selected_tests,omitempty"` + TotalSelectedTestClass int `json:"total_selected_test_classes,omitempty"` + CPUTimeSaved int64 `json:"cpu_time_saved,omitempty"` + BuildTool string `json:"build_tool,omitempty"` + Language string `json:"language,omitempty"` + Errors []string `json:"errors,omitempty"` + } + + CacheIntelligenceMetaData struct { + CacheSize int `json:"cache_size,omitempty"` + IsNonDefaultPath bool `json:"is_non_default_path,omitempty"` + IsCustomKeys bool `json:"is_custom_keys,omitempty"` + Errors []string `json:"errors,omitempty"` + } + + DlcMetadata struct { + TotalLayers int `json:"total_layers,omitempty"` + LayersRestored int `json:"layers_restored,omitempty"` + Errors []string `json:"errors,omitempty"` + } + OutputV2 struct { Key string `json:"key,omitempty"` Value string `json:"value"` @@ -127,6 +169,7 @@ type ( Artifact []byte `json:"artifact,omitempty"` OutputV2 []*OutputV2 `json:"outputV2,omitempty"` OptimizationState string `json:"optimization_state,omitempty"` + TelemetryData *TelemetryData `json:"telemetry_data,omitempty"` } StreamOutputRequest struct { @@ -213,6 +256,7 @@ type ( Artifact []byte `json:"artifact,omitempty"` Outputs []*OutputV2 `json:"outputs,omitempty"` OptimizationState string `json:"optimization_state,omitempty"` + TelemetryData *TelemetryData `json:"telemetry_data,omitempty"` } ) diff --git a/pipeline/runtime/run.go b/pipeline/runtime/run.go index 7960d77b..0d542024 100644 --- a/pipeline/runtime/run.go +++ b/pipeline/runtime/run.go @@ -26,7 +26,7 @@ const ( ) func executeRunStep(ctx context.Context, f RunFunc, r *api.StartStepRequest, out io.Writer, tiConfig *tiCfg.Cfg) ( //nolint:gocritic,gocyclo,funlen - *runtime.State, map[string]string, map[string]string, []byte, []*api.OutputV2, string, error) { + *runtime.State, map[string]string, map[string]string, []byte, []*api.OutputV2, *api.TelemetryData, string, error) { start := time.Now() step := toStep(r) step.Command = r.Run.Command @@ -36,9 +36,10 @@ func executeRunStep(ctx context.Context, f RunFunc, r *api.StartStepRequest, out optimizationState := types.DISABLED exportEnvFile := fmt.Sprintf("%s/%s-export.env", pipeline.SharedVolPath, step.ID) step.Envs["DRONE_ENV"] = exportEnvFile + telemetryData := &api.TelemetryData{} if (len(r.OutputVars) > 0 || len(r.Outputs) > 0) && (len(step.Entrypoint) == 0 || len(step.Command) == 0) { - return nil, nil, nil, nil, nil, string(optimizationState), fmt.Errorf("output variable should not be set for unset entrypoint or command") + return nil, nil, nil, nil, nil, nil, string(optimizationState), fmt.Errorf("output variable should not be set for unset entrypoint or command") } if r.ScratchDir != "" { @@ -98,14 +99,14 @@ func executeRunStep(ctx context.Context, f RunFunc, r *api.StartStepRequest, out timeTakenMs := time.Since(start).Milliseconds() reportStart := time.Now() - if rerr := report.ParseAndUploadTests(ctx, r.TestReport, r.WorkingDir, step.Name, log, reportStart, tiConfig, r.Envs); rerr != nil { + if rerr := report.ParseAndUploadTests(ctx, r.TestReport, r.WorkingDir, step.Name, log, reportStart, tiConfig, &telemetryData.TestIntelligenceMetaData, r.Envs); rerr != nil { logrus.WithContext(ctx).WithError(rerr).WithField("step", step.Name).Errorln("failed to upload report") log.Errorf("Failed to upload report. Time taken: %s", time.Since(reportStart)) } // Parse and upload savings to TI if tiConfig.GetParseSavings() { - optimizationState = savings.ParseAndUploadSavings(ctx, r.WorkingDir, log, step.Name, checkStepSuccess(exited, err), timeTakenMs, tiConfig, r.Envs) + optimizationState = savings.ParseAndUploadSavings(ctx, r.WorkingDir, log, step.Name, checkStepSuccess(exited, err), timeTakenMs, tiConfig, r.Envs, telemetryData) } useCINewGodotEnvVersion := false @@ -190,11 +191,11 @@ func executeRunStep(ctx context.Context, f RunFunc, r *api.StartStepRequest, out } } - return exited, outputs, exportEnvs, artifact, outputsV2, string(optimizationState), finalErr + return exited, outputs, exportEnvs, artifact, outputsV2, telemetryData, string(optimizationState), finalErr } if len(summaryOutputsV2) == 0 || !report.TestSummaryAsOutputEnabled(r.Envs) { - return exited, nil, exportEnvs, artifact, nil, string(optimizationState), err + return exited, nil, exportEnvs, artifact, nil, telemetryData, string(optimizationState), err } // even if the step failed, we still want to return the summary outputs - return exited, summaryOutputs, exportEnvs, artifact, summaryOutputsV2, string(optimizationState), err + return exited, summaryOutputs, exportEnvs, artifact, summaryOutputsV2, telemetryData, string(optimizationState), err } diff --git a/pipeline/runtime/runtest.go b/pipeline/runtime/runtest.go index 8a8c943f..5bdf2ea4 100644 --- a/pipeline/runtime/runtest.go +++ b/pipeline/runtime/runtest.go @@ -33,7 +33,7 @@ var ( ) func executeRunTestStep(ctx context.Context, f RunFunc, r *api.StartStepRequest, out io.Writer, tiConfig *tiCfg.Cfg) ( //nolint:gocritic,gocyclo - *runtime.State, map[string]string, map[string]string, []byte, []*api.OutputV2, string, error) { + *runtime.State, map[string]string, map[string]string, []byte, []*api.OutputV2, *api.TelemetryData, string, error) { log := &logrus.Logger{ Out: out, Level: logrus.InfoLevel, @@ -44,9 +44,10 @@ func executeRunTestStep(ctx context.Context, f RunFunc, r *api.StartStepRequest, start := time.Now() optimizationState := types.DISABLED - cmd, err := instrumentation.GetCmd(ctx, &r.RunTest, r.Name, r.WorkingDir, log, r.Envs, tiConfig) + telemetryData := &api.TelemetryData{} + cmd, err := instrumentation.GetCmd(ctx, &r.RunTest, r.Name, r.WorkingDir, log, r.Envs, tiConfig, &telemetryData.TestIntelligenceMetaData) if err != nil { - return nil, nil, nil, nil, nil, string(optimizationState), err + return nil, nil, nil, nil, nil, nil, string(optimizationState), err } instrumentation.InjectReportInformation(r) @@ -59,7 +60,7 @@ func executeRunTestStep(ctx context.Context, f RunFunc, r *api.StartStepRequest, step.Envs["DRONE_ENV"] = exportEnvFile if (len(r.OutputVars) > 0 || len(r.Outputs) > 0) && (len(step.Entrypoint) == 0 || len(step.Command) == 0) { - return nil, nil, nil, nil, nil, string(optimizationState), fmt.Errorf("output variable should not be set for unset entrypoint or command") + return nil, nil, nil, nil, nil, nil, string(optimizationState), fmt.Errorf("output variable should not be set for unset entrypoint or command") } outputFile := fmt.Sprintf("%s/%s-output.env", pipeline.SharedVolPath, step.ID) @@ -74,7 +75,7 @@ func executeRunTestStep(ctx context.Context, f RunFunc, r *api.StartStepRequest, exited, err := f(ctx, step, out, false, false) timeTakenMs := time.Since(start).Milliseconds() - collectionErr := collectRunTestData(ctx, log, r, start, step.Name, tiConfig) + collectionErr := collectRunTestData(ctx, log, r, start, step.Name, tiConfig, telemetryData) if err == nil { // Fail the step if run was successful but error during collection err = collectionErr @@ -82,7 +83,7 @@ func executeRunTestStep(ctx context.Context, f RunFunc, r *api.StartStepRequest, // Parse and upload savings to TI if tiConfig.GetParseSavings() { - optimizationState = savings.ParseAndUploadSavings(ctx, r.WorkingDir, log, step.Name, checkStepSuccess(exited, err), timeTakenMs, tiConfig, r.Envs) + optimizationState = savings.ParseAndUploadSavings(ctx, r.WorkingDir, log, step.Name, checkStepSuccess(exited, err), timeTakenMs, tiConfig, r.Envs, telemetryData) } useCINewGodotEnvVersion := false @@ -125,36 +126,36 @@ func executeRunTestStep(ctx context.Context, f RunFunc, r *api.StartStepRequest, outputsV2 = append(outputsV2, summaryOutputV2...) } // when outputvars are defined and step has suceeded, fetchErr takes priority - return exited, outputs, exportEnvs, artifact, outputsV2, string(optimizationState), fetchErr + return exited, outputs, exportEnvs, artifact, outputsV2, telemetryData, string(optimizationState), fetchErr } if report.TestSummaryAsOutputEnabled(r.Envs) { - return exited, summaryOutputs, exportEnvs, artifact, summaryOutputV2, string(optimizationState), err + return exited, summaryOutputs, exportEnvs, artifact, summaryOutputV2, telemetryData, string(optimizationState), err } } else if len(r.OutputVars) > 0 { if exited != nil && exited.Exited && exited.ExitCode == 0 { if len(summaryOutputV2) != 0 && report.TestSummaryAsOutputEnabled(r.Envs) { // when step has failed return the actual error - return exited, outputs, exportEnvs, artifact, summaryOutputV2, string(optimizationState), err + return exited, outputs, exportEnvs, artifact, summaryOutputV2, telemetryData, string(optimizationState), err } // when outputvars are defined and step has suceeded, fetchErr takes priority - return exited, outputs, exportEnvs, artifact, nil, string(optimizationState), fetchErr + return exited, outputs, exportEnvs, artifact, nil, telemetryData, string(optimizationState), fetchErr } if len(outputs) != 0 && len(summaryOutputV2) != 0 && report.TestSummaryAsOutputEnabled(r.Envs) { // when step has failed return the actual error - return exited, summaryOutputs, exportEnvs, artifact, summaryOutputV2, string(optimizationState), err + return exited, summaryOutputs, exportEnvs, artifact, summaryOutputV2, telemetryData, string(optimizationState), err } } if len(outputs) != 0 && len(summaryOutputV2) != 0 && report.TestSummaryAsOutputEnabled(r.Envs) { // when there is no output vars requested, fetchErr will have non nil value // In that case return err, which reflects pipeline error - return exited, summaryOutputs, exportEnvs, artifact, summaryOutputV2, string(optimizationState), err + return exited, summaryOutputs, exportEnvs, artifact, summaryOutputV2, telemetryData, string(optimizationState), err } - return exited, nil, exportEnvs, artifact, nil, string(optimizationState), err + return exited, nil, exportEnvs, artifact, nil, telemetryData, string(optimizationState), err } // collectRunTestData collects callgraph and test reports after executing the step -func collectRunTestData(ctx context.Context, log *logrus.Logger, r *api.StartStepRequest, start time.Time, stepName string, tiConfig *tiCfg.Cfg) error { +func collectRunTestData(ctx context.Context, log *logrus.Logger, r *api.StartStepRequest, start time.Time, stepName string, tiConfig *tiCfg.Cfg, telemetryData *api.TelemetryData) error { cgStart := time.Now() cgErr := collectCgFn(ctx, stepName, time.Since(start).Milliseconds(), log, cgStart, tiConfig, cgDir) if cgErr != nil { @@ -163,7 +164,7 @@ func collectRunTestData(ctx context.Context, log *logrus.Logger, r *api.StartSte } reportStart := time.Now() - crErr := collectTestReportsFn(ctx, r.TestReport, r.WorkingDir, stepName, log, reportStart, tiConfig, r.Envs) + crErr := collectTestReportsFn(ctx, r.TestReport, r.WorkingDir, stepName, log, reportStart, tiConfig, &telemetryData.TestIntelligenceMetaData, r.Envs) if crErr != nil { log.WithField("error", crErr).Errorln(fmt.Sprintf("Failed to upload report. Time taken: %s", time.Since(reportStart))) } diff --git a/pipeline/runtime/runtest_test.go b/pipeline/runtime/runtest_test.go index cb3c2a52..1729160d 100644 --- a/pipeline/runtime/runtest_test.go +++ b/pipeline/runtime/runtest_test.go @@ -22,6 +22,8 @@ func Test_CollectRunTestData(t *testing.T) { "", "", "", "", "", "", "", "", "", false, false) + telemetryData := api.TelemetryData{} + tests := []struct { name string cgErr error @@ -58,10 +60,10 @@ func Test_CollectRunTestData(t *testing.T) { collectCgFn = func(ctx context.Context, stepID string, timeMs int64, log *logrus.Logger, start time.Time, tiConfig *tiCfg.Cfg, dir string) error { return tc.cgErr } - collectTestReportsFn = func(ctx context.Context, report api.TestReport, workDir, stepID string, log *logrus.Logger, start time.Time, tiConfig *tiCfg.Cfg, envs map[string]string) error { + collectTestReportsFn = func(ctx context.Context, report api.TestReport, workDir, stepID string, log *logrus.Logger, start time.Time, tiConfig *tiCfg.Cfg, testMetadata *api.TestIntelligenceMetaData, envs map[string]string) error { return tc.crErr } - err := collectRunTestData(ctx, log, &apiReq, time.Now(), stepName, &tiConfig) + err := collectRunTestData(ctx, log, &apiReq, time.Now(), stepName, &tiConfig, &telemetryData) assert.Equal(t, tc.collectionErr, err) }) } diff --git a/pipeline/runtime/runtestsV2.go b/pipeline/runtime/runtestsV2.go index dd859d66..ce33c089 100644 --- a/pipeline/runtime/runtestsV2.go +++ b/pipeline/runtime/runtestsV2.go @@ -20,6 +20,7 @@ import ( "github.com/harness/lite-engine/pipeline" tiCfg "github.com/harness/lite-engine/ti/config" "github.com/harness/lite-engine/ti/instrumentation" + "github.com/harness/lite-engine/ti/instrumentation/common" "github.com/harness/lite-engine/ti/instrumentation/csharp" "github.com/harness/lite-engine/ti/instrumentation/java" "github.com/harness/lite-engine/ti/instrumentation/python" @@ -51,7 +52,7 @@ const ( //nolint:gocritic,gocyclo func executeRunTestsV2Step(ctx context.Context, f RunFunc, r *api.StartStepRequest, out io.Writer, - tiConfig *tiCfg.Cfg) (*runtime.State, map[string]string, map[string]string, []byte, []*api.OutputV2, string, error) { + tiConfig *tiCfg.Cfg) (*runtime.State, map[string]string, map[string]string, []byte, []*api.OutputV2, *api.TelemetryData, string, error) { start := time.Now() log := logrus.New() log.Out = out @@ -59,10 +60,11 @@ func executeRunTestsV2Step(ctx context.Context, f RunFunc, r *api.StartStepReque step := toStep(r) setTiEnvVariables(step, tiConfig) step.Entrypoint = r.RunTestsV2.Entrypoint + telemetryData := &api.TelemetryData{} - preCmd, err := SetupRunTestV2(ctx, &r.RunTestsV2, step.Name, r.WorkingDir, log, r.Envs, tiConfig) + preCmd, err := SetupRunTestV2(ctx, &r.RunTestsV2, step.Name, r.WorkingDir, log, r.Envs, tiConfig, &telemetryData.TestIntelligenceMetaData) if err != nil { - return nil, nil, nil, nil, nil, string(optimizationState), err + return nil, nil, nil, nil, nil, nil, string(optimizationState), err } command := r.RunTestsV2.Command[0] if preCmd != "" { @@ -74,7 +76,7 @@ func executeRunTestsV2Step(ctx context.Context, f RunFunc, r *api.StartStepReque step.Envs["DRONE_ENV"] = exportEnvFile if (len(r.OutputVars) > 0 || len(r.Outputs) > 0) && (len(step.Entrypoint) == 0 || len(step.Command) == 0) { - return nil, nil, nil, nil, nil, string(optimizationState), fmt.Errorf("output variable should not be set for unset entrypoint or command") + return nil, nil, nil, nil, nil, nil, string(optimizationState), fmt.Errorf("output variable should not be set for unset entrypoint or command") } outputFile := fmt.Sprintf("%s/%s-output.env", pipeline.SharedVolPath, step.ID) @@ -95,13 +97,13 @@ func executeRunTestsV2Step(ctx context.Context, f RunFunc, r *api.StartStepReque exited, err := f(ctx, step, out, r.LogDrone, false) timeTakenMs := time.Since(start).Milliseconds() - collectionErr := collectTestReportsAndCg(ctx, log, r, start, step.Name, tiConfig) + collectionErr := collectTestReportsAndCg(ctx, log, r, start, step.Name, tiConfig, telemetryData) if err == nil { err = collectionErr } if tiConfig.GetParseSavings() { - optimizationState = savings.ParseAndUploadSavings(ctx, r.WorkingDir, log, step.Name, checkStepSuccess(exited, err), timeTakenMs, tiConfig, r.Envs) + optimizationState = savings.ParseAndUploadSavings(ctx, r.WorkingDir, log, step.Name, checkStepSuccess(exited, err), timeTakenMs, tiConfig, r.Envs, telemetryData) } useCINewGodotEnvVersion := false @@ -143,26 +145,26 @@ func executeRunTestsV2Step(ctx context.Context, f RunFunc, r *api.StartStepReque if report.TestSummaryAsOutputEnabled(r.Envs) { outputsV2 = append(outputsV2, summaryOutputsV2...) } - return exited, outputs, exportEnvs, artifact, outputsV2, string(optimizationState), err + return exited, outputs, exportEnvs, artifact, outputsV2, telemetryData, string(optimizationState), err } else if len(r.OutputVars) > 0 { // only return err when output vars are expected if report.TestSummaryAsOutputEnabled(r.Envs) { - return exited, summaryOutputs, exportEnvs, artifact, summaryOutputsV2, string(optimizationState), err + return exited, summaryOutputs, exportEnvs, artifact, summaryOutputsV2, telemetryData, string(optimizationState), err } - return exited, outputs, exportEnvs, artifact, nil, string(optimizationState), err + return exited, outputs, exportEnvs, artifact, nil, telemetryData, string(optimizationState), err } if len(summaryOutputsV2) != 0 && report.TestSummaryAsOutputEnabled(r.Envs) { - return exited, outputs, exportEnvs, artifact, summaryOutputsV2, string(optimizationState), nil + return exited, outputs, exportEnvs, artifact, summaryOutputsV2, telemetryData, string(optimizationState), nil } - return exited, outputs, exportEnvs, artifact, nil, string(optimizationState), nil + return exited, outputs, exportEnvs, artifact, nil, telemetryData, string(optimizationState), nil } if len(summaryOutputsV2) != 0 && report.TestSummaryAsOutputEnabled(r.Envs) { - return exited, summaryOutputs, exportEnvs, artifact, summaryOutputsV2, string(optimizationState), err + return exited, summaryOutputs, exportEnvs, artifact, summaryOutputsV2, telemetryData, string(optimizationState), err } - return exited, nil, exportEnvs, artifact, nil, string(optimizationState), err + return exited, nil, exportEnvs, artifact, nil, telemetryData, string(optimizationState), err } -func SetupRunTestV2(ctx context.Context, config *api.RunTestsV2Config, stepID, workspace string, log *logrus.Logger, envs map[string]string, tiConfig *tiCfg.Cfg) (string, error) { +func SetupRunTestV2(ctx context.Context, config *api.RunTestsV2Config, stepID, workspace string, log *logrus.Logger, envs map[string]string, tiConfig *tiCfg.Cfg, testMetadata *api.TestIntelligenceMetaData) (string, error) { agentPaths := make(map[string]string) fs := filesystem.New() tmpFilePath := tiConfig.GetDataDir() @@ -213,7 +215,7 @@ func SetupRunTestV2(ctx context.Context, config *api.RunTestsV2Config, stepID, w if err != nil || pythonArtifactDir == "" { return preCmd, fmt.Errorf("failed to set config file or env variable to inject agent, %s", err) } - err = createSelectedTestFile(ctx, fs, stepID, workspace, log, tiConfig, tmpFilePath, envs, config, filterfilePath) + err = createSelectedTestFile(ctx, fs, stepID, workspace, log, tiConfig, tmpFilePath, envs, config, filterfilePath, testMetadata) if err != nil { return preCmd, fmt.Errorf("error while creating filter file %s", err) } @@ -575,7 +577,7 @@ func downloadDotNetAgent(ctx context.Context, path, dotNetAgentV2Url string, fs // This is nothing but filterfile where all the tests selected will be stored func createSelectedTestFile(ctx context.Context, fs filesystem.FileSystem, stepID, workspace string, log *logrus.Logger, - tiConfig *tiCfg.Cfg, tmpFilepath string, envs map[string]string, runV2Config *api.RunTestsV2Config, filterFilePath string) error { + tiConfig *tiCfg.Cfg, tmpFilepath string, envs map[string]string, runV2Config *api.RunTestsV2Config, filterFilePath string, testMetadata *api.TestIntelligenceMetaData) error { isManualExecution := instrumentation.IsManualExecution(tiConfig) resp, isFilterFilePresent := getTestsSelection(ctx, fs, stepID, workspace, log, isManualExecution, tiConfig, envs, runV2Config) if tiConfig.GetParseSavings() { @@ -601,6 +603,8 @@ func createSelectedTestFile(ctx context.Context, fs filesystem.FileSystem, stepI log.WithError(err).Errorln("failed to populate items in filterfile") return err } + testMetadata.TotalSelectedTests = resp.SelectedTests + testMetadata.TotalSelectedTestClass = common.CountDistinctClasses(resp.Tests) return nil } @@ -646,7 +650,7 @@ func writetoBazelrcFile(log *logrus.Logger, fs filesystem.FileSystem) error { return nil } -func collectTestReportsAndCg(ctx context.Context, log *logrus.Logger, r *api.StartStepRequest, start time.Time, stepName string, tiConfig *tiCfg.Cfg) error { +func collectTestReportsAndCg(ctx context.Context, log *logrus.Logger, r *api.StartStepRequest, start time.Time, stepName string, tiConfig *tiCfg.Cfg, telemetryData *api.TelemetryData) error { cgStart := time.Now() cgErr := collectCgFn(ctx, stepName, time.Since(start).Milliseconds(), log, cgStart, tiConfig, outDir) @@ -661,7 +665,7 @@ func collectTestReportsAndCg(ctx context.Context, log *logrus.Logger, r *api.Sta } reportStart := time.Now() - crErr := collectTestReportsFn(ctx, r.TestReport, r.WorkingDir, stepName, log, reportStart, tiConfig, r.Envs) + crErr := collectTestReportsFn(ctx, r.TestReport, r.WorkingDir, stepName, log, reportStart, tiConfig, &telemetryData.TestIntelligenceMetaData, r.Envs) if crErr != nil { log.WithField("error", crErr).Errorln(fmt.Sprintf("Failed to upload report. Time taken: %s", time.Since(reportStart))) } diff --git a/pipeline/runtime/runtestsV2_test.go b/pipeline/runtime/runtestsV2_test.go index 464464d5..1fa0a473 100644 --- a/pipeline/runtime/runtestsV2_test.go +++ b/pipeline/runtime/runtestsV2_test.go @@ -62,10 +62,10 @@ func Test_CollectRunTestsV2Data(t *testing.T) { collectCgFn = func(ctx context.Context, stepID string, timeMs int64, log *logrus.Logger, start time.Time, tiConfig *tiCfg.Cfg, dir string) error { return tc.cgErr } - collectTestReportsFn = func(ctx context.Context, report api.TestReport, workDir, stepID string, log *logrus.Logger, start time.Time, tiConfig *tiCfg.Cfg, envs map[string]string) error { + collectTestReportsFn = func(ctx context.Context, report api.TestReport, workDir, stepID string, log *logrus.Logger, start time.Time, tiConfig *tiCfg.Cfg, testMetadata *api.TestIntelligenceMetaData, envs map[string]string) error { return tc.crErr } - err := collectTestReportsAndCg(ctx, log, &apiReq, time.Now(), stepName, &tiConfig) + err := collectTestReportsAndCg(ctx, log, &apiReq, time.Now(), stepName, &tiConfig, &api.TelemetryData{}) assert.Equal(t, tc.collectionErr, err) }) } @@ -73,16 +73,17 @@ func Test_CollectRunTestsV2Data(t *testing.T) { func Test_createSelectedTestFile(t *testing.T) { type args struct { - ctx context.Context - fs filesystem.FileSystem - stepID string - workspace string - log *logrus.Logger - tiConfig *tiCfg.Cfg - tmpFilepath string - envs map[string]string - runV2Config *api.RunTestsV2Config - filterFilePath string + ctx context.Context + fs filesystem.FileSystem + stepID string + workspace string + log *logrus.Logger + tiConfig *tiCfg.Cfg + tmpFilepath string + envs map[string]string + runV2Config *api.RunTestsV2Config + filterFilePath string + testIntelligenceMetaData *api.TestIntelligenceMetaData } tests := []struct { name string @@ -102,7 +103,8 @@ func Test_createSelectedTestFile(t *testing.T) { tt.args.tmpFilepath, tt.args.envs, tt.args.runV2Config, - tt.args.filterFilePath); (err != nil) != tt.wantErr { + tt.args.filterFilePath, + tt.args.testIntelligenceMetaData); (err != nil) != tt.wantErr { t.Errorf("createSelectedTestFile() error = %v, wantErr %v", err, tt.wantErr) } }) diff --git a/pipeline/runtime/step_executor.go b/pipeline/runtime/step_executor.go index b0e4fbd8..371e62db 100644 --- a/pipeline/runtime/step_executor.go +++ b/pipeline/runtime/step_executor.go @@ -43,6 +43,7 @@ type StepStatus struct { Artifact []byte OutputV2 []*api.OutputV2 OptimizationState string + TelemetryData *api.TelemetryData } const ( @@ -88,9 +89,9 @@ func (e *StepExecutor) StartStep(ctx context.Context, r *api.StartStepRequest) e go func() { wr := getLogStreamWriter(r) - state, outputs, envs, artifact, outputV2, optimizationState, stepErr := e.executeStep(ctx, r, wr) + state, outputs, envs, artifact, outputV2, telemetrydata, optimizationState, stepErr := e.executeStep(ctx, r, wr) status := StepStatus{Status: Complete, State: state, StepErr: stepErr, Outputs: outputs, Envs: envs, - Artifact: artifact, OutputV2: outputV2, OptimizationState: optimizationState} + Artifact: artifact, OutputV2: outputV2, OptimizationState: optimizationState, TelemetryData: telemetrydata} e.mu.Lock() e.stepStatus[r.ID] = status channels := e.stepWaitCh[r.ID] @@ -118,9 +119,9 @@ func (e *StepExecutor) StartStepWithStatusUpdate(ctx context.Context, r *api.Sta setPrevStepExportEnvs(r) } wr = getLogStreamWriter(r) - state, outputs, envs, artifact, outputV2, optimizationState, stepErr := e.executeStep(ctx, r, wr) + state, outputs, envs, artifact, outputV2, telemetryData, optimizationState, stepErr := e.executeStep(ctx, r, wr) status := StepStatus{Status: Complete, State: state, StepErr: stepErr, Outputs: outputs, Envs: envs, - Artifact: artifact, OutputV2: outputV2, OptimizationState: optimizationState} + Artifact: artifact, OutputV2: outputV2, OptimizationState: optimizationState, TelemetryData: telemetryData} pollResponse := convertStatus(status) if r.StageRuntimeID != "" && len(pollResponse.Envs) > 0 { pipeline.GetEnvState().Add(r.StageRuntimeID, pollResponse.Envs) @@ -257,7 +258,7 @@ func (e *StepExecutor) executeStepDrone(r *api.StartStepRequest) (*runtime.State r.Kind = api.Run // only this kind is supported - exited, _, _, _, _, _, err := run(ctx, e.engine.Run, r, stepLog, pipeline.GetState().GetTIConfig()) + exited, _, _, _, _, _, _, err := run(ctx, e.engine.Run, r, stepLog, pipeline.GetState().GetTIConfig()) if ctx.Err() == context.Canceled || ctx.Err() == context.DeadlineExceeded { logr.WithError(err).Warnln("step execution canceled") return nil, ctx.Err() @@ -289,10 +290,10 @@ func (e *StepExecutor) executeStepDrone(r *api.StartStepRequest) (*runtime.State } func (e *StepExecutor) executeStep(ctx context.Context, r *api.StartStepRequest, wr logstream.Writer) (*runtime.State, map[string]string, //nolint:gocritic - map[string]string, []byte, []*api.OutputV2, string, error) { + map[string]string, []byte, []*api.OutputV2, *api.TelemetryData, string, error) { if r.LogDrone { state, err := e.executeStepDrone(r) - return state, nil, nil, nil, nil, "", err + return state, nil, nil, nil, nil, nil, "", err } // If TI Config has been passed in the step request, use that insetad of relying on the one in the pipeline state var tiConfig *tiCfg.Cfg @@ -314,7 +315,7 @@ func executeStepHelper( //nolint:gocritic f RunFunc, wr logstream.Writer, tiCfg *tiCfg.Cfg) (*runtime.State, map[string]string, - map[string]string, []byte, []*api.OutputV2, string, error) { + map[string]string, []byte, []*api.OutputV2, *api.TelemetryData, string, error) { // if the step is configured as a daemon, it is detached // from the main process and executed separately. // We do here only for non-container step. @@ -329,7 +330,7 @@ func executeStepHelper( //nolint:gocritic run(ctx, f, r, wr, tiCfg) //nolint:errcheck wr.Close() }() - return &runtime.State{Exited: false}, nil, nil, nil, nil, "", nil + return &runtime.State{Exited: false}, nil, nil, nil, nil, nil, "", nil } var result error @@ -341,7 +342,7 @@ func executeStepHelper( //nolint:gocritic defer cancel() } - exited, outputs, envs, artifact, outputV2, optimizationState, err := + exited, outputs, envs, artifact, outputV2, telemetryData, optimizationState, err := run(ctx, f, r, wr, tiCfg) if err != nil { result = multierror.Append(result, err) @@ -360,7 +361,7 @@ func executeStepHelper( //nolint:gocritic // DeadlineExceeded error this indicates the step was timed out. switch ctx.Err() { case context.Canceled, context.DeadlineExceeded: - return nil, nil, nil, nil, nil, "", ctx.Err() + return nil, nil, nil, nil, nil, nil, "", ctx.Err() } if exited != nil { @@ -376,11 +377,11 @@ func executeStepHelper( //nolint:gocritic logrus.WithContext(ctx).WithField("id", r.ID).Infof("received exit code %d\n", exited.ExitCode) } } - return exited, outputs, envs, artifact, outputV2, optimizationState, result + return exited, outputs, envs, artifact, outputV2, telemetryData, optimizationState, result } func run(ctx context.Context, f RunFunc, r *api.StartStepRequest, out io.Writer, tiConfig *tiCfg.Cfg) ( //nolint:gocritic - *runtime.State, map[string]string, map[string]string, []byte, []*api.OutputV2, string, error) { + *runtime.State, map[string]string, map[string]string, []byte, []*api.OutputV2, *api.TelemetryData, string, error) { if r.Kind == api.Run { return executeRunStep(ctx, f, r, out, tiConfig) } @@ -500,6 +501,7 @@ func convertStatus(status StepStatus) *api.PollStepResponse { //nolint:gocritic Artifact: status.Artifact, OutputV2: status.OutputV2, OptimizationState: status.OptimizationState, + TelemetryData: status.TelemetryData, } stepErr := status.StepErr @@ -527,10 +529,10 @@ func convertStatus(status StepStatus) *api.PollStepResponse { //nolint:gocritic func convertPollResponse(r *api.PollStepResponse, envs map[string]string) api.VMTaskExecutionResponse { if r.Error == "" { - return api.VMTaskExecutionResponse{CommandExecutionStatus: api.Success, OutputVars: r.Outputs, Artifact: r.Artifact, Outputs: r.OutputV2, OptimizationState: r.OptimizationState} + return api.VMTaskExecutionResponse{CommandExecutionStatus: api.Success, OutputVars: r.Outputs, Artifact: r.Artifact, Outputs: r.OutputV2, OptimizationState: r.OptimizationState, TelemetryData: r.TelemetryData} } if report.TestSummaryAsOutputEnabled(envs) { - return api.VMTaskExecutionResponse{CommandExecutionStatus: api.Failure, OutputVars: r.Outputs, Outputs: r.OutputV2, ErrorMessage: r.Error, OptimizationState: r.OptimizationState} + return api.VMTaskExecutionResponse{CommandExecutionStatus: api.Failure, OutputVars: r.Outputs, Outputs: r.OutputV2, ErrorMessage: r.Error, OptimizationState: r.OptimizationState, TelemetryData: r.TelemetryData} } return api.VMTaskExecutionResponse{CommandExecutionStatus: api.Failure, ErrorMessage: r.Error, OptimizationState: r.OptimizationState} } diff --git a/pipeline/runtime/step_executor_stateless.go b/pipeline/runtime/step_executor_stateless.go index cbb18d1c..182693d9 100644 --- a/pipeline/runtime/step_executor_stateless.go +++ b/pipeline/runtime/step_executor_stateless.go @@ -45,9 +45,9 @@ func (e *StepExecutorStateless) Run( e.stepStatus = StepStatus{Status: Running} - state, outputs, envs, artifact, outputV2, optimizationState, stepErr := e.executeStep(ctx, r, cfg, writer) + state, outputs, envs, artifact, outputV2, telemetryData, optimizationState, stepErr := e.executeStep(ctx, r, cfg, writer) e.stepStatus = StepStatus{Status: Complete, State: state, StepErr: stepErr, Outputs: outputs, Envs: envs, - Artifact: artifact, OutputV2: outputV2, OptimizationState: optimizationState} + Artifact: artifact, OutputV2: outputV2, OptimizationState: optimizationState, TelemetryData: telemetryData} pollResponse := convertStatus(e.stepStatus) return convertPollResponse(pollResponse, r.Envs), nil } @@ -58,7 +58,7 @@ func (e *StepExecutorStateless) executeStep( //nolint:gocritic cfg *spec.PipelineConfig, writer logstream.Writer, ) (*runtime.State, map[string]string, - map[string]string, []byte, []*api.OutputV2, string, error) { + map[string]string, []byte, []*api.OutputV2, *api.TelemetryData, string, error) { runFunc := func(ctx context.Context, step *spec.Step, output io.Writer, isDrone bool, isHosted bool) (*runtime.State, error) { return engine.RunStep(ctx, engine.Opts{}, step, output, cfg, isDrone, isHosted) } diff --git a/ti/instrumentation/common/helper.go b/ti/instrumentation/common/helper.go index 8e3a7780..0824e8ad 100644 --- a/ti/instrumentation/common/helper.go +++ b/ti/instrumentation/common/helper.go @@ -1,6 +1,7 @@ package common import ( + "github.com/harness/ti-client/types" ti "github.com/harness/ti-client/types" "github.com/mattn/go-zglob" ) @@ -69,3 +70,14 @@ func GetUniqueTestStrings(tests []ti.RunnableTest) []string { } return ut } + +// countDistinctClasses counts the number of distinct classes in the given tests +func CountDistinctClasses(tests []types.RunnableTest) int { + uniqueClasses := make(map[string]bool) // Map to track unique class names + + for _, test := range tests { + uniqueClasses[test.Class] = true // Add class to map (duplicates will be ignored) + } + + return len(uniqueClasses) // Return the count of unique keys in the map +} diff --git a/ti/instrumentation/instrumentation.go b/ti/instrumentation/instrumentation.go index f2cf7a9c..d3b82c2d 100644 --- a/ti/instrumentation/instrumentation.go +++ b/ti/instrumentation/instrumentation.go @@ -246,7 +246,7 @@ func computeSelectedTests(ctx context.Context, config *api.RunTestConfig, log *l config.RunOnlySelectedTests = true } -func GetCmd(ctx context.Context, config *api.RunTestConfig, stepID, workspace string, log *logrus.Logger, envs map[string]string, cfg *tiCfg.Cfg) (string, error) { +func GetCmd(ctx context.Context, config *api.RunTestConfig, stepID, workspace string, log *logrus.Logger, envs map[string]string, cfg *tiCfg.Cfg, testMetadata *api.TestIntelligenceMetaData) (string, error) { fs := filesystem.New() tmpFilePath := cfg.GetDataDir() @@ -308,6 +308,8 @@ func GetCmd(ctx context.Context, config *api.RunTestConfig, stepID, workspace st if err != nil { return "", err } + testMetadata.TotalSelectedTests = selection.SelectedTests + testMetadata.TotalSelectedTestClass = common.CountDistinctClasses(selection.Tests) if cfg.GetIgnoreInstr() { log.Infoln("Ignoring instrumentation and not attaching agent") diff --git a/ti/report/parser/junit/junit.go b/ti/report/parser/junit/junit.go index 75414893..0078725d 100644 --- a/ti/report/parser/junit/junit.go +++ b/ti/report/parser/junit/junit.go @@ -52,6 +52,7 @@ func ParseTests(paths []string, log *logrus.Logger, envs map[string]string) []*t totalTests += testsInFile fileMap[file] = testsInFile } + log.Infoln("Number of cases parsed in each file: ", fileMap) log.WithField("num_cases", totalTests).Infoln(fmt.Sprintf("Parsed %d test cases", totalTests)) return tests diff --git a/ti/report/report.go b/ti/report/report.go index 7d2427c9..daa76bd5 100644 --- a/ti/report/report.go +++ b/ti/report/report.go @@ -19,7 +19,7 @@ import ( "github.com/sirupsen/logrus" ) -func ParseAndUploadTests(ctx context.Context, report api.TestReport, workDir, stepID string, log *logrus.Logger, start time.Time, tiConfig *tiCfg.Cfg, envs map[string]string) error { +func ParseAndUploadTests(ctx context.Context, report api.TestReport, workDir, stepID string, log *logrus.Logger, start time.Time, tiConfig *tiCfg.Cfg, testMetadata *api.TestIntelligenceMetaData, envs map[string]string) error { if report.Kind != api.Junit { return fmt.Errorf("unknown report type: %s", report.Kind) } @@ -50,10 +50,22 @@ func ParseAndUploadTests(ctx context.Context, report api.TestReport, workDir, st return err } logrus.WithContext(ctx).Infoln(fmt.Sprintf("Completed TI service request to write report for step %s, took %.2f seconds", stepID, time.Since(startTime).Seconds())) + //Write tests telemetry data, total test, total test classes,selected test, cselected classes, + testMetadata.TotalTests = len(tests) + testMetadata.TotalTestClasses = countDistinctClasses(tests) log.Infoln(fmt.Sprintf("Successfully collected test reports in %s time", time.Since(start))) return nil } +func countDistinctClasses(testCases []*types.TestCase) int { + uniqueClasses := make(map[string]bool) + + for _, testCase := range testCases { + uniqueClasses[testCase.ClassName] = true + } + + return len(uniqueClasses) +} func SaveReportSummaryToOutputs(ctx context.Context, tiConfig *tiCfg.Cfg, stepID string, outputs map[string]string, log *logrus.Logger, envs map[string]string) error { if !TestSummaryAsOutputEnabled(envs) { return nil diff --git a/ti/savings/cache/gradle/helper.go b/ti/savings/cache/gradle/helper.go index ae7e7e00..904b2f0f 100644 --- a/ti/savings/cache/gradle/helper.go +++ b/ti/savings/cache/gradle/helper.go @@ -6,6 +6,7 @@ import ( "strconv" "strings" + "github.com/harness/ti-client/types" gradleTypes "github.com/harness/ti-client/types/cache/gradle" "golang.org/x/net/html" ) @@ -293,3 +294,21 @@ func (n *JsonNode) populateFrom(htmlNode *html.Node) { //nolint:gocyclo n.Text = textBuffer.String() } } + +func GetMetadataFromGradleMetrics(metrics types.SavingsRequest) (int, int) { + totalTasks := 0 + cachedTasks := 0 + + for _, profile := range metrics.GradleMetrics.Profiles { + for _, project := range profile.Projects { + for _, task := range project.Tasks { + totalTasks++ + if task.State == "FROM-CACHE" { + cachedTasks++ + } + } + } + } + + return totalTasks, cachedTasks +} diff --git a/ti/savings/savings.go b/ti/savings/savings.go index 87692058..dc512bf2 100644 --- a/ti/savings/savings.go +++ b/ti/savings/savings.go @@ -5,8 +5,10 @@ import ( "strconv" "time" + "github.com/harness/lite-engine/api" tiCfg "github.com/harness/lite-engine/ti/config" "github.com/harness/lite-engine/ti/savings/cache" + "github.com/harness/lite-engine/ti/savings/cache/gradle" "github.com/harness/lite-engine/ti/savings/dlc" "github.com/harness/ti-client/types" "github.com/sirupsen/logrus" @@ -15,7 +17,7 @@ import ( const restoreCacheHarnessStepID = "restore-cache-harness" func ParseAndUploadSavings(ctx context.Context, workspace string, log *logrus.Logger, stepID string, stepSuccess bool, cmdTimeTaken int64, - tiConfig *tiCfg.Cfg, envs map[string]string) types.IntelligenceExecutionState { + tiConfig *tiCfg.Cfg, envs map[string]string, telemetryData *api.TelemetryData) types.IntelligenceExecutionState { states := make([]types.IntelligenceExecutionState, 0) // Cache Savings start := time.Now() @@ -31,6 +33,9 @@ func ParseAndUploadSavings(ctx context.Context, workspace string, log *logrus.Lo log.Infof("Successfully uploaded savings for feature %s in %0.2f seconds", types.BUILD_CACHE, time.Since(tiStart).Seconds()) } + totaltasks, cachedtasks := gradle.GetMetadataFromGradleMetrics(savingsRequest) + telemetryData.BuildIntelligenceMetaData.BuildTasks = totaltasks + telemetryData.BuildIntelligenceMetaData.TasksRestored = cachedtasks } // TI Savings @@ -60,6 +65,8 @@ func ParseAndUploadSavings(ctx context.Context, workspace string, log *logrus.Lo log.Infof("Successfully uploaded savings for feature %s in %0.2f seconds", types.DLC, time.Since(tiStart).Seconds()) } + telemetryData.DlcMetadata.TotalLayers = savingsRequest.DlcMetrics.TotalLayers + telemetryData.DlcMetadata.LayersRestored = savingsRequest.DlcMetrics.Cached } } }