Skip to content

Commit

Permalink
feat: [CI-15495]: Send telemetry data via step response
Browse files Browse the repository at this point in the history
  • Loading branch information
@smjt-h
smjt-h committed Jan 9, 2025
1 parent 3dfdc11 commit e2b6417
Show file tree
Hide file tree
Showing 14 changed files with 185 additions and 76 deletions.
44 changes: 44 additions & 0 deletions api/api.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -105,6 +105,48 @@ type (
Files []*spec.File `json:"files,omitempty"`
StepStatus StepStatusConfig `json:"step_status,omitempty"`
}

TelemetryData struct {
BuildIntelligenceMetaData BuildIntelligenceMetaData `json:"build_intelligence_data,omitempty"`
TestIntelligenceMetaData TestIntelligenceMetaData `json:"test_intelligence_data,omitempty"`
CacheIntelligenceMetaData CacheIntelligenceMetaData `json:"cache_intelligence_data,omitempty"`
DlcMetadata DlcMetadata `json:"dlc_metadata,omitempty"`
Errors []string `json:"errors,omitempty"`
}

BuildIntelligenceMetaData struct {
BuildTasks int `json:"build_tasks,omitempty"`
TasksRestored int `json:"tasks_restored,omitempty"`
StepType string `json:"step_type,omitempty"`
BuildTool string `json:"build_tool,omitempty"`
Language string `json:"language,omitempty"`
Errors []string `json:"errors,omitempty"`
}

TestIntelligenceMetaData struct {
TotalTests int `json:"total_tests,omitempty"`
TotalTestClasses int `json:"total_test_classes,omitempty"`
TotalSelectedTests int `json:"total_selected_tests,omitempty"`
TotalSelectedTestClass int `json:"total_selected_test_classes,omitempty"`
CPUTimeSaved int64 `json:"cpu_time_saved,omitempty"`
BuildTool string `json:"build_tool,omitempty"`
Language string `json:"language,omitempty"`
Errors []string `json:"errors,omitempty"`
}

CacheIntelligenceMetaData struct {
CacheSize int `json:"cache_size,omitempty"`
IsNonDefaultPath bool `json:"is_non_default_path,omitempty"`
IsCustomKeys bool `json:"is_custom_keys,omitempty"`
Errors []string `json:"errors,omitempty"`
}

DlcMetadata struct {
TotalLayers int `json:"total_layers,omitempty"`
LayersRestored int `json:"layers_restored,omitempty"`
Errors []string `json:"errors,omitempty"`
}

OutputV2 struct {
Key string `json:"key,omitempty"`
Value string `json:"value"`
Expand All @@ -127,6 +169,7 @@ type (
Artifact []byte `json:"artifact,omitempty"`
OutputV2 []*OutputV2 `json:"outputV2,omitempty"`
OptimizationState string `json:"optimization_state,omitempty"`
TelemetryData *TelemetryData `json:"telemetry_data,omitempty"`
}

StreamOutputRequest struct {
Expand Down Expand Up @@ -213,6 +256,7 @@ type (
Artifact []byte `json:"artifact,omitempty"`
Outputs []*OutputV2 `json:"outputs,omitempty"`
OptimizationState string `json:"optimization_state,omitempty"`
TelemetryData *TelemetryData `json:"telemetry_data,omitempty"`
}
)

Expand Down
15 changes: 8 additions & 7 deletions pipeline/runtime/run.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ const (
)

func executeRunStep(ctx context.Context, f RunFunc, r *api.StartStepRequest, out io.Writer, tiConfig *tiCfg.Cfg) ( //nolint:gocritic,gocyclo,funlen
*runtime.State, map[string]string, map[string]string, []byte, []*api.OutputV2, string, error) {
*runtime.State, map[string]string, map[string]string, []byte, []*api.OutputV2, *api.TelemetryData, string, error) {
start := time.Now()
step := toStep(r)
step.Command = r.Run.Command
Expand All @@ -36,9 +36,10 @@ func executeRunStep(ctx context.Context, f RunFunc, r *api.StartStepRequest, out
optimizationState := types.DISABLED
exportEnvFile := fmt.Sprintf("%s/%s-export.env", pipeline.SharedVolPath, step.ID)
step.Envs["DRONE_ENV"] = exportEnvFile
telemetryData := &api.TelemetryData{}

if (len(r.OutputVars) > 0 || len(r.Outputs) > 0) && (len(step.Entrypoint) == 0 || len(step.Command) == 0) {
return nil, nil, nil, nil, nil, string(optimizationState), fmt.Errorf("output variable should not be set for unset entrypoint or command")
return nil, nil, nil, nil, nil, nil, string(optimizationState), fmt.Errorf("output variable should not be set for unset entrypoint or command")
}

if r.ScratchDir != "" {
Expand Down Expand Up @@ -98,14 +99,14 @@ func executeRunStep(ctx context.Context, f RunFunc, r *api.StartStepRequest, out
timeTakenMs := time.Since(start).Milliseconds()

reportStart := time.Now()
if rerr := report.ParseAndUploadTests(ctx, r.TestReport, r.WorkingDir, step.Name, log, reportStart, tiConfig, r.Envs); rerr != nil {
if rerr := report.ParseAndUploadTests(ctx, r.TestReport, r.WorkingDir, step.Name, log, reportStart, tiConfig, &telemetryData.TestIntelligenceMetaData, r.Envs); rerr != nil {
logrus.WithContext(ctx).WithError(rerr).WithField("step", step.Name).Errorln("failed to upload report")
log.Errorf("Failed to upload report. Time taken: %s", time.Since(reportStart))
}

// Parse and upload savings to TI
if tiConfig.GetParseSavings() {
optimizationState = savings.ParseAndUploadSavings(ctx, r.WorkingDir, log, step.Name, checkStepSuccess(exited, err), timeTakenMs, tiConfig, r.Envs)
optimizationState = savings.ParseAndUploadSavings(ctx, r.WorkingDir, log, step.Name, checkStepSuccess(exited, err), timeTakenMs, tiConfig, r.Envs, telemetryData)
}

useCINewGodotEnvVersion := false
Expand Down Expand Up @@ -190,11 +191,11 @@ func executeRunStep(ctx context.Context, f RunFunc, r *api.StartStepRequest, out
}
}

return exited, outputs, exportEnvs, artifact, outputsV2, string(optimizationState), finalErr
return exited, outputs, exportEnvs, artifact, outputsV2, telemetryData, string(optimizationState), finalErr
}
if len(summaryOutputsV2) == 0 || !report.TestSummaryAsOutputEnabled(r.Envs) {
return exited, nil, exportEnvs, artifact, nil, string(optimizationState), err
return exited, nil, exportEnvs, artifact, nil, telemetryData, string(optimizationState), err
}
// even if the step failed, we still want to return the summary outputs
return exited, summaryOutputs, exportEnvs, artifact, summaryOutputsV2, string(optimizationState), err
return exited, summaryOutputs, exportEnvs, artifact, summaryOutputsV2, telemetryData, string(optimizationState), err
}
31 changes: 16 additions & 15 deletions pipeline/runtime/runtest.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ var (
)

func executeRunTestStep(ctx context.Context, f RunFunc, r *api.StartStepRequest, out io.Writer, tiConfig *tiCfg.Cfg) ( //nolint:gocritic,gocyclo
*runtime.State, map[string]string, map[string]string, []byte, []*api.OutputV2, string, error) {
*runtime.State, map[string]string, map[string]string, []byte, []*api.OutputV2, *api.TelemetryData, string, error) {
log := &logrus.Logger{
Out: out,
Level: logrus.InfoLevel,
Expand All @@ -44,9 +44,10 @@ func executeRunTestStep(ctx context.Context, f RunFunc, r *api.StartStepRequest,

start := time.Now()
optimizationState := types.DISABLED
cmd, err := instrumentation.GetCmd(ctx, &r.RunTest, r.Name, r.WorkingDir, log, r.Envs, tiConfig)
telemetryData := &api.TelemetryData{}
cmd, err := instrumentation.GetCmd(ctx, &r.RunTest, r.Name, r.WorkingDir, log, r.Envs, tiConfig, &telemetryData.TestIntelligenceMetaData)
if err != nil {
return nil, nil, nil, nil, nil, string(optimizationState), err
return nil, nil, nil, nil, nil, nil, string(optimizationState), err
}

instrumentation.InjectReportInformation(r)
Expand All @@ -59,7 +60,7 @@ func executeRunTestStep(ctx context.Context, f RunFunc, r *api.StartStepRequest,
step.Envs["DRONE_ENV"] = exportEnvFile

if (len(r.OutputVars) > 0 || len(r.Outputs) > 0) && (len(step.Entrypoint) == 0 || len(step.Command) == 0) {
return nil, nil, nil, nil, nil, string(optimizationState), fmt.Errorf("output variable should not be set for unset entrypoint or command")
return nil, nil, nil, nil, nil, nil, string(optimizationState), fmt.Errorf("output variable should not be set for unset entrypoint or command")
}

outputFile := fmt.Sprintf("%s/%s-output.env", pipeline.SharedVolPath, step.ID)
Expand All @@ -74,15 +75,15 @@ func executeRunTestStep(ctx context.Context, f RunFunc, r *api.StartStepRequest,

exited, err := f(ctx, step, out, false, false)
timeTakenMs := time.Since(start).Milliseconds()
collectionErr := collectRunTestData(ctx, log, r, start, step.Name, tiConfig)
collectionErr := collectRunTestData(ctx, log, r, start, step.Name, tiConfig, telemetryData)
if err == nil {
// Fail the step if run was successful but error during collection
err = collectionErr
}

// Parse and upload savings to TI
if tiConfig.GetParseSavings() {
optimizationState = savings.ParseAndUploadSavings(ctx, r.WorkingDir, log, step.Name, checkStepSuccess(exited, err), timeTakenMs, tiConfig, r.Envs)
optimizationState = savings.ParseAndUploadSavings(ctx, r.WorkingDir, log, step.Name, checkStepSuccess(exited, err), timeTakenMs, tiConfig, r.Envs, telemetryData)
}

useCINewGodotEnvVersion := false
Expand Down Expand Up @@ -125,36 +126,36 @@ func executeRunTestStep(ctx context.Context, f RunFunc, r *api.StartStepRequest,
outputsV2 = append(outputsV2, summaryOutputV2...)
}
// when outputvars are defined and step has suceeded, fetchErr takes priority
return exited, outputs, exportEnvs, artifact, outputsV2, string(optimizationState), fetchErr
return exited, outputs, exportEnvs, artifact, outputsV2, telemetryData, string(optimizationState), fetchErr
}
if report.TestSummaryAsOutputEnabled(r.Envs) {
return exited, summaryOutputs, exportEnvs, artifact, summaryOutputV2, string(optimizationState), err
return exited, summaryOutputs, exportEnvs, artifact, summaryOutputV2, telemetryData, string(optimizationState), err
}
} else if len(r.OutputVars) > 0 {
if exited != nil && exited.Exited && exited.ExitCode == 0 {
if len(summaryOutputV2) != 0 && report.TestSummaryAsOutputEnabled(r.Envs) {
// when step has failed return the actual error
return exited, outputs, exportEnvs, artifact, summaryOutputV2, string(optimizationState), err
return exited, outputs, exportEnvs, artifact, summaryOutputV2, telemetryData, string(optimizationState), err
}
// when outputvars are defined and step has suceeded, fetchErr takes priority
return exited, outputs, exportEnvs, artifact, nil, string(optimizationState), fetchErr
return exited, outputs, exportEnvs, artifact, nil, telemetryData, string(optimizationState), fetchErr
}
if len(outputs) != 0 && len(summaryOutputV2) != 0 && report.TestSummaryAsOutputEnabled(r.Envs) {
// when step has failed return the actual error
return exited, summaryOutputs, exportEnvs, artifact, summaryOutputV2, string(optimizationState), err
return exited, summaryOutputs, exportEnvs, artifact, summaryOutputV2, telemetryData, string(optimizationState), err
}
}
if len(outputs) != 0 && len(summaryOutputV2) != 0 && report.TestSummaryAsOutputEnabled(r.Envs) {
// when there is no output vars requested, fetchErr will have non nil value
// In that case return err, which reflects pipeline error
return exited, summaryOutputs, exportEnvs, artifact, summaryOutputV2, string(optimizationState), err
return exited, summaryOutputs, exportEnvs, artifact, summaryOutputV2, telemetryData, string(optimizationState), err
}

return exited, nil, exportEnvs, artifact, nil, string(optimizationState), err
return exited, nil, exportEnvs, artifact, nil, telemetryData, string(optimizationState), err
}

// collectRunTestData collects callgraph and test reports after executing the step
func collectRunTestData(ctx context.Context, log *logrus.Logger, r *api.StartStepRequest, start time.Time, stepName string, tiConfig *tiCfg.Cfg) error {
func collectRunTestData(ctx context.Context, log *logrus.Logger, r *api.StartStepRequest, start time.Time, stepName string, tiConfig *tiCfg.Cfg, telemetryData *api.TelemetryData) error {
cgStart := time.Now()
cgErr := collectCgFn(ctx, stepName, time.Since(start).Milliseconds(), log, cgStart, tiConfig, cgDir)
if cgErr != nil {
Expand All @@ -163,7 +164,7 @@ func collectRunTestData(ctx context.Context, log *logrus.Logger, r *api.StartSte
}

reportStart := time.Now()
crErr := collectTestReportsFn(ctx, r.TestReport, r.WorkingDir, stepName, log, reportStart, tiConfig, r.Envs)
crErr := collectTestReportsFn(ctx, r.TestReport, r.WorkingDir, stepName, log, reportStart, tiConfig, &telemetryData.TestIntelligenceMetaData, r.Envs)
if crErr != nil {
log.WithField("error", crErr).Errorln(fmt.Sprintf("Failed to upload report. Time taken: %s", time.Since(reportStart)))
}
Expand Down
6 changes: 4 additions & 2 deletions pipeline/runtime/runtest_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,8 @@ func Test_CollectRunTestData(t *testing.T) {
"", "", "", "", "", "", "", "",
"", false, false)

telemetryData := api.TelemetryData{}

tests := []struct {
name string
cgErr error
Expand Down Expand Up @@ -58,10 +60,10 @@ func Test_CollectRunTestData(t *testing.T) {
collectCgFn = func(ctx context.Context, stepID string, timeMs int64, log *logrus.Logger, start time.Time, tiConfig *tiCfg.Cfg, dir string) error {
return tc.cgErr
}
collectTestReportsFn = func(ctx context.Context, report api.TestReport, workDir, stepID string, log *logrus.Logger, start time.Time, tiConfig *tiCfg.Cfg, envs map[string]string) error {
collectTestReportsFn = func(ctx context.Context, report api.TestReport, workDir, stepID string, log *logrus.Logger, start time.Time, tiConfig *tiCfg.Cfg, testMetadata *api.TestIntelligenceMetaData, envs map[string]string) error {
return tc.crErr
}
err := collectRunTestData(ctx, log, &apiReq, time.Now(), stepName, &tiConfig)
err := collectRunTestData(ctx, log, &apiReq, time.Now(), stepName, &tiConfig, &telemetryData)
assert.Equal(t, tc.collectionErr, err)
})
}
Expand Down
Loading

0 comments on commit e2b6417

Please sign in to comment.