From bf8b5f3a86c61ec97d283b1b6e50a184085cfcf3 Mon Sep 17 00:00:00 2001 From: sindunuragarp Date: Mon, 13 Nov 2023 15:30:42 +0100 Subject: [PATCH] move all metrics init logic out of main --- cmd/actionsmetricsserver/main.go | 12 +-- pkg/actionsmetrics/event_reader.go | 4 +- pkg/actionsmetrics/metrics.go | 128 ++++++++++++++++------------- 3 files changed, 73 insertions(+), 71 deletions(-) diff --git a/cmd/actionsmetricsserver/main.go b/cmd/actionsmetricsserver/main.go index 7a39a16db1..f32bf5f7e4 100644 --- a/cmd/actionsmetricsserver/main.go +++ b/cmd/actionsmetricsserver/main.go @@ -48,9 +48,7 @@ var ( ) const ( - webhookSecretTokenEnvName = "GITHUB_WEBHOOK_SECRET_TOKEN" - prometheusRunBucketIntervalsName = "PROMETHEUS_RUN_BUCKET_INTERVALS" - prometheusQueueBucketIntervalsName = "PROMETHEUS_QUEUE_BUCKET_INTERVALS" + webhookSecretTokenEnvName = "GITHUB_WEBHOOK_SECRET_TOKEN" ) func init() { @@ -75,10 +73,6 @@ func main() { logFormat string ghClient *github.Client - - // List of histogram buckets that we want to see in metrics - runBucketsList actionsmetrics.BucketsSlice - queueBucketsList actionsmetrics.BucketsSlice ) var c github.Config @@ -89,8 +83,6 @@ func main() { } webhookSecretTokenEnv = os.Getenv(webhookSecretTokenEnvName) - runBucketsList.Set(os.Getenv(prometheusRunBucketIntervalsName)) - queueBucketsList.Set(os.Getenv(prometheusQueueBucketIntervalsName)) flag.StringVar(&webhookAddr, "webhook-addr", ":8000", "The address the metric endpoint binds to.") flag.StringVar(&metricsAddr, "metrics-addr", ":8080", "The address the metric endpoint binds to.") @@ -121,8 +113,6 @@ func main() { webhookSecretToken = webhookSecretTokenEnv } - actionsmetrics.InitializeMetrics(runBucketsList, queueBucketsList) - if webhookSecretToken == "" { logger.Info(fmt.Sprintf("-github-webhook-secret-token and %s are missing or empty. Create one following https://docs.github.com/en/developers/webhooks-and-events/securing-your-webhooks and specify it via the flag or the envvar", webhookSecretTokenEnvName)) } diff --git a/pkg/actionsmetrics/event_reader.go b/pkg/actionsmetrics/event_reader.go index 78bd73a643..606891fe5e 100644 --- a/pkg/actionsmetrics/event_reader.go +++ b/pkg/actionsmetrics/event_reader.go @@ -134,7 +134,7 @@ func (reader *EventReader) ProcessWorkflowJobEvent(ctx context.Context, event in log.Info("reading workflow_job logs") } - githubWorkflowJobQueueHistogram.With(labels).Observe(parseResult.QueueTime.Seconds()) + githubWorkflowJobQueueDurationSeconds.With(labels).Observe(parseResult.QueueTime.Seconds()) case "completed": githubWorkflowJobsCompletedTotal.With(labels).Inc() @@ -200,7 +200,7 @@ func (reader *EventReader) ProcessWorkflowJobEvent(ctx context.Context, event in } if runTimeSeconds != nil { - githubWorkflowJobRunHistogram.With(extraLabel("job_conclusion", *e.WorkflowJob.Conclusion, labels)).Observe(*runTimeSeconds) + githubWorkflowJobRunDurationSeconds.With(extraLabel("job_conclusion", *e.WorkflowJob.Conclusion, labels)).Observe(*runTimeSeconds) } } } diff --git a/pkg/actionsmetrics/metrics.go b/pkg/actionsmetrics/metrics.go index 77f4744e52..972a6f9f5e 100644 --- a/pkg/actionsmetrics/metrics.go +++ b/pkg/actionsmetrics/metrics.go @@ -6,47 +6,46 @@ package actionsmetrics import ( "fmt" - "github.com/prometheus/client_golang/prometheus" - "sigs.k8s.io/controller-runtime/pkg/metrics" + "os" "strconv" "strings" -) -type BucketsSlice []float64 + "github.com/prometheus/client_golang/prometheus" + "sigs.k8s.io/controller-runtime/pkg/metrics" +) -func (i *BucketsSlice) String() string { - return fmt.Sprintf("%v", *i) -} +const ( + prometheusRunBucketIntervalsName = "PROMETHEUS_RUN_BUCKET_INTERVALS" + prometheusQueueBucketIntervalsName = "PROMETHEUS_QUEUE_BUCKET_INTERVALS" +) -func (i *BucketsSlice) Set(value string) error { - valuesStr := strings.Split(value, ",") - for _, str := range valuesStr { - // Convert the string to float64. - val, err := strconv.ParseFloat(str, 64) +func init() { + queueBuckets := defaultRuntimeBuckets + if _, ok := os.LookupEnv(prometheusQueueBucketIntervalsName); ok { + buckets, err := parseBucketsString(os.Getenv(prometheusQueueBucketIntervalsName)) if err != nil { - return err + fmt.Fprintf(os.Stderr, "Warning: Failed to parse %s, using default buckets: %s\n", prometheusQueueBucketIntervalsName, err) + } else { + queueBuckets = buckets } - *i = append(*i, val) } - return nil -} -var githubWorkflowJobQueueHistogram *prometheus.HistogramVec -var githubWorkflowJobRunHistogram *prometheus.HistogramVec + runBuckets := defaultRuntimeBuckets + if _, ok := os.LookupEnv(prometheusRunBucketIntervalsName); ok { + buckets, err := parseBucketsString(os.Getenv(prometheusRunBucketIntervalsName)) + if err != nil { + fmt.Fprintf(os.Stderr, "Warning: Failed to parse %s, using default buckets: %s\n", prometheusRunBucketIntervalsName, err) + } else { + runBuckets = buckets + } + } -func initMetrics(runBuckets, queueBuckets []float64) { - githubWorkflowJobRunHistogram = githubWorkflowJobRunDurationSeconds(runBuckets) - githubWorkflowJobQueueHistogram = githubWorkflowJobQueueDurationSeconds(queueBuckets) + githubWorkflowJobQueueDurationSeconds = initGithubWorkflowJobQueueDurationSeconds(queueBuckets) + githubWorkflowJobRunDurationSeconds = initGithubWorkflowJobRunDurationSeconds(runBuckets) - if len(runBuckets) == 0 { - githubWorkflowJobRunHistogram = githubWorkflowJobRunDurationSeconds(DefaultRuntimeBuckets) - } - if len(queueBuckets) == 0 { - githubWorkflowJobQueueHistogram = githubWorkflowJobQueueDurationSeconds(DefaultRuntimeBuckets) - } metrics.Registry.MustRegister( - githubWorkflowJobQueueHistogram, - githubWorkflowJobRunHistogram, + githubWorkflowJobQueueDurationSeconds, + githubWorkflowJobRunDurationSeconds, githubWorkflowJobConclusionsTotal, githubWorkflowJobsQueuedTotal, githubWorkflowJobsStartedTotal, @@ -55,34 +54,8 @@ func initMetrics(runBuckets, queueBuckets []float64) { ) } -func githubWorkflowJobQueueDurationSeconds(buckets []float64) *prometheus.HistogramVec { - return prometheus.NewHistogramVec( - prometheus.HistogramOpts{ - Name: "github_workflow_job_queue_duration_seconds", - Help: "Queue times for workflow jobs in seconds", - Buckets: buckets, - }, - metricLabels(), - ) -} - -func githubWorkflowJobRunDurationSeconds(buckets []float64) *prometheus.HistogramVec { - return prometheus.NewHistogramVec( - prometheus.HistogramOpts{ - Name: "github_workflow_job_run_duration_seconds", - Help: "Run times for workflow jobs in seconds", - Buckets: buckets, - }, - metricLabels("job_conclusion"), - ) -} - -func InitializeMetrics(runBuckets, queueBuckets []float64) { - initMetrics(runBuckets, queueBuckets) -} - var ( - DefaultRuntimeBuckets = []float64{ + defaultRuntimeBuckets = []float64{ 0.01, 0.05, 0.1, @@ -135,9 +108,48 @@ func metricLabels(extras ...string) []string { return append(append([]string{}, commonLabels...), extras...) } +func parseBucketsString(value string) ([]float64, error) { + valuesStr := strings.Split(value, ",") + buckets := make([]float64, 0, len(valuesStr)) + + for _, str := range valuesStr { + val, err := strconv.ParseFloat(str, 64) + if err != nil { + return nil, err + } + buckets = append(buckets, val) + } + + return buckets, nil +} + +func initGithubWorkflowJobQueueDurationSeconds(buckets []float64) *prometheus.HistogramVec { + return prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Name: "github_workflow_job_queue_duration_seconds", + Help: "Queue times for workflow jobs in seconds", + Buckets: buckets, + }, + metricLabels(), + ) +} + +func initGithubWorkflowJobRunDurationSeconds(buckets []float64) *prometheus.HistogramVec { + return prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Name: "github_workflow_job_run_duration_seconds", + Help: "Run times for workflow jobs in seconds", + Buckets: buckets, + }, + metricLabels("job_conclusion"), + ) +} + var ( - commonLabels = []string{"runs_on", "job_name", "organization", "repository", "repository_full_name", "owner", "workflow_name", "head_branch"} - githubWorkflowJobConclusionsTotal = prometheus.NewCounterVec( + commonLabels = []string{"runs_on", "job_name", "organization", "repository", "repository_full_name", "owner", "workflow_name", "head_branch"} + githubWorkflowJobQueueDurationSeconds *prometheus.HistogramVec + githubWorkflowJobRunDurationSeconds *prometheus.HistogramVec + githubWorkflowJobConclusionsTotal = prometheus.NewCounterVec( prometheus.CounterOpts{ Name: "github_workflow_job_conclusions_total", Help: "Conclusions for tracked workflow jobs",