ACCDT-1238: Parametrize actions metrics histogram buckets size

cmd/actionsmetricsserver/main.go

@@ -48,7 +48,9 @@
 )
 
 const (
-	webhookSecretTokenEnvName = "GITHUB_WEBHOOK_SECRET_TOKEN"
+	webhookSecretTokenEnvName          = "GITHUB_WEBHOOK_SECRET_TOKEN"
+	prometheusRunBucketIntervalsName   = "PROMETHEUS_RUN_BUCKET_INTERVALS"
+	prometheusQueueBucketIntervalsName = "PROMETHEUS_QUEUE_BUCKET_INTERVALS"
 )
 
 func init() {
@@ -73,6 +75,10 @@
 		logFormat string
 
 		ghClient *github.Client
+
+		// List of histogram buckets that we want to see in metrics
+		runBucketsList   actionsmetrics.BucketsSlice
+		queueBucketsList actionsmetrics.BucketsSlice
 	)
 
 	var c github.Config
@@ -83,6 +89,8 @@
 	}
 
 	webhookSecretTokenEnv = os.Getenv(webhookSecretTokenEnvName)
+	runBucketsList.Set(os.Getenv(prometheusRunBucketIntervalsName))
+	queueBucketsList.Set(os.Getenv(prometheusQueueBucketIntervalsName))
 
 	flag.StringVar(&webhookAddr, "webhook-addr", ":8000", "The address the metric endpoint binds to.")
 	flag.StringVar(&metricsAddr, "metrics-addr", ":8080", "The address the metric endpoint binds to.")
@@ -113,6 +121,8 @@
 		webhookSecretToken = webhookSecretTokenEnv
 	}
 
+	actionsmetrics.InitializeMetrics(runBucketsList, queueBucketsList)
+
 	if webhookSecretToken == "" {
 		logger.Info(fmt.Sprintf("-github-webhook-secret-token and %s are missing or empty. Create one following https://docs.github.com/en/developers/webhooks-and-events/securing-your-webhooks and specify it via the flag or the envvar", webhookSecretTokenEnvName))
 	}

pkg/actionsmetrics/event_reader.go

@@ -134,7 +134,7 @@ func (reader *EventReader) ProcessWorkflowJobEvent(ctx context.Context, event in
 			log.Info("reading workflow_job logs")
 		}
 
-		githubWorkflowJobQueueDurationSeconds.With(labels).Observe(parseResult.QueueTime.Seconds())
+		githubWorkflowJobQueueHistogram.With(labels).Observe(parseResult.QueueTime.Seconds())
 
 	case "completed":
 		githubWorkflowJobsCompletedTotal.With(labels).Inc()
@@ -200,7 +200,7 @@ func (reader *EventReader) ProcessWorkflowJobEvent(ctx context.Context, event in
 		}
 
 		if runTimeSeconds != nil {
-			githubWorkflowJobRunDurationSeconds.With(extraLabel("job_conclusion", *e.WorkflowJob.Conclusion, labels)).Observe(*runTimeSeconds)
+			githubWorkflowJobRunHistogram.With(extraLabel("job_conclusion", *e.WorkflowJob.Conclusion, labels)).Observe(*runTimeSeconds)
 		}
 	}
 }

pkg/actionsmetrics/metrics.go

@@ -5,14 +5,48 @@
 package actionsmetrics
 
 import (
+	"fmt"
 	"github.com/prometheus/client_golang/prometheus"
 	"sigs.k8s.io/controller-runtime/pkg/metrics"
+	"strconv"
+	"strings"
 )
 
-func init() {
+type BucketsSlice []float64
+
+func (i *BucketsSlice) String() string {
+	return fmt.Sprintf("%v", *i)
+}
+
+func (i *BucketsSlice) Set(value string) error {
+	valuesStr := strings.Split(value, ",")
+	for _, str := range valuesStr {
+		// Convert the string to float64.
+		val, err := strconv.ParseFloat(str, 64)
+		if err != nil {
+			return err
+		}
+		*i = append(*i, val)
+	}
+	return nil
+}
+
+var githubWorkflowJobQueueHistogram *prometheus.HistogramVec
+var githubWorkflowJobRunHistogram *prometheus.HistogramVec
+
+func initMetrics(runBuckets, queueBuckets []float64) {
+	githubWorkflowJobRunHistogram = githubWorkflowJobRunDurationSeconds(runBuckets)
+	githubWorkflowJobQueueHistogram = githubWorkflowJobQueueDurationSeconds(queueBuckets)
+
+	if len(runBuckets) == 0 {
+		githubWorkflowJobRunHistogram = githubWorkflowJobRunDurationSeconds(DefaultRuntimeBuckets)
+	}
+	if len(queueBuckets) == 0 {
+		githubWorkflowJobQueueHistogram = githubWorkflowJobQueueDurationSeconds(DefaultRuntimeBuckets)
+	}
 	metrics.Registry.MustRegister(
-		githubWorkflowJobQueueDurationSeconds,
-		githubWorkflowJobRunDurationSeconds,
+		githubWorkflowJobQueueHistogram,
+		githubWorkflowJobRunHistogram,
 		githubWorkflowJobConclusionsTotal,
 		githubWorkflowJobsQueuedTotal,
 		githubWorkflowJobsStartedTotal,
@@ -21,8 +55,34 @@ func init() {
 	)
 }
 
+func githubWorkflowJobQueueDurationSeconds(buckets []float64) *prometheus.HistogramVec {
+	return prometheus.NewHistogramVec(
+		prometheus.HistogramOpts{
+			Name:    "github_workflow_job_queue_duration_seconds",
+			Help:    "Queue times for workflow jobs in seconds",
+			Buckets: buckets,
+		},
+		metricLabels(),
+	)
+}
+
+func githubWorkflowJobRunDurationSeconds(buckets []float64) *prometheus.HistogramVec {
+	return prometheus.NewHistogramVec(
+		prometheus.HistogramOpts{
+			Name:    "github_workflow_job_run_duration_seconds",
+			Help:    "Run times for workflow jobs in seconds",
+			Buckets: buckets,
+		},
+		metricLabels("job_conclusion"),
+	)
+}
+
+func InitializeMetrics(runBuckets, queueBuckets []float64) {
+	initMetrics(runBuckets, queueBuckets)
+}
+
 var (
-	runtimeBuckets []float64 = []float64{
+	DefaultRuntimeBuckets = []float64{
 		0.01,
 		0.05,
 		0.1,
@@ -76,23 +136,7 @@ func metricLabels(extras ...string) []string {
 }
 
 var (
-	commonLabels                          = []string{"runs_on", "job_name", "organization", "repository", "repository_full_name", "owner", "workflow_name", "head_branch"}
-	githubWorkflowJobQueueDurationSeconds = prometheus.NewHistogramVec(
-		prometheus.HistogramOpts{
-			Name:    "github_workflow_job_queue_duration_seconds",
-			Help:    "Queue times for workflow jobs in seconds",
-			Buckets: runtimeBuckets,
-		},
-		metricLabels(),
-	)
-	githubWorkflowJobRunDurationSeconds = prometheus.NewHistogramVec(
-		prometheus.HistogramOpts{
-			Name:    "github_workflow_job_run_duration_seconds",
-			Help:    "Run times for workflow jobs in seconds",
-			Buckets: runtimeBuckets,
-		},
-		metricLabels("job_conclusion"),
-	)
+	commonLabels                      = []string{"runs_on", "job_name", "organization", "repository", "repository_full_name", "owner", "workflow_name", "head_branch"}
 	githubWorkflowJobConclusionsTotal = prometheus.NewCounterVec(
 		prometheus.CounterOpts{
 			Name: "github_workflow_job_conclusions_total",