Skip to content

Commit

Permalink
move all metrics init logic out of main
Browse files Browse the repository at this point in the history
  • Loading branch information
sindunuragarp committed Nov 13, 2023
1 parent 3219cfe commit bf8b5f3
Show file tree
Hide file tree
Showing 3 changed files with 73 additions and 71 deletions.
12 changes: 1 addition & 11 deletions cmd/actionsmetricsserver/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -48,9 +48,7 @@ var (
)

const (
webhookSecretTokenEnvName = "GITHUB_WEBHOOK_SECRET_TOKEN"
prometheusRunBucketIntervalsName = "PROMETHEUS_RUN_BUCKET_INTERVALS"
prometheusQueueBucketIntervalsName = "PROMETHEUS_QUEUE_BUCKET_INTERVALS"
webhookSecretTokenEnvName = "GITHUB_WEBHOOK_SECRET_TOKEN"
)

func init() {
Expand All @@ -75,10 +73,6 @@ func main() {
logFormat string

ghClient *github.Client

// List of histogram buckets that we want to see in metrics
runBucketsList actionsmetrics.BucketsSlice
queueBucketsList actionsmetrics.BucketsSlice
)

var c github.Config
Expand All @@ -89,8 +83,6 @@ func main() {
}

webhookSecretTokenEnv = os.Getenv(webhookSecretTokenEnvName)
runBucketsList.Set(os.Getenv(prometheusRunBucketIntervalsName))
queueBucketsList.Set(os.Getenv(prometheusQueueBucketIntervalsName))

flag.StringVar(&webhookAddr, "webhook-addr", ":8000", "The address the metric endpoint binds to.")
flag.StringVar(&metricsAddr, "metrics-addr", ":8080", "The address the metric endpoint binds to.")
Expand Down Expand Up @@ -121,8 +113,6 @@ func main() {
webhookSecretToken = webhookSecretTokenEnv
}

actionsmetrics.InitializeMetrics(runBucketsList, queueBucketsList)

if webhookSecretToken == "" {
logger.Info(fmt.Sprintf("-github-webhook-secret-token and %s are missing or empty. Create one following https://docs.github.com/en/developers/webhooks-and-events/securing-your-webhooks and specify it via the flag or the envvar", webhookSecretTokenEnvName))
}
Expand Down
4 changes: 2 additions & 2 deletions pkg/actionsmetrics/event_reader.go
Original file line number Diff line number Diff line change
Expand Up @@ -134,7 +134,7 @@ func (reader *EventReader) ProcessWorkflowJobEvent(ctx context.Context, event in
log.Info("reading workflow_job logs")
}

githubWorkflowJobQueueHistogram.With(labels).Observe(parseResult.QueueTime.Seconds())
githubWorkflowJobQueueDurationSeconds.With(labels).Observe(parseResult.QueueTime.Seconds())

case "completed":
githubWorkflowJobsCompletedTotal.With(labels).Inc()
Expand Down Expand Up @@ -200,7 +200,7 @@ func (reader *EventReader) ProcessWorkflowJobEvent(ctx context.Context, event in
}

if runTimeSeconds != nil {
githubWorkflowJobRunHistogram.With(extraLabel("job_conclusion", *e.WorkflowJob.Conclusion, labels)).Observe(*runTimeSeconds)
githubWorkflowJobRunDurationSeconds.With(extraLabel("job_conclusion", *e.WorkflowJob.Conclusion, labels)).Observe(*runTimeSeconds)
}
}
}
Expand Down
128 changes: 70 additions & 58 deletions pkg/actionsmetrics/metrics.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,47 +6,46 @@ package actionsmetrics

import (
"fmt"
"github.com/prometheus/client_golang/prometheus"
"sigs.k8s.io/controller-runtime/pkg/metrics"
"os"
"strconv"
"strings"
)

type BucketsSlice []float64
"github.com/prometheus/client_golang/prometheus"
"sigs.k8s.io/controller-runtime/pkg/metrics"
)

func (i *BucketsSlice) String() string {
return fmt.Sprintf("%v", *i)
}
const (
prometheusRunBucketIntervalsName = "PROMETHEUS_RUN_BUCKET_INTERVALS"
prometheusQueueBucketIntervalsName = "PROMETHEUS_QUEUE_BUCKET_INTERVALS"
)

func (i *BucketsSlice) Set(value string) error {
valuesStr := strings.Split(value, ",")
for _, str := range valuesStr {
// Convert the string to float64.
val, err := strconv.ParseFloat(str, 64)
func init() {
queueBuckets := defaultRuntimeBuckets
if _, ok := os.LookupEnv(prometheusQueueBucketIntervalsName); ok {
buckets, err := parseBucketsString(os.Getenv(prometheusQueueBucketIntervalsName))
if err != nil {
return err
fmt.Fprintf(os.Stderr, "Warning: Failed to parse %s, using default buckets: %s\n", prometheusQueueBucketIntervalsName, err)
} else {
queueBuckets = buckets
}
*i = append(*i, val)
}
return nil
}

var githubWorkflowJobQueueHistogram *prometheus.HistogramVec
var githubWorkflowJobRunHistogram *prometheus.HistogramVec
runBuckets := defaultRuntimeBuckets
if _, ok := os.LookupEnv(prometheusRunBucketIntervalsName); ok {
buckets, err := parseBucketsString(os.Getenv(prometheusRunBucketIntervalsName))
if err != nil {
fmt.Fprintf(os.Stderr, "Warning: Failed to parse %s, using default buckets: %s\n", prometheusRunBucketIntervalsName, err)
} else {
runBuckets = buckets
}
}

func initMetrics(runBuckets, queueBuckets []float64) {
githubWorkflowJobRunHistogram = githubWorkflowJobRunDurationSeconds(runBuckets)
githubWorkflowJobQueueHistogram = githubWorkflowJobQueueDurationSeconds(queueBuckets)
githubWorkflowJobQueueDurationSeconds = initGithubWorkflowJobQueueDurationSeconds(queueBuckets)
githubWorkflowJobRunDurationSeconds = initGithubWorkflowJobRunDurationSeconds(runBuckets)

if len(runBuckets) == 0 {
githubWorkflowJobRunHistogram = githubWorkflowJobRunDurationSeconds(DefaultRuntimeBuckets)
}
if len(queueBuckets) == 0 {
githubWorkflowJobQueueHistogram = githubWorkflowJobQueueDurationSeconds(DefaultRuntimeBuckets)
}
metrics.Registry.MustRegister(
githubWorkflowJobQueueHistogram,
githubWorkflowJobRunHistogram,
githubWorkflowJobQueueDurationSeconds,
githubWorkflowJobRunDurationSeconds,
githubWorkflowJobConclusionsTotal,
githubWorkflowJobsQueuedTotal,
githubWorkflowJobsStartedTotal,
Expand All @@ -55,34 +54,8 @@ func initMetrics(runBuckets, queueBuckets []float64) {
)
}

func githubWorkflowJobQueueDurationSeconds(buckets []float64) *prometheus.HistogramVec {
return prometheus.NewHistogramVec(
prometheus.HistogramOpts{
Name: "github_workflow_job_queue_duration_seconds",
Help: "Queue times for workflow jobs in seconds",
Buckets: buckets,
},
metricLabels(),
)
}

func githubWorkflowJobRunDurationSeconds(buckets []float64) *prometheus.HistogramVec {
return prometheus.NewHistogramVec(
prometheus.HistogramOpts{
Name: "github_workflow_job_run_duration_seconds",
Help: "Run times for workflow jobs in seconds",
Buckets: buckets,
},
metricLabels("job_conclusion"),
)
}

func InitializeMetrics(runBuckets, queueBuckets []float64) {
initMetrics(runBuckets, queueBuckets)
}

var (
DefaultRuntimeBuckets = []float64{
defaultRuntimeBuckets = []float64{
0.01,
0.05,
0.1,
Expand Down Expand Up @@ -135,9 +108,48 @@ func metricLabels(extras ...string) []string {
return append(append([]string{}, commonLabels...), extras...)
}

func parseBucketsString(value string) ([]float64, error) {
valuesStr := strings.Split(value, ",")
buckets := make([]float64, 0, len(valuesStr))

for _, str := range valuesStr {
val, err := strconv.ParseFloat(str, 64)
if err != nil {
return nil, err
}
buckets = append(buckets, val)
}

return buckets, nil
}

func initGithubWorkflowJobQueueDurationSeconds(buckets []float64) *prometheus.HistogramVec {
return prometheus.NewHistogramVec(
prometheus.HistogramOpts{
Name: "github_workflow_job_queue_duration_seconds",
Help: "Queue times for workflow jobs in seconds",
Buckets: buckets,
},
metricLabels(),
)
}

func initGithubWorkflowJobRunDurationSeconds(buckets []float64) *prometheus.HistogramVec {
return prometheus.NewHistogramVec(
prometheus.HistogramOpts{
Name: "github_workflow_job_run_duration_seconds",
Help: "Run times for workflow jobs in seconds",
Buckets: buckets,
},
metricLabels("job_conclusion"),
)
}

var (
commonLabels = []string{"runs_on", "job_name", "organization", "repository", "repository_full_name", "owner", "workflow_name", "head_branch"}
githubWorkflowJobConclusionsTotal = prometheus.NewCounterVec(
commonLabels = []string{"runs_on", "job_name", "organization", "repository", "repository_full_name", "owner", "workflow_name", "head_branch"}
githubWorkflowJobQueueDurationSeconds *prometheus.HistogramVec
githubWorkflowJobRunDurationSeconds *prometheus.HistogramVec
githubWorkflowJobConclusionsTotal = prometheus.NewCounterVec(
prometheus.CounterOpts{
Name: "github_workflow_job_conclusions_total",
Help: "Conclusions for tracked workflow jobs",
Expand Down

0 comments on commit bf8b5f3

Please sign in to comment.