-
Notifications
You must be signed in to change notification settings - Fork 1
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #128 from uselagoon/metrics
feat: add initial lagoon metrics
- Loading branch information
Showing
18 changed files
with
258 additions
and
1 deletion.
There are no files selected for viewing
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
Oops, something went wrong.
File renamed without changes.
File renamed without changes.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,92 @@ | ||
| package v1beta1 | ||
|
|
||
| import ( | ||
| "github.com/prometheus/client_golang/prometheus" | ||
| "github.com/prometheus/client_golang/prometheus/promauto" | ||
| ) | ||
|
|
||
| var ( | ||
| // general counters for builds | ||
| buildsRunningGauge = promauto.NewGauge(prometheus.GaugeOpts{ | ||
| Name: "lagoon_builds_running_current", | ||
| Help: "The total number of Lagoon builds running", | ||
| }) | ||
| buildsPendingGauge = promauto.NewGauge(prometheus.GaugeOpts{ | ||
| Name: "lagoon_builds_pending_current", | ||
| Help: "The total number of Lagoon builds pending", | ||
| }) | ||
| buildsStartedCounter = promauto.NewCounter(prometheus.CounterOpts{ | ||
| Name: "lagoon_builds_started_total", | ||
| Help: "The total number of Lagoon builds started", | ||
| }) | ||
| buildsCompletedCounter = promauto.NewCounter(prometheus.CounterOpts{ | ||
| Name: "lagoon_builds_completed_total", | ||
| Help: "The total number of Lagoon builds completed", | ||
| }) | ||
| buildsFailedCounter = promauto.NewCounter(prometheus.CounterOpts{ | ||
| Name: "lagoon_builds_failed_total", | ||
| Help: "The total number of Lagoon builds failed", | ||
| }) | ||
| buildsCancelledCounter = promauto.NewCounter(prometheus.CounterOpts{ | ||
| Name: "lagoon_builds_cancelled_total", | ||
| Help: "The total number of Lagoon builds cancelled", | ||
| }) | ||
|
|
||
| // general counters for tasks | ||
| tasksRunningGauge = promauto.NewGauge(prometheus.GaugeOpts{ | ||
| Name: "lagoon_tasks_running_current", | ||
| Help: "The total number of Lagoon tasks running", | ||
| }) | ||
| tasksStartedCounter = promauto.NewCounter(prometheus.CounterOpts{ | ||
| Name: "lagoon_tasks_started_total", | ||
| Help: "The total number of Lagoon tasks started", | ||
| }) | ||
| tasksCompletedCounter = promauto.NewCounter(prometheus.CounterOpts{ | ||
| Name: "lagoon_tasks_completed_total", | ||
| Help: "The total number of Lagoon tasks completed", | ||
| }) | ||
| tasksFailedCounter = promauto.NewCounter(prometheus.CounterOpts{ | ||
| Name: "lagoon_tasks_failed_total", | ||
| Help: "The total number of Lagoon tasks failed", | ||
| }) | ||
| tasksCancelledCounter = promauto.NewCounter(prometheus.CounterOpts{ | ||
| Name: "lagoon_tasks_cancelled_total", | ||
| Help: "The total number of Lagoon tasks cancelled", | ||
| }) | ||
|
|
||
| // buildStatus will count the build transisiton steps | ||
| // when the build step changes, the count is removed and the new step metric is created | ||
| // this is useful to gauge how long particular steps take in a build | ||
| buildStatus = promauto.NewGaugeVec(prometheus.GaugeOpts{ | ||
| Name: "lagoon_build_status", | ||
| Help: "The status of running Lagoon builds", | ||
| }, | ||
| []string{ | ||
| "build_name", | ||
| "build_namespace", | ||
| "build_step", | ||
| }, | ||
| ) | ||
|
|
||
| // RunningStatus will count when a build or task is running | ||
| // when the build or task is complete, the count is removed | ||
| // this is useful to gauge how long a build or task runs for | ||
| buildRunningStatus = promauto.NewGaugeVec(prometheus.GaugeOpts{ | ||
| Name: "lagoon_build_running_status", | ||
| Help: "The duration of running Lagoon builds", | ||
| }, | ||
| []string{ | ||
| "build_name", | ||
| "build_namespace", | ||
| }, | ||
| ) | ||
| taskRunningStatus = promauto.NewGaugeVec(prometheus.GaugeOpts{ | ||
| Name: "lagoon_task_running_status", | ||
| Help: "The duration of running Lagoon tasks", | ||
| }, | ||
| []string{ | ||
| "task_name", | ||
| "task_namespace", | ||
| }, | ||
| ) | ||
| ) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,51 @@ | ||
| package v1beta1 | ||
|
|
||
| import ( | ||
| "context" | ||
| "fmt" | ||
|
|
||
| corev1 "k8s.io/api/core/v1" | ||
| "sigs.k8s.io/controller-runtime/pkg/client" | ||
| ) | ||
|
|
||
| func (r *LagoonMonitorReconciler) calculateBuildMetrics(ctx context.Context) error { | ||
| listOption := (&client.ListOptions{}).ApplyOptions([]client.ListOption{ | ||
| client.MatchingLabels(map[string]string{ | ||
| "lagoon.sh/jobType": "build", | ||
| "lagoon.sh/controller": r.ControllerNamespace, | ||
| }), | ||
| }) | ||
| buildPods := &corev1.PodList{} | ||
| if err := r.List(ctx, buildPods, listOption); err != nil { | ||
| return fmt.Errorf("Unable to list builds in the cluster, there may be none or something went wrong: %v", err) | ||
| } | ||
| runningBuilds := float64(0) | ||
| for _, buildPod := range buildPods.Items { | ||
| if buildPod.Status.Phase == corev1.PodRunning { | ||
| runningBuilds = runningBuilds + 1 | ||
| } | ||
| } | ||
| buildsRunningGauge.Set(runningBuilds) | ||
| return nil | ||
| } | ||
|
|
||
| func (r *LagoonMonitorReconciler) calculateTaskMetrics(ctx context.Context) error { | ||
| listOption := (&client.ListOptions{}).ApplyOptions([]client.ListOption{ | ||
| client.MatchingLabels(map[string]string{ | ||
| "lagoon.sh/jobType": "task", | ||
| "lagoon.sh/controller": r.ControllerNamespace, | ||
| }), | ||
| }) | ||
| taskPods := &corev1.PodList{} | ||
| if err := r.List(ctx, taskPods, listOption); err != nil { | ||
| return fmt.Errorf("Unable to list tasks in the cluster, there may be none or something went wrong: %v", err) | ||
| } | ||
| runningTasks := float64(0) | ||
| for _, taskPod := range taskPods.Items { | ||
| if taskPod.Status.Phase == corev1.PodRunning { | ||
| runningTasks = runningTasks + 1 | ||
| } | ||
| } | ||
| tasksRunningGauge.Set(runningTasks) | ||
| return nil | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,30 @@ | ||
| package metrics | ||
|
|
||
| import ( | ||
| "fmt" | ||
| "net/http" | ||
| "time" | ||
|
|
||
| "github.com/go-logr/logr" | ||
| "github.com/prometheus/client_golang/prometheus/promhttp" | ||
| ) | ||
|
|
||
| // NewServer returns a *http.Server serving prometheus metrics in a new | ||
| // goroutine. | ||
| // Caller should defer Shutdown() for cleanup. | ||
| func NewServer(log logr.Logger, addr string) *http.Server { | ||
| mux := http.NewServeMux() | ||
| mux.Handle("/metrics", promhttp.Handler()) | ||
| s := http.Server{ | ||
| Addr: addr, | ||
| Handler: mux, | ||
| ReadTimeout: 16 * time.Second, | ||
| WriteTimeout: 16 * time.Second, | ||
| } | ||
| go func() { | ||
| if err := s.ListenAndServe(); err != http.ErrServerClosed { | ||
| log.Error(fmt.Errorf("metrics server did not shut down cleanly"), err.Error()) | ||
| } | ||
| }() | ||
| return &s | ||
| } |
Oops, something went wrong.