Skip to content

Commit

Permalink
Enable waku prometheus metrics
Browse files Browse the repository at this point in the history
  • Loading branch information
@snormore
snormore committed Sep 28, 2023
1 parent 456eaf1 commit 9b6ab70
Show file tree
Hide file tree
Showing 11 changed files with 225 additions and 351 deletions.
2 changes: 0 additions & 2 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,6 @@ require (
github.com/xmtp/go-msgio v0.2.1-0.20220510223757-25a701b79cd3
github.com/xmtp/proto/v3 v3.27.0
github.com/yoheimuta/protolint v0.39.0
go.opencensus.io v0.24.0
go.uber.org/zap v1.24.0
golang.org/x/sync v0.3.0
google.golang.org/grpc v1.53.0
Expand Down Expand Up @@ -76,7 +75,6 @@ require (
github.com/godbus/dbus/v5 v5.1.0 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang/glog v1.0.0 // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/golang/protobuf v1.5.3 // indirect
github.com/golang/snappy v0.0.4 // indirect
github.com/google/gopacket v1.1.19 // indirect
Expand Down
13 changes: 0 additions & 13 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -357,8 +357,6 @@ github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4er
github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE=
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E=
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
Expand Down Expand Up @@ -405,7 +403,6 @@ github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE=
Expand Down Expand Up @@ -433,7 +430,6 @@ github.com/google/pprof v0.0.0-20230705174524-200ffdc848b8 h1:n6vlPhxsA+BW/XsS5+
github.com/google/pprof v0.0.0-20230705174524-200ffdc848b8/go.mod h1:Jh3hGz2jkYak8qXPD19ryItVnUgpgeqzdkY/D0EaeuA=
github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/uuid v1.1.5/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
Expand Down Expand Up @@ -1041,9 +1037,7 @@ github.com/status-im/keycard-go v0.0.0-20190316090335-8537d3370df4/go.mod h1:RZL
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c=
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
github.com/stretchr/testify v0.0.0-20151208002404-e3a8ff8ce365/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.2.0/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
Expand All @@ -1052,10 +1046,7 @@ github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81P
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals=
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
github.com/swaggest/swgui v1.6.2 h1:DR+ioYt11YrXMaEmLcgaOEFSZ/8QW30uYYE/Ck41cPA=
Expand Down Expand Up @@ -1171,8 +1162,6 @@ go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0=
go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo=
go.opentelemetry.io/otel v0.11.0/go.mod h1:G8UCk+KooF2HLkgo8RHX9epABH/aRGYET7gQOqBVdB0=
go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
Expand Down Expand Up @@ -1329,7 +1318,6 @@ golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc/go.mod h1:/O7V0waA8r7cgGh81R
golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
golang.org/x/net v0.0.0-20200904194848-62affa334b73/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20210220033124-5f55cee0dc0d/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
Expand Down Expand Up @@ -1627,7 +1615,6 @@ google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8
google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60=
google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk=
google.golang.org/grpc v1.32.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc=
google.golang.org/grpc v1.53.0 h1:LAv2ds7cmFV/XTS3XG1NneeENYrXGmorPxsBbptIjNc=
google.golang.org/grpc v1.53.0/go.mod h1:OnIrk0ipVdj4N5d9IUoFUx72/VlD7+jUsHwZgwSMQpw=
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
Expand Down
98 changes: 36 additions & 62 deletions pkg/e2e/metrics.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,94 +4,68 @@ import (
"context"
"time"

"github.com/prometheus/client_golang/prometheus"
"github.com/xmtp/xmtp-node-go/pkg/metrics"
"go.opencensus.io/stats"
"go.opencensus.io/stats/view"
metricstag "go.opencensus.io/tag"
"go.uber.org/zap"
)

var (
successfulRuns = stats.Int64("successful_runs", "Number of successful runs", stats.UnitDimensionless)
failedRuns = stats.Int64("failed_runs", "Number of failed runs", stats.UnitDimensionless)
runDurationSeconds = stats.Float64("run_duration_seconds", "Duration of the run in seconds", stats.UnitSeconds)
testNameTagKey = "test"
testStatusTagKey = "status"

testNameTagKey = metricstag.MustNewKey("test")
testStatusTagKey = metricstag.MustNewKey("status")

views = []*view.View{
{
Name: "xmtpd_e2e_successful_runs",
Measure: successfulRuns,
Description: "Number of successful runs",
Aggregation: view.Count(),
TagKeys: []metricstag.Key{testNameTagKey},
successfulRuns = prometheus.NewCounterVec(
prometheus.CounterOpts{
Name: "xmtpd_e2e_successful_runs",
Help: "Number of successful runs",
},
{
Name: "xmtpd_e2e_failed_runs",
Measure: failedRuns,
Description: "Number of failed runs",
Aggregation: view.Count(),
TagKeys: []metricstag.Key{testNameTagKey},
[]string{testNameTagKey},
)
failedRuns = prometheus.NewCounterVec(
prometheus.CounterOpts{
Name: "xmtpd_e2e_failed_runs",
Help: "Number of failed runs",
},
{
Name: "xmtpd_e2e_run_duration_seconds",
Measure: runDurationSeconds,
Description: "Duration of the run in seconds",
Aggregation: view.Distribution(append(floatRange(30), 40, 50, 60, 90, 120, 300)...),
TagKeys: []metricstag.Key{testNameTagKey, testStatusTagKey},
[]string{testNameTagKey},
)
runDuration = prometheus.NewHistogramVec(
prometheus.HistogramOpts{
Name: "xmtpd_e2e_run_duration_seconds",
Help: "Duration of the run in seconds",
Buckets: append(floatRange(30), 40, 50, 60, 90, 120, 300),
},
}
[]string{testNameTagKey, testStatusTagKey},
)
)

func (r *Runner) withMetricsServer(fn func() error) error {
metrics := metrics.NewMetricsServer("0.0.0.0", 8008, r.log)
metrics.Start(context.Background())
metrics, err := metrics.NewMetricsServer(context.Background(), "0.0.0.0", 8008, r.log, r.prom)
if err != nil {
return err
}
defer func() {
err := metrics.Stop(r.ctx)
err := metrics.Close()
if err != nil {
r.log.Error("stopping metrics server", zap.Error(err))
}
}()

err := view.Register(views...)
if err != nil {
return err
}
r.prom.MustRegister(successfulRuns)
r.prom.MustRegister(failedRuns)
r.prom.MustRegister(runDuration)

return fn()
}

func recordSuccessfulRun(ctx context.Context, tags ...tag) error {
return recordWithTags(ctx, tags, successfulRuns.M(1))
func recordSuccessfulRun(ctx context.Context, testName string) {
successfulRuns.WithLabelValues(testName).Inc()
}

func recordFailedRun(ctx context.Context, tags ...tag) error {
return recordWithTags(ctx, tags, failedRuns.M(1))
func recordFailedRun(ctx context.Context, testName string) {
failedRuns.WithLabelValues(testName).Inc()
}

func recordRunDuration(ctx context.Context, duration time.Duration, tags ...tag) error {
return recordWithTags(ctx, tags, runDurationSeconds.M(duration.Seconds()))
}

type tag struct {
key metricstag.Key
value string
}

func newTag(key metricstag.Key, value string) tag {
return tag{
key: key,
value: value,
}
}

func recordWithTags(ctx context.Context, tags []tag, ms ...stats.Measurement) error {
mutators := make([]metricstag.Mutator, len(tags))
for i, tag := range tags {
mutators[i] = metricstag.Upsert(tag.key, tag.value)
}
return stats.RecordWithTags(ctx, mutators, ms...)
func recordRunDuration(ctx context.Context, duration time.Duration, testName, testStatus string) {
runDuration.WithLabelValues(testName, testStatus).Observe(duration.Seconds())
}

func floatRange(n int) []float64 {
Expand Down
32 changes: 7 additions & 25 deletions pkg/e2e/runner.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (
"net/http"
"time"

"github.com/prometheus/client_golang/prometheus"
"go.uber.org/zap"
"golang.org/x/sync/errgroup"

Expand All @@ -15,6 +16,7 @@ type Runner struct {
ctx context.Context
log *zap.Logger
config *Config
prom *prometheus.Registry
suite *Suite
}

Expand All @@ -23,6 +25,7 @@ func NewRunner(ctx context.Context, log *zap.Logger, config *Config) *Runner {
ctx: ctx,
log: log,
config: config,
prom: prometheus.NewRegistry(),
suite: NewSuite(ctx, log, config),
}
}
Expand Down Expand Up @@ -61,7 +64,6 @@ func (r *Runner) Start() error {
}

func (r *Runner) runTest(test *Test) error {
nameTag := newTag(testNameTagKey, test.Name)
started := time.Now().UTC()
log := r.log.With(zap.String("test", test.Name))

Expand All @@ -70,35 +72,15 @@ func (r *Runner) runTest(test *Test) error {
duration := ended.Sub(started)
log = log.With(zap.Duration("duration", duration))
if err != nil {
recordErr := recordFailedRun(r.ctx, nameTag)
if recordErr != nil {
log.Error("recording failed run metric", zap.Error(recordErr))
}
recordFailedRun(r.ctx, test.Name)
log.Error("test failed", zap.Error(err))

statusTag := newTag(testStatusTagKey, "failed")
err = recordRunDuration(r.ctx, duration, nameTag, statusTag)
if err != nil {
log.Error("recording run duration", zap.Error(err))
return err
}

recordRunDuration(r.ctx, duration, test.Name, "failed")
return err
}
log.Info("test passed")

err = recordSuccessfulRun(r.ctx, nameTag)
if err != nil {
log.Error("recording successful run metric", zap.Error(err))
return err
}

statusTag := newTag(testStatusTagKey, "success")
err = recordRunDuration(r.ctx, duration, nameTag, statusTag)
if err != nil {
log.Error("recording run duration", zap.Error(err))
return err
}
recordSuccessfulRun(r.ctx, test.Name)
recordRunDuration(r.ctx, duration, test.Name, "success")

return nil
}
55 changes: 20 additions & 35 deletions pkg/metrics/api-limits.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,46 +3,31 @@ package metrics
import (
"context"

"go.opencensus.io/stats"
"go.opencensus.io/stats/view"
"go.opencensus.io/tag"
"go.uber.org/zap"
"github.com/prometheus/client_golang/prometheus"
)

var bucketsNameKey = newTagKey("name")
var bucketsNameKey = "name"

var ratelimiterBucketsGaugeMeasure = stats.Int64("ratelimiter_buckets", "size of ratelimiter buckets map", stats.UnitDimensionless)
var ratelimiterBucketsGaugeView = &view.View{
Name: "xmtp_ratelimiter_buckets",
Measure: ratelimiterBucketsGaugeMeasure,
Description: "Size of rate-limiter buckets maps",
Aggregation: view.LastValue(),
TagKeys: []tag.Key{bucketsNameKey},
}
var ratelimiterBuckets = prometheus.NewGaugeVec(
prometheus.GaugeOpts{
Name: "xmtp_ratelimiter_buckets",
Help: "Size of rate-limiter buckets maps",
},
[]string{bucketsNameKey},
)

func EmitRatelimiterBucketsSize(ctx context.Context, log *zap.Logger, name string, size int) {
err := recordWithTags(ctx, []tag.Mutator{tag.Insert(bucketsNameKey, name)}, ratelimiterBucketsGaugeMeasure.M(int64(size)))
if err != nil {
log.Warn("recording metric",
zap.String("metric", ratelimiterBucketsGaugeMeasure.Name()),
zap.Error(err))
}
func EmitRatelimiterBucketsSize(ctx context.Context, name string, size int) {
ratelimiterBuckets.WithLabelValues(name).Set(float64(size))
}

var ratelimiterBucketsDeletedCounterMeasure = stats.Int64("xmtp_ratelimiter_entries_deleted", "Count of deleted entries from ratelimiter buckets map", stats.UnitDimensionless)
var ratelimiterBucketsDeletedCounterView = &view.View{
Name: "xmtp_ratelimiter_entries_deleted",
Measure: ratelimiterBucketsDeletedCounterMeasure,
Description: "Count of deleted entries from rate-limiter buckets maps",
Aggregation: view.Count(),
TagKeys: []tag.Key{bucketsNameKey},
}
var ratelimiterBucketsDeleted = prometheus.NewGaugeVec(
prometheus.GaugeOpts{
Name: "xmtp_ratelimiter_entries_deleted",
Help: "Count of deleted entries from rate-limiter buckets maps",
},
[]string{bucketsNameKey},
)

func EmitRatelimiterDeletedEntries(ctx context.Context, log *zap.Logger, name string, count int) {
err := recordWithTags(ctx, []tag.Mutator{tag.Insert(bucketsNameKey, name)}, ratelimiterBucketsDeletedCounterMeasure.M(int64(count)))
if err != nil {
log.Warn("recording metric",
zap.String("metric", ratelimiterBucketsDeletedCounterMeasure.Name()),
zap.Error(err))
}
func EmitRatelimiterDeletedEntries(ctx context.Context, name string, count int) {
ratelimiterBucketsDeleted.WithLabelValues(name).Set(float64(count))
}
Loading

0 comments on commit 9b6ab70

Please sign in to comment.