Skip to content
This repository has been archived by the owner on Jun 20, 2024. It is now read-only.

Commit

Permalink
feat: per HTTP handler metrics
Browse files Browse the repository at this point in the history
This adds generic HTTP metrics per handler/method/code:
- ipfs_http_requests_inflight (gauge)
  - partially addresses #112 (outgoing tracked in caboose)
- ipfs_http_requests_total (counter)
- ipfs_http_request_duration_seconds (summary)
  - partially addresses #120
- ipfs_http_request_size_bytes (summary)
- ipfs_http_response_size_bytes (summary)
  • Loading branch information
lidel committed May 25, 2023
1 parent 6d54974 commit b86a827
Show file tree
Hide file tree
Showing 5 changed files with 110 additions and 47 deletions.
10 changes: 5 additions & 5 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ require (
github.com/mitchellh/go-server-timing v1.0.1
github.com/multiformats/go-multicodec v0.8.1
github.com/multiformats/go-multihash v0.2.1
github.com/prometheus/client_golang v1.14.0
github.com/prometheus/client_golang v1.15.1
github.com/rs/dnscache v0.0.0-20211102005908-e0241e321417
github.com/spf13/cobra v1.6.1
github.com/stretchr/testify v1.8.2
Expand All @@ -36,7 +36,7 @@ require (
)

require (
github.com/alecthomas/units v0.0.0-20210927113745-59d0afb8317a // indirect
github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect
github.com/asecurityteam/rolling v0.0.0-20230418204413-b4052899307d // indirect
github.com/benbjohnson/clock v1.3.0 // indirect
github.com/beorn7/perks v1.0.1 // indirect
Expand All @@ -61,7 +61,7 @@ require (
github.com/godbus/dbus/v5 v5.1.0 // indirect
github.com/golang/gddo v0.0.0-20180823221919-9d8ff1c67be5 // indirect
github.com/golang/mock v1.6.0 // indirect
github.com/golang/protobuf v1.5.2 // indirect
github.com/golang/protobuf v1.5.3 // indirect
github.com/google/gopacket v1.1.19 // indirect
github.com/google/pprof v0.0.0-20221203041831-ce31453925ec // indirect
github.com/google/uuid v1.3.0 // indirect
Expand Down Expand Up @@ -127,7 +127,7 @@ require (
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/polydawn/refmt v0.89.0 // indirect
github.com/prometheus/client_model v0.3.0 // indirect
github.com/prometheus/common v0.39.0 // indirect
github.com/prometheus/common v0.42.0 // indirect
github.com/prometheus/procfs v0.9.0 // indirect
github.com/quic-go/qpack v0.4.0 // indirect
github.com/quic-go/qtls-go1-19 v0.2.1 // indirect
Expand Down Expand Up @@ -171,7 +171,7 @@ require (
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f // indirect
google.golang.org/grpc v1.53.0 // indirect
google.golang.org/protobuf v1.28.1 // indirect
google.golang.org/protobuf v1.30.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
lukechampine.com/blake3 v1.1.7 // indirect
nhooyr.io/websocket v1.8.7 // indirect
Expand Down
19 changes: 10 additions & 9 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -41,8 +41,8 @@ git.apache.org/thrift.git v0.0.0-20180902110319-2566ecd5d999/go.mod h1:fPE2ZNJGy
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
github.com/alecthomas/units v0.0.0-20210927113745-59d0afb8317a h1:E/8AP5dFtMhl5KPJz66Kt9G0n+7Sn41Fy1wv9/jHOrc=
github.com/alecthomas/units v0.0.0-20210927113745-59d0afb8317a/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE=
github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 h1:s6gZFSlWYmbqAuRjVTiNNhvNRfY2Wxp9nhfyel4rklc=
github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE=
github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c=
github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
github.com/asecurityteam/rolling v0.0.0-20230418204413-b4052899307d h1:OD9AM8JZUHPrkEa9/SgQW2cCX6Om8d6n0akPS5leWJA=
Expand Down Expand Up @@ -199,8 +199,9 @@ github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QD
github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw=
github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=
github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
Expand Down Expand Up @@ -513,15 +514,15 @@ github.com/polydawn/refmt v0.0.0-20190807091052-3d65705ee9f1/go.mod h1:uIp+gprXx
github.com/polydawn/refmt v0.89.0 h1:ADJTApkvkeBZsN0tBTx8QjpD9JkmxbKp0cxfr9qszm4=
github.com/polydawn/refmt v0.89.0/go.mod h1:/zvteZs/GwLtCgZ4BL6CBsk9IKIlexP43ObX9AxTqTw=
github.com/prometheus/client_golang v0.8.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
github.com/prometheus/client_golang v1.14.0 h1:nJdhIvne2eSX/XRAFV9PcvFFRbrjbcTUj0VP62TMhnw=
github.com/prometheus/client_golang v1.14.0/go.mod h1:8vpkKitgIVNcqrRBWh1C4TIUQgYNtG/XQE4E/Zae36Y=
github.com/prometheus/client_golang v1.15.1 h1:8tXpTmJbyH5lydzFPoxSIJ0J46jdh3tylbvM1xCv0LI=
github.com/prometheus/client_golang v1.15.1/go.mod h1:e9yaBhRPU2pPNsZwE+JdQl0KEt1N9XgF6zxWmaC0xOk=
github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4=
github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w=
github.com/prometheus/common v0.0.0-20180801064454-c7de2306084e/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
github.com/prometheus/common v0.39.0 h1:oOyhkDq05hPZKItWVBkJ6g6AtGxi+fy7F4JvUV8uhsI=
github.com/prometheus/common v0.39.0/go.mod h1:6XBZ7lYdLCbkAVhwRsWTZn+IN5AB9F/NXd5w0BbEX0Y=
github.com/prometheus/common v0.42.0 h1:EKsfXEYo4JpWMHH5cg+KOUWeuJSov1Id8zGR8eeI1YM=
github.com/prometheus/common v0.42.0/go.mod h1:xBwqVerjNdUDjgODMpudtOMwlOwf2SaTr1yjz4b7Zbc=
github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
github.com/prometheus/procfs v0.9.0 h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJfhI=
github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY=
Expand Down Expand Up @@ -1038,8 +1039,8 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba
google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w=
google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng=
google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
Expand Down
29 changes: 10 additions & 19 deletions handlers.go
Original file line number Diff line number Diff line change
Expand Up @@ -117,10 +117,17 @@ func makeGatewayHandler(bs bstore.Blockstore, kuboRPC []string, port int, blockC
}

gwHandler := gateway.NewHandler(gwConf, gwAPI)
ipfsHandler := withHTTPMetrics(gwHandler, "ipfs")
ipnsHandler := withHTTPMetrics(gwHandler, "ipns")

mux := http.NewServeMux()
mux.Handle("/ipfs/", gwHandler)
mux.Handle("/ipns/", gwHandler)
mux.Handle("/api/v0/", newKuboRPCHandler(kuboRPC))
mux.Handle("/ipfs/", ipfsHandler)
mux.Handle("/ipns/", ipnsHandler)

// TODO: below is legacy which we want to remove, measuring this separately
// allows us to decide when is the time to do it.
legacyKuboRpcHandler := withHTTPMetrics(newKuboRPCHandler(kuboRPC), "legacyKuboRpc")
mux.Handle("/api/v0/", legacyKuboRpcHandler)

// Note: in the future we may want to make this more configurable.
noDNSLink := false
Expand All @@ -146,26 +153,10 @@ func makeGatewayHandler(bs bstore.Blockstore, kuboRPC []string, port int, blockC
}
}

// Creates metrics handler for total response size. Matches the same metrics
// from Kubo:
// https://github.com/ipfs/kubo/blob/e550d9e4761ea394357c413c02ade142c0dea88c/core/corehttp/metrics.go#L79-L152
sum := prometheus.NewSummaryVec(prometheus.SummaryOpts{
Namespace: "ipfs",
Subsystem: "http",
Name: "response_size_bytes",
Help: "The HTTP response sizes in bytes.",
Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},
}, []string{"code"})
err = prometheus.Register(sum)
if err != nil {
return nil, err
}

// Construct the HTTP handler for the gateway.
handler := withConnect(mux)
handler = http.Handler(gateway.WithHostname(handler, gwAPI, publicGateways, noDNSLink))
handler = servertiming.Middleware(handler, nil)
handler = promhttp.InstrumentHandlerResponseSize(sum, handler)

// Add logging.
handler = withRequestLogger(handler)
Expand Down
85 changes: 85 additions & 0 deletions metrics.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,85 @@
package main

import (
"net/http"

"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promhttp"
)

func registerVersionMetric(version string) {
m := prometheus.NewGauge(prometheus.GaugeOpts{
Namespace: "ipfs",
Subsystem: "bifrost_gateway",
Name: "info",
Help: "Information about bifrost-gateway instance.",
ConstLabels: prometheus.Labels{"version": version},
})
prometheus.MustRegister(m)
m.Set(1)
}

// withHTTPMetrics collects metrics around HTTP request/response count, duration, and size
// per specific handler. Allows us to track them separately for /ipns and /ipfs.
func withHTTPMetrics(handler http.Handler, handlerName string) http.Handler {

// TODO: move HTTP metrics below to separate file
// HTTP metric template names match Kubo:
// https://github.com/ipfs/kubo/blob/e550d9e4761ea394357c413c02ade142c0dea88c/core/corehttp/metrics.go#L79-L152
// This allows Kubo users to migrate to bifrost-gateway and compare global totals.
opts := prometheus.SummaryOpts{
Namespace: "ipfs",
Subsystem: "http",
Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},
ConstLabels: prometheus.Labels{"handler": handlerName},
}
// Dynamic labels 'method or 'code' are auto-filled
// by https://pkg.go.dev/github.com/prometheus/client_golang/prometheus/promhttp#InstrumentHandlerResponseSize
labels := []string{"method", "code"}

reqWip := prometheus.NewGauge(
prometheus.GaugeOpts{
Namespace: opts.Namespace,
Subsystem: opts.Subsystem,
Name: "requests_inflight",
Help: "Tracks the number of HTTP client requests currently in progress.",
ConstLabels: opts.ConstLabels,
},
)
prometheus.MustRegister(reqWip)

reqCnt := prometheus.NewCounterVec(
prometheus.CounterOpts{
Namespace: opts.Namespace,
Subsystem: opts.Subsystem,
Name: "requests_total",
Help: "Total number of HTTP requests made.",
ConstLabels: opts.ConstLabels,
},
labels,
)
prometheus.MustRegister(reqCnt)

opts.Name = "request_duration_seconds"
opts.Help = "The HTTP request latencies in seconds."
reqDur := prometheus.NewSummaryVec(opts, labels)
prometheus.MustRegister(reqDur)

opts.Name = "request_size_bytes"
opts.Help = "The HTTP request sizes in bytes."
reqSz := prometheus.NewSummaryVec(opts, labels)
prometheus.MustRegister(reqSz)

opts.Name = "response_size_bytes"
opts.Help = "The HTTP response sizes in bytes."
resSz := prometheus.NewSummaryVec(opts, labels)
prometheus.MustRegister(resSz)

handler = promhttp.InstrumentHandlerInFlight(reqWip, handler)
handler = promhttp.InstrumentHandlerCounter(reqCnt, handler)
handler = promhttp.InstrumentHandlerDuration(reqDur, handler)
handler = promhttp.InstrumentHandlerRequestSize(reqSz, handler)
handler = promhttp.InstrumentHandlerResponseSize(resSz, handler)

return handler
}
14 changes: 0 additions & 14 deletions version.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,6 @@ package main
import (
"runtime/debug"
"time"

"github.com/prometheus/client_golang/prometheus"
)

var name = "bifrost-gateway"
Expand Down Expand Up @@ -39,15 +37,3 @@ func buildVersion() string {
}
return "dev-build"
}

func registerVersionMetric(version string) {
m := prometheus.NewGauge(prometheus.GaugeOpts{
Namespace: "ipfs",
Subsystem: "bifrost_gateway",
Name: "info",
Help: "Information about bifrost-gateway instance.",
ConstLabels: prometheus.Labels{"version": version},
})
prometheus.MustRegister(m)
m.Set(1)
}

0 comments on commit b86a827

Please sign in to comment.