Skip to content

Commit

Permalink
Allow filtering of exported metrics (#40)
Browse files Browse the repository at this point in the history 
* Allow filtering of exported metrics

* Fix tests for new constructor parameters
  • Loading branch information
@peterbourgon
peterbourgon authored Jan 14, 2020
1 parent d55d903 commit 15ff168
Show file tree
Hide file tree
Showing 11 changed files with 348 additions and 127 deletions.
61 changes: 27 additions & 34 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,53 +53,46 @@ fastly-exporter -token XXX
This will collect real-time stats for all Fastly services visible to your
token, and make them available as Prometheus metrics on 127.0.0.1:8080/metrics.

### Advanced

```
USAGE
fastly-exporter [flags]
FLAGS
-api-refresh 1m0s how often to poll api.fastly.com for updated service metadata
-api-timeout 15s HTTP client timeout for api.fastly.com requests (5–60s)
-debug false Log debug information
-endpoint http://127.0.0.1:8080/metrics Prometheus /metrics endpoint
-name-exclude-regex ... if set, ignore any service whose name matches this regex
-name-include-regex ... if set, only include services whose names match this regex
-namespace fastly Prometheus namespace
-rt-timeout 45s HTTP client timeout for rt.fastly.com requests (45–120s)
-service ... if set, only include this service ID (repeatable)
-shard ... if set, only include services whose hashed IDs modulo m equal n-1 (format 'n/m')
-subsystem rt Prometheus subsystem
-token ... Fastly API token (required; also via FASTLY_API_TOKEN)
-version false print version information and exit
```
### Filtering services

By default, all services available to your token will be exported. You can
specify an explicit set of service IDs by using the `-service xxx` flag.
(Service IDs are available at the top of your [Fastly dashboard][db].) You can
also include only those services whose name matches a regex by using the
`-name-include-regex '^Production'` flag, or reject any service whose name
matches a regex by using the `-name-exclude-regex '.*TEST.*'` flag.
specify an explicit set of service IDs to export by using the `-service xxx`
flag. (Service IDs are available at the top of your [Fastly dashboard][db].) You
can also include only those services whose name matches a regex by using the
`-service-whitelist '^Production'` flag, or skip any service whose name matches
a regex by using the `-service-blacklist '.*TEST.*'` flag.

[db]: https://manage.fastly.com/services/all

For tokens with access to a lot of services, it's possible to "shard" the
services among different instances of the fastly-exporter by using the `-shard`
flag. For example, to shard all services between 3 exporters, you would start
each exporter as
services among different instances of the fastly-exporter by using the
`-service-shard` flag. For example, to shard all services between 3 exporters,
you would start each exporter as

```
fastly-exporter [common flags] -shard 1/3
fastly-exporter [common flags] -shard 2/3
fastly-exporter [common flags] -shard 3/3
```

Flags which restrict the services that are exported combine with AND semantics.
That is, `-service A -service B -name-include-regex 'Foo'` would only export
data for service A and/or B if their names also matched "Foo". Or, specifying
`-name-include-regex 'Prod' -name-exclude-regex '^test-'` would only export data
for services whose names contained "Prod" and did not start with "test-".
### Filtering exported metrics

By default, all metrics provided by the Fastly real-time stats API are exported
as Prometheus metrics. You can export only those metrics whose name matches a
regex by using the `-metric-whitelist bytes_total$` flag, or elide any metric
whose name matches a regex by using the `-metric-blacklist imgopto` flag.

### Filter semantics

All flags that restrict services or metrics are repeatable. Repeating the same
flag causes its condition to be combined with OR semantics. For example,
`-service A -service B` would include both services A and B (but not service C).
Or, `-service-blacklist Test -service-blacklist Staging` would skip any service
whose name contained Test or Staging.

Different flags (for the same filter target) combine with AND semantics. For
example, `-metric-whitelist 'bytes_total$' -metric-blacklist imgopto` would only
export metrics whose names ended in bytes_total, but didn't include imgopto.

### Docker

Expand Down
97 changes: 59 additions & 38 deletions cmd/fastly-exporter/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@ import (
"net/url"
"os"
"os/signal"
"regexp"
"strconv"
"strings"
"time"
Expand All @@ -18,6 +17,7 @@ import (
"github.com/go-kit/kit/log/level"
"github.com/oklog/run"
"github.com/peterbourgon/fastly-exporter/pkg/api"
"github.com/peterbourgon/fastly-exporter/pkg/filter"
"github.com/peterbourgon/fastly-exporter/pkg/prom"
"github.com/peterbourgon/fastly-exporter/pkg/rt"
"github.com/peterbourgon/usage"
Expand All @@ -30,21 +30,28 @@ var programVersion = "dev"
func main() {
fs := flag.NewFlagSet("fastly-exporter", flag.ExitOnError)
var (
token = fs.String("token", "", "Fastly API token (required; also via FASTLY_API_TOKEN)")
addr = fs.String("endpoint", "http://127.0.0.1:8080/metrics", "Prometheus /metrics endpoint")
namespace = fs.String("namespace", "fastly", "Prometheus namespace")
subsystem = fs.String("subsystem", "rt", "Prometheus subsystem")
serviceIDs = stringslice{}
includeStr = fs.String("name-include-regex", "", "if set, only include services whose names match this regex")
excludeStr = fs.String("name-exclude-regex", "", "if set, ignore any service whose name matches this regex")
shard = fs.String("shard", "", "if set, only include services whose hashed IDs modulo m equal n-1 (format 'n/m')")
token = fs.String("token", "", "Fastly API token (required; also via FASTLY_API_TOKEN)")
addr = fs.String("endpoint", "http://127.0.0.1:8080/metrics", "Prometheus /metrics endpoint")
namespace = fs.String("namespace", "fastly", "Prometheus namespace")
subsystem = fs.String("subsystem", "rt", "Prometheus subsystem")
serviceShard = fs.String("service-shard", "", "if set, only include services whose hashed IDs modulo m equal n-1 (format 'n/m')")
serviceIDs = stringslice{}
serviceWhitelist = stringslice{}
serviceBlacklist = stringslice{}
metricWhitelist = stringslice{}
metricBlacklist = stringslice{}

apiRefresh = fs.Duration("api-refresh", time.Minute, "how often to poll api.fastly.com for updated service metadata (15s–10m)")
apiTimeout = fs.Duration("api-timeout", 15*time.Second, "HTTP client timeout for api.fastly.com requests (5–60s)")
rtTimeout = fs.Duration("rt-timeout", 45*time.Second, "HTTP client timeout for rt.fastly.com requests (45–120s)")
debug = fs.Bool("debug", false, "Log debug information")
versionFlag = fs.Bool("version", false, "print version information and exit")
)
fs.Var(&serviceIDs, "service", "if set, only include this service ID (repeatable)")
fs.Var(&serviceWhitelist, "service-whitelist", "if set, only include services whose names match this regex (repeatable)")
fs.Var(&serviceBlacklist, "service-blacklist", "if set, don't include services whose names match this regex (repeatable)")
fs.Var(&metricWhitelist, "metric-whitelist", "if set, only export metrics whose names match this regex (repeatable)")
fs.Var(&metricBlacklist, "metric-blacklist", "if set, don't export metrics whose names match this regex (repeatable)")
fs.Usage = usage.For(fs, "fastly-exporter [flags]")
fs.Parse(os.Args[1:])

Expand Down Expand Up @@ -108,50 +115,72 @@ func main() {
}
}

var include, exclude *regexp.Regexp
var serviceNameFilter filter.Filter
{
var err error
if *includeStr != "" {
if include, err = regexp.Compile(*includeStr); err != nil {
level.Error(logger).Log("err", "-name-include-regex invalid", "msg", err)
for _, expr := range serviceWhitelist {
if err := serviceNameFilter.Whitelist(expr); err != nil {
level.Error(logger).Log("err", "invalid -service-whitelist", "msg", err)
os.Exit(1)
}
level.Info(logger).Log("filter", "services", "type", "name whitelist", "expr", expr)
}
for _, expr := range serviceBlacklist {
if err := serviceNameFilter.Blacklist(expr); err != nil {
level.Error(logger).Log("err", "invalid -service-blacklist", "msg", err)
os.Exit(1)
}
level.Info(logger).Log("filter", "services", "type", "name blacklist", "expr", expr)
}
}

var metricNameFilter filter.Filter
{
for _, expr := range metricWhitelist {
if err := metricNameFilter.Whitelist(expr); err != nil {
level.Error(logger).Log("err", "invalid -metric-whitelist", "msg", err)
os.Exit(1)
}
level.Info(logger).Log("filter", "metrics", "type", "name whitelist", "expr", expr)

}
if *excludeStr != "" {
if exclude, err = regexp.Compile(*excludeStr); err != nil {
level.Error(logger).Log("err", "-name-exclude-regex invalid", "msg", err)
for _, expr := range metricBlacklist {
if err := metricNameFilter.Blacklist(expr); err != nil {
level.Error(logger).Log("err", "invalid -metric-blacklist", "msg", err)
os.Exit(1)
}
level.Info(logger).Log("filter", "metrics", "type", "name blacklist", "expr", expr)
}
}

var shardN, shardM uint64
{
if *shard != "" {
toks := strings.SplitN(*shard, "/", 2)
if *serviceShard != "" {
toks := strings.SplitN(*serviceShard, "/", 2)
if len(toks) != 2 {
level.Error(logger).Log("err", "-shard must be of the format 'n/m'")
level.Error(logger).Log("err", "-service-shard must be of the format 'n/m'")
os.Exit(1)
}
var err error
shardN, err = strconv.ParseUint(toks[0], 10, 64)
if err != nil {
level.Error(logger).Log("err", "-shard must be of the format 'n/m'")
level.Error(logger).Log("err", "-service-shard must be of the format 'n/m'")
os.Exit(1)
}
if shardN <= 0 {
level.Error(logger).Log("err", "first part of -shard flag should be greater than zero")
level.Error(logger).Log("err", "first part of -service-shard flag should be greater than zero")
os.Exit(1)
}
shardM, err = strconv.ParseUint(toks[1], 10, 64)
if err != nil {
level.Error(logger).Log("err", "-shard must be of the format 'n/m'")
level.Error(logger).Log("err", "-service-shard must be of the format 'n/m'")
os.Exit(1)
}
if shardN > shardM {
level.Error(logger).Log("err", fmt.Sprintf("-shard with n=%d m=%d is invalid: n must be less than or equal to m", shardN, shardM))
level.Error(logger).Log("err", fmt.Sprintf("-service-shard with n=%d m=%d is invalid: n must be less than or equal to m", shardN, shardM))
os.Exit(1)
}
level.Info(logger).Log("filter", "services", "type", "by shard", "n", shardN, "m", shardM)

}
}

Expand All @@ -163,7 +192,7 @@ func main() {
var metrics *prom.Metrics
{
var err error
metrics, err = prom.NewMetrics(*namespace, *subsystem, registry)
metrics, err = prom.NewMetrics(*namespace, *subsystem, metricNameFilter, registry)
if err != nil {
level.Error(logger).Log("err", err)
os.Exit(1)
Expand All @@ -177,25 +206,17 @@ func main() {

var apiCacheOptions []api.CacheOption
{
apiCacheOptions = append(apiCacheOptions, api.WithLogger(apiLogger))
apiCacheOptions = append(apiCacheOptions,
api.WithLogger(apiLogger),
api.WithNameFilter(serviceNameFilter),
)

if len(serviceIDs) > 0 {
level.Info(apiLogger).Log("filtering_on", "explicit service IDs", "count", len(serviceIDs))
level.Info(logger).Log("filter", "services", "type", "explicit service IDs", "count", len(serviceIDs))
apiCacheOptions = append(apiCacheOptions, api.WithExplicitServiceIDs(serviceIDs...))
}

if include != nil {
level.Info(apiLogger).Log("filtering_on", "service name include regex", "regex", include.String())
apiCacheOptions = append(apiCacheOptions, api.WithNameIncluding(include))
}

if exclude != nil {
level.Info(apiLogger).Log("filtering_on", "service name exclude regex", "regex", exclude.String())
apiCacheOptions = append(apiCacheOptions, api.WithNameExcluding(exclude))
}

if shardM > 0 {
level.Info(apiLogger).Log("filtering_on", "shard allocation", "shard", *shard, "n", shardN, "m", shardM)
apiCacheOptions = append(apiCacheOptions, api.WithShard(shardN, shardM))
}
}
Expand Down
53 changes: 20 additions & 33 deletions pkg/api/cache.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,14 +4,14 @@ import (
"encoding/json"
"fmt"
"net/http"
"regexp"
"sort"
"sync"
"time"

"github.com/cespare/xxhash"
"github.com/go-kit/kit/log"
"github.com/go-kit/kit/log/level"
"github.com/peterbourgon/fastly-exporter/pkg/filter"
"github.com/pkg/errors"
)

Expand All @@ -32,12 +32,11 @@ type Service struct {
// Cache polls api.fastly.com/service to keep metadata about
// one or more service IDs up-to-date.
type Cache struct {
token string
whitelist stringset
include *regexp.Regexp
exclude *regexp.Regexp
shard shardSlice
logger log.Logger
token string
serviceIDs stringSet
nameFilter filter.Filter
shard shardSlice
logger log.Logger

mtx sync.RWMutex
services map[string]Service
Expand Down Expand Up @@ -65,20 +64,13 @@ type CacheOption func(*Cache)
// provided service IDs. By default, all service IDs available to the provided
// token are allowed.
func WithExplicitServiceIDs(ids ...string) CacheOption {
return func(c *Cache) { c.whitelist = newStringSet(ids) }
return func(c *Cache) { c.serviceIDs = newStringSet(ids) }
}

// WithNameIncluding restricts the cache to fetch metadata only for the services
// whose names match the provided regexp. By default, no name filtering occurs.
func WithNameIncluding(re *regexp.Regexp) CacheOption {
return func(c *Cache) { c.include = re }
}

// WithNameExcluding restricts the cache to fetch metadata only for the services
// whose names do not match the provided regexp. By default, no name filtering
// occurs.
func WithNameExcluding(re *regexp.Regexp) CacheOption {
return func(c *Cache) { c.exclude = re }
// WithNameFilter restricts the cache to fetch metadata only for the services
// whose names pass the provided filter. By default, no name filtering occurs.
func WithNameFilter(f filter.Filter) CacheOption {
return func(c *Cache) { c.nameFilter = f }
}

// WithShard restricts the cache to fetch metadata only for those services whose
Expand Down Expand Up @@ -139,18 +131,13 @@ func (c *Cache) Refresh(client HTTPClient) error {
"service_version", s.Version,
))

if reject := !c.whitelist.empty() && !c.whitelist.has(s.ID); reject {
debug.Log("result", "rejected", "reason", "not in service ID whitelist")
continue
}

if reject := c.include != nil && !c.include.MatchString(s.Name); reject {
debug.Log("result", "rejected", "reason", "service name didn't match include regex")
if reject := !c.serviceIDs.empty() && !c.serviceIDs.has(s.ID); reject {
debug.Log("result", "rejected", "reason", "service ID not explicitly allowed")
continue
}

if reject := c.exclude != nil && c.exclude.MatchString(s.Name); reject {
debug.Log("result", "rejected", "reason", "service name matched exclude regex")
if reject := !c.nameFilter.Allow(s.Name); reject {
debug.Log("result", "rejected", "reason", "service name rejected by name filter")
continue
}

Expand Down Expand Up @@ -208,21 +195,21 @@ func (c *Cache) Metadata(id string) (name string, version int, found bool) {
//
//

type stringset map[string]struct{}
type stringSet map[string]struct{}

func newStringSet(initial []string) stringset {
ss := stringset{}
func newStringSet(initial []string) stringSet {
ss := stringSet{}
for _, s := range initial {
ss[s] = struct{}{}
}
return ss
}

func (ss stringset) empty() bool {
func (ss stringSet) empty() bool {
return len(ss) == 0
}

func (ss stringset) has(s string) bool {
func (ss stringSet) has(s string) bool {
_, ok := ss[s]
return ok
}
Expand Down
Loading

0 comments on commit 15ff168

Please sign in to comment.