From a435c71dec0756edb689ea55295420014f0c6be7 Mon Sep 17 00:00:00 2001 From: Matt DeBoer Date: Tue, 10 Jan 2017 11:31:28 -0800 Subject: [PATCH] add 'namespace' to allow parallel tests add failing test case which shows old metrics are not cleared code cleanup code cleanup assure new gagues are exported properly recreate gauges on each scrape to clear stale values include tests for stale metrics of all types clear containers as well --- containers.go | 20 +-- containers_test.go | 4 +- exporter.go | 11 +- exporter_test.go | 306 +++++++++++++++++++++++++++++++++++---------- main.go | 6 +- 5 files changed, 267 insertions(+), 80 deletions(-) diff --git a/containers.go b/containers.go index 0ad268e..137a8ff 100644 --- a/containers.go +++ b/containers.go @@ -18,12 +18,14 @@ const ( ) type CounterContainer struct { - counters map[string]*prometheus.CounterVec + counters map[string]*prometheus.CounterVec + namespace string } -func NewCounterContainer() *CounterContainer { +func NewCounterContainer(namespace string) *CounterContainer { return &CounterContainer{ - counters: make(map[string]*prometheus.CounterVec), + counters: make(map[string]*prometheus.CounterVec), + namespace: namespace, } } @@ -33,7 +35,7 @@ func (c *CounterContainer) Fetch(name, help string, labels ...string) (*promethe if !exists { counter = prometheus.NewCounterVec(prometheus.CounterOpts{ - Namespace: namespace, + Namespace: c.namespace, Name: name, Help: help, }, labels) @@ -45,12 +47,14 @@ func (c *CounterContainer) Fetch(name, help string, labels ...string) (*promethe } type GaugeContainer struct { - gauges map[string]*prometheus.GaugeVec + gauges map[string]*prometheus.GaugeVec + namespace string } -func NewGaugeContainer() *GaugeContainer { +func NewGaugeContainer(namespace string) *GaugeContainer { return &GaugeContainer{ - gauges: make(map[string]*prometheus.GaugeVec), + gauges: make(map[string]*prometheus.GaugeVec), + namespace: namespace, } } @@ -60,7 +64,7 @@ func (c *GaugeContainer) Fetch(name, help string, labels ...string) (*prometheus if !exists { gauge = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: namespace, + Namespace: c.namespace, Name: name, Help: help, }, labels) diff --git a/containers_test.go b/containers_test.go index 837dab0..26201cb 100644 --- a/containers_test.go +++ b/containers_test.go @@ -32,7 +32,7 @@ func Test_container_key(t *testing.T) { } func Test_container_fetch_counter(t *testing.T) { - container := NewCounterContainer() + container := NewCounterContainer("marathon") _, new := container.Fetch("foo", "") if !new { @@ -52,7 +52,7 @@ func Test_container_fetch_counter(t *testing.T) { } func Test_container_fetch_gauge(t *testing.T) { - container := NewGaugeContainer() + container := NewGaugeContainer("marathon") _, new := container.Fetch("foo", "") if !new { diff --git a/exporter.go b/exporter.go index e7b133a..af6aa04 100644 --- a/exporter.go +++ b/exporter.go @@ -12,7 +12,7 @@ import ( "github.com/prometheus/common/log" ) -const namespace = "marathon" +const defaultNamespace = "marathon" type Exporter struct { scraper Scraper @@ -71,6 +71,9 @@ func (e *Exporter) scrape(ch chan<- prometheus.Metric) { } }(time.Now()) + // Rebuild gauges & coutners to avoid stale values + e.Gauges = NewGaugeContainer(e.Gauges.namespace) + e.Counters = NewCounterContainer(e.Counters.namespace) if err = e.exportApps(ch); err != nil { return } @@ -422,11 +425,11 @@ func (e *Exporter) scrapeTimer(key string, json *gabs.Container) (bool, error) { return new, nil } -func NewExporter(s Scraper) *Exporter { +func NewExporter(s Scraper, namespace string) *Exporter { return &Exporter{ scraper: s, - Counters: NewCounterContainer(), - Gauges: NewGaugeContainer(), + Counters: NewCounterContainer(namespace), + Gauges: NewGaugeContainer(namespace), duration: prometheus.NewGauge(prometheus.GaugeOpts{ Namespace: namespace, Subsystem: "exporter", diff --git a/exporter_test.go b/exporter_test.go index a9a38a4..b9d5a90 100644 --- a/exporter_test.go +++ b/exporter_test.go @@ -6,6 +6,7 @@ import ( "net/http" "net/http/httptest" "regexp" + "runtime" "strings" "testing" @@ -20,12 +21,58 @@ type testScraper struct { results string } +type testExporter struct { + exporter *Exporter + server *httptest.Server +} + func (s *testScraper) Scrape(path string) ([]byte, error) { return []byte(s.results), nil } +func newTestExporter(namespace string) *testExporter { + exporter := NewExporter(&testScraper{`{}`}, namespace) + + prometheus.MustRegister(exporter) + server := httptest.NewServer(prometheus.UninstrumentedHandler()) + return &testExporter{ + exporter: exporter, + server: server, + } +} + +func (te *testExporter) close() { + prometheus.Unregister(te.exporter) + te.server.Close() +} + +func (te *testExporter) export(json string) ([]byte, error) { + + te.exporter.scraper = &testScraper{json} + response, err := http.Get(te.server.URL) + if err != nil { + return nil, err + } + + defer response.Body.Close() + body, err := ioutil.ReadAll(response.Body) + if err != nil { + return nil, err + } + + return body, nil +} + +func getFunctionName() string { + pc := make([]uintptr, 1) + runtime.Callers(2, pc) + f := runtime.FuncForPC(pc[0]) + parts := strings.Split(f.Name(), ".") + return parts[len(parts)-1] +} + func export(json string) ([]byte, error) { - exporter := NewExporter(&testScraper{json}) + exporter := NewExporter(&testScraper{json}, "marathon") prometheus.MustRegister(exporter) defer prometheus.Unregister(exporter) @@ -60,8 +107,32 @@ func Test_export_version(t *testing.T) { } } +func assertResultsContain(t *testing.T, results []byte, patterns ...string) { + for _, pattern := range patterns { + re := regexp.MustCompile(pattern) + if !re.Match(results) { + t.Errorf("No metric matching pattern: %s\n", re) + } + } +} + +func assertResultsDoNotContain(t *testing.T, results []byte, patterns ...string) { + for _, pattern := range patterns { + re := regexp.MustCompile(pattern) + if re.Match(results) { + t.Errorf("Metric matching pattern: '%s' should not exist\n", re) + } + } +} + func Test_export_counters(t *testing.T) { - results, err := export(`{ + + fName := getFunctionName() + te := newTestExporter(fName) + defer te.close() + + // First pass + results, err := te.export(`{ "counters": { "foo_count": {"count": 1}, "bar_count": {"count": 2} @@ -71,19 +142,36 @@ func Test_export_counters(t *testing.T) { t.Fatal(err) } - //t.Log(string(results)) - for _, re := range []*regexp.Regexp{ - regexp.MustCompile("marathon_foo_count 1"), - regexp.MustCompile("marathon_bar_count 2"), - } { - if !re.Match(results) { - t.Errorf("No counter matching pattern: %s\n", re) + assertResultsContain(t, results, + fName+"_foo_count 1", + fName+"_bar_count 2") + + // Second pass; 'bar' metric no longer present + results, err = te.export(`{ + "counters": { + "foo_count": {"count": 1}, + "baz_count": {"count": 3} } + }`) + if err != nil { + t.Fatal(err) } + + assertResultsContain(t, results, + fName+"_foo_count 1", + fName+"_baz_count 3") + + assertResultsDoNotContain(t, results, + fName+"_bar_count 2") } func Test_export_gauges(t *testing.T) { - results, err := export(`{ + + fName := getFunctionName() + te := newTestExporter(fName) + defer te.close() + + results, err := te.export(`{ "gauges": { "foo_value": {"value": 1}, "bar_value": {"value": 2} @@ -94,19 +182,36 @@ func Test_export_gauges(t *testing.T) { t.Fatal(err) } - //t.Log(string(results)) - for _, re := range []*regexp.Regexp{ - regexp.MustCompile("marathon_foo_value 1"), - regexp.MustCompile("marathon_bar_value 2"), - } { - if !re.Match(results) { - t.Errorf("No gauge matching pattern: %s\n", re) + assertResultsContain(t, results, + fName+"_foo_value 1", + fName+"_bar_value 2") + + results, err = te.export(`{ + "gauges": { + "foo_value": {"value": 1}, + "baz_value": {"value": 3} } + }`) + + if err != nil { + t.Fatal(err) } + + assertResultsContain(t, results, + fName+"_foo_value 1", + fName+"_baz_value 3") + + assertResultsDoNotContain(t, results, + fName+"_bar_value 2") } func Test_export_meters(t *testing.T) { - results, err := export(`{ + + fName := getFunctionName() + te := newTestExporter(fName) + defer te.close() + + results, err := te.export(`{ "meters": { "foo_meter": {"count":1,"m1_rate":1,"m5_rate":1,"m15_rate":1,"mean_rate":1,"units":"foos/bar"}, "bar_meter": {"count":2,"m1_rate":2,"m5_rate":2,"m15_rate":2,"mean_rate":2,"units":"foos/bar"} @@ -117,21 +222,40 @@ func Test_export_meters(t *testing.T) { t.Fatal(err) } - //t.Log(string(results)) - for _, re := range []*regexp.Regexp{ - regexp.MustCompile("marathon_foo_meter_count 1"), - regexp.MustCompile("marathon_foo_meter{rate=\"(1m|5m|15m|mean)\"} 1"), - regexp.MustCompile("marathon_bar_meter_count 2"), - regexp.MustCompile("marathon_bar_meter{rate=\"(1m|5m|15m|mean)\"} 2"), - } { - if !re.Match(results) { - t.Errorf("No meter metric matching pattern: %s\n", re) + assertResultsContain(t, results, + fName+"_foo_meter_count 1", + fName+"_foo_meter{rate=\"(1m|5m|15m|mean)\"} 1", + fName+"_bar_meter_count 2", + fName+"_bar_meter{rate=\"(1m|5m|15m|mean)\"} 2") + + results, err = te.export(`{ + "meters": { + "foo_meter": {"count":1,"m1_rate":1,"m5_rate":1,"m15_rate":1,"mean_rate":1,"units":"foos/bar"}, + "baz_meter": {"count":2,"m1_rate":2,"m5_rate":2,"m15_rate":2,"mean_rate":2,"units":"foos/bar"} } + }`) + + if err != nil { + t.Fatal(err) } + + assertResultsContain(t, results, + fName+"_foo_meter_count 1", + fName+"_foo_meter{rate=\"(1m|5m|15m|mean)\"} 1", + fName+"_baz_meter_count 2", + fName+"_baz_meter{rate=\"(1m|5m|15m|mean)\"} 2") + + assertResultsDoNotContain(t, results, + fName+"_bar_meter") } func Test_export_histograms(t *testing.T) { - results, err := export(`{ + + fName := getFunctionName() + te := newTestExporter(fName) + defer te.close() + + results, err := te.export(`{ "histograms": { "foo_histogram": {"count":1,"p50":1,"p75":1,"p95":1,"p98":1,"p99":1,"p999":1,"max":1,"mean":1,"min":1,"stddev":1}, "bar_histogram": {"count":2,"p50":2,"p75":2,"p95":2,"p98":2,"p99":2,"p999":2,"max":2,"mean":2,"min":2,"stddev":2} @@ -142,29 +266,57 @@ func Test_export_histograms(t *testing.T) { t.Fatal(err) } - //t.Log(string(results)) - for _, re := range []*regexp.Regexp{ - regexp.MustCompile("marathon_foo_histogram_count 1"), - regexp.MustCompile("marathon_foo_histogram_max 1"), - regexp.MustCompile("marathon_foo_histogram_min 1"), - regexp.MustCompile("marathon_foo_histogram_mean 1"), - regexp.MustCompile("marathon_foo_histogram_stddev 1"), - regexp.MustCompile("marathon_foo_histogram{percentile=\"0\\.\\d+\"} 1"), - regexp.MustCompile("marathon_bar_histogram_count 2"), - regexp.MustCompile("marathon_bar_histogram_max 2"), - regexp.MustCompile("marathon_bar_histogram_min 2"), - regexp.MustCompile("marathon_bar_histogram_mean 2"), - regexp.MustCompile("marathon_bar_histogram_stddev 2"), - regexp.MustCompile("marathon_bar_histogram{percentile=\"0\\.\\d+\"} 2"), - } { - if !re.Match(results) { - t.Errorf("No histogram metric matching pattern: %s\n", re) + assertResultsContain(t, results, + fName+"_foo_histogram_count 1", + fName+"_foo_histogram_max 1", + fName+"_foo_histogram_min 1", + fName+"_foo_histogram_mean 1", + fName+"_foo_histogram_stddev 1", + fName+"_foo_histogram{percentile=\"0\\.\\d+\"} 1", + fName+"_bar_histogram_count 2", + fName+"_bar_histogram_max 2", + fName+"_bar_histogram_min 2", + fName+"_bar_histogram_mean 2", + fName+"_bar_histogram_stddev 2", + fName+"_bar_histogram{percentile=\"0\\.\\d+\"} 2") + + results, err = te.export(`{ + "histograms": { + "foo_histogram": {"count":1,"p50":1,"p75":1,"p95":1,"p98":1,"p99":1,"p999":1,"max":1,"mean":1,"min":1,"stddev":1}, + "baz_histogram": {"count":2,"p50":2,"p75":2,"p95":2,"p98":2,"p99":2,"p999":2,"max":2,"mean":2,"min":2,"stddev":2} } + }`) + + if err != nil { + t.Fatal(err) } + + assertResultsContain(t, results, + fName+"_foo_histogram_count 1", + fName+"_foo_histogram_max 1", + fName+"_foo_histogram_min 1", + fName+"_foo_histogram_mean 1", + fName+"_foo_histogram_stddev 1", + fName+"_foo_histogram{percentile=\"0\\.\\d+\"} 1", + fName+"_baz_histogram_count 2", + fName+"_baz_histogram_max 2", + fName+"_baz_histogram_min 2", + fName+"_baz_histogram_mean 2", + fName+"_baz_histogram_stddev 2", + fName+"_baz_histogram{percentile=\"0\\.\\d+\"} 2") + + assertResultsDoNotContain(t, results, + fName+"_bar_histogram") + } func Test_export_timers(t *testing.T) { - results, err := export(`{ + + fName := getFunctionName() + te := newTestExporter(fName) + defer te.close() + + results, err := te.export(`{ "timers": { "foo_timer": {"count":1,"p50":1,"p75":1,"p95":1,"p98":1,"p99":1,"p999":1,"max":1,"mean":1,"min":1,"stddev":1,"m1_rate":1,"m5_rate":1,"m15_rate":1,"mean_rate":1,"duration_units":"foos","rate_units":"bars/foo"}, "bar_timer": {"count":2,"p50":2,"p75":2,"p95":2,"p98":2,"p99":2,"p999":2,"max":2,"mean":2,"min":2,"stddev":2,"m1_rate":2,"m5_rate":2,"m15_rate":2,"mean_rate":2,"duration_units":"bars","rate_units":"foos/bar"} @@ -175,25 +327,49 @@ func Test_export_timers(t *testing.T) { t.Fatal(err) } - //t.Log(string(results)) - for _, re := range []*regexp.Regexp{ - regexp.MustCompile("marathon_foo_timer_count 1"), - regexp.MustCompile("marathon_foo_timer_max 1"), - regexp.MustCompile("marathon_foo_timer_min 1"), - regexp.MustCompile("marathon_foo_timer_mean 1"), - regexp.MustCompile("marathon_foo_timer_stddev 1"), - regexp.MustCompile("marathon_foo_timer{percentile=\"0\\.\\d+\"} 1"), - regexp.MustCompile("marathon_foo_timer_rate{rate=\"(1m|5m|15m|mean)\"} 1"), - regexp.MustCompile("marathon_bar_timer_count 2"), - regexp.MustCompile("marathon_bar_timer_max 2"), - regexp.MustCompile("marathon_bar_timer_min 2"), - regexp.MustCompile("marathon_bar_timer_mean 2"), - regexp.MustCompile("marathon_bar_timer_stddev 2"), - regexp.MustCompile("marathon_bar_timer{percentile=\"0\\.\\d+\"} 2"), - regexp.MustCompile("marathon_bar_timer_rate{rate=\"(1m|5m|15m|mean)\"} 2"), - } { - if !re.Match(results) { - t.Errorf("No timer metric matching pattern: %s\n", re) + assertResultsContain(t, results, + fName+"_foo_timer_count 1", + fName+"_foo_timer_max 1", + fName+"_foo_timer_min 1", + fName+"_foo_timer_mean 1", + fName+"_foo_timer_stddev 1", + fName+"_foo_timer{percentile=\"0\\.\\d+\"} 1", + fName+"_foo_timer_rate{rate=\"(1m|5m|15m|mean)\"} 1", + fName+"_bar_timer_count 2", + fName+"_bar_timer_max 2", + fName+"_bar_timer_min 2", + fName+"_bar_timer_mean 2", + fName+"_bar_timer_stddev 2", + fName+"_bar_timer{percentile=\"0\\.\\d+\"} 2", + fName+"_bar_timer_rate{rate=\"(1m|5m|15m|mean)\"} 2") + + results, err = te.export(`{ + "timers": { + "foo_timer": {"count":1,"p50":1,"p75":1,"p95":1,"p98":1,"p99":1,"p999":1,"max":1,"mean":1,"min":1,"stddev":1,"m1_rate":1,"m5_rate":1,"m15_rate":1,"mean_rate":1,"duration_units":"foos","rate_units":"bars/foo"}, + "baz_timer": {"count":2,"p50":2,"p75":2,"p95":2,"p98":2,"p99":2,"p999":2,"max":2,"mean":2,"min":2,"stddev":2,"m1_rate":2,"m5_rate":2,"m15_rate":2,"mean_rate":2,"duration_units":"bars","rate_units":"foos/bar"} } + }`) + + if err != nil { + t.Fatal(err) } + + assertResultsContain(t, results, + fName+"_foo_timer_count 1", + fName+"_foo_timer_max 1", + fName+"_foo_timer_min 1", + fName+"_foo_timer_mean 1", + fName+"_foo_timer_stddev 1", + fName+"_foo_timer{percentile=\"0\\.\\d+\"} 1", + fName+"_foo_timer_rate{rate=\"(1m|5m|15m|mean)\"} 1", + fName+"_baz_timer_count 2", + fName+"_baz_timer_max 2", + fName+"_baz_timer_min 2", + fName+"_baz_timer_mean 2", + fName+"_baz_timer_stddev 2", + fName+"_baz_timer{percentile=\"0\\.\\d+\"} 2", + fName+"_baz_timer_rate{rate=\"(1m|5m|15m|mean)\"} 2") + + assertResultsDoNotContain(t, results, + fName+"_bar_timer") } diff --git a/main.go b/main.go index 8d4bde0..a9752de 100644 --- a/main.go +++ b/main.go @@ -25,6 +25,10 @@ var ( marathonUri = flag.String( "marathon.uri", "http://marathon.mesos:8080", "URI of Marathon") + + namespace = flag.String( + "namespace", "marathon", + "Namespace -- used to prefix all metric names with '{namespace}_'") ) func marathonConnect(uri *url.URL) error { @@ -83,7 +87,7 @@ func main() { time.Sleep(retryTimeout) } - exporter := NewExporter(&scraper{uri}) + exporter := NewExporter(&scraper{uri}, *namespace) prometheus.MustRegister(exporter) http.Handle(*metricsPath, prometheus.Handler())