From 39a9e377c6f387b47a2e7db07d79b4170715d44c Mon Sep 17 00:00:00 2001 From: Parsa Date: Thu, 28 Apr 2022 18:05:39 +0430 Subject: [PATCH] feature: count json objects with same value if we want to count json objects with same values we can use countbylabel type in the metric configuration Signed-off-by: Parsa Signed-off-by: Parsa --- README.md | 11 +++++++++++ config/config.go | 17 +++++++++++------ examples/config.yml | 9 +++++++++ exporter/collector.go | 39 ++++++++++++++++++++++++++++++++++++++- exporter/util.go | 18 ++++++++++++++++++ test/config/good.yml | 7 +++++++ test/response/good.txt | 4 ++++ 7 files changed, 98 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index adbbdf20..eff1d0b2 100644 --- a/README.md +++ b/README.md @@ -59,6 +59,15 @@ metrics: count: '{.count}' # dynamic value boolean: '{.some_boolean}' +- name: example_count + type: countbylabel + help: Example of count json labels and + path: '{.values[*].state}' + labels: + environment: beta # static label + state: '{}' # dynamic label + mincount: 1 + headers: X-Dummy: my-test-header @@ -68,6 +77,8 @@ Serving HTTP on 0.0.0.0 port 8000 ... $ ./json_exporter --config.file examples/config.yml & $ curl "http://localhost:7979/probe?target=http://localhost:8000/examples/data.json" | grep ^example +example_count{environment="beta",state="ACTIVE"} 2 +example_count{environment="beta",state="INACTIVE"} 1 example_global_value{environment="beta",location="planet-mars"} 1234 example_value_active{environment="beta",id="id-A"} 1 example_value_active{environment="beta",id="id-C"} 1 diff --git a/config/config.go b/config/config.go index 5d320a60..e809a384 100644 --- a/config/config.go +++ b/config/config.go @@ -22,12 +22,13 @@ import ( // Metric contains values that define a metric type Metric struct { - Name string - Path string - Labels map[string]string - Type MetricType - Help string - Values map[string]string + Name string + Path string + Labels map[string]string + Type MetricType + Help string + Values map[string]string + Mincount int } type MetricType string @@ -35,6 +36,7 @@ type MetricType string const ( ValueScrape MetricType = "value" // default ObjectScrape MetricType = "object" + CountScrape MetricType = "countbylabel" ) // Config contains metrics and headers defining a configuration @@ -69,6 +71,9 @@ func LoadConfig(configPath string) (Config, error) { if config.Metrics[i].Help == "" { config.Metrics[i].Help = config.Metrics[i].Name } + if !(config.Metrics[i].Mincount > 0) { + config.Metrics[i].Mincount = 1 + } } return config, nil diff --git a/examples/config.yml b/examples/config.yml index b4b28f1f..119bf103 100644 --- a/examples/config.yml +++ b/examples/config.yml @@ -19,6 +19,15 @@ metrics: count: '{.count}' # dynamic value boolean: '{.some_boolean}' +- name: example_count + type: countbylabel + help: Example of count json labels and + path: '{.values[*].state}' + labels: + environment: beta # static label + state: '{}' # dynamic label + mincount: 1 + headers: X-Dummy: my-test-header diff --git a/exporter/collector.go b/exporter/collector.go index 7d61bf70..55259789 100644 --- a/exporter/collector.go +++ b/exporter/collector.go @@ -34,6 +34,7 @@ type JSONMetric struct { KeyJSONPath string ValueJSONPath string LabelsJSONPaths []string + Mincount int } func (mc JSONMetricCollector) Describe(ch chan<- *prometheus.Desc) { @@ -44,7 +45,7 @@ func (mc JSONMetricCollector) Describe(ch chan<- *prometheus.Desc) { func (mc JSONMetricCollector) Collect(ch chan<- prometheus.Metric) { for _, m := range mc.JSONMetrics { - if m.ValueJSONPath == "" { // ScrapeType is 'value' + if m.ValueJSONPath == "" && m.Mincount == 0 { // ScrapeType is 'value' value, err := extractValue(mc.Logger, mc.Data, m.KeyJSONPath, false) if err != nil { level.Error(mc.Logger).Log("msg", "Failed to extract value for metric", "path", m.KeyJSONPath, "err", err, "metric", m.Desc) @@ -63,6 +64,42 @@ func (mc JSONMetricCollector) Collect(ch chan<- prometheus.Metric) { level.Error(mc.Logger).Log("msg", "Failed to convert extracted value to float64", "path", m.KeyJSONPath, "value", value, "err", err, "metric", m.Desc) continue } + } else if m.Mincount > 0 { // ScrapeType is 'countbylabel' + values, err := extractValue(mc.Logger, mc.Data, m.KeyJSONPath, true) + if err != nil { + level.Error(mc.Logger).Log("msg", "Failed to extract json objects for metric", "err", err, "metric", m.Desc) + continue + } + + var jsonData []interface{} + counts := make(map[interface{}]int) + + if err := json.Unmarshal([]byte(values), &jsonData); err == nil { + for _, data := range jsonData { + counts[data]++ + } + for data, count := range counts { + if count >= m.Mincount { + jdata, err := json.Marshal(data) + if err != nil { + level.Error(mc.Logger).Log("msg", "Failed to marshal data to json", "path", m.ValueJSONPath, "err", err, "metric", m.Desc, "data", data) + continue + } + if err != nil { + level.Error(mc.Logger).Log("msg", "Failed to extract value for metric", "path", m.ValueJSONPath, "err", err, "metric", m.Desc) + continue + } + + ch <- prometheus.MustNewConstMetric( + m.Desc, + prometheus.UntypedValue, + float64(count), + extractLabels(mc.Logger, jdata, m.LabelsJSONPaths)..., + ) + } + } + } + } else { // ScrapeType is 'object' values, err := extractValue(mc.Logger, mc.Data, m.KeyJSONPath, true) if err != nil { diff --git a/exporter/util.go b/exporter/util.go index aa3e8989..e80c9da0 100644 --- a/exporter/util.go +++ b/exporter/util.go @@ -104,6 +104,24 @@ func CreateMetricsList(c config.Config) ([]JSONMetric, error) { } metrics = append(metrics, jsonMetric) } + case config.CountScrape: + var variableLabels, variableLabelsValues []string + for k, v := range metric.Labels { + variableLabels = append(variableLabels, k) + variableLabelsValues = append(variableLabelsValues, v) + } + jsonMetric := JSONMetric{ + Desc: prometheus.NewDesc( + metric.Name, + metric.Help, + variableLabels, + nil, + ), + KeyJSONPath: metric.Path, + Mincount: metric.Mincount, + LabelsJSONPaths: variableLabelsValues, + } + metrics = append(metrics, jsonMetric) default: return nil, fmt.Errorf("Unknown metric type: '%s', for metric: '%s'", metric.Type, metric.Name) } diff --git a/test/config/good.yml b/test/config/good.yml index dd6017f4..8d49efb8 100644 --- a/test/config/good.yml +++ b/test/config/good.yml @@ -19,3 +19,10 @@ metrics: count: '{.count}' # dynamic value boolean: '{.some_boolean}' +- name: example_count + type: countbylabel + help: Example of count object from a json + path: '{.values[*].state}' + labels: + environment: beta # static label + state: '{}' # dynamic label diff --git a/test/response/good.txt b/test/response/good.txt index dc7538f8..1895353e 100644 --- a/test/response/good.txt +++ b/test/response/good.txt @@ -1,3 +1,7 @@ +# HELP example_count Example of count object from a json +# TYPE example_count untyped +example_count{environment="beta",state="ACTIVE"} 2 +example_count{environment="beta",state="INACTIVE"} 1 # HELP example_global_value Example of a top-level global value scrape in the json # TYPE example_global_value untyped example_global_value{environment="beta",location="planet-mars"} 1234