From f4d2d10d7428ec413ecb41caeb94dbf14f36b688 Mon Sep 17 00:00:00 2001 From: Parsa Date: Wed, 6 Sep 2023 22:35:30 +0430 Subject: [PATCH] feature: count json objects with same value if we want to count json objects with same values we can use countbylabel type in the metric configuration Signed-off-by: Parsa --- README.md | 4 ++++ config/config.go | 6 +++++- examples/config.yml | 10 +++++++++- exporter/collector.go | 37 +++++++++++++++++++++++++++++++++++++ exporter/util.go | 22 ++++++++++++++++++++++ test/config/good.yml | 8 ++++++++ test/response/good.txt | 4 ++++ 7 files changed, 89 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index f514a85d..f1ea67a1 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,10 @@ Serving HTTP on :: port 8000 (http://[::]:8000/) ... ## TEST with 'default' module $ curl "http://localhost:7979/probe?module=default&target=http://localhost:8000/examples/data.json" +# HELP example_count Example of count object from a json +# TYPE example_count untyped +example_count{environment="beta",state="ACTIVE"} 2 +example_count{environment="beta",state="INACTIVE"} 1 # HELP example_global_value Example of a top-level global value scrape in the json # TYPE example_global_value untyped example_global_value{environment="beta",location="planet-mars"} 1234 diff --git a/config/config.go b/config/config.go index 6cd0accb..156a7412 100644 --- a/config/config.go +++ b/config/config.go @@ -30,6 +30,7 @@ type Metric struct { EpochTimestamp string Help string Values map[string]string + MinimumCount int } type ScrapeType string @@ -37,6 +38,7 @@ type ScrapeType string const ( ValueScrape ScrapeType = "value" // default ObjectScrape ScrapeType = "object" + CountScrape ScrapeType = "countbylabel" ) type ValueType string @@ -89,8 +91,10 @@ func LoadConfig(configPath string) (Config, error) { if module.Metrics[i].ValueType == "" { module.Metrics[i].ValueType = ValueTypeUntyped } + if !(module.Metrics[i].MinimumCount > 0) { + module.Metrics[i].MinimumCount = 1 + } } } - return config, nil } diff --git a/examples/config.yml b/examples/config.yml index 9d0745c0..959a4f0d 100644 --- a/examples/config.yml +++ b/examples/config.yml @@ -30,6 +30,15 @@ modules: active: 1 # static value count: '{.count}' # dynamic value boolean: '{.some_boolean}' + - name: example_count + type: countbylabel + help: Example of count json labels + path: '{.values[*].state}' + labels: + environment: beta # static label + state: '{}' # dynamic label + # The minimum count of labels required to expose the metric for each label. Defaults to 1. + minimumCount: 1 animals: metrics: @@ -66,4 +75,3 @@ modules: # content: | # {"time_diff": "{{ duration `95` }}","anotherVar": "{{ .myVal | first }}"} # templatize: true - diff --git a/exporter/collector.go b/exporter/collector.go index 4effc10f..55899c10 100644 --- a/exporter/collector.go +++ b/exporter/collector.go @@ -39,6 +39,7 @@ type JSONMetric struct { LabelsJSONPaths []string ValueType prometheus.ValueType EpochTimestampJSONPath string + MinimumCount int } func (mc JSONMetricCollector) Describe(ch chan<- *prometheus.Desc) { @@ -70,6 +71,42 @@ func (mc JSONMetricCollector) Collect(ch chan<- prometheus.Metric) { continue } + case config.CountScrape: + values, err := extractValue(mc.Logger, mc.Data, m.KeyJSONPath, true) + if err != nil { + level.Error(mc.Logger).Log("msg", "Failed to extract json objects for metric", "err", err, "metric", m.Desc) + continue + } + + var jsonData []interface{} + counts := make(map[interface{}]int) + + if err := json.Unmarshal([]byte(values), &jsonData); err == nil { + for _, data := range jsonData { + counts[data]++ + } + for data, count := range counts { + if count >= m.MinimumCount { + jdata, err := json.Marshal(data) + if err != nil { + level.Error(mc.Logger).Log("msg", "Failed to marshal data to json", "path", m.ValueJSONPath, "err", err, "metric", m.Desc, "data", data) + continue + } + if err != nil { + level.Error(mc.Logger).Log("msg", "Failed to extract value for metric", "path", m.ValueJSONPath, "err", err, "metric", m.Desc) + continue + } + + ch <- prometheus.MustNewConstMetric( + m.Desc, + prometheus.UntypedValue, + float64(count), + extractLabels(mc.Logger, jdata, m.LabelsJSONPaths)..., + ) + } + } + } + case config.ObjectScrape: values, err := extractValue(mc.Logger, mc.Data, m.KeyJSONPath, true) if err != nil { diff --git a/exporter/util.go b/exporter/util.go index 8374ddce..3eece0b0 100644 --- a/exporter/util.go +++ b/exporter/util.go @@ -133,6 +133,28 @@ func CreateMetricsList(c config.Module) ([]JSONMetric, error) { } metrics = append(metrics, jsonMetric) } + case config.CountScrape: + var variableLabels, variableLabelsValues []string + for k, v := range metric.Labels { + variableLabels = append(variableLabels, k) + variableLabelsValues = append(variableLabelsValues, v) + } + jsonMetric := JSONMetric{ + Type: config.CountScrape, + Desc: prometheus.NewDesc( + metric.Name, + metric.Help, + variableLabels, + nil, + ), + KeyJSONPath: metric.Path, + MinimumCount: metric.MinimumCount, + LabelsJSONPaths: variableLabelsValues, + ValueType: valueType, + EpochTimestampJSONPath: metric.EpochTimestamp, + } + fmt.Println(jsonMetric) + metrics = append(metrics, jsonMetric) default: return nil, fmt.Errorf("Unknown metric type: '%s', for metric: '%s'", metric.Type, metric.Name) } diff --git a/test/config/good.yml b/test/config/good.yml index 9bde7320..78fd8b59 100644 --- a/test/config/good.yml +++ b/test/config/good.yml @@ -22,3 +22,11 @@ modules: active: 1 # static value count: '{.count}' # dynamic value boolean: '{.some_boolean}' + + - name: example_count + type: countbylabel + help: Example of count object from a json + path: '{.values[*].state}' + labels: + environment: beta # static label + state: '{}' # dynamic label diff --git a/test/response/good.txt b/test/response/good.txt index d9b1ca1e..40aec1b7 100644 --- a/test/response/good.txt +++ b/test/response/good.txt @@ -1,3 +1,7 @@ +# HELP example_count Example of count object from a json +# TYPE example_count untyped +example_count{environment="beta",state="ACTIVE"} 2 +example_count{environment="beta",state="INACTIVE"} 1 # HELP example_global_value Example of a top-level global value scrape in the json # TYPE example_global_value gauge example_global_value{environment="beta",location="planet-mars"} 1234