From ca1304003c13e88cb6fd26ef9112437c0dcd1bbd Mon Sep 17 00:00:00 2001 From: ngrebels <110986489+ngrebels@users.noreply.github.com> Date: Wed, 5 Oct 2022 12:32:05 -0400 Subject: [PATCH] Support Value Conversions (#172) * Bump k8s.io/client-go from 0.24.2 to 0.24.3 (#171) Bumps [k8s.io/client-go](https://github.com/kubernetes/client-go) from 0.24.2 to 0.24.3. - [Release notes](https://github.com/kubernetes/client-go/releases) - [Changelog](https://github.com/kubernetes/client-go/blob/master/CHANGELOG.md) - [Commits](https://github.com/kubernetes/client-go/compare/v0.24.2...v0.24.3) --- updated-dependencies: - dependency-name: k8s.io/client-go dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Signed-off-by: ngrebels Signed-off-by: Yao Hong Kok * Bump github.com/prometheus/common from 0.35.0 to 0.37.0 (#170) Bumps [github.com/prometheus/common](https://github.com/prometheus/common) from 0.35.0 to 0.37.0. - [Release notes](https://github.com/prometheus/common/releases) - [Commits](https://github.com/prometheus/common/compare/v0.35.0...v0.37.0) --- updated-dependencies: - dependency-name: github.com/prometheus/common dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Signed-off-by: ngrebels Signed-off-by: Yao Hong Kok * Added a value converter for dynamic values and associated tests Signed-off-by: ngrebels Signed-off-by: Yao Hong Kok * Refactored into functions and created a type Signed-off-by: ngrebels Signed-off-by: Yao Hong Kok * value converter: added example Signed-off-by: ngrebels Signed-off-by: Yao Hong Kok * Remove underscore from variable name Signed-off-by: Yao Hong Kok * Fix formatting error from merging Signed-off-by: Yao Hong Kok Signed-off-by: ngrebels Signed-off-by: Yao Hong Kok Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Yao Hong Kok --- README.md | 14 ++++ config/config.go | 3 + examples/config.yml | 13 ++++ exporter/collector.go | 18 +++++ exporter/util.go | 23 ++++++ test/config/test-converter.yml | 116 +++++++++++++++++++++++++++++++ test/response/test-converter.txt | 29 ++++++++ test/serve/test-converter.json | 16 +++++ 8 files changed, 232 insertions(+) create mode 100644 test/config/test-converter.yml create mode 100644 test/response/test-converter.txt create mode 100644 test/serve/test-converter.json diff --git a/README.md b/README.md index b2e0e265..8d694baa 100644 --- a/README.md +++ b/README.md @@ -60,6 +60,18 @@ modules: active: 1 # static value count: '{.count}' # dynamic value boolean: '{.some_boolean}' + + - name: example_convert + type: object + path: '{.values[0,1]}' + labels: + state: '{.state}' + values: + state: '{.state}' + valueconverter: + '{.state}': #convert value 'state' into a number + active: 1 + inactive: 2 headers: X-Dummy: my-test-header @@ -70,6 +82,8 @@ Serving HTTP on 0.0.0.0 port 8000 ... $ ./json_exporter --config.file examples/config.yml & $ curl "http://localhost:7979/probe?module=default&target=http://localhost:8000/examples/data.json" | grep ^example +example_convert_state{state="ACTIVE"} 1 +example_convert_state{state="INACTIVE"} 2 example_global_value{environment="beta",location="planet-mars"} 1234 example_value_active{environment="beta",id="id-A"} 1 example_value_active{environment="beta",id="id-C"} 1 diff --git a/config/config.go b/config/config.go index 8a5322de..b290cdd8 100644 --- a/config/config.go +++ b/config/config.go @@ -30,8 +30,11 @@ type Metric struct { EpochTimestamp string Help string Values map[string]string + ValueConverter ValueConverterType } +type ValueConverterType map[string]map[string]string + type ScrapeType string const ( diff --git a/examples/config.yml b/examples/config.yml index 70e43a75..77f36dfb 100644 --- a/examples/config.yml +++ b/examples/config.yml @@ -25,6 +25,19 @@ modules: active: 1 # static value count: '{.count}' # dynamic value boolean: '{.some_boolean}' + + - name: example_convert + type: object + path: '{.values[0,1]}' + labels: + state: '{.state}' + values: + state: '{.state}' + valueconverter: + '{.state}': #convert value 'state' in JSON into a number + active: 1 + inactive: 2 + headers: X-Dummy: my-test-header diff --git a/exporter/collector.go b/exporter/collector.go index 4effc10f..2ef76b0e 100644 --- a/exporter/collector.go +++ b/exporter/collector.go @@ -16,6 +16,7 @@ package exporter import ( "bytes" "encoding/json" + "strings" "time" "github.com/go-kit/log" @@ -38,6 +39,7 @@ type JSONMetric struct { ValueJSONPath string LabelsJSONPaths []string ValueType prometheus.ValueType + ValueConverter config.ValueConverterType EpochTimestampJSONPath string } @@ -86,11 +88,14 @@ func (mc JSONMetricCollector) Collect(ch chan<- prometheus.Metric) { continue } value, err := extractValue(mc.Logger, jdata, m.ValueJSONPath, false) + if err != nil { level.Error(mc.Logger).Log("msg", "Failed to extract value for metric", "path", m.ValueJSONPath, "err", err, "metric", m.Desc) continue } + value = convertValueIfNeeded(m, value) + if floatValue, err := SanitizeValue(value); err == nil { metric := prometheus.MustNewConstMetric( m.Desc, @@ -161,6 +166,19 @@ func extractLabels(logger log.Logger, data []byte, paths []string) []string { return labels } +// Returns the conversion of the dynamic value- if it exists in the ValueConverter configuration +func convertValueIfNeeded(m JSONMetric, value string) string { + if m.ValueConverter != nil { + if valueMappings, hasPathKey := m.ValueConverter[m.ValueJSONPath]; hasPathKey { + value = strings.ToLower(value) + + if _, hasValueKey := valueMappings[value]; hasValueKey { + value = valueMappings[value] + } + } + } + return value +} func timestampMetric(logger log.Logger, m JSONMetric, data []byte, pm prometheus.Metric) prometheus.Metric { if m.EpochTimestampJSONPath == "" { return pm diff --git a/exporter/util.go b/exporter/util.go index d684fac3..d3dbc739 100644 --- a/exporter/util.go +++ b/exporter/util.go @@ -118,6 +118,9 @@ func CreateMetricsList(c config.Module) ([]JSONMetric, error) { variableLabels = append(variableLabels, k) variableLabelsValues = append(variableLabelsValues, v) } + + var valueConverters config.ValueConverterType = initializeValueConverter(metric) + jsonMetric := JSONMetric{ Type: config.ObjectScrape, Desc: prometheus.NewDesc( @@ -130,6 +133,7 @@ func CreateMetricsList(c config.Module) ([]JSONMetric, error) { ValueJSONPath: valuePath, LabelsJSONPaths: variableLabelsValues, ValueType: valueType, + ValueConverter: valueConverters, EpochTimestampJSONPath: metric.EpochTimestamp, } metrics = append(metrics, jsonMetric) @@ -245,3 +249,22 @@ func renderBody(logger log.Logger, body config.Body, tplValues url.Values) (meth } return } + +// Initializes and returns a ValueConverter object. nil if there aren't any conversions +func initializeValueConverter(metric config.Metric) config.ValueConverterType { + var valueConverters config.ValueConverterType + + //convert all keys to lowercase + if metric.ValueConverter != nil { + valueConverters = make(config.ValueConverterType) + for valuesKey, innerMap := range metric.ValueConverter { + //make the mappings for each value key lowercase + valueConverters[valuesKey] = make(map[string]string) + for conversionFrom, conversionTo := range innerMap { + valueConverters[valuesKey][strings.ToLower(conversionFrom)] = conversionTo + } + } + } + + return valueConverters +} diff --git a/test/config/test-converter.yml b/test/config/test-converter.yml new file mode 100644 index 00000000..0b87b517 --- /dev/null +++ b/test/config/test-converter.yml @@ -0,0 +1,116 @@ +--- +#the following tests use ./valueconverter.json + +modules: + default: + metrics: + #State should be converted to 1 for the metric value + #Test uppercase key + - name: test1 + path: "{$}" + help: Testing Single Value Converter on Type Object + type: object + labels: + name: '{.name}' + values: + state: '{.state}' + valueconverter: + '{.state}': + ACTIVE: 1 + + #State should be converted to 1 for the metric value + #Test lowercase key + - name: test2 + path: "{$}" + help: Testing Single Value Converter on Type Object + type: object + labels: + name: '{.name}' + values: + state: '{.state}' + valueconverter: + '{.state}': + active: 1 + + #There should be two JSONs returned: a metric for 'state' with value 1, and a metric with value 12 + - name: test3 + path: "{$}" + help: Testing Multi Diff Value Converter on Type Object + type: object + labels: + name: '{.name}' + values: + state: '{.state}' + active: 12 + valueconverter: + '{.state}': + ACTIVE: 1 + + #Nothing should be returned. This should be an error since 'state' can't be converted to an int + - name: test4 + path: "{$}" + help: Testing Value with missing conversion + type: object + labels: + name: '{.name}' + values: + state: '{.state}' + + #Test nested JSON. It should return with both values but with 12 as the metric value + - name: test5 + path: '{.values[*]}' + help: Testing Conversion with Missing value + type: object + labels: + name: '{.name}' + values: + active: 12 + valueconverter: + '{.state}': + ACTIVE: 1 + + #Test nested JSON. + #It should return with both values but 'state' should be converted + - name: test6 + path: '{.values[*]}' + help: Testing Value with Multiple Conversions + type: object + labels: + name: '{.name}' + values: + state: '{.state}' + valueconverter: + '{.state}': + ACTIVE: 1 + inactive: 2 + + #Test nested JSON. + #However, it should only return with state value + #There is a missing key: 'down' in valueconverter + - name: test7 + path: '{.values[*]}' + help: Testing Value with Multiple Conversions and Missing Key, Value + type: object + labels: + name: '{.name}' + values: + state: '{.state}' + valueconverter: + '{.state}': + ACTIVE: 1 + + #Two metrics should be returned + #State should be mapped to 1. + #ResponseCode should be 2 because it already has a value in the JSON. + - name: test8 + path: "{$}" + help: Testing Multi Diff Value Converter on Type Object + type: object + labels: + name: '{.name}' + values: + stat: '{.state}' + active: '{.responseCode}' + valueconverter: + '{.state}': + ACTIVE: 1 \ No newline at end of file diff --git a/test/response/test-converter.txt b/test/response/test-converter.txt new file mode 100644 index 00000000..8e7e5927 --- /dev/null +++ b/test/response/test-converter.txt @@ -0,0 +1,29 @@ +# HELP test1_state Testing Single Value Converter on Type Object +# TYPE test1_state untyped +test1_state{name="Test Converter"} 1 +# HELP test2_state Testing Single Value Converter on Type Object +# TYPE test2_state untyped +test2_state{name="Test Converter"} 1 +# HELP test3_active Testing Multi Diff Value Converter on Type Object +# TYPE test3_active untyped +test3_active{name="Test Converter"} 12 +# HELP test3_state Testing Multi Diff Value Converter on Type Object +# TYPE test3_state untyped +test3_state{name="Test Converter"} 1 +# HELP test5_active Testing Conversion with Missing value +# TYPE test5_active untyped +test5_active{name="id-A"} 12 +test5_active{name="id-B"} 12 +# HELP test6_state Testing Value with Multiple Conversions +# TYPE test6_state untyped +test6_state{name="id-A"} 1 +test6_state{name="id-B"} 2 +# HELP test7_state Testing Value with Multiple Conversions and Missing Key, Value +# TYPE test7_state untyped +test7_state{name="id-A"} 1 +# HELP test8_active Testing Multi Diff Value Converter on Type Object +# TYPE test8_active untyped +test8_active{name="Test Converter"} 2 +# HELP test8_stat Testing Multi Diff Value Converter on Type Object +# TYPE test8_stat untyped +test8_stat{name="Test Converter"} 1 diff --git a/test/serve/test-converter.json b/test/serve/test-converter.json new file mode 100644 index 00000000..c6fd58b9 --- /dev/null +++ b/test/serve/test-converter.json @@ -0,0 +1,16 @@ +{ + "name": "Test Converter", + "state": "ACTIVE", + "responseCode": 2, + "values": [ + { + "name": "id-A", + "state": "ACTIVE" + }, + { + "name": "id-B", + "state": "INACTIVE" + } + ] +} + \ No newline at end of file