Skip to content

Commit

Permalink
Feat: Add data transformations.
Browse files Browse the repository at this point in the history
Signed-off-by: Matt Kruse <[email protected]>
  • Loading branch information
MattK360 committed Oct 1, 2024
1 parent 03abaa7 commit 778510a
Show file tree
Hide file tree
Showing 12 changed files with 337 additions and 14 deletions.
14 changes: 14 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,20 @@ animal_population{name="deer",predator="false"} 456
animal_population{name="lion",predator="true"} 123
animal_population{name="pigeon",predator="false"} 789

## Test transform module
$ curl "http://localhost:7979/probe?module=transform&target=http://localhost:8000/examples/transform-data.json"

# HELP origin_health Health of each origin in the pool
# TYPE origin_health untyped
origin_health{address="10.0.0.1",endpoint_name="origin3",pool_id="2",pool_name="pool2"} 1
origin_health{address="10.0.0.1",endpoint_name="origin4",pool_id="3",pool_name="pool3"} 0
origin_health{address="127.0.0.1",endpoint_name="origin1",pool_id="1",pool_name="pool1"} 1
origin_health{address="192.168.1.1",endpoint_name="origin2",pool_id="1",pool_name="pool1"} 0
# HELP pool_health Health of the pools
# TYPE pool_health untyped
pool_health{pool_id="1",pool_name="pool1"} 1
pool_health{pool_id="2",pool_name="pool2"} 1
pool_health{pool_id="3",pool_name="pool3"} 0

## TEST through prometheus:

Expand Down
18 changes: 10 additions & 8 deletions config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
package config

import (
"github.com/prometheus-community/json_exporter/transformers"
"os"

pconfig "github.com/prometheus/common/config"
Expand All @@ -22,14 +23,15 @@ import (

// Metric contains values that define a metric
type Metric struct {
Name string
Path string
Labels map[string]string
Type ScrapeType
ValueType ValueType
EpochTimestamp string
Help string
Values map[string]string
Name string
Path string
Labels map[string]string
Type ScrapeType
ValueType ValueType
EpochTimestamp string
Help string
Values map[string]string
Transformations []transformers.TransformationConfig
}

type ScrapeType string
Expand Down
31 changes: 31 additions & 0 deletions examples/config.yml
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,37 @@ modules:
values:
population: '{ .population }'

transform:
metrics:
- name: origin
transformations:
- type: jq
query: |-
.result[] | .name as $poolName | .id as $poolId | .origins[] | ( .name as $name | .healthy as $endpointHealth | {endpointName: $name, endpointHealthy: .healthy, poolName: $poolName, address:.address, poolId:$poolId, address:.address} )
help: Health of each origin in the pool
path: '{ [*] }'
type: object
labels:
pool_id: '{.poolId}'
pool_name: '{.poolName}'
address: '{.address}'
endpoint_name: '{.endpointName}'
values:
health: '{.endpointHealthy}' # Extract only the `healthy` field


- name: pool
type: object
help: Health of the pools
path: '{.result[*]}'
labels:
pool_name: '{.name}'
pool_id: '{.id}'
values:
health: '{.healthy}'



## HTTP connection configurations can be set in 'modules.<module_name>.http_client_config' field. For full http client config parameters, ref: https://pkg.go.dev/github.com/prometheus/common/config?tab=doc#HTTPClientConfig
#
# http_client_config:
Expand Down
15 changes: 15 additions & 0 deletions examples/prometheus.yml
Original file line number Diff line number Diff line change
Expand Up @@ -32,3 +32,18 @@ scrape_configs:
## Location of the json exporter's real <hostname>:<port>
replacement: host.docker.internal:7979 # equivalent to "localhost:7979"

## Gather metrics from transform-data JSON source using the 'transform' module
- job_name: json_transform
metrics_path: /probe
params:
module: [transform] # Use the 'transform' module
static_configs:
- targets:
- http://localhost:8000/examples/transform-data.json
relabel_configs:
- source_labels: [__address__]
target_label: __param_target
- source_labels: [__param_target]
target_label: instance
- target_label: __address__
replacement: host.docker.internal:7979 # json_exporter instance
45 changes: 45 additions & 0 deletions examples/transform-data.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
{
"result": [
{
"name": "pool1",
"id": "1",
"healthy": true,
"origins": [
{
"name": "origin1",
"address": "127.0.0.1",
"healthy": true
},
{
"name": "origin2",
"address": "192.168.1.1",
"healthy": false
}
]
},
{
"name": "pool2",
"id": "2",
"healthy": true,
"origins": [
{
"name": "origin3",
"address": "10.0.0.1",
"healthy": true
}
]
},
{
"name": "pool3",
"id": "3",
"healthy": false,
"origins": [
{
"name": "origin4",
"address": "10.0.0.1",
"healthy": false
}
]
}
]
}
26 changes: 20 additions & 6 deletions exporter/collector.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ package exporter
import (
"bytes"
"encoding/json"
"github.com/prometheus-community/json_exporter/transformers"
"time"

"github.com/go-kit/log"
Expand All @@ -39,6 +40,7 @@ type JSONMetric struct {
LabelsJSONPaths []string
ValueType prometheus.ValueType
EpochTimestampJSONPath string
Transformers []transformers.Transformer
}

func (mc JSONMetricCollector) Describe(ch chan<- *prometheus.Desc) {
Expand All @@ -49,11 +51,22 @@ func (mc JSONMetricCollector) Describe(ch chan<- *prometheus.Desc) {

func (mc JSONMetricCollector) Collect(ch chan<- prometheus.Metric) {
for _, m := range mc.JSONMetrics {

jsonData := mc.Data
for _, transformer := range m.Transformers {
transformedData, err := transformer.Transform(jsonData)
if err != nil {
level.Error(mc.Logger).Log("msg", "Transformation failed", "err", err)
continue
}
jsonData = transformedData
}

switch m.Type {
case config.ValueScrape:
value, err := extractValue(mc.Logger, mc.Data, m.KeyJSONPath, false)
value, err := extractValue(mc.Logger, jsonData, m.KeyJSONPath, false)
if err != nil {
level.Error(mc.Logger).Log("msg", "Failed to extract value for metric", "path", m.KeyJSONPath, "err", err, "metric", m.Desc)
level.Error(mc.Logger).Log("msg", "Failed to extract value for metric", "path", m.KeyJSONPath, "err", err, "metric", m.Desc, "data", jsonData)
continue
}

Expand All @@ -62,16 +75,17 @@ func (mc JSONMetricCollector) Collect(ch chan<- prometheus.Metric) {
m.Desc,
m.ValueType,
floatValue,
extractLabels(mc.Logger, mc.Data, m.LabelsJSONPaths)...,
extractLabels(mc.Logger, jsonData, m.LabelsJSONPaths)...,
)
ch <- timestampMetric(mc.Logger, m, mc.Data, metric)
ch <- timestampMetric(mc.Logger, m, jsonData, metric)
} else {
level.Error(mc.Logger).Log("msg", "Failed to convert extracted value to float64", "path", m.KeyJSONPath, "value", value, "err", err, "metric", m.Desc)
level.Error(mc.Logger).Log("msg", "Failed to convert extracted value to float64", "path", m.KeyJSONPath, "value", value, "err", err, "metric", m.Desc) // was getting error here!
continue
}

case config.ObjectScrape:
values, err := extractValue(mc.Logger, mc.Data, m.KeyJSONPath, true)
values, err := extractValue(mc.Logger, jsonData, m.KeyJSONPath, true)

if err != nil {
level.Error(mc.Logger).Log("msg", "Failed to extract json objects for metric", "err", err, "metric", m.Desc)
continue
Expand Down
19 changes: 19 additions & 0 deletions exporter/util.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ import (
"context"
"errors"
"fmt"
"github.com/prometheus-community/json_exporter/transformers"
"io"
"math"
"net/http"
Expand Down Expand Up @@ -80,6 +81,22 @@ func CreateMetricsList(c config.Module) ([]JSONMetric, error) {
valueType prometheus.ValueType
)
for _, metric := range c.Metrics {
metricTransfomers := []transformers.Transformer{}
for _, tConfig := range metric.Transformations {
transformer, err := transformers.NewTransformer(tConfig) // Use the package reference here
if err != nil {
return nil, err
}
metricTransfomers = append(metricTransfomers, transformer)
}
switch metric.ValueType {
case config.ValueTypeGauge:
valueType = prometheus.GaugeValue
case config.ValueTypeCounter:
valueType = prometheus.CounterValue
default:
valueType = prometheus.UntypedValue
}
switch metric.ValueType {
case config.ValueTypeGauge:
valueType = prometheus.GaugeValue
Expand Down Expand Up @@ -107,6 +124,7 @@ func CreateMetricsList(c config.Module) ([]JSONMetric, error) {
LabelsJSONPaths: variableLabelsValues,
ValueType: valueType,
EpochTimestampJSONPath: metric.EpochTimestamp,
Transformers: metricTransfomers, // Add transformers here
}
metrics = append(metrics, jsonMetric)
case config.ObjectScrape:
Expand All @@ -130,6 +148,7 @@ func CreateMetricsList(c config.Module) ([]JSONMetric, error) {
LabelsJSONPaths: variableLabelsValues,
ValueType: valueType,
EpochTimestampJSONPath: metric.EpochTimestamp,
Transformers: metricTransfomers, // Add transformers here
}
metrics = append(metrics, jsonMetric)
}
Expand Down
2 changes: 2 additions & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,8 @@ require (
github.com/google/uuid v1.3.0 // indirect
github.com/huandu/xstrings v1.3.3 // indirect
github.com/imdario/mergo v0.3.11 // indirect
github.com/itchyny/gojq v0.12.16 // indirect
github.com/itchyny/timefmt-go v0.1.6 // indirect
github.com/jpillora/backoff v1.0.0 // indirect
github.com/mitchellh/copystructure v1.0.0 // indirect
github.com/mitchellh/reflectwalk v1.0.0 // indirect
Expand Down
4 changes: 4 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,10 @@ github.com/huandu/xstrings v1.3.3 h1:/Gcsuc1x8JVbJ9/rlye4xZnVAbEkGauT8lbebqcQws4
github.com/huandu/xstrings v1.3.3/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE=
github.com/imdario/mergo v0.3.11 h1:3tnifQM4i+fbajXKBHXWEH+KvNHqojZ778UH75j3bGA=
github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA=
github.com/itchyny/gojq v0.12.16 h1:yLfgLxhIr/6sJNVmYfQjTIv0jGctu6/DgDoivmxTr7g=
github.com/itchyny/gojq v0.12.16/go.mod h1:6abHbdC2uB9ogMS38XsErnfqJ94UlngIJGlRAIj4jTM=
github.com/itchyny/timefmt-go v0.1.6 h1:ia3s54iciXDdzWzwaVKXZPbiXzxxnv1SPGFfM/myJ5Q=
github.com/itchyny/timefmt-go v0.1.6/go.mod h1:RRDZYC5s9ErkjQvTvvU7keJjxUYzIISJGxm9/mAERQg=
github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA=
github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
Expand Down
55 changes: 55 additions & 0 deletions transformers/jq_transformer.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
package transformers

import (
"encoding/json"
"github.com/itchyny/gojq"
)

// JQTransformer struct for jq transformation
type JQTransformer struct {
Query string
}

// NewJQTransformer creates a new JQTransformer with a given query
func NewJQTransformer(query string) JQTransformer {
return JQTransformer{Query: query}
}

// Transform applies the jq filter transformation to the input data
func (jq JQTransformer) Transform(data []byte) ([]byte, error) {
return applyJQFilter(data, jq.Query)
}

// applyJQFilter uses gojq to apply a jq transformation to the input data
func applyJQFilter(jsonData []byte, jqQuery string) ([]byte, error) {
var input interface{}
if err := json.Unmarshal(jsonData, &input); err != nil {
return nil, err
}

query, err := gojq.Parse(jqQuery)
if err != nil {
return nil, err
}

iter := query.Run(input)
var results []interface{}
for {
v, ok := iter.Next()
if !ok {
break
}
if err, ok := v.(error); ok {
return nil, err
}
results = append(results, v)
}

// Convert the transformed result back to JSON []byte
transformedJSON, err := json.Marshal(results)
if err != nil {
return nil, err
}

return transformedJSON, nil
}
21 changes: 21 additions & 0 deletions transformers/transformers.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
package transformers

import "fmt"

type Transformer interface {
Transform(data []byte) ([]byte, error)
}

type TransformationConfig struct {
Type string
Query string
}

func NewTransformer(config TransformationConfig) (Transformer, error) {
switch config.Type {
case "jq":
return NewJQTransformer(config.Query), nil
default:
return nil, fmt.Errorf("unsupported transformer type: %s", config.Type)
}
}
Loading

0 comments on commit 778510a

Please sign in to comment.