WIP: Initial support for CEL

This adds initial support for queries using the common expression
language (CEL).

Signed-off-by: Manuel Rüger <[email protected]>

config/config.go

@@ -23,6 +23,7 @@ import (
 // Metric contains values that define a metric
 type Metric struct {
 	Name           string
+	Engine         EngineType
 	Path           string
 	Labels         map[string]string
 	Type           ScrapeType
@@ -44,7 +45,14 @@ type ValueType string
 const (
 	ValueTypeGauge   ValueType = "gauge"
 	ValueTypeCounter ValueType = "counter"
-	ValueTypeUntyped ValueType = "untyped"
+	ValueTypeUntyped ValueType = "untyped" // default
+)
+
+type EngineType string
+
+const (
+	EngineTypeJSONPath EngineType = "jsonpath" // default
+	EngineTypeCEL      EngineType = "cel"
 )
 
 // Config contains multiple modules.
@@ -89,6 +97,9 @@ func LoadConfig(configPath string) (Config, error) {
 			if module.Metrics[i].ValueType == "" {
 				module.Metrics[i].ValueType = ValueTypeUntyped
 			}
+			if module.Metrics[i].Engine == "" {
+				module.Metrics[i].Engine = EngineTypeJSONPath
+			}
 		}
 	}
 

examples/config.yml

@@ -9,7 +9,15 @@ modules:
       help: Example of a top-level global value scrape in the json
       labels:
         environment: beta # static label
-        location: 'planet-{.location}'          # dynamic label
+        location: 'planet-{ .location }' # dynamic label
+    - name: example_cel_global_value
+      engine: cel
+      path: '.counter'
+      help: Example of a top-level global value scrape in the json using cel
+      valuetype: 'gauge'
+      labels:
+        environment: "\"beta\"" # static label. Quotes need to be escaped for CEL
+        location: "\"planet-\"+.location" # dynamic label. Quotes need to be escaped for CEL
     - name: example_timestamped_value
       type: object
       path: '{ .values[?(@.state == "INACTIVE")] }'

exporter/collector.go

@@ -16,10 +16,12 @@ package exporter
 import (
 	"bytes"
 	"encoding/json"
+	"fmt"
 	"time"
 
 	"github.com/go-kit/log"
 	"github.com/go-kit/log/level"
+	"github.com/google/cel-go/cel"
 	"github.com/prometheus-community/json_exporter/config"
 	"github.com/prometheus/client_golang/prometheus"
 	"k8s.io/client-go/util/jsonpath"
@@ -34,6 +36,7 @@ type JSONMetricCollector struct {
 type JSONMetric struct {
 	Desc                   *prometheus.Desc
 	Type                   config.ScrapeType
+	EngineType             config.EngineType
 	KeyJSONPath            string
 	ValueJSONPath          string
 	LabelsJSONPaths        []string
@@ -51,7 +54,8 @@ func (mc JSONMetricCollector) Collect(ch chan<- prometheus.Metric) {
 	for _, m := range mc.JSONMetrics {
 		switch m.Type {
 		case config.ValueScrape:
-			value, err := extractValue(mc.Logger, mc.Data, m.KeyJSONPath, false)
+			level.Debug(mc.Logger).Log("msg", "Extracting value for metric", "path", m.KeyJSONPath, "metric", m.Desc)
+			value, err := extractValue(mc.Logger, m.EngineType, mc.Data, m.KeyJSONPath, false)
 			if err != nil {
 				level.Error(mc.Logger).Log("msg", "Failed to extract value for metric", "path", m.KeyJSONPath, "err", err, "metric", m.Desc)
 				continue
@@ -62,7 +66,7 @@ func (mc JSONMetricCollector) Collect(ch chan<- prometheus.Metric) {
 					m.Desc,
 					m.ValueType,
 					floatValue,
-					extractLabels(mc.Logger, mc.Data, m.LabelsJSONPaths)...,
+					extractLabels(mc.Logger, m.EngineType, mc.Data, m.LabelsJSONPaths)...,
 				)
 				ch <- timestampMetric(mc.Logger, m, mc.Data, metric)
 			} else {
@@ -71,7 +75,8 @@ func (mc JSONMetricCollector) Collect(ch chan<- prometheus.Metric) {
 			}
 
 		case config.ObjectScrape:
-			values, err := extractValue(mc.Logger, mc.Data, m.KeyJSONPath, true)
+			level.Debug(mc.Logger).Log("msg", "Extracting object for metric", "path", m.KeyJSONPath, "metric", m.Desc)
+			values, err := extractValue(mc.Logger, m.EngineType, mc.Data, m.KeyJSONPath, true)
 			if err != nil {
 				level.Error(mc.Logger).Log("msg", "Failed to extract json objects for metric", "err", err, "metric", m.Desc)
 				continue
@@ -85,7 +90,7 @@ func (mc JSONMetricCollector) Collect(ch chan<- prometheus.Metric) {
 						level.Error(mc.Logger).Log("msg", "Failed to marshal data to json", "path", m.ValueJSONPath, "err", err, "metric", m.Desc, "data", data)
 						continue
 					}
-					value, err := extractValue(mc.Logger, jdata, m.ValueJSONPath, false)
+					value, err := extractValue(mc.Logger, m.EngineType, jdata, m.ValueJSONPath, false)
 					if err != nil {
 						level.Error(mc.Logger).Log("msg", "Failed to extract value for metric", "path", m.ValueJSONPath, "err", err, "metric", m.Desc)
 						continue
@@ -96,7 +101,7 @@ func (mc JSONMetricCollector) Collect(ch chan<- prometheus.Metric) {
 							m.Desc,
 							m.ValueType,
 							floatValue,
-							extractLabels(mc.Logger, jdata, m.LabelsJSONPaths)...,
+							extractLabels(mc.Logger, m.EngineType, jdata, m.LabelsJSONPaths)...,
 						)
 						ch <- timestampMetric(mc.Logger, m, jdata, metric)
 					} else {
@@ -115,8 +120,19 @@ func (mc JSONMetricCollector) Collect(ch chan<- prometheus.Metric) {
 	}
 }
 
+func extractValue(logger log.Logger, engine config.EngineType, data []byte, path string, enableJSONOutput bool) (string, error) {
+	switch engine {
+	case config.EngineTypeJSONPath:
+		return extractValueJSONPath(logger, data, path, enableJSONOutput)
+	case config.EngineTypeCEL:
+		return extractValueCEL(logger, data, path)
+	default:
+		return "", fmt.Errorf("Unknown engine type: %s", engine)
+	}
+}
+
 // Returns the last matching value at the given json path
-func extractValue(logger log.Logger, data []byte, path string, enableJSONOutput bool) (string, error) {
+func extractValueJSONPath(logger log.Logger, data []byte, path string, enableJSONOutput bool) (string, error) {
 	var jsonData interface{}
 	buf := new(bytes.Buffer)
 
@@ -148,11 +164,61 @@ func extractValue(logger log.Logger, data []byte, path string, enableJSONOutput
 	return buf.String(), nil
 }
 
+// Returns the last matching value at the given json path
+func extractValueCEL(logger log.Logger, data []byte, expression string) (string, error) {
+
+	var jsonData map[string]any
+
+	err := json.Unmarshal(data, &jsonData)
+	if err != nil {
+		level.Error(logger).Log("msg", "Failed to unmarshal data to json", "err", err, "data", data)
+		return "", err
+	}
+
+	inputVars := make([]cel.EnvOption, 0, len(jsonData))
+	for k := range jsonData {
+		inputVars = append(inputVars, cel.Variable(k, cel.DynType))
+	}
+
+	env, err := cel.NewEnv(inputVars...)
+
+	if err != nil {
+		level.Error(logger).Log("msg", "Failed to set up CEL environment", "err", err, "data", data)
+		return "", err
+	}
+
+	ast, issues := env.Compile(expression)
+	if issues != nil && issues.Err() != nil {
+		level.Error(logger).Log("CEL type-check error", issues.Err(), "expression", expression)
+		return "", err
+	}
+	prg, err := env.Program(ast)
+	if err != nil {
+		level.Error(logger).Log("CEL program construction error", err)
+		return "", err
+	}
+
+	out, _, err := prg.Eval(jsonData)
+	if err != nil {
+		level.Error(logger).Log("msg", "Failed to evaluate cel query", "err", err, "expression", expression, "data", jsonData)
+		return "", err
+	}
+
+	result := out.ConvertToType(cel.StringType)
+
+	// Since we are finally going to extract only float64, unquote if necessary
+	if res, err := jsonpath.UnquoteExtend(result.Value().(string)); err == nil {
+		return res, nil
+	}
+
+	return result.Value().(string), nil
+}
+
 // Returns the list of labels created from the list of provided json paths
-func extractLabels(logger log.Logger, data []byte, paths []string) []string {
+func extractLabels(logger log.Logger, engine config.EngineType, data []byte, paths []string) []string {
 	labels := make([]string, len(paths))
 	for i, path := range paths {
-		if result, err := extractValue(logger, data, path, false); err == nil {
+		if result, err := extractValue(logger, engine, data, path, false); err == nil {
 			labels[i] = result
 		} else {
 			level.Error(logger).Log("msg", "Failed to extract label value", "err", err, "path", path, "data", data)
@@ -165,7 +231,7 @@ func timestampMetric(logger log.Logger, m JSONMetric, data []byte, pm prometheus
 	if m.EpochTimestampJSONPath == "" {
 		return pm
 	}
-	ts, err := extractValue(logger, data, m.EpochTimestampJSONPath, false)
+	ts, err := extractValue(logger, m.EngineType, data, m.EpochTimestampJSONPath, false)
 	if err != nil {
 		level.Error(logger).Log("msg", "Failed to extract timestamp for metric", "path", m.KeyJSONPath, "err", err, "metric", m.Desc)
 		return pm

exporter/util.go

@@ -103,6 +103,7 @@ func CreateMetricsList(c config.Module) ([]JSONMetric, error) {
 					variableLabels,
 					nil,
 				),
+				EngineType:             metric.Engine,
 				KeyJSONPath:            metric.Path,
 				LabelsJSONPaths:        variableLabelsValues,
 				ValueType:              valueType,
@@ -125,6 +126,7 @@ func CreateMetricsList(c config.Module) ([]JSONMetric, error) {
 						variableLabels,
 						nil,
 					),
+					EngineType:             metric.Engine,
 					KeyJSONPath:            metric.Path,
 					ValueJSONPath:          valuePath,
 					LabelsJSONPaths:        variableLabelsValues,
@@ -134,7 +136,7 @@ func CreateMetricsList(c config.Module) ([]JSONMetric, error) {
 				metrics = append(metrics, jsonMetric)
 			}
 		default:
-			return nil, fmt.Errorf("Unknown metric type: '%s', for metric: '%s'", metric.Type, metric.Name)
+			return nil, fmt.Errorf("unknown metric type: '%s', for metric: '%s'", metric.Type, metric.Name)
 		}
 	}
 	return metrics, nil

go.mod

@@ -6,6 +6,7 @@ require (
 	github.com/Masterminds/sprig/v3 v3.2.3
 	github.com/alecthomas/kingpin/v2 v2.3.2
 	github.com/go-kit/log v0.2.1
+	github.com/google/cel-go v0.18.2
 	github.com/prometheus/client_golang v1.17.0
 	github.com/prometheus/common v0.45.0
 	github.com/prometheus/exporter-toolkit v0.10.0
@@ -17,6 +18,7 @@ require (
 	github.com/Masterminds/goutils v1.1.1 // indirect
 	github.com/Masterminds/semver/v3 v3.2.0 // indirect
 	github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect
+	github.com/antlr4-go/antlr/v4 v4.13.0 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/coreos/go-systemd/v22 v22.5.0 // indirect
@@ -35,13 +37,17 @@ require (
 	github.com/prometheus/procfs v0.11.1 // indirect
 	github.com/shopspring/decimal v1.2.0 // indirect
 	github.com/spf13/cast v1.3.1 // indirect
+	github.com/stoewer/go-strcase v1.2.0 // indirect
 	github.com/xhit/go-str2duration/v2 v2.1.0 // indirect
 	golang.org/x/crypto v0.14.0 // indirect
+	golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc // indirect
 	golang.org/x/net v0.17.0 // indirect
 	golang.org/x/oauth2 v0.12.0 // indirect
 	golang.org/x/sync v0.3.0 // indirect
 	golang.org/x/sys v0.13.0 // indirect
 	golang.org/x/text v0.13.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20230803162519-f966b187b2e5 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20230803162519-f966b187b2e5 // indirect
 	google.golang.org/protobuf v1.31.0 // indirect
 )

go.sum

@@ -8,6 +8,8 @@ github.com/alecthomas/kingpin/v2 v2.3.2 h1:H0aULhgmSzN8xQ3nX1uxtdlTHYoPLu5AhHxWr
 github.com/alecthomas/kingpin/v2 v2.3.2/go.mod h1:0gyi0zQnjuFk8xrkNKamJoyUo382HRL7ATRpFZCw6tE=
 github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 h1:s6gZFSlWYmbqAuRjVTiNNhvNRfY2Wxp9nhfyel4rklc=
 github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE=
+github.com/antlr4-go/antlr/v4 v4.13.0 h1:lxCg3LAv+EUK6t1i0y1V6/SLeUi0eKEKdhQAlS8TVTI=
+github.com/antlr4-go/antlr/v4 v4.13.0/go.mod h1:pfChB/xh/Unjila75QW7+VU4TSnWnnk9UTnmpPaOR2g=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
@@ -27,6 +29,8 @@ github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5y
 github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
 github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=
 github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
+github.com/google/cel-go v0.18.2 h1:L0B6sNBSVmt0OyECi8v6VOS74KOc9W/tLiWKfZABvf4=
+github.com/google/cel-go v0.18.2/go.mod h1:kWcIzTsPX0zmQ+H3TirHstLLf9ep5QTsZBN9u4dOYLg=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
 github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
@@ -66,6 +70,8 @@ github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXY
 github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=
 github.com/spf13/cast v1.3.1 h1:nFm6S0SMdyzrzcmThSipiEubIDy8WEXKNZ0UOgiRpng=
 github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
+github.com/stoewer/go-strcase v1.2.0 h1:Z2iHWqGXH00XYgqDmNgQbIBxf3wrNq0F3feEy0ainaU=
+github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
@@ -79,6 +85,8 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y
 golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4=
 golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc=
 golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=
+golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc h1:mCRnTeVUjcrhlRmO0VK8a6k6Rrf6TF9htwo2pJVSjIU=
+golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
 golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
@@ -118,6 +126,10 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c=
 google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
+google.golang.org/genproto/googleapis/api v0.0.0-20230803162519-f966b187b2e5 h1:nIgk/EEq3/YlnmVVXVnm14rC2oxgs1o0ong4sD/rd44=
+google.golang.org/genproto/googleapis/api v0.0.0-20230803162519-f966b187b2e5/go.mod h1:5DZzOUPCLYL3mNkQ0ms0F3EuUNZ7py1Bqeq6sxzI7/Q=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20230803162519-f966b187b2e5 h1:eSaPbMR4T7WfH9FvABk36NBMacoTUKdWCvV0dx+KfOg=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20230803162519-f966b187b2e5/go.mod h1:zBEcrKX2ZOcEkHWxBPAIvYUWOKKMIhYcmNiUIu2ji3I=
 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
 google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8=