Add "ignore_missing_values" option to metrics.

cmd/main_test.go

@@ -164,6 +164,43 @@ func TestCorrectResponse(t *testing.T) {
 	}
 }
 
+func TestIgnoreMissingValues(t *testing.T) {
+	tests := []struct {
+		ConfigFile    string
+		ServeFile     string
+		Module        string
+		ShouldSucceed bool
+	}{
+		{"../test/config/ignore_missing_values.yml", "/serve/good.json", "missing_value_ok", true},
+		{"../test/config/ignore_missing_values.yml", "/serve/good.json", "missing_value_not_ok", false},
+		{"../test/config/ignore_missing_values.yml", "/serve/good.json", "missing_object_value_ok", true},
+		{"../test/config/ignore_missing_values.yml", "/serve/good.json", "missing_object_value_not_ok", false},
+	}
+
+	target := httptest.NewServer(http.FileServer(http.Dir("../test")))
+	defer target.Close()
+
+	for i, test := range tests {
+		c, err := config.LoadConfig(test.ConfigFile)
+		if err != nil {
+			t.Fatalf("Failed to load config file %s", test.ConfigFile)
+		}
+
+		req := httptest.NewRequest("GET", "http://example.com/foo"+"?module="+test.Module+"&target="+target.URL+test.ServeFile, nil)
+		recorder := httptest.NewRecorder()
+		logBuffer := strings.Builder{}
+		probeHandler(recorder, req, log.NewLogfmtLogger(&logBuffer), c)
+
+		if test.ShouldSucceed && logBuffer.Len() > 0 {
+			t.Fatalf("Ignore missing values test %d (module: %s) fails unexpectedly.\nLOG:\n%s", i, test.Module, logBuffer.String())
+		}
+
+		if !test.ShouldSucceed && logBuffer.Len() == 0 {
+			t.Fatalf("Ignore missing values test %d (module: %s) succeeded unexpectedly.", i, test.Module)
+		}
+	}
+}
+
 func TestBasicAuth(t *testing.T) {
 	username := "myUser"
 	password := "mySecretPassword"

config/config.go

@@ -22,14 +22,15 @@ import (
 
 // Metric contains values that define a metric
 type Metric struct {
-	Name           string
-	Path           string
-	Labels         map[string]string
-	Type           ScrapeType
-	ValueType      ValueType
-	EpochTimestamp string
-	Help           string
-	Values         map[string]string
+	Name                string
+	Path                string
+	Labels              map[string]string
+	Type                ScrapeType
+	ValueType           ValueType
+	EpochTimestamp      string
+	Help                string
+	Values              map[string]string
+	IgnoreMissingValues bool `yaml:"ignore_missing_values,omitempty"`
 }
 
 type ScrapeType string

examples/config.yml

@@ -30,6 +30,13 @@ modules:
         active: 1         # static value
         count: '{.count}' # dynamic value
         boolean: '{.some_boolean}'
+    - name: example_missing_value
+      path: '{ .missing_value }'
+      ignore_missing_values: true
+      help: >-
+        Example of ignoring a missing value.
+        This metric will not be reported if the path .missing_value is not
+        present, and an error message will not be generated.
 
   animals:
     metrics:

exporter/collector.go

@@ -39,6 +39,7 @@ type JSONMetric struct {
 	LabelsJSONPaths        []string
 	ValueType              prometheus.ValueType
 	EpochTimestampJSONPath string
+	IgnoreMissingValues    bool
 }
 
 func (mc JSONMetricCollector) Describe(ch chan<- *prometheus.Desc) {
@@ -51,12 +52,16 @@ func (mc JSONMetricCollector) Collect(ch chan<- prometheus.Metric) {
 	for _, m := range mc.JSONMetrics {
 		switch m.Type {
 		case config.ValueScrape:
-			value, err := extractValue(mc.Logger, mc.Data, m.KeyJSONPath, false)
+			value, missing, err := extractValue(mc.Logger, mc.Data, m.KeyJSONPath, false, m.IgnoreMissingValues)
 			if err != nil {
 				level.Error(mc.Logger).Log("msg", "Failed to extract value for metric", "path", m.KeyJSONPath, "err", err, "metric", m.Desc)
 				continue
 			}
 
+			if missing {
+				continue
+			}
+
 			if floatValue, err := SanitizeValue(value); err == nil {
 				metric := prometheus.MustNewConstMetric(
 					m.Desc,
@@ -71,12 +76,16 @@ func (mc JSONMetricCollector) Collect(ch chan<- prometheus.Metric) {
 			}
 
 		case config.ObjectScrape:
-			values, err := extractValue(mc.Logger, mc.Data, m.KeyJSONPath, true)
+			values, missing, err := extractValue(mc.Logger, mc.Data, m.KeyJSONPath, true, m.IgnoreMissingValues)
 			if err != nil {
 				level.Error(mc.Logger).Log("msg", "Failed to extract json objects for metric", "err", err, "metric", m.Desc)
 				continue
 			}
 
+			if missing {
+				continue
+			}
+
 			var jsonData []interface{}
 			if err := json.Unmarshal([]byte(values), &jsonData); err == nil {
 				for _, data := range jsonData {
@@ -85,12 +94,16 @@ func (mc JSONMetricCollector) Collect(ch chan<- prometheus.Metric) {
 						level.Error(mc.Logger).Log("msg", "Failed to marshal data to json", "path", m.ValueJSONPath, "err", err, "metric", m.Desc, "data", data)
 						continue
 					}
-					value, err := extractValue(mc.Logger, jdata, m.ValueJSONPath, false)
+					value, missing, err := extractValue(mc.Logger, jdata, m.ValueJSONPath, false, m.IgnoreMissingValues)
 					if err != nil {
 						level.Error(mc.Logger).Log("msg", "Failed to extract value for metric", "path", m.ValueJSONPath, "err", err, "metric", m.Desc)
 						continue
 					}
 
+					if missing {
+						continue
+					}
+
 					if floatValue, err := SanitizeValue(value); err == nil {
 						metric := prometheus.MustNewConstMetric(
 							m.Desc,
@@ -115,8 +128,8 @@ func (mc JSONMetricCollector) Collect(ch chan<- prometheus.Metric) {
 	}
 }
 
-// Returns the last matching value at the given json path
-func extractValue(logger log.Logger, data []byte, path string, enableJSONOutput bool) (string, error) {
+// Returns the last matching value at the given json path and a flag if the path was missing
+func extractValue(logger log.Logger, data []byte, path string, enableJSONOutput bool, ignoreMissingValues bool) (string, bool, error) {
 	var jsonData interface{}
 	buf := new(bytes.Buffer)
 
@@ -125,34 +138,42 @@ func extractValue(logger log.Logger, data []byte, path string, enableJSONOutput
 		j.EnableJSONOutput(true)
 	}
 
+	if ignoreMissingValues {
+		j.AllowMissingKeys(true)
+	}
+
 	if err := json.Unmarshal(data, &jsonData); err != nil {
 		level.Error(logger).Log("msg", "Failed to unmarshal data to json", "err", err, "data", data)
-		return "", err
+		return "", false, err
 	}
 
 	if err := j.Parse(path); err != nil {
 		level.Error(logger).Log("msg", "Failed to parse jsonpath", "err", err, "path", path, "data", data)
-		return "", err
+		return "", false, err
 	}
 
 	if err := j.Execute(buf, jsonData); err != nil {
 		level.Error(logger).Log("msg", "Failed to execute jsonpath", "err", err, "path", path, "data", data)
-		return "", err
+		return "", false, err
+	}
+
+	if buf.Len() == 0 && ignoreMissingValues {
+		return "", true, nil
 	}
 
 	// Since we are finally going to extract only float64, unquote if necessary
 	if res, err := jsonpath.UnquoteExtend(buf.String()); err == nil {
-		return res, nil
+		return res, false, nil
 	}
 
-	return buf.String(), nil
+	return buf.String(), false, nil
 }
 
 // Returns the list of labels created from the list of provided json paths
 func extractLabels(logger log.Logger, data []byte, paths []string) []string {
 	labels := make([]string, len(paths))
 	for i, path := range paths {
-		if result, err := extractValue(logger, data, path, false); err == nil {
+		if result, _, err := extractValue(logger, data, path, false, false); err == nil {
 			labels[i] = result
 		} else {
 			level.Error(logger).Log("msg", "Failed to extract label value", "err", err, "path", path, "data", data)
@@ -165,11 +186,16 @@ func timestampMetric(logger log.Logger, m JSONMetric, data []byte, pm prometheus
 	if m.EpochTimestampJSONPath == "" {
 		return pm
 	}
-	ts, err := extractValue(logger, data, m.EpochTimestampJSONPath, false)
+	ts, missing, err := extractValue(logger, data, m.EpochTimestampJSONPath, false, m.IgnoreMissingValues)
 	if err != nil {
 		level.Error(logger).Log("msg", "Failed to extract timestamp for metric", "path", m.KeyJSONPath, "err", err, "metric", m.Desc)
 		return pm
 	}
+
+	if missing {
+		return pm
+	}
+
 	epochTime, err := SanitizeIntValue(ts)
 	if err != nil {
 		level.Error(logger).Log("msg", "Failed to parse timestamp for metric", "path", m.KeyJSONPath, "err", err, "metric", m.Desc)

exporter/util.go

@@ -107,6 +107,7 @@ func CreateMetricsList(c config.Module) ([]JSONMetric, error) {
 				LabelsJSONPaths:        variableLabelsValues,
 				ValueType:              valueType,
 				EpochTimestampJSONPath: metric.EpochTimestamp,
+				IgnoreMissingValues:    metric.IgnoreMissingValues,
 			}
 			metrics = append(metrics, jsonMetric)
 		case config.ObjectScrape:
@@ -130,6 +131,7 @@ func CreateMetricsList(c config.Module) ([]JSONMetric, error) {
 					LabelsJSONPaths:        variableLabelsValues,
 					ValueType:              valueType,
 					EpochTimestampJSONPath: metric.EpochTimestamp,
+					IgnoreMissingValues:    metric.IgnoreMissingValues,
 				}
 				metrics = append(metrics, jsonMetric)
 			}

test/config/ignore_missing_values.yml

@@ -0,0 +1,29 @@
+---
+modules:
+  missing_value_ok:
+    metrics:
+      - name: example_global_value_missing
+        ignore_missing_values: true
+        path: "{ .missing_value }"
+
+  missing_value_not_ok:
+    metrics:
+      - name: example_global_value_missing
+        path: "{ .missing_value }"
+
+  missing_object_value_ok:
+    metrics:
+      - name: example_value_in_object_missing
+        type: object
+        ignore_missing_values: true
+        path: "{.values[0]}"
+        values:
+          missing_value: "{ .missing_value }"
+
+  missing_object_value_not_ok:
+    metrics:
+      - name: example_value_in_object_missing
+        type: object
+        path: "{.values[0]}"
+        values:
+          missing_value: "{ .missing_value }"