fix(render): return error on partial targets fetch

cmd/carbonapi/http/render_handler.go

@@ -348,18 +348,18 @@ func renderHandler(w http.ResponseWriter, r *http.Request) {
 	var body []byte
 
 	returnCode := http.StatusOK
-	if len(results) == 0 {
+	if len(results) == 0 || (len(errors) > 0 && config.Config.Upstreams.RequireSuccessAll) {
 		// Obtain error code from the errors
 		// In case we have only "Not Found" errors, result should be 404
 		// Otherwise it should be 500
 		returnCode, errMsgs := helper.MergeHttpErrorMap(errors)
 		logger.Debug("error response or no response", zap.Strings("error", errMsgs))
 		// Allow override status code for 404-not-found replies.
-		if returnCode == 404 {
+		if returnCode == http.StatusNotFound {
 			returnCode = config.Config.NotFoundStatusCode
 		}
 
-		if returnCode == 400 || returnCode == http.StatusForbidden || returnCode >= 500 {
+		if returnCode == http.StatusBadRequest || returnCode == http.StatusNotFound || returnCode == http.StatusForbidden || returnCode >= 500 {
 			setError(w, accessLogDetails, strings.Join(errMsgs, ","), returnCode, uid.String())
 			logAsError = true
 			return

cmd/mockbackend/e2etesting.go

@@ -313,6 +313,7 @@ func e2eTest(logger *zap.Logger, noapp, breakOnError bool) bool {
 			failed = true
 			logger.Error("test failed",
 				zap.Errors("failures", failures),
+				zap.String("url", t.URL), zap.String("type", t.Type), zap.String("body", t.Body),
 			)
 			for _, v := range runningApps {
 				if !v.IsRunning() {

cmd/mockbackend/find.go

@@ -72,15 +72,33 @@ func (cfg *listener) findHandler(wr http.ResponseWriter, req *http.Request) {
 		Metrics: []carbonapi_v3_pb.GlobResponse{},
 	}
 
+	returnCode := http.StatusOK
 	if query[0] != "*" {
 		for m := range cfg.Listener.Expressions {
 			globMatches := []carbonapi_v3_pb.GlobMatch{}
 
-			for _, metric := range cfg.Expressions[m].Data {
-				globMatches = append(globMatches, carbonapi_v3_pb.GlobMatch{
-					Path:   metric.MetricName,
-					IsLeaf: true,
-				})
+			returnCode = http.StatusNotFound
+			if response, ok := cfg.Expressions[m]; ok {
+				if response.ReplyDelayMS > 0 {
+					delay := time.Duration(response.ReplyDelayMS) * time.Millisecond
+					time.Sleep(delay)
+				}
+				if response.HttpCode == http.StatusNotFound {
+					returnCode = http.StatusNotFound
+				} else if response.HttpCode != 0 && response.HttpCode != http.StatusOK {
+					// return first error
+					returnCode = response.HttpCode
+					http.Error(wr, http.StatusText(returnCode), returnCode)
+					return
+				} else {
+					returnCode = http.StatusOK
+					for _, metric := range response.Data {
+						globMatches = append(globMatches, carbonapi_v3_pb.GlobMatch{
+							Path:   metric.MetricName,
+							IsLeaf: true,
+						})
+					}
+				}
 			}
 			multiGlobs.Metrics = append(multiGlobs.Metrics,
 				carbonapi_v3_pb.GlobResponse{
@@ -96,8 +114,18 @@ func (cfg *listener) findHandler(wr http.ResponseWriter, req *http.Request) {
 				delay := time.Duration(response.ReplyDelayMS) * time.Millisecond
 				time.Sleep(delay)
 			}
-			for _, metric := range response.Data {
-				returnMap[metric.MetricName] = struct{}{}
+			if response.HttpCode == http.StatusNotFound {
+				returnCode = http.StatusNotFound
+			} else if response.HttpCode != 0 && response.HttpCode != http.StatusOK {
+				// return first error
+				returnCode = response.HttpCode
+				http.Error(wr, http.StatusText(returnCode), returnCode)
+				return
+			} else {
+				returnCode = http.StatusOK
+				for _, metric := range response.Data {
+					returnMap[metric.MetricName] = struct{}{}
+				}
 			}
 		}
 
@@ -123,6 +151,12 @@ func (cfg *listener) findHandler(wr http.ResponseWriter, req *http.Request) {
 		})
 	}
 
+	if returnCode == http.StatusNotFound {
+		// return 404 when no data
+		http.Error(wr, http.StatusText(returnCode), returnCode)
+		return
+	}
+
 	logger.Info("will return", zap.Any("response", multiGlobs))
 
 	var b []byte

cmd/mockbackend/http_common.go

@@ -7,6 +7,7 @@ import (
 )
 
 type Response struct {
+	HttpCode       int      `yaml:"httpCode"` // return error if one of expresssions not success
 	ReplyDelayMS   int      `yaml:"replyDelayMS"`
 	PathExpression string   `yaml:"pathExpression"`
 	Data           []Metric `yaml:"data"`
@@ -86,6 +87,7 @@ func copyResponse(src Response) Response {
 	dst := Response{
 		PathExpression: src.PathExpression,
 		ReplyDelayMS:   src.ReplyDelayMS,
+		HttpCode:       src.HttpCode,
 		Data:           make([]Metric, len(src.Data)),
 	}
 

cmd/mockbackend/render.go

@@ -97,68 +97,84 @@ func (cfg *listener) renderHandler(wr http.ResponseWriter, req *http.Request) {
 		Expressions: copyMap(cfg.Expressions),
 	}
 
+	returnCode := http.StatusOK
+
 	for _, target := range targets {
-		response, ok := newCfg.Expressions[target]
-		if !ok {
-			wr.WriteHeader(http.StatusNotFound)
-			_, _ = wr.Write([]byte("Not found"))
-			return
-		}
-		if response.ReplyDelayMS > 0 {
-			delay := time.Duration(response.ReplyDelayMS) * time.Millisecond
-			logger.Info("will add extra delay",
-				zap.Duration("delay", delay),
-			)
-			time.Sleep(delay)
-		}
-		for _, m := range response.Data {
-			step := m.Step
-			if step == 0 {
-				step = 1
-			}
-			startTime := m.StartTime
-			if startTime == 0 {
-				startTime = step
+		if response, ok := newCfg.Expressions[target]; ok {
+			if response.ReplyDelayMS > 0 {
+				delay := time.Duration(response.ReplyDelayMS) * time.Millisecond
+				logger.Info("will add extra delay",
+					zap.Duration("delay", delay),
+				)
+				time.Sleep(delay)
 			}
-			isAbsent := make([]bool, 0, len(m.Values))
-			protov2Values := make([]float64, 0, len(m.Values))
-			for i := range m.Values {
-				if math.IsNaN(m.Values[i]) {
-					isAbsent = append(isAbsent, true)
-					protov2Values = append(protov2Values, 0.0)
-				} else {
-					isAbsent = append(isAbsent, false)
-					protov2Values = append(protov2Values, m.Values[i])
-				}
-			}
-			fr2 := carbonapi_v2_pb.FetchResponse{
-				Name:      m.MetricName,
-				StartTime: int32(startTime),
-				StopTime:  int32(startTime + step*len(protov2Values)),
-				StepTime:  int32(step),
-				Values:    protov2Values,
-				IsAbsent:  isAbsent,
+
+			if response.HttpCode == http.StatusNotFound {
+				continue
 			}
 
-			fr3 := carbonapi_v3_pb.FetchResponse{
-				Name:                    m.MetricName,
-				PathExpression:          target,
-				ConsolidationFunc:       "avg",
-				StartTime:               int64(startTime),
-				StopTime:                int64(startTime + step*len(m.Values)),
-				StepTime:                int64(step),
-				XFilesFactor:            0,
-				HighPrecisionTimestamps: false,
-				Values:                  m.Values,
-				RequestStartTime:        1,
-				RequestStopTime:         int64(startTime + step*len(m.Values)),
+			if response.HttpCode != 0 && response.HttpCode != http.StatusOK {
+				http.Error(wr, http.StatusText(response.HttpCode), response.HttpCode)
+				return
 			}
 
-			multiv2.Metrics = append(multiv2.Metrics, fr2)
-			multiv3.Metrics = append(multiv3.Metrics, fr3)
+			returnCode = http.StatusOK
+
+			for _, m := range response.Data {
+				step := m.Step
+				if step == 0 {
+					step = 1
+				}
+				startTime := m.StartTime
+				if startTime == 0 {
+					startTime = step
+				}
+				isAbsent := make([]bool, 0, len(m.Values))
+				protov2Values := make([]float64, 0, len(m.Values))
+				for i := range m.Values {
+					if math.IsNaN(m.Values[i]) {
+						isAbsent = append(isAbsent, true)
+						protov2Values = append(protov2Values, 0.0)
+					} else {
+						isAbsent = append(isAbsent, false)
+						protov2Values = append(protov2Values, m.Values[i])
+					}
+				}
+				fr2 := carbonapi_v2_pb.FetchResponse{
+					Name:      m.MetricName,
+					StartTime: int32(startTime),
+					StopTime:  int32(startTime + step*len(protov2Values)),
+					StepTime:  int32(step),
+					Values:    protov2Values,
+					IsAbsent:  isAbsent,
+				}
+
+				fr3 := carbonapi_v3_pb.FetchResponse{
+					Name:                    m.MetricName,
+					PathExpression:          target,
+					ConsolidationFunc:       "avg",
+					StartTime:               int64(startTime),
+					StopTime:                int64(startTime + step*len(m.Values)),
+					StepTime:                int64(step),
+					XFilesFactor:            0,
+					HighPrecisionTimestamps: false,
+					Values:                  m.Values,
+					RequestStartTime:        1,
+					RequestStopTime:         int64(startTime + step*len(m.Values)),
+				}
+
+				multiv2.Metrics = append(multiv2.Metrics, fr2)
+				multiv3.Metrics = append(multiv3.Metrics, fr3)
+			}
 		}
 	}
 
+	if returnCode == http.StatusNotFound {
+		wr.WriteHeader(http.StatusNotFound)
+		_, _ = wr.Write([]byte("Not found"))
+		return
+	}
+
 	if cfg.Listener.ShuffleResults {
 		rand.Shuffle(len(multiv2.Metrics), func(i, j int) {
 			multiv2.Metrics[i], multiv2.Metrics[j] = multiv2.Metrics[j], multiv2.Metrics[i]

cmd/mockbackend/testcases/render_error/carbonapi.yaml

@@ -0,0 +1,57 @@
+listen: "localhost:8081"
+expvar:
+  enabled: true
+  pprofEnabled: false
+  listen: ""
+concurency: 1000
+notFoundStatusCode: 200
+cache:
+   type: "mem"
+   size_mb: 0
+   defaultTimeoutSec: 60
+cpus: 0
+tz: ""
+maxBatchSize: 0
+graphite:
+    host: ""
+    interval: "60s"
+    prefix: "carbon.api"
+    pattern: "{prefix}.{fqdn}"
+idleConnections: 10
+pidFile: ""
+upstreams:
+    buckets: 10
+    timeouts:
+        find: "2s"
+        render: "5s"
+        connect: "200ms"
+    concurrencyLimitPerServer: 0
+    keepAliveInterval: "30s"
+    maxIdleConnsPerHost: 100
+    backendsv2:
+        backends:
+          -
+            groupName: "mock-001"
+            protocol: "auto"
+            lbMethod: "all"
+            maxTries: 3
+            maxBatchSize: 0
+            keepAliveInterval: "10s"
+            concurrencyLimit: 0
+            forceAttemptHTTP2: true
+            maxIdleConnsPerHost: 1000
+            timeouts:
+                find: "3s"
+                render: "5s"
+                connect: "200ms"
+            servers:
+                - "http://127.0.0.1:9070"
+    graphite09compat: false
+expireDelaySec: 10
+logger:
+    - logger: ""
+      file: "stderr"
+      level: "debug"
+      encoding: "console"
+      encodingTime: "iso8601"
+      encodingDuration: "seconds"