Skip to content

Commit

Permalink
Merge branch 'main' into dependabot/github_actions/docker/setup-qemu-…
Browse files Browse the repository at this point in the history
…action-3
  • Loading branch information
Civil authored Feb 12, 2024
2 parents 0d5d7ae + dd35d90 commit b3dbb4d
Show file tree
Hide file tree
Showing 20 changed files with 342 additions and 48 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,8 @@ jobs:
strategy:
matrix:
go:
- ^1.20
- ^1.21
- ^1.22
- ^1
steps:

Expand Down
27 changes: 14 additions & 13 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -121,7 +121,7 @@ Golang compatibility matrix:
| 1.18 | 0.16.0-patch2 |
| 1.20 | 0.16.1 |

Overall rule of thumb is that carbonapi supports last 2 major go versions. E.x. at this moment Go 1.19 and 1.18 are supported.
Overall rule of thumb is that carbonapi supports last 2 major go versions. E.x. at this moment Go 1.22 and 1.21 are supported.

You can verify current versions that are being tested in [CI Configuration](https://github.com/go-graphite/carbonapi/blob/main/.github/workflows/tests.yml#L14).

Expand Down Expand Up @@ -156,18 +156,19 @@ Internal Metrics
----------------------------------
The internal metrics are configured inside the [graphite](https://github.com/go-graphite/carbonapi/blob/main/doc/configuration.md#graphite) subsection and sent to your destinated host on an specified interval. The metrics are:

cache_items - if caching is enabled, this metric will contain many metrics are stored in cache
cache_size - configured query cache size in bytes
request_cache_hits - how many requests were served from cache. (this is for requests to /render endpoint)
request_cache_misses - how many requests were not in cache. (this is for requests to /render endpoint)
request_cache_overhead_ns - how much time in ns it took to talk to cache (that is useful to assess if cache actually helps you in terms of latency) (this is for requests to /render endpoint)
find_requests - requests server by endpoint /metrics/find
requests - requests served by endpoint /render
requests_in_XX_to_XX - request response times in percentiles
timeouts - number of timeouts while fetching from backend
backend_cache_hits - how many requests were not read from backend
backend_cache_misses - how many requests were not found in the backend

| Metric Name | Description |
| ----------- | ----------- |
| cache_items | if caching is enabled, this metric will contain many metrics are stored in cache |
| cache_size | configured query cache size in bytes |
| request_cache_hits | how many requests were served from cache. (this is for requests to /render endpoint) |
| request_cache_misses | how many requests were not in cache. (this is for requests to /render endpoint) |
| request_cache_overhead_ns | how much time in ns it took to talk to cache (that is useful to assess if cache actually helps you in terms of latency) (this is for |requests to /render endpoint)
| find_requests | requests server by endpoint /metrics/find |
| requests | requests served by endpoint /render |
| requests_in_XX_to_XX | request response times in percentiles |
| timeouts | number of timeouts while fetching from backend |
| backend_cache_hits | how many requests were not read from backend |
| backend_cache_misses | how many requests were not found in the backend |

OSX Build Notes
---------------
Expand Down
3 changes: 2 additions & 1 deletion cmd/carbonapi/config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -107,7 +107,8 @@ type ConfigType struct {
TruncateTimeMap map[time.Duration]time.Duration `mapstructure:"truncateTime"`
TruncateTime []DurationTruncate `mapstructure:"-" json:"-"` // produce from TruncateTimeMap and sort in reverse order

CombineMultipleTargetsInOne bool `mapstructure:"combineMultipleTargetsInOne"`
MaxQueryLength uint64 `mapstructure:"maxQueryLength"`
CombineMultipleTargetsInOne bool `mapstructure:"combineMultipleTargetsInOne"`

ResponseCache cache.BytesCache `mapstructure:"-" json:"-"`
BackendCache cache.BytesCache `mapstructure:"-" json:"-"`
Expand Down
22 changes: 14 additions & 8 deletions cmd/carbonapi/config/init.go
Original file line number Diff line number Diff line change
Expand Up @@ -330,7 +330,7 @@ func createCache(logger *zap.Logger, cacheName string, cacheConfig *CacheConfig)
}
}

func SetUpViper(logger *zap.Logger, configPath *string, viperPrefix string) {
func SetUpViper(logger *zap.Logger, configPath *string, exactConfig bool, viperPrefix string) {
if *configPath != "" {
b, err := os.ReadFile(*configPath)
if err != nil {
Expand Down Expand Up @@ -386,24 +386,30 @@ func SetUpViper(logger *zap.Logger, configPath *string, viperPrefix string) {
viper.SetDefault("upstreams.internalRoutingCache", "600s")
viper.SetDefault("upstreams.buckets", 10)
viper.SetDefault("upstreams.sumBuckets", false)
viper.SetDefault("upstreams.bucketsWeight", []int64{})
viper.SetDefault("upstreams.bucketsNames", []string{})
viper.SetDefault("upstreams.bucketsWidth", []int64{})
viper.SetDefault("upstreams.bucketsLabels", []string{})
viper.SetDefault("upstreams.slowLogThreshold", "1s")
viper.SetDefault("upstreams.timeouts.global", "10s")
viper.SetDefault("upstreams.timeouts.afterStarted", "2s")
viper.SetDefault("upstreams.timeouts.find", "2s")
viper.SetDefault("upstreams.timeouts.render", "10s")
viper.SetDefault("upstreams.timeouts.connect", "200ms")
viper.SetDefault("upstreams.concurrencyLimit", 0)
viper.SetDefault("upstreams.concurrencyLimitPerServer", 0)
viper.SetDefault("upstreams.keepAliveInterval", "30s")
viper.SetDefault("upstreams.maxIdleConnsPerHost", 100)
viper.SetDefault("upstreams.scaleToCommonStep", true)
viper.SetDefault("upstreams.graphite09compat", false)
viper.SetDefault("graphite09compat", false)
viper.SetDefault("expireDelaySec", 600)
viper.SetDefault("useCachingDNSResolver", false)
viper.SetDefault("logger", map[string]string{})
viper.SetDefault("combineMultipleTargetsInOne", false)
viper.AutomaticEnv()

err := viper.Unmarshal(&Config)
var err error
if exactConfig {
err = viper.UnmarshalExact(&Config)
} else {
err = viper.Unmarshal(&Config)
}

if err != nil {
logger.Fatal("failed to parse config",
zap.Error(err),
Expand Down
6 changes: 6 additions & 0 deletions cmd/carbonapi/http/expand_handler.go
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,12 @@ func expandHandler(w http.ResponseWriter, r *http.Request) {
return
}

if queryLengthLimitExceeded(query, config.Config.MaxQueryLength) {
setError(w, &accessLogDetails, "query length limit exceeded", http.StatusBadRequest, uid.String())
logAsError = true
return
}

var pv3Request pbv3.MultiGlobRequest
pv3Request.Metrics = query
pv3Request.StartTime = from64
Expand Down
6 changes: 6 additions & 0 deletions cmd/carbonapi/http/find_handlers.go
Original file line number Diff line number Diff line change
Expand Up @@ -221,6 +221,12 @@ func findHandler(w http.ResponseWriter, r *http.Request) {
return
}

if queryLengthLimitExceeded(query, config.Config.MaxQueryLength) {
setError(w, &accessLogDetails, "query length limit exceeded", http.StatusBadRequest, uid.String())
logAsError = true
return
}

if format == completerFormat {
var replacer = strings.NewReplacer("/", ".")
for i := range query {
Expand Down
20 changes: 20 additions & 0 deletions cmd/carbonapi/http/helper.go
Original file line number Diff line number Diff line change
Expand Up @@ -282,3 +282,23 @@ func timestampTruncate(ts int64, duration time.Duration, durations []config.Dura
}
return ts
}

func setError(w http.ResponseWriter, accessLogDetails *carbonapipb.AccessLogDetails, msg string, status int, carbonapiUUID string) {
w.Header().Set(ctxHeaderUUID, carbonapiUUID)
http.Error(w, http.StatusText(status)+": "+msg, status)
accessLogDetails.Reason = msg
accessLogDetails.HTTPCode = int32(status)
}

func queryLengthLimitExceeded(query []string, maxLength uint64) bool {
if maxLength > 0 {
var queryLengthSum uint64 = 0
for _, q := range query {
queryLengthSum += uint64(len(q))
}
if queryLengthSum > maxLength {
return true
}
}
return false
}
53 changes: 53 additions & 0 deletions cmd/carbonapi/http/helper_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package http

import (
"fmt"
"strings"
"testing"
"time"

Expand Down Expand Up @@ -68,3 +69,55 @@ func Test_timestampTruncate(t *testing.T) {
})
}
}

func Test_queryLengthLimitExceeded(t *testing.T) {
tests := []struct {
query []string
maxLength uint64
want bool
}{
{
query: []string{"a.b.c.d.e", "a.a.a.a.a.a.a.a.a.a.a.a"},
maxLength: 20,
want: true,
},
{
query: []string{"a.b.c", "a.b.d"},
maxLength: 10,
want: false,
},
{
query: []string{"a.b.c", "a.b.d"},
maxLength: 9,
want: true,
},
{
query: []string{"a.b.c.d.e", "a.a.a.a.a.a.a.a.a.a.a.a"},
maxLength: 0,
want: false,
},
{
query: []string{"a.b.b.c.*", "a.d.e", "a.b.c", "a.b.e", "a.a.a.b", "a.f.s.x.w.g"},
maxLength: 30,
want: true,
},
{
query: []string{"a.b.c.d.e", "a.b.c.d.f", "b.a.*"},
maxLength: 10000,
want: false,
},
{
query: []string{},
maxLength: 0,
want: false,
},
}

for _, tt := range tests {
t.Run(fmt.Sprintf("queryLengthLimitExceeded([%s], %d) -> %t", strings.Join(tt.query, ", "), tt.maxLength, tt.want), func(t *testing.T) {
if got := queryLengthLimitExceeded(tt.query, tt.maxLength); got != tt.want {
t.Errorf("queryLengthLimitExceeded() = %t, want %t", got, tt.want)
}
})
}
}
2 changes: 1 addition & 1 deletion cmd/carbonapi/http/main_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -114,7 +114,7 @@ func init() {
logger := zapwriter.Logger("main")

cfgFile := ""
config.SetUpViper(logger, &cfgFile, "CARBONAPI_")
config.SetUpViper(logger, &cfgFile, false, "CARBONAPI_")
config.Config.Upstreams.Backends = []string{"dummy"}
config.SetUpConfigUpstreams(logger)
config.SetUpConfig(logger, "(test)")
Expand Down
13 changes: 6 additions & 7 deletions cmd/carbonapi/http/render_handler.go
Original file line number Diff line number Diff line change
Expand Up @@ -42,13 +42,6 @@ func cleanupParams(r *http.Request) {
r.Form.Del("_t") // Used by jquery.graphite.js
}

func setError(w http.ResponseWriter, accessLogDetails *carbonapipb.AccessLogDetails, msg string, status int, carbonapiUUID string) {
w.Header().Set(ctxHeaderUUID, carbonapiUUID)
http.Error(w, http.StatusText(status)+": "+msg, status)
accessLogDetails.Reason = msg
accessLogDetails.HTTPCode = int32(status)
}

func getCacheTimeout(logger *zap.Logger, r *http.Request, now32, until32 int64, duration time.Duration, cacheConfig *config.CacheConfig) int32 {
if tstr := r.FormValue("cacheTimeout"); tstr != "" {
t, err := strconv.Atoi(tstr)
Expand Down Expand Up @@ -232,6 +225,12 @@ func renderHandler(w http.ResponseWriter, r *http.Request) {
}
}

if queryLengthLimitExceeded(targets, config.Config.MaxQueryLength) {
setError(w, accessLogDetails, "total target length limit exceeded", http.StatusBadRequest, uid.String())
logAsError = true
return
}

if useCache {
tc := time.Now()
response, err := config.Config.ResponseCache.Get(responseCacheKey)
Expand Down
6 changes: 6 additions & 0 deletions cmd/carbonapi/http/tags_handler.go
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,12 @@ func tagHandler(w http.ResponseWriter, r *http.Request) {
q.Del("pretty")
rawQuery := q.Encode()

if queryLengthLimitExceeded(r.Form["query"], config.Config.MaxQueryLength) {
setError(w, accessLogDetails, "query length limit exceeded", http.StatusBadRequest, uuid.String())
logAsError = true
return
}

// TODO(civil): Implement caching
var res []string
if strings.HasSuffix(r.URL.Path, "tags") || strings.HasSuffix(r.URL.Path, "tags/") {
Expand Down
3 changes: 2 additions & 1 deletion cmd/carbonapi/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ func main() {

configPath := flag.String("config", "", "Path to the `config file`.")
checkConfig := flag.Bool("check-config", false, "Check config file and exit.")
exactConfig := flag.Bool("exact-config", false, "Ensure that all config params are contained in the target struct.")
envPrefix := flag.String("envprefix", "CARBONAPI", "Prefix for environment variables override")
if *envPrefix == "(empty)" {
*envPrefix = ""
Expand All @@ -43,7 +44,7 @@ func main() {
logger.Warn("empty prefix is not recommended due to possible collisions with OS environment variables")
}
flag.Parse()
config.SetUpViper(logger, configPath, *envPrefix)
config.SetUpViper(logger, configPath, *exactConfig, *envPrefix)
if *checkConfig {
os.Exit(0)
}
Expand Down
61 changes: 61 additions & 0 deletions cmd/mockbackend/testcases/pr817/carbonapi.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
listen: "localhost:8081"
expvar:
enabled: true
pprofEnabled: false
listen: ""
concurency: 1000
notFoundStatusCode: 200
cache:
type: "mem"
size_mb: 0
defaultTimeoutSec: 60
cpus: 0
tz: ""
maxBatchSize: 500
maxQueryLength: 20
combineMultipleTargetsInOne: true
graphite:
host: ""
interval: "60s"
prefix: "carbon.api"
pattern: "{prefix}.{fqdn}"
idleConnections: 10
pidFile: ""
upstreams:
buckets: 10
timeouts:
find: "2s"
render: "10s"
connect: "200ms"
concurrencyLimitPerServer: 100
keepAliveInterval: "30s"
maxIdleConnsPerHost: 100
doMultipleRequestsIfSplit: true
backendsv2:
backends:
-
groupName: "mock-001"
protocol: "auto"
lbMethod: "all"
maxTries: 3
maxBatchSize: 500
keepAliveInterval: "10s"
concurrencyLimit: 0
forceAttemptHTTP2: true
maxIdleConnsPerHost: 1000
doMultipleRequestsIfSplit: true
timeouts:
find: "15s"
render: "50s"
connect: "200ms"
servers:
- "http://127.0.0.1:9070"
graphite09compat: false
expireDelaySec: 10
logger:
- logger: ""
file: "stderr"
level: "debug"
encoding: "console"
encodingTime: "iso8601"
encodingDuration: "seconds"
Loading

0 comments on commit b3dbb4d

Please sign in to comment.