From c50fd6f0368e095471c167dab61127c2ba75ac05 Mon Sep 17 00:00:00 2001 From: Thomas Mertz Date: Wed, 19 Jul 2023 09:10:46 +0200 Subject: [PATCH 1/5] caches metric can be missalloccated --- cmd/bucky/inconsistent.go | 6 ------ 1 file changed, 6 deletions(-) diff --git a/cmd/bucky/inconsistent.go b/cmd/bucky/inconsistent.go index 39af8a6a..0f55745b 100644 --- a/cmd/bucky/inconsistent.go +++ b/cmd/bucky/inconsistent.go @@ -7,7 +7,6 @@ import ( "net" "os" "sort" - "strings" "time" ) @@ -63,11 +62,6 @@ func InconsistentMetrics(hostports []string, regex string) (map[string][]string, } for _, m := range metrics { - if strings.HasPrefix(m, "carbon.agents.") { - // These metrics are inserted into the stream after hashing - // is done. They will never be consistent and shouldn't be. - continue - } if Cluster.Hash.GetNode(m).Server != host { results[server] = append(results[server], m) } From 2a2be87e7231704edb9fa5eb44ec0933f4e61932 Mon Sep 17 00:00:00 2001 From: Thorsieger Date: Wed, 19 Jul 2023 18:44:12 +0200 Subject: [PATCH 2/5] add parameter to check or not cache metrics --- cmd/bucky/inconsistent.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/cmd/bucky/inconsistent.go b/cmd/bucky/inconsistent.go index 0f55745b..df912889 100644 --- a/cmd/bucky/inconsistent.go +++ b/cmd/bucky/inconsistent.go @@ -35,6 +35,10 @@ Use bucky rebalance to correct.` "Force the remote daemons to rebuild their cache.") c.Flag.BoolVar(&listRegexMode, "r", false, "Filter by a regular expression.") + c.Flag.BoolVar(&listCacheMetrics, "list-cache-metrics", true, + "Filter carbon cache metrics.") + c.Flag.StringVar(&listCacheMetricsPrefix, "cache-metric-prefix", "carbon.agents.", + "cache metric prefix") } func InconsistentMetrics(hostports []string, regex string) (map[string][]string, error) { @@ -62,6 +66,9 @@ func InconsistentMetrics(hostports []string, regex string) (map[string][]string, } for _, m := range metrics { + if !listCacheMetrics && strings.HasPrefix(m, listCacheMetricsPrefix) { + continue + } if Cluster.Hash.GetNode(m).Server != host { results[server] = append(results[server], m) } From 82a5d1902ff1840f3b1c0e5c437924a3fd82cd10 Mon Sep 17 00:00:00 2001 From: Thorsieger Date: Wed, 19 Jul 2023 18:52:31 +0200 Subject: [PATCH 3/5] fix flag names --- cmd/bucky/inconsistent.go | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/cmd/bucky/inconsistent.go b/cmd/bucky/inconsistent.go index df912889..26f5b6c1 100644 --- a/cmd/bucky/inconsistent.go +++ b/cmd/bucky/inconsistent.go @@ -7,11 +7,15 @@ import ( "net" "os" "sort" + "strings" "time" ) // import "github.com/go-graphite/buckytools/hashing" +var inconsistentCacheMetrics bool +var inconsistentCacheMetricsPrefix string + func init() { usage := "[options]" short := "Find metrics not in correct locations." @@ -35,9 +39,9 @@ Use bucky rebalance to correct.` "Force the remote daemons to rebuild their cache.") c.Flag.BoolVar(&listRegexMode, "r", false, "Filter by a regular expression.") - c.Flag.BoolVar(&listCacheMetrics, "list-cache-metrics", true, + c.Flag.BoolVar(&inconsistentCacheMetrics, "list-cache-metrics", true, "Filter carbon cache metrics.") - c.Flag.StringVar(&listCacheMetricsPrefix, "cache-metric-prefix", "carbon.agents.", + c.Flag.StringVar(&inconsistentCacheMetricsPrefix, "cache-metric-prefix", "carbon.agents.", "cache metric prefix") } @@ -66,7 +70,7 @@ func InconsistentMetrics(hostports []string, regex string) (map[string][]string, } for _, m := range metrics { - if !listCacheMetrics && strings.HasPrefix(m, listCacheMetricsPrefix) { + if !inconsistentCacheMetrics && strings.HasPrefix(m, inconsistentCacheMetricsPrefix) { continue } if Cluster.Hash.GetNode(m).Server != host { From 5fc4c79deeef0d1cc62c9f7ef01cb1c605e0f641 Mon Sep 17 00:00:00 2001 From: Thomas Mertz Date: Thu, 20 Jul 2023 08:49:42 +0200 Subject: [PATCH 4/5] add options to rebalance --- cmd/bucky/inconsistent.go | 12 +++++------- cmd/bucky/rebalance.go | 2 ++ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/cmd/bucky/inconsistent.go b/cmd/bucky/inconsistent.go index 26f5b6c1..5075a8d0 100644 --- a/cmd/bucky/inconsistent.go +++ b/cmd/bucky/inconsistent.go @@ -13,8 +13,8 @@ import ( // import "github.com/go-graphite/buckytools/hashing" -var inconsistentCacheMetrics bool -var inconsistentCacheMetricsPrefix string +var inconsistentInconsistentCacheMetrics bool +var inconsistentGoCarbonMetricsPrefix string func init() { usage := "[options]" @@ -39,10 +39,8 @@ Use bucky rebalance to correct.` "Force the remote daemons to rebuild their cache.") c.Flag.BoolVar(&listRegexMode, "r", false, "Filter by a regular expression.") - c.Flag.BoolVar(&inconsistentCacheMetrics, "list-cache-metrics", true, - "Filter carbon cache metrics.") - c.Flag.StringVar(&inconsistentCacheMetricsPrefix, "cache-metric-prefix", "carbon.agents.", - "cache metric prefix") + c.Flag.BoolVar(&inconsistentInconsistentCacheMetrics, "inconsistent-cache-metrics", false, "Search for go-carbon inconsistent metrics.") + c.Flag.StringVar(&inconsistentGoCarbonMetricsPrefix, "go-carbon-prefix", "carbon.agents.", "go-carbon metric prefix") } func InconsistentMetrics(hostports []string, regex string) (map[string][]string, error) { @@ -70,7 +68,7 @@ func InconsistentMetrics(hostports []string, regex string) (map[string][]string, } for _, m := range metrics { - if !inconsistentCacheMetrics && strings.HasPrefix(m, inconsistentCacheMetricsPrefix) { + if !inconsistentInconsistentCacheMetrics && strings.HasPrefix(m, inconsistentGoCarbonMetricsPrefix) { continue } if Cluster.Hash.GetNode(m).Server != host { diff --git a/cmd/bucky/rebalance.go b/cmd/bucky/rebalance.go index b852bc70..6844dac0 100644 --- a/cmd/bucky/rebalance.go +++ b/cmd/bucky/rebalance.go @@ -56,6 +56,8 @@ Set -offload=true to speed up rebalance.` c.Flag.BoolVar(&listForce, "f", false, "Force the remote daemons to rebuild their cache.") c.Flag.StringVar(&rebalanceConfig.allowedDsts, "allowed-dsts", "", "Only copy/rebalance metrics to the allowed destinations (ip1:port,ip2:port). By default (i.e. empty), all dsts are allowed.") c.Flag.StringVar(&rebalanceConfig.allowedMetricRegex, "r", "", "Only copy/rebalance metrics matching regex. By default (i.e. empty), all metrics are allowed.") + c.Flag.BoolVar(&inconsistentInconsistentCacheMetrics, "inconsistent-cache-metrics", false, "Search for go-carbon inconsistent metrics.") + c.Flag.StringVar(&inconsistentGoCarbonMetricsPrefix, "go-carbon-prefix", "carbon.agents.", "go-carbon metric prefix") } // countMap returns the number of metrics in a server -> metrics mapping From ab8c2ce4528cb9626a461349be5ddf3ebf7db661 Mon Sep 17 00:00:00 2001 From: Thomas Mertz Date: Mon, 24 Jul 2023 11:23:52 +0200 Subject: [PATCH 5/5] reverse flag logic and add comments --- cmd/bucky/inconsistent.go | 8 +++++--- cmd/bucky/rebalance.go | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/cmd/bucky/inconsistent.go b/cmd/bucky/inconsistent.go index 5075a8d0..fb1103a7 100644 --- a/cmd/bucky/inconsistent.go +++ b/cmd/bucky/inconsistent.go @@ -13,7 +13,7 @@ import ( // import "github.com/go-graphite/buckytools/hashing" -var inconsistentInconsistentCacheMetrics bool +var inconsistentFilterCacheMetrics bool var inconsistentGoCarbonMetricsPrefix string func init() { @@ -39,7 +39,7 @@ Use bucky rebalance to correct.` "Force the remote daemons to rebuild their cache.") c.Flag.BoolVar(&listRegexMode, "r", false, "Filter by a regular expression.") - c.Flag.BoolVar(&inconsistentInconsistentCacheMetrics, "inconsistent-cache-metrics", false, "Search for go-carbon inconsistent metrics.") + c.Flag.BoolVar(&inconsistentFilterCacheMetrics, "filter-cache-metrics", true, "Filter go-carbon inconsistent metrics.") c.Flag.StringVar(&inconsistentGoCarbonMetricsPrefix, "go-carbon-prefix", "carbon.agents.", "go-carbon metric prefix") } @@ -68,7 +68,9 @@ func InconsistentMetrics(hostports []string, regex string) (map[string][]string, } for _, m := range metrics { - if !inconsistentInconsistentCacheMetrics && strings.HasPrefix(m, inconsistentGoCarbonMetricsPrefix) { + if inconsistentFilterCacheMetrics && strings.HasPrefix(m, inconsistentGoCarbonMetricsPrefix) { + // Graphite cache metrics (usually "carbon.agents.*" metrics) can be inserted into the stream + // after hashing has been completed (if written locally). In this case, they will never be consistent and shouldn't be. continue } if Cluster.Hash.GetNode(m).Server != host { diff --git a/cmd/bucky/rebalance.go b/cmd/bucky/rebalance.go index 6844dac0..736187ca 100644 --- a/cmd/bucky/rebalance.go +++ b/cmd/bucky/rebalance.go @@ -56,7 +56,7 @@ Set -offload=true to speed up rebalance.` c.Flag.BoolVar(&listForce, "f", false, "Force the remote daemons to rebuild their cache.") c.Flag.StringVar(&rebalanceConfig.allowedDsts, "allowed-dsts", "", "Only copy/rebalance metrics to the allowed destinations (ip1:port,ip2:port). By default (i.e. empty), all dsts are allowed.") c.Flag.StringVar(&rebalanceConfig.allowedMetricRegex, "r", "", "Only copy/rebalance metrics matching regex. By default (i.e. empty), all metrics are allowed.") - c.Flag.BoolVar(&inconsistentInconsistentCacheMetrics, "inconsistent-cache-metrics", false, "Search for go-carbon inconsistent metrics.") + c.Flag.BoolVar(&inconsistentFilterCacheMetrics, "filter-cache-metrics", true, "Filter go-carbon inconsistent metrics.") c.Flag.StringVar(&inconsistentGoCarbonMetricsPrefix, "go-carbon-prefix", "carbon.agents.", "go-carbon metric prefix") }