From 6b91a7a52ec42f57e79bdf81e30932ad1c378e3a Mon Sep 17 00:00:00 2001 From: Andrey Vazhenin Date: Tue, 24 Dec 2019 14:04:13 +0300 Subject: [PATCH 1/4] DMP-3004: added storage-level metrics --- namespaces.go | 77 +++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 71 insertions(+), 6 deletions(-) diff --git a/namespaces.go b/namespaces.go index baeadf1..7df0bf0 100644 --- a/namespaces.go +++ b/namespaces.go @@ -90,8 +90,6 @@ var ( counter("client_write_error", "client write error"), counter("client_write_success", "client write success"), counter("client_write_timeout", "client write timeout"), - counter("defrag_reads", "defrag reads"), - counter("defrag_writes", "defrag writes"), counter("evicted_objects", "evicted objects"), counter("expired_objects", "expired objects"), counter("fail_generation", "fail generation"), @@ -166,7 +164,6 @@ var ( counter("xdr_write_success", "xdr write success"), counter("xdr_write_timeout", "xdr write timeout"), gauge("available_bin_names", "available bin names"), - // broken gauge("defrag_q", "defrag queue"), gauge("device_available_pct", "device available pct"), gauge("device_compression_ratio", "device compression ratio"), gauge("device_free_pct", "device free pct"), @@ -216,7 +213,6 @@ var ( gauge("prole_objects", "prole objects"), gauge("prole_tombstones", "prole tombstones"), gauge("replication-factor", "replication factor"), - // broken gauge("shadow_write_q", "shadow write queue"), gauge("stop_writes", "stop writes"), gauge("stop-writes-pct", "stop writes pct"), gauge("tombstones", "tombstones"), @@ -225,10 +221,17 @@ var ( gauge("dead_partitions", "dead partitions"), gauge("unavailable_partitions", "unavailable partitions"), gauge("rack-id", "rack id"), - // gauge("write_q", "write queue"), // device-level stats don't appear to work // and this plugin thinks "storage-engine.device[0].write_q" is malformed. } + NamespaceStorageMetrics = []metric{ + counter("defrag_reads", "defrag reads"), + counter("defrag_writes", "defrag writes"), + gauge("shadow_write_q", "shadow write queue"), + gauge("defrag_q", "defrag queue"), + gauge("write_q", "write queue"), + + } ) type nsCollector cmetrics @@ -246,6 +249,18 @@ func newNSCollector() nsCollector { ), } } + for _, m := range NamespaceStorageMetrics { + ns[m.aeroName] = cmetric{ + typ: m.typ, + desc: prometheus.NewDesc( + promkey(systemNamespace, m.aeroName), + m.desc, + []string{"namespace", "mount"}, + nil, + ), + } + } + return ns } @@ -255,6 +270,45 @@ func (nc nsCollector) describe(ch chan<- *prometheus.Desc) { } } +func (nc nsCollector) parseStorage(s string, d string) (string, error) { + r := "" + for _, l := range strings.Split(s, ";") { + for _, v := range strings.Split(l, ":") { + kv := strings.SplitN(v, "=", 2) + if len(kv) > 1 { + if strings.HasPrefix(kv[0], d) { + //todo: optimize + kv[0] = strings.Replace(kv[0] + ".", d, "", 1) + kv[0] = strings.Replace(kv[0], ".", "", -1) + } + r += kv[0] + "=" + kv[1] + ";" + } + } + } + return r, nil +} + +func (nc nsCollector) splitInfo(s map[string]string, ns string) (map[string]string, map[string]string, map[string]string) { + ns_metrics := map[string]string{} + ns_storage_metrics := map[string]string{} + ns_storage_devices := map[string]string{} + + for _, l := range strings.Split(s["namespace/"+ns], ";") { + for _, v := range strings.Split(l, ":") { + kv := strings.SplitN(v, "=", 2) + if strings.HasPrefix(kv[0], "storage-engine") { + ns_storage_metrics["namespace/"+ns] += v + ";" + if strings.HasSuffix(kv[0], "]") { + ns_storage_devices[kv[1]] = kv[0] + } + } else { + ns_metrics["namespace/"+ns] += v + ";" + } + } + } + return ns_storage_metrics, ns_metrics, ns_storage_devices +} + func (nc nsCollector) collect(conn *as.Connection) ([]prometheus.Metric, error) { info, err := as.RequestInfo(conn, "namespaces") if err != nil { @@ -266,10 +320,21 @@ func (nc nsCollector) collect(conn *as.Connection) ([]prometheus.Metric, error) if err != nil { return nil, err } + + nsinfo_storage, nsinfo_standart, nsinfo_devices := nc.splitInfo(nsinfo,ns) + metrics = append( metrics, - infoCollect(cmetrics(nc), nsinfo["namespace/"+ns], ns)..., + infoCollect(cmetrics(nc), nsinfo_standart["namespace/"+ns], ns)..., ) + + for name, metric := range nsinfo_devices { + nsinfo_storage["namespace/"+ns], err = nc.parseStorage(nsinfo_storage["namespace/"+ns], metric) + metrics = append( + metrics, + infoCollect(cmetrics(nc), nsinfo_storage["namespace/"+ns], ns, name)..., + ) + } } return metrics, nil } From 0500d2ab2fc9cc83a8cf9e070bbd51e4614f752c Mon Sep 17 00:00:00 2001 From: Andrey Vazhenin Date: Tue, 24 Dec 2019 14:51:15 +0300 Subject: [PATCH 2/4] DMP-3004: use buffers, fix lint, optimize map[string]string{} --- namespaces.go | 39 ++++++++++++++++++++++----------------- 1 file changed, 22 insertions(+), 17 deletions(-) diff --git a/namespaces.go b/namespaces.go index 7df0bf0..9bb5220 100644 --- a/namespaces.go +++ b/namespaces.go @@ -2,6 +2,7 @@ package main import ( "strings" + "bytes" as "github.com/aerospike/aerospike-client-go" "github.com/prometheus/client_golang/prometheus" @@ -271,7 +272,7 @@ func (nc nsCollector) describe(ch chan<- *prometheus.Desc) { } func (nc nsCollector) parseStorage(s string, d string) (string, error) { - r := "" + buf := bytes.Buffer{} for _, l := range strings.Split(s, ";") { for _, v := range strings.Split(l, ":") { kv := strings.SplitN(v, "=", 2) @@ -281,32 +282,36 @@ func (nc nsCollector) parseStorage(s string, d string) (string, error) { kv[0] = strings.Replace(kv[0] + ".", d, "", 1) kv[0] = strings.Replace(kv[0], ".", "", -1) } - r += kv[0] + "=" + kv[1] + ";" + buf.WriteString(kv[0] + "=" + kv[1] + ";") } } } + r := buf.String() return r, nil } -func (nc nsCollector) splitInfo(s map[string]string, ns string) (map[string]string, map[string]string, map[string]string) { - ns_metrics := map[string]string{} - ns_storage_metrics := map[string]string{} - ns_storage_devices := map[string]string{} +func (nc nsCollector) splitInfo(s string) (string, string, map[string]string) { + nsStorageMounts := map[string]string{} - for _, l := range strings.Split(s["namespace/"+ns], ";") { + bufStandardMetrics := bytes.Buffer{} + bufStorageMetrics := bytes.Buffer{} + + for _, l := range strings.Split(s, ";") { for _, v := range strings.Split(l, ":") { kv := strings.SplitN(v, "=", 2) if strings.HasPrefix(kv[0], "storage-engine") { - ns_storage_metrics["namespace/"+ns] += v + ";" + bufStorageMetrics.WriteString(v + ";") if strings.HasSuffix(kv[0], "]") { - ns_storage_devices[kv[1]] = kv[0] + nsStorageMounts[kv[1]] = kv[0] } } else { - ns_metrics["namespace/"+ns] += v + ";" + bufStandardMetrics.WriteString(v + ";") } } } - return ns_storage_metrics, ns_metrics, ns_storage_devices + nsStandardMetrics := bufStandardMetrics.String() + nsStorageMetrics := bufStorageMetrics.String() + return nsStorageMetrics, nsStandardMetrics, nsStorageMounts } func (nc nsCollector) collect(conn *as.Connection) ([]prometheus.Metric, error) { @@ -316,23 +321,23 @@ func (nc nsCollector) collect(conn *as.Connection) ([]prometheus.Metric, error) } var metrics []prometheus.Metric for _, ns := range strings.Split(info["namespaces"], ";") { - nsinfo, err := as.RequestInfo(conn, "namespace/"+ns) + nsInfo, err := as.RequestInfo(conn, "namespace/"+ns) if err != nil { return nil, err } - nsinfo_storage, nsinfo_standart, nsinfo_devices := nc.splitInfo(nsinfo,ns) + nsInfoStorage, nsInfoStandard, nsInfoStorageDevices := nc.splitInfo(nsInfo["namespace/"+ns]) metrics = append( metrics, - infoCollect(cmetrics(nc), nsinfo_standart["namespace/"+ns], ns)..., + infoCollect(cmetrics(nc), nsInfoStandard, ns)..., ) - for name, metric := range nsinfo_devices { - nsinfo_storage["namespace/"+ns], err = nc.parseStorage(nsinfo_storage["namespace/"+ns], metric) + for mountName, metricName := range nsInfoStorageDevices { + nsInfoStorage, err = nc.parseStorage(nsInfoStorage, metricName) metrics = append( metrics, - infoCollect(cmetrics(nc), nsinfo_storage["namespace/"+ns], ns, name)..., + infoCollect(cmetrics(nc), nsInfoStorage, ns, mountName)..., ) } } From 2714d7869b0cdbd3262956a8c6c38b3bb2588620 Mon Sep 17 00:00:00 2001 From: Andrey Vazhenin Date: Tue, 24 Dec 2019 14:54:16 +0300 Subject: [PATCH 3/4] DMP-3004: fix linting --- namespaces.go | 51 +++++++++++++++++++++++++-------------------------- 1 file changed, 25 insertions(+), 26 deletions(-) diff --git a/namespaces.go b/namespaces.go index 9bb5220..afacc3f 100644 --- a/namespaces.go +++ b/namespaces.go @@ -1,8 +1,8 @@ package main import ( - "strings" "bytes" + "strings" as "github.com/aerospike/aerospike-client-go" "github.com/prometheus/client_golang/prometheus" @@ -226,12 +226,11 @@ var ( // and this plugin thinks "storage-engine.device[0].write_q" is malformed. } NamespaceStorageMetrics = []metric{ - counter("defrag_reads", "defrag reads"), + counter("defrag_reads", "defrag reads"), counter("defrag_writes", "defrag writes"), - gauge("shadow_write_q", "shadow write queue"), + gauge("shadow_write_q", "shadow write queue"), gauge("defrag_q", "defrag queue"), - gauge("write_q", "write queue"), - + gauge("write_q", "write queue"), } ) @@ -273,37 +272,37 @@ func (nc nsCollector) describe(ch chan<- *prometheus.Desc) { func (nc nsCollector) parseStorage(s string, d string) (string, error) { buf := bytes.Buffer{} - for _, l := range strings.Split(s, ";") { + for _, l := range strings.Split(s, ";") { for _, v := range strings.Split(l, ":") { kv := strings.SplitN(v, "=", 2) if len(kv) > 1 { - if strings.HasPrefix(kv[0], d) { - //todo: optimize - kv[0] = strings.Replace(kv[0] + ".", d, "", 1) - kv[0] = strings.Replace(kv[0], ".", "", -1) - } + if strings.HasPrefix(kv[0], d) { + //todo: optimize + kv[0] = strings.Replace(kv[0]+".", d, "", 1) + kv[0] = strings.Replace(kv[0], ".", "", -1) + } buf.WriteString(kv[0] + "=" + kv[1] + ";") } } } r := buf.String() - return r, nil + return r, nil } func (nc nsCollector) splitInfo(s string) (string, string, map[string]string) { - nsStorageMounts := map[string]string{} + nsStorageMounts := map[string]string{} - bufStandardMetrics := bytes.Buffer{} + bufStandardMetrics := bytes.Buffer{} bufStorageMetrics := bytes.Buffer{} - for _, l := range strings.Split(s, ";") { + for _, l := range strings.Split(s, ";") { for _, v := range strings.Split(l, ":") { kv := strings.SplitN(v, "=", 2) if strings.HasPrefix(kv[0], "storage-engine") { bufStorageMetrics.WriteString(v + ";") - if strings.HasSuffix(kv[0], "]") { - nsStorageMounts[kv[1]] = kv[0] - } + if strings.HasSuffix(kv[0], "]") { + nsStorageMounts[kv[1]] = kv[0] + } } else { bufStandardMetrics.WriteString(v + ";") } @@ -311,7 +310,7 @@ func (nc nsCollector) splitInfo(s string) (string, string, map[string]string) { } nsStandardMetrics := bufStandardMetrics.String() nsStorageMetrics := bufStorageMetrics.String() - return nsStorageMetrics, nsStandardMetrics, nsStorageMounts + return nsStorageMetrics, nsStandardMetrics, nsStorageMounts } func (nc nsCollector) collect(conn *as.Connection) ([]prometheus.Metric, error) { @@ -333,13 +332,13 @@ func (nc nsCollector) collect(conn *as.Connection) ([]prometheus.Metric, error) infoCollect(cmetrics(nc), nsInfoStandard, ns)..., ) - for mountName, metricName := range nsInfoStorageDevices { - nsInfoStorage, err = nc.parseStorage(nsInfoStorage, metricName) - metrics = append( - metrics, - infoCollect(cmetrics(nc), nsInfoStorage, ns, mountName)..., - ) - } + for mountName, metricName := range nsInfoStorageDevices { + nsInfoStorage, err = nc.parseStorage(nsInfoStorage, metricName) + metrics = append( + metrics, + infoCollect(cmetrics(nc), nsInfoStorage, ns, mountName)..., + ) + } } return metrics, nil } From ec56f3435fde628661d05fb6683dfbc0827db89f Mon Sep 17 00:00:00 2001 From: Andrey Vazhenin Date: Sat, 28 Dec 2019 21:44:31 +0300 Subject: [PATCH 4/4] DMP-3004: added notes, fix the err checking --- namespaces.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/namespaces.go b/namespaces.go index afacc3f..e893678 100644 --- a/namespaces.go +++ b/namespaces.go @@ -271,6 +271,11 @@ func (nc nsCollector) describe(ch chan<- *prometheus.Desc) { } func (nc nsCollector) parseStorage(s string, d string) (string, error) { + // the function remove the storage prefix metrics for each device: + // d is storage-engine.device[ix] + // s is all storage metrics that has been scraped + // storage-engine.device[ix].age -> age + // https://www.aerospike.com/docs/reference/metrics/#storage-engine.device[ix].age buf := bytes.Buffer{} for _, l := range strings.Split(s, ";") { for _, v := range strings.Split(l, ":") { @@ -334,6 +339,11 @@ func (nc nsCollector) collect(conn *as.Connection) ([]prometheus.Metric, error) for mountName, metricName := range nsInfoStorageDevices { nsInfoStorage, err = nc.parseStorage(nsInfoStorage, metricName) + + if err != nil { + return nil, err + } + metrics = append( metrics, infoCollect(cmetrics(nc), nsInfoStorage, ns, mountName)...,