From 19c500963e31a99d1242e943be8c471265af706a Mon Sep 17 00:00:00 2001 From: Ilya Mashchenko Date: Thu, 19 Dec 2024 22:23:41 +0200 Subject: [PATCH] feat(go.d): add NATS collector (#19252) add go.d/nats --- src/go/plugin/go.d/README.md | 9 +- src/go/plugin/go.d/collector/init.go | 1 + src/go/plugin/go.d/collector/nats/charts.go | 161 ++++++++++ src/go/plugin/go.d/collector/nats/collect.go | 85 ++++++ .../plugin/go.d/collector/nats/collector.go | 113 +++++++ .../go.d/collector/nats/collector_test.go | 282 ++++++++++++++++++ .../go.d/collector/nats/config_schema.json | 191 ++++++++++++ .../plugin/go.d/collector/nats/metadata.yml | 248 +++++++++++++++ src/go/plugin/go.d/collector/nats/restapi.go | 76 +++++ .../go.d/collector/nats/testdata/config.json | 22 ++ .../go.d/collector/nats/testdata/config.yaml | 19 ++ .../nats/testdata/v2.10.24/healthz-ok.json | 3 + .../nats/testdata/v2.10.24/varz.json | 75 +++++ src/go/plugin/go.d/config/go.d.conf | 1 + src/go/plugin/go.d/config/go.d/nats.conf | 6 + src/go/plugin/go.d/config/go.d/sd/docker.conf | 7 + .../go.d/config/go.d/sd/net_listeners.conf | 7 + 17 files changed, 1302 insertions(+), 4 deletions(-) create mode 100644 src/go/plugin/go.d/collector/nats/charts.go create mode 100644 src/go/plugin/go.d/collector/nats/collect.go create mode 100644 src/go/plugin/go.d/collector/nats/collector.go create mode 100644 src/go/plugin/go.d/collector/nats/collector_test.go create mode 100644 src/go/plugin/go.d/collector/nats/config_schema.json create mode 100644 src/go/plugin/go.d/collector/nats/metadata.yml create mode 100644 src/go/plugin/go.d/collector/nats/restapi.go create mode 100644 src/go/plugin/go.d/collector/nats/testdata/config.json create mode 100644 src/go/plugin/go.d/collector/nats/testdata/config.yaml create mode 100644 src/go/plugin/go.d/collector/nats/testdata/v2.10.24/healthz-ok.json create mode 100644 src/go/plugin/go.d/collector/nats/testdata/v2.10.24/varz.json create mode 100644 src/go/plugin/go.d/config/go.d/nats.conf diff --git a/src/go/plugin/go.d/README.md b/src/go/plugin/go.d/README.md index a398b97f89721e..0a65a090f6ed00 100644 --- a/src/go/plugin/go.d/README.md +++ b/src/go/plugin/go.d/README.md @@ -14,8 +14,8 @@ All capabilities are set automatically during Netdata installation using the [official installation method](/packaging/installer/methods/kickstart.md). -| Capability | Required by | -|:--------------------|:-------------------------------------------------------------------------------------------------------:| +| Capability | Required by | +|:--------------------|:---------------------------------------------------------------------------------------------------------:| | CAP_NET_RAW | [Ping](https://github.com/netdata/netdata/tree/master/src/go/plugin/go.d/collector/ping#readme) | | CAP_NET_ADMIN | [Wireguard](https://github.com/netdata/netdata/tree/master/src/go/plugin/go.d/collector/wireguard#readme) | | CAP_DAC_READ_SEARCH | [Filecheck](https://github.com/netdata/netdata/tree/master/src/go/plugin/go.d/collector/filecheck#readme) | @@ -25,8 +25,8 @@ All capabilities are set automatically during Netdata installation using the [of
Data Collection Modules -| Name | Monitors | -|:-------------------------------------------------------------------------------------------------------------------|:-----------------------------:| +| Name | Monitors | +|:---------------------------------------------------------------------------------------------------------------------|:-----------------------------:| | [adaptec_raid](https://github.com/netdata/netdata/tree/master/src/go/plugin/go.d/collector/adaptecraid) | Adaptec Hardware RAID | | [activemq](https://github.com/netdata/netdata/tree/master/src/go/plugin/go.d/collector/activemq) | ActiveMQ | | [ap](https://github.com/netdata/netdata/tree/master/src/go/plugin/go.d/collector/ap) | Wireless AP | @@ -84,6 +84,7 @@ All capabilities are set automatically during Netdata installation using the [of | [mongoDB](https://github.com/netdata/netdata/tree/master/src/go/plugin/go.d/collector/mongodb) | MongoDB | | [monit](https://github.com/netdata/netdata/tree/master/src/go/plugin/go.d/collector/monit) | Monit | | [mysql](https://github.com/netdata/netdata/tree/master/src/go/plugin/go.d/collector/mysql) | MySQL | +| [nats](https://github.com/netdata/netdata/tree/master/src/go/plugin/go.d/collector/nats) | NATS | | [nginx](https://github.com/netdata/netdata/tree/master/src/go/plugin/go.d/collector/nginx) | NGINX | | [nginxplus](https://github.com/netdata/netdata/tree/master/src/go/plugin/go.d/collector/nginxplus) | NGINX Plus | | [nginxunit](https://github.com/netdata/netdata/tree/master/src/go/plugin/go.d/collector/nginxunit) | NGINX Unit | diff --git a/src/go/plugin/go.d/collector/init.go b/src/go/plugin/go.d/collector/init.go index a5c6b645edab59..eea4e6f064dc9b 100644 --- a/src/go/plugin/go.d/collector/init.go +++ b/src/go/plugin/go.d/collector/init.go @@ -61,6 +61,7 @@ import ( _ "github.com/netdata/netdata/go/plugins/plugin/go.d/collector/mongodb" _ "github.com/netdata/netdata/go/plugins/plugin/go.d/collector/monit" _ "github.com/netdata/netdata/go/plugins/plugin/go.d/collector/mysql" + _ "github.com/netdata/netdata/go/plugins/plugin/go.d/collector/nats" _ "github.com/netdata/netdata/go/plugins/plugin/go.d/collector/nginx" _ "github.com/netdata/netdata/go/plugins/plugin/go.d/collector/nginxplus" _ "github.com/netdata/netdata/go/plugins/plugin/go.d/collector/nginxunit" diff --git a/src/go/plugin/go.d/collector/nats/charts.go b/src/go/plugin/go.d/collector/nats/charts.go new file mode 100644 index 00000000000000..68674e3312f34b --- /dev/null +++ b/src/go/plugin/go.d/collector/nats/charts.go @@ -0,0 +1,161 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package nats + +import ( + "fmt" + + "github.com/netdata/netdata/go/plugins/plugin/go.d/agent/module" +) + +const ( + prioServerTraffic = module.Priority + iota + prioServerMessages + prioServerConnectionsCurrent + prioServerConnectionsRate + prioHttpEndpointRequests + prioServerHealthProbeStatus + prioServerCpuUsage + prioServerMemoryUsage + prioServerUptime +) + +var serverCharts = func() module.Charts { + charts := module.Charts{ + chartServerConnectionsCurrent.Copy(), + chartServerConnectionsRate.Copy(), + chartServerTraffic.Copy(), + chartServerMessages.Copy(), + chartServerHealthProbeStatus.Copy(), + chartServerCpuUsage.Copy(), + chartServerMemUsage.Copy(), + chartServerUptime.Copy(), + } + charts = append(charts, httpEndpointCharts()...) + return charts +}() + +var ( + chartServerTraffic = module.Chart{ + ID: "server_traffic", + Title: "Server Traffic", + Units: "bytes/s", + Fam: "traffic", + Ctx: "nats.server_traffic", + Priority: prioServerTraffic, + Type: module.Area, + Dims: module.Dims{ + {ID: "in_bytes", Name: "in", Algo: module.Incremental}, + {ID: "out_bytes", Name: "out", Mul: -1, Algo: module.Incremental}, + }, + } + chartServerMessages = module.Chart{ + ID: "server_messages", + Title: "Server Messages", + Units: "messages/s", + Fam: "traffic", + Ctx: "nats.server_messages", + Priority: prioServerMessages, + Dims: module.Dims{ + {ID: "in_msgs", Name: "in", Algo: module.Incremental}, + {ID: "out_msgs", Name: "out", Mul: -1, Algo: module.Incremental}, + }, + } + chartServerConnectionsCurrent = module.Chart{ + ID: "server_connections_current", + Title: "Server Current Connections", + Units: "connections", + Fam: "connections", + Ctx: "nats.server_connections_current", + Priority: prioServerConnectionsCurrent, + Dims: module.Dims{ + {ID: "connections", Name: "active"}, + }, + } + chartServerConnectionsRate = module.Chart{ + ID: "server_connections_rate", + Title: "Server Connections", + Units: "connections/s", + Fam: "connections", + Ctx: "nats.server_connections_rate", + Priority: prioServerConnectionsRate, + Dims: module.Dims{ + {ID: "total_connections", Name: "connections", Algo: module.Incremental}, + }, + } + chartServerHealthProbeStatus = module.Chart{ + ID: "server_health_probe_status", + Title: "Server Health Probe Status", + Units: "status", + Fam: "health", + Ctx: "nats.server_health_probe_status", + Priority: prioServerHealthProbeStatus, + Dims: module.Dims{ + {ID: "healthz_status_ok", Name: "ok"}, + {ID: "healthz_status_error", Name: "error"}, + }, + } + chartServerCpuUsage = module.Chart{ + ID: "server_cpu_usage", + Title: "Server CPU Usage", + Units: "percent", + Fam: "rusage", + Ctx: "nats.server_cpu_usage", + Priority: prioServerCpuUsage, + Type: module.Area, + Dims: module.Dims{ + {ID: "cpu", Name: "used"}, + }, + } + chartServerMemUsage = module.Chart{ + ID: "server_mem_usage", + Title: "Server Memory Usage", + Units: "bytes", + Fam: "rusage", + Ctx: "nats.server_mem_usage", + Priority: prioServerMemoryUsage, + Type: module.Area, + Dims: module.Dims{ + {ID: "mem", Name: "used"}, + }, + } + chartServerUptime = module.Chart{ + ID: "server_uptime", + Title: "Server Uptime", + Units: "seconds", + Fam: "uptime", + Ctx: "nats.server_uptime", + Priority: prioServerUptime, + Dims: module.Dims{ + {ID: "uptime", Name: "uptime"}, + }, + } +) + +func httpEndpointCharts() module.Charts { + var charts module.Charts + for _, path := range httpEndpoints { + chart := httpEndpointRequestsChartTmpl.Copy() + chart.ID = fmt.Sprintf(chart.ID, path) + chart.Labels = []module.Label{ + {Key: "http_endpoint", Value: path}, + } + for _, dim := range chart.Dims { + dim.ID = fmt.Sprintf(dim.ID, path) + } + charts = append(charts, chart) + } + return charts +} + +var httpEndpointRequestsChartTmpl = module.Chart{ + ID: "http_endpoint_%s_requests", + Title: "HTTP Endpoint Requests", + Units: "requests/s", + Fam: "http requests", + Ctx: "nats.http_endpoint_requests", + Priority: prioHttpEndpointRequests, + Dims: module.Dims{ + {ID: "http_endpoint_%s_req", Name: "requests", Algo: module.Incremental}, + }, +} diff --git a/src/go/plugin/go.d/collector/nats/collect.go b/src/go/plugin/go.d/collector/nats/collect.go new file mode 100644 index 00000000000000..d14b2e74e49f3b --- /dev/null +++ b/src/go/plugin/go.d/collector/nats/collect.go @@ -0,0 +1,85 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package nats + +import ( + "fmt" + "net/http" + + "github.com/netdata/netdata/go/plugins/plugin/go.d/pkg/metrix" + "github.com/netdata/netdata/go/plugins/plugin/go.d/pkg/web" +) + +const ( + urlPathVarz = "/varz" + urlPathHealthz = "/healthz" +) + +func (c *Collector) collect() (map[string]int64, error) { + mx := make(map[string]int64) + + if err := c.collectVarz(mx); err != nil { + return nil, err + } + if err := c.collectHealthz(mx); err != nil { + return nil, err + } + + return mx, nil +} + +func (c *Collector) collectVarz(mx map[string]int64) error { + // https://docs.nats.io/running-a-nats-service/nats_admin/monitoring#general-information + req, err := web.NewHTTPRequestWithPath(c.RequestConfig, urlPathVarz) + if err != nil { + return err + } + + var resp varzResponse + if err := web.DoHTTP(c.httpClient).RequestJSON(req, &resp); err != nil { + return err + } + + mx["uptime"] = int64(resp.Now.Sub(resp.Start).Seconds()) + mx["in_msgs"] = resp.InMsgs + mx["out_msgs"] = resp.OutMsgs + mx["in_bytes"] = resp.InBytes + mx["out_bytes"] = resp.OutBytes + mx["slow_consumers"] = resp.SlowConsumers + mx["subscriptions"] = int64(resp.Subscriptions) + mx["connections"] = int64(resp.Connections) + mx["total_connections"] = int64(resp.TotalConnections) + mx["routes"] = int64(resp.Routes) + mx["remotes"] = int64(resp.Remotes) + mx["cpu"] = int64(resp.CPU) + mx["mem"] = resp.Mem + + for _, path := range httpEndpoints { + v := resp.HTTPReqStats[path] + mx[fmt.Sprintf("http_endpoint_%s_req", path)] = int64(v) + } + + return nil +} + +func (c *Collector) collectHealthz(mx map[string]int64) error { + // https://docs.nats.io/running-a-nats-service/nats_admin/monitoring#health + req, err := web.NewHTTPRequestWithPath(c.RequestConfig, urlPathHealthz) + if err != nil { + return err + } + + var resp healthzResponse + client := web.DoHTTP(c.httpClient).OnNokCode(func(resp *http.Response) (bool, error) { return true, nil }) + if err := client.RequestJSON(req, &resp); err != nil { + return err + } + if resp.Status == nil { + return fmt.Errorf("healthz response missing status") + } + + mx["healthz_status_ok"] = metrix.Bool(*resp.Status == "ok") + mx["healthz_status_error"] = metrix.Bool(*resp.Status != "ok") + + return nil +} diff --git a/src/go/plugin/go.d/collector/nats/collector.go b/src/go/plugin/go.d/collector/nats/collector.go new file mode 100644 index 00000000000000..847163f8aa9c78 --- /dev/null +++ b/src/go/plugin/go.d/collector/nats/collector.go @@ -0,0 +1,113 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package nats + +import ( + "context" + _ "embed" + "errors" + "fmt" + "net/http" + "time" + + "github.com/netdata/netdata/go/plugins/plugin/go.d/agent/module" + "github.com/netdata/netdata/go/plugins/plugin/go.d/pkg/confopt" + "github.com/netdata/netdata/go/plugins/plugin/go.d/pkg/web" +) + +//go:embed "config_schema.json" +var configSchema string + +func init() { + module.Register("nats", module.Creator{ + Create: func() module.Module { return New() }, + JobConfigSchema: configSchema, + Config: func() any { return &Config{} }, + }) +} + +func New() *Collector { + return &Collector{ + Config: Config{ + HTTPConfig: web.HTTPConfig{ + RequestConfig: web.RequestConfig{ + URL: "http://127.0.0.1:8222", + }, + ClientConfig: web.ClientConfig{ + Timeout: confopt.Duration(time.Second), + }, + }, + }, + charts: serverCharts.Copy(), + } +} + +type Config struct { + Vnode string `yaml:"vnode,omitempty" json:"vnode"` + UpdateEvery int `yaml:"update_every,omitempty" json:"update_every"` + web.HTTPConfig `yaml:",inline" json:""` +} + +type Collector struct { + module.Base + Config `yaml:",inline" json:""` + + charts *module.Charts + + httpClient *http.Client +} + +func (c *Collector) Configuration() any { + return c.Config +} + +func (c *Collector) Init(context.Context) error { + if c.URL == "" { + return errors.New("URL required but not set") + } + + httpClient, err := web.NewHTTPClient(c.ClientConfig) + if err != nil { + return fmt.Errorf("init HTTP client: %v", err) + } + c.httpClient = httpClient + + c.Debugf("using URL %s", c.URL) + c.Debugf("using timeout: %s", c.Timeout) + + return nil +} + +func (c *Collector) Check(context.Context) error { + mx, err := c.collect() + if err != nil { + return err + } + if len(mx) == 0 { + return errors.New("no metrics collected") + + } + return nil +} + +func (c *Collector) Charts() *module.Charts { + return c.charts +} + +func (c *Collector) Collect(context.Context) map[string]int64 { + mx, err := c.collect() + if err != nil { + c.Error(err) + } + + if len(mx) == 0 { + return nil + } + return mx +} + +func (c *Collector) Cleanup(context.Context) { + if c.httpClient != nil { + c.httpClient.CloseIdleConnections() + } +} diff --git a/src/go/plugin/go.d/collector/nats/collector_test.go b/src/go/plugin/go.d/collector/nats/collector_test.go new file mode 100644 index 00000000000000..c49f6f56bdf109 --- /dev/null +++ b/src/go/plugin/go.d/collector/nats/collector_test.go @@ -0,0 +1,282 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package nats + +import ( + "context" + "net/http" + "net/http/httptest" + "os" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/netdata/netdata/go/plugins/plugin/go.d/agent/module" + "github.com/netdata/netdata/go/plugins/plugin/go.d/pkg/web" +) + +var ( + dataConfigJSON, _ = os.ReadFile("testdata/config.json") + dataConfigYAML, _ = os.ReadFile("testdata/config.yaml") + + dataVer210Varz, _ = os.ReadFile("testdata/v2.10.24/varz.json") + dataVer210HealthzOk, _ = os.ReadFile("testdata/v2.10.24/healthz-ok.json") +) + +func Test_testDataIsValid(t *testing.T) { + for name, data := range map[string][]byte{ + "dataConfigJSON": dataConfigJSON, + "dataConfigYAML": dataConfigYAML, + "dataVer210Varz": dataVer210Varz, + "dataVer210HealthzOk": dataVer210HealthzOk, + } { + require.NotNil(t, data, name) + } +} + +func TestCollector_ConfigurationSerialize(t *testing.T) { + module.TestConfigurationSerialize(t, &Collector{}, dataConfigJSON, dataConfigYAML) +} + +func TestCollector_Init(t *testing.T) { + tests := map[string]struct { + wantFail bool + config Config + }{ + "success with default": { + wantFail: false, + config: New().Config, + }, + "fail when URL not set": { + wantFail: true, + config: Config{ + HTTPConfig: web.HTTPConfig{ + RequestConfig: web.RequestConfig{URL: ""}, + }, + }, + }, + } + + for name, test := range tests { + t.Run(name, func(t *testing.T) { + collr := New() + collr.Config = test.config + + if test.wantFail { + assert.Error(t, collr.Init(context.Background())) + } else { + assert.NoError(t, collr.Init(context.Background())) + } + }) + } +} + +func TestCollector_Check(t *testing.T) { + tests := map[string]struct { + wantFail bool + prepare func(t *testing.T) (nu *Collector, cleanup func()) + }{ + "success on valid response": { + wantFail: false, + prepare: caseOk, + }, + "fail on unexpected JSON response": { + wantFail: true, + prepare: caseUnexpectedJsonResponse, + }, + "fail on invalid data response": { + wantFail: true, + prepare: caseInvalidDataResponse, + }, + "fail on connection refused": { + wantFail: true, + prepare: caseConnectionRefused, + }, + "fail on 404 response": { + wantFail: true, + prepare: case404, + }, + } + + for name, test := range tests { + t.Run(name, func(t *testing.T) { + collr, cleanup := test.prepare(t) + defer cleanup() + + if test.wantFail { + assert.Error(t, collr.Check(context.Background())) + } else { + assert.NoError(t, collr.Check(context.Background())) + } + }) + } +} + +func TestCollector_Charts(t *testing.T) { + assert.NotNil(t, New().Charts()) +} + +func TestCollector_Collect(t *testing.T) { + tests := map[string]struct { + prepare func(t *testing.T) (nu *Collector, cleanup func()) + wantNumOfCharts int + wantMetrics map[string]int64 + }{ + "success on valid response": { + prepare: caseOk, + wantNumOfCharts: len(serverCharts), + wantMetrics: map[string]int64{ + "connections": 0, + "cpu": 0, + "healthz_status_error": 0, + "healthz_status_ok": 1, + "http_endpoint_/_req": 3, + "http_endpoint_/accountz_req": 2, + "http_endpoint_/accstatz_req": 2, + "http_endpoint_/connz_req": 2, + "http_endpoint_/gatewayz_req": 2, + "http_endpoint_/healthz_req": 2017, + "http_endpoint_/ipqueuesz_req": 0, + "http_endpoint_/jsz_req": 3, + "http_endpoint_/leafz_req": 2, + "http_endpoint_/raftz_req": 0, + "http_endpoint_/routez_req": 2, + "http_endpoint_/stacksz_req": 0, + "http_endpoint_/subsz_req": 1, + "http_endpoint_/varz_req": 3750, + "in_bytes": 0, + "in_msgs": 0, + "mem": 21725184, + "out_bytes": 0, + "out_msgs": 0, + "remotes": 0, + "routes": 0, + "slow_consumers": 0, + "subscriptions": 57, + "total_connections": 0, + "uptime": 27513, + }, + }, + "fail on unexpected JSON response": { + prepare: caseUnexpectedJsonResponse, + wantMetrics: nil, + }, + "fail on invalid data response": { + prepare: caseInvalidDataResponse, + wantMetrics: nil, + }, + "fail on connection refused": { + prepare: caseConnectionRefused, + wantMetrics: nil, + }, + "fail on 404 response": { + prepare: case404, + wantMetrics: nil, + }, + } + + for name, test := range tests { + t.Run(name, func(t *testing.T) { + collr, cleanup := test.prepare(t) + defer cleanup() + + _ = collr.Check(context.Background()) + + mx := collr.Collect(context.Background()) + + require.Equal(t, test.wantMetrics, mx) + + if len(test.wantMetrics) > 0 { + assert.Equal(t, test.wantNumOfCharts, len(*collr.Charts()), "want charts") + + module.TestMetricsHasAllChartsDims(t, collr.Charts(), mx) + } + }) + } +} + +func caseOk(t *testing.T) (*Collector, func()) { + t.Helper() + srv := httptest.NewServer(http.HandlerFunc( + func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case urlPathVarz: + _, _ = w.Write(dataVer210Varz) + case urlPathHealthz: + _, _ = w.Write(dataVer210HealthzOk) + default: + w.WriteHeader(http.StatusNotFound) + } + })) + collr := New() + collr.URL = srv.URL + require.NoError(t, collr.Init(context.Background())) + + return collr, srv.Close +} + +func caseUnexpectedJsonResponse(t *testing.T) (*Collector, func()) { + t.Helper() + resp := ` +{ + "elephant": { + "burn": false, + "mountain": true, + "fog": false, + "skin": -1561907625, + "burst": "anyway", + "shadow": 1558616893 + }, + "start": "ever", + "base": 2093056027, + "mission": -2007590351, + "victory": 999053756, + "die": false +} +` + srv := httptest.NewServer(http.HandlerFunc( + func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte(resp)) + })) + collr := New() + collr.URL = srv.URL + require.NoError(t, collr.Init(context.Background())) + + return collr, srv.Close +} + +func caseInvalidDataResponse(t *testing.T) (*Collector, func()) { + t.Helper() + srv := httptest.NewServer(http.HandlerFunc( + func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte("hello and\n goodbye")) + })) + collr := New() + collr.URL = srv.URL + require.NoError(t, collr.Init(context.Background())) + + return collr, srv.Close +} + +func caseConnectionRefused(t *testing.T) (*Collector, func()) { + t.Helper() + collr := New() + collr.URL = "http://127.0.0.1:65001" + require.NoError(t, collr.Init(context.Background())) + + return collr, func() {} +} + +func case404(t *testing.T) (*Collector, func()) { + t.Helper() + srv := httptest.NewServer(http.HandlerFunc( + func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNotFound) + })) + collr := New() + collr.URL = srv.URL + require.NoError(t, collr.Init(context.Background())) + + return collr, srv.Close +} diff --git a/src/go/plugin/go.d/collector/nats/config_schema.json b/src/go/plugin/go.d/collector/nats/config_schema.json new file mode 100644 index 00000000000000..1cc4d7ac028f01 --- /dev/null +++ b/src/go/plugin/go.d/collector/nats/config_schema.json @@ -0,0 +1,191 @@ +{ + "jsonSchema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "NATS collector configuration.", + "type": "object", + "properties": { + "update_every": { + "title": "Update every", + "description": "Data collection interval, measured in seconds.", + "type": "integer", + "minimum": 1, + "default": 1 + }, + "url": { + "title": "URL", + "description": "The URL of the NATS [monitoring endpoint](https://docs.nats.io/running-a-nats-service/nats_admin/monitoring#enabling-monitoring).", + "type": "string", + "default": "http://127.0.0.1:8222", + "format": "uri" + }, + "timeout": { + "title": "Timeout", + "description": "The timeout in seconds for the HTTP request.", + "type": "number", + "minimum": 0.5, + "default": 1 + }, + "not_follow_redirects": { + "title": "Not follow redirects", + "description": "If set, the client will not follow HTTP redirects automatically.", + "type": "boolean" + }, + "vnode": { + "title": "Vnode", + "description": "Associates this data collection job with a [Virtual Node](https://learn.netdata.cloud/docs/netdata-agent/configuration/organize-systems-metrics-and-alerts#virtual-nodes).", + "type": "string" + }, + "username": { + "title": "Username", + "description": "The username for basic authentication.", + "type": "string", + "sensitive": true + }, + "password": { + "title": "Password", + "description": "The password for basic authentication.", + "type": "string", + "sensitive": true + }, + "proxy_url": { + "title": "Proxy URL", + "description": "The URL of the proxy server.", + "type": "string" + }, + "proxy_username": { + "title": "Proxy username", + "description": "The username for proxy authentication.", + "type": "string", + "sensitive": true + }, + "proxy_password": { + "title": "Proxy password", + "description": "The password for proxy authentication.", + "type": "string", + "sensitive": true + }, + "headers": { + "title": "Headers", + "description": "Additional HTTP headers to include in the request.", + "type": [ + "object", + "null" + ], + "additionalProperties": { + "type": "string" + } + }, + "tls_skip_verify": { + "title": "Skip TLS verification", + "description": "If set, TLS certificate verification will be skipped.", + "type": "boolean" + }, + "tls_ca": { + "title": "TLS CA", + "description": "The path to the CA certificate file for TLS verification.", + "type": "string", + "pattern": "^$|^/" + }, + "tls_cert": { + "title": "TLS certificate", + "description": "The path to the client certificate file for TLS authentication.", + "type": "string", + "pattern": "^$|^/" + }, + "tls_key": { + "title": "TLS key", + "description": "The path to the client key file for TLS authentication.", + "type": "string", + "pattern": "^$|^/" + }, + "body": { + "title": "Body", + "type": "string" + }, + "method": { + "title": "Method", + "type": "string" + } + }, + "required": [ + "url" + ], + "patternProperties": { + "^name$": {} + } + }, + "uiSchema": { + "uiOptions": { + "fullPage": true + }, + "body": { + "ui:widget": "hidden" + }, + "method": { + "ui:widget": "hidden" + }, + "vnode": { + "ui:placeholder": "To use this option, first create a Virtual Node and then reference its name here." + }, + "timeout": { + "ui:help": "Accepts decimals for precise control (e.g., type 1.5 for 1.5 seconds)." + }, + "username": { + "ui:widget": "password" + }, + "proxy_username": { + "ui:widget": "password" + }, + "password": { + "ui:widget": "password" + }, + "proxy_password": { + "ui:widget": "password" + }, + "ui:flavour": "tabs", + "ui:options": { + "tabs": [ + { + "title": "Base", + "fields": [ + "update_every", + "url", + "timeout", + "not_follow_redirects", + "vnode" + ] + }, + { + "title": "Auth", + "fields": [ + "username", + "password" + ] + }, + { + "title": "TLS", + "fields": [ + "tls_skip_verify", + "tls_ca", + "tls_cert", + "tls_key" + ] + }, + { + "title": "Proxy", + "fields": [ + "proxy_url", + "proxy_username", + "proxy_password" + ] + }, + { + "title": "Headers", + "fields": [ + "headers" + ] + } + ] + } + } +} diff --git a/src/go/plugin/go.d/collector/nats/metadata.yml b/src/go/plugin/go.d/collector/nats/metadata.yml new file mode 100644 index 00000000000000..1cd392fda44390 --- /dev/null +++ b/src/go/plugin/go.d/collector/nats/metadata.yml @@ -0,0 +1,248 @@ +plugin_name: go.d.plugin +modules: + - meta: + id: collector-go.d.plugin-nats + plugin_name: go.d.plugin + module_name: nats + monitored_instance: + name: NATS + link: https://nats.io/ + categories: + - data-collection.message-brokers + icon_filename: nats.svg + related_resources: + integrations: + list: [] + alternative_monitored_instances: [] + info_provided_to_referring_integrations: + description: "" + keywords: + - nats + - messaging + - broker + most_popular: false + overview: + data_collection: + metrics_description: | + This collector monitors the activity and performance of NATS servers. + method_description: | + It sends HTTP requests to the NATS HTTP server's dedicated [monitoring port](https://docs.nats.io/running-a-nats-service/nats_admin/monitoring#monitoring-nats). + default_behavior: + auto_detection: + description: | + The collector can automatically detect NATS instances running on: + + - localhost that are listening on port 8222 + - within Docker containers + limits: + description: "" + performance_impact: + description: "" + additional_permissions: + description: "" + multi_instance: true + supported_platforms: + include: [] + exclude: [] + setup: + prerequisites: + list: + - title: Enable NATS monitoring + description: | + See [Enable monitoring](https://docs.nats.io/running-a-nats-service/nats_admin/monitoring#enabling-monitoring). + configuration: + file: + name: go.d/nats.conf + options: + description: | + The following options can be defined globally: update_every, autodetection_retry. + folding: + title: Config options + enabled: true + list: + - name: update_every + description: Data collection frequency. + default_value: 1 + required: false + - name: autodetection_retry + description: Recheck interval in seconds. Zero means no recheck will be scheduled. + default_value: 0 + required: false + - name: url + description: Server URL. + default_value: http://127.0.0.1:8222 + required: true + - name: timeout + description: HTTP request timeout. + default_value: 1 + required: false + - name: username + description: Username for basic HTTP authentication. + default_value: "" + required: false + - name: password + description: Password for basic HTTP authentication. + default_value: "" + required: false + - name: proxy_url + description: Proxy URL. + default_value: "" + required: false + - name: proxy_username + description: Username for proxy basic HTTP authentication. + default_value: "" + required: false + - name: proxy_password + description: Password for proxy basic HTTP authentication. + default_value: "" + required: false + - name: method + description: HTTP request method. + default_value: GET + required: false + - name: body + description: HTTP request body. + default_value: "" + required: false + - name: headers + description: HTTP request headers. + default_value: "" + required: false + - name: not_follow_redirects + description: Redirect handling policy. Controls whether the client follows redirects. + default_value: false + required: false + - name: tls_skip_verify + description: Server certificate chain and hostname validation policy. Controls whether the client performs this check. + default_value: false + required: false + - name: tls_ca + description: Certification authority that the client uses when verifying the server's certificates. + default_value: "" + required: false + - name: tls_cert + description: Client TLS certificate. + default_value: "" + required: false + - name: tls_key + description: Client TLS key. + default_value: "" + required: false + examples: + folding: + title: Config + enabled: true + list: + - name: Basic + description: A basic example configuration. + folding: + enabled: false + config: | + jobs: + - name: local + url: http://127.0.0.1:8222 + - name: HTTP authentication + description: Basic HTTP authentication. + config: | + jobs: + - name: local + url: http://127.0.0.1:8222 + username: username + password: password + - name: HTTPS with self-signed certificate + description: NATS with enabled HTTPS and self-signed certificate. + config: | + jobs: + - name: local + url: http://127.0.0.1:8222 + tls_skip_verify: yes + - name: Multi-instance + description: | + > **Note**: When you define multiple jobs, their names must be unique. + + Collecting metrics from local and remote instances. + config: | + jobs: + - name: local + url: http://127.0.0.1:8222 + + - name: remote + url: http://192.0.2.1:8222 + troubleshooting: + problems: + list: [] + alerts: [] + metrics: + folding: + title: Metrics + enabled: false + description: "" + availability: [] + scopes: + - name: server + description: These metrics refer to NATS servers. + labels: [] + metrics: + - name: nats.server_traffic + description: Server Traffic + unit: bytes/s + chart_type: area + dimensions: + - name: in + - name: out + - name: nats.server_messages + description: Server Messages + unit: messages/s + chart_type: line + dimensions: + - name: in + - name: out + - name: nats.server_connections_current + description: Server Current Connections + unit: connections + chart_type: line + dimensions: + - name: active + - name: nats.server_connections_rate + description: Server Connections + unit: connections/s + chart_type: line + dimensions: + - name: connections + - name: nats.server_health_probe_status + description: Server Health Probe Status + unit: status + chart_type: line + dimensions: + - name: ok + - name: error + - name: nats.server_cpu_usage + description: Server CPU Usage + unit: percent + chart_type: area + dimensions: + - name: used + - name: nats.server_mem_usage + description: Server Memory Usage + unit: bytes + chart_type: area + dimensions: + - name: used + - name: nats.server_uptime + description: Server Uptime + unit: seconds + chart_type: line + dimensions: + - name: uptime + - name: http endpoint + description: These metrics refer to HTTP endpoints. + labels: + - name: http_endpoint + description: "HTTP endpoint path." + metrics: + - name: nats.http_endpoint_requests + description: HTTP Endpoint Requests + unit: requests/s + chart_type: line + dimensions: + - name: requests diff --git a/src/go/plugin/go.d/collector/nats/restapi.go b/src/go/plugin/go.d/collector/nats/restapi.go new file mode 100644 index 00000000000000..94043602e2e20c --- /dev/null +++ b/src/go/plugin/go.d/collector/nats/restapi.go @@ -0,0 +1,76 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package nats + +import ( + "time" +) + +// https://docs.nats.io/running-a-nats-service/nats_admin/monitoring + +// https://github.com/nats-io/nats-server/blob/v2.10.24/server/monitor.go#L1164 +type varzResponse struct { + ID string `json:"server_id"` + Name string `json:"server_name"` + Version string `json:"version"` + Proto int `json:"proto"` + Host string `json:"host"` + Port int `json:"port"` + IP string `json:"ip,omitempty"` + MaxConn int `json:"max_connections"` + MaxSubs int `json:"max_subscriptions,omitempty"` + PingInterval time.Duration `json:"ping_interval"` + MaxPingsOut int `json:"ping_max"` + HTTPHost string `json:"http_host"` + HTTPPort int `json:"http_port"` + HTTPBasePath string `json:"http_base_path"` + HTTPSPort int `json:"https_port"` + AuthTimeout float64 `json:"auth_timeout"` + MaxControlLine int32 `json:"max_control_line"` + MaxPayload int `json:"max_payload"` + MaxPending int64 `json:"max_pending"` + TLSTimeout float64 `json:"tls_timeout"` + WriteDeadline time.Duration `json:"write_deadline"` + Start time.Time `json:"start"` + Now time.Time `json:"now"` + Uptime string `json:"uptime"` + Mem int64 `json:"mem"` + Cores int `json:"cores"` + MaxProcs int `json:"gomaxprocs"` + CPU float64 `json:"cpu"` + Connections int `json:"connections"` + TotalConnections uint64 `json:"total_connections"` + Routes int `json:"routes"` + Remotes int `json:"remotes"` + Leafs int `json:"leafnodes"` + InMsgs int64 `json:"in_msgs"` + OutMsgs int64 `json:"out_msgs"` + InBytes int64 `json:"in_bytes"` + OutBytes int64 `json:"out_bytes"` + SlowConsumers int64 `json:"slow_consumers"` + Subscriptions uint32 `json:"subscriptions"` + HTTPReqStats map[string]uint64 `json:"http_req_stats"` +} + +// //https://github.com/nats-io/nats-server/blob/v2.10.24/server/server.go#L2851 +var httpEndpoints = []string{ + "/", + "/varz", + "/connz", + "/routez", + "/gatewayz", + "/leafz", + "/subsz", + "/stacksz", + "/accountz", + "/accstatz", + "/jsz", + "/healthz", + "/ipqueuesz", + "/raftz", +} + +// https://github.com/nats-io/nats-server/blob/v2.10.24/server/monitor.go#L3125 +type healthzResponse struct { + Status *string `json:"status"` +} diff --git a/src/go/plugin/go.d/collector/nats/testdata/config.json b/src/go/plugin/go.d/collector/nats/testdata/config.json new file mode 100644 index 00000000000000..dee7e030c9dec2 --- /dev/null +++ b/src/go/plugin/go.d/collector/nats/testdata/config.json @@ -0,0 +1,22 @@ +{ + "vnode": "ok", + "update_every": 123, + "url": "ok", + "body": "ok", + "method": "ok", + "headers": { + "ok": "ok" + }, + "username": "ok", + "password": "ok", + "proxy_url": "ok", + "proxy_username": "ok", + "proxy_password": "ok", + "timeout": 123.123, + "not_follow_redirects": true, + "tls_ca": "ok", + "tls_cert": "ok", + "tls_key": "ok", + "tls_skip_verify": true, + "force_http2": true +} diff --git a/src/go/plugin/go.d/collector/nats/testdata/config.yaml b/src/go/plugin/go.d/collector/nats/testdata/config.yaml new file mode 100644 index 00000000000000..d74c39ba07579b --- /dev/null +++ b/src/go/plugin/go.d/collector/nats/testdata/config.yaml @@ -0,0 +1,19 @@ +vnode: "ok" +update_every: 123 +url: "ok" +body: "ok" +method: "ok" +headers: + ok: "ok" +username: "ok" +password: "ok" +proxy_url: "ok" +proxy_username: "ok" +proxy_password: "ok" +timeout: 123.123 +not_follow_redirects: yes +tls_ca: "ok" +tls_cert: "ok" +tls_key: "ok" +tls_skip_verify: yes +force_http2: yes diff --git a/src/go/plugin/go.d/collector/nats/testdata/v2.10.24/healthz-ok.json b/src/go/plugin/go.d/collector/nats/testdata/v2.10.24/healthz-ok.json new file mode 100644 index 00000000000000..bc4e01029df63a --- /dev/null +++ b/src/go/plugin/go.d/collector/nats/testdata/v2.10.24/healthz-ok.json @@ -0,0 +1,3 @@ +{ + "status": "ok" +} diff --git a/src/go/plugin/go.d/collector/nats/testdata/v2.10.24/varz.json b/src/go/plugin/go.d/collector/nats/testdata/v2.10.24/varz.json new file mode 100644 index 00000000000000..8d31549b67e61d --- /dev/null +++ b/src/go/plugin/go.d/collector/nats/testdata/v2.10.24/varz.json @@ -0,0 +1,75 @@ +{ + "server_id": "NASZPQXJ3BIJOGQHV5ZEWGI6EH3YRQPI2Z5GJRA4AZ47TC4PX4OJGY63", + "server_name": "NASZPQXJ3BIJOGQHV5ZEWGI6EH3YRQPI2Z5GJRA4AZ47TC4PX4OJGY63", + "version": "2.10.24", + "proto": 1, + "git_commit": "1d6f7ea", + "go": "go1.23.4", + "host": "0.0.0.0", + "port": 4222, + "max_connections": 65536, + "ping_interval": 120000000000, + "ping_max": 2, + "http_host": "0.0.0.0", + "http_port": 8222, + "http_base_path": "", + "https_port": 0, + "auth_timeout": 2, + "max_control_line": 4096, + "max_payload": 1048576, + "max_pending": 67108864, + "cluster": { + "name": "my_cluster", + "addr": "0.0.0.0", + "cluster_port": 6222, + "auth_timeout": 2, + "tls_timeout": 2, + "pool_size": 3 + }, + "gateway": {}, + "leaf": {}, + "mqtt": {}, + "websocket": {}, + "jetstream": {}, + "tls_timeout": 2, + "write_deadline": 10000000000, + "start": "2024-12-19T11:51:48.038140697Z", + "now": "2024-12-19T19:30:21.110744698Z", + "uptime": "7h38m33s", + "mem": 21725184, + "cores": 16, + "gomaxprocs": 16, + "cpu": 0, + "connections": 0, + "total_connections": 0, + "routes": 0, + "remotes": 0, + "leafnodes": 0, + "in_msgs": 0, + "out_msgs": 0, + "in_bytes": 0, + "out_bytes": 0, + "slow_consumers": 0, + "subscriptions": 57, + "http_req_stats": { + "/": 3, + "/accountz": 2, + "/accstatz": 2, + "/connz": 2, + "/gatewayz": 2, + "/healthz": 2017, + "/jsz": 3, + "/leafz": 2, + "/routez": 2, + "/subsz": 1, + "/varz": 3750 + }, + "config_load_time": "2024-12-19T11:51:48.038140697Z", + "system_account": "$SYS", + "slow_consumer_stats": { + "clients": 0, + "routes": 0, + "gateways": 0, + "leafs": 0 + } +} diff --git a/src/go/plugin/go.d/config/go.d.conf b/src/go/plugin/go.d/config/go.d.conf index a2435e20541e7d..c9f59060c085b7 100644 --- a/src/go/plugin/go.d/config/go.d.conf +++ b/src/go/plugin/go.d/config/go.d.conf @@ -70,6 +70,7 @@ modules: # mongodb: yes # monit: yes # mysql: yes +# nats: yes # nginx: yes # nginxplus: yes # nginxunit: yes diff --git a/src/go/plugin/go.d/config/go.d/nats.conf b/src/go/plugin/go.d/config/go.d/nats.conf new file mode 100644 index 00000000000000..f19e0de1a90dd1 --- /dev/null +++ b/src/go/plugin/go.d/config/go.d/nats.conf @@ -0,0 +1,6 @@ +## All available configuration options, their descriptions and default values: +## https://github.com/netdata/netdata/tree/master/src/go/plugin/go.d/collector/nats#readme + +#jobs: +# - name: local +# url: http://127.0.0.1:8222 diff --git a/src/go/plugin/go.d/config/go.d/sd/docker.conf b/src/go/plugin/go.d/config/go.d/sd/docker.conf index e8bac9b1e81d11..b7f80cb8c660d5 100644 --- a/src/go/plugin/go.d/config/go.d/sd/docker.conf +++ b/src/go/plugin/go.d/config/go.d/sd/docker.conf @@ -58,6 +58,8 @@ classify: expr: '{{ or (eq .PrivatePort "27017") (match "sp" .Image "mongo mongo:* */mongodb */mongodb:* */mongodb-community-server */mongodb-community-server:*") }}' - tags: "mysql" expr: '{{ or (eq .PrivatePort "3306") (match "sp" .Image "mysql mysql:* */mysql */mysql:* mariadb mariadb:* */mariadb */mariadb:* percona percona:* */percona-mysql */percona-mysql:*") }}' + - tags: "nats" + expr: '{{ and (eq .PrivatePort "8222") (match "sp" .Image "nats nats:*") }}' - tags: "nginx" expr: '{{ match "sp" .Image "nginx nginx:*" }}' - tags: "nginxunit" @@ -197,6 +199,11 @@ compose: module: mysql name: docker_{{.Name}} dsn: netdata@tcp({{.Address}})/ + - selector: "nats" + template: | + - module: nats + name: docker_{{.Name}} + url: http://{{.Address}} - selector: "nginx" template: | - module: nginx diff --git a/src/go/plugin/go.d/config/go.d/sd/net_listeners.conf b/src/go/plugin/go.d/config/go.d/sd/net_listeners.conf index 6a7f7d4a7f6da7..14681e00698351 100644 --- a/src/go/plugin/go.d/config/go.d/sd/net_listeners.conf +++ b/src/go/plugin/go.d/config/go.d/sd/net_listeners.conf @@ -92,6 +92,8 @@ classify: expr: '{{ or (eq .Port "2812") (eq .Comm "monit") }}' - tags: "mysql" expr: '{{ or (eq .Port "3306") (eq .Comm "mysqld" "mariadbd") }}' + - tags: "nats" + expr: '{{ and (eq .Port "8222") (eq .Comm "nats-server") }}' - tags: "nginx" expr: '{{ and (eq .Port "80" "8080") (eq .Comm "nginx") }}' - tags: "nginxunit" @@ -393,6 +395,11 @@ compose: - module: mysql name: local dsn: netdata@tcp({{.Address}})/ + - selector: "nats" + template: | + - module: nats + name: local + url: http://{{.Address}} - selector: "nginx" template: | - module: nginx