From d290766df9809b70dcfbe98ff0ff7b20393402a9 Mon Sep 17 00:00:00 2001 From: Ilya Mashchenko Date: Thu, 16 May 2024 16:21:57 +0300 Subject: [PATCH] add go.d/rspamd (#17679) --- src/go/collectors/go.d.plugin/README.md | 14 +- .../discoverer/netlisteners/netlisteners.go | 4 +- .../collectors/go.d.plugin/config/go.d.conf | 1 + .../go.d.plugin/config/go.d/rspamd.conf | 6 + .../config/go.d/sd/net_listeners.conf | 9 +- src/go/collectors/go.d.plugin/modules/init.go | 1 + .../go.d.plugin/modules/rspamd/charts.go | 110 ++++++++ .../go.d.plugin/modules/rspamd/collect.go | 94 +++++++ .../modules/rspamd/config_schema.json | 177 ++++++++++++ .../go.d.plugin/modules/rspamd/metadata.yaml | 221 +++++++++++++++ .../go.d.plugin/modules/rspamd/rspamd.go | 113 ++++++++ .../go.d.plugin/modules/rspamd/rspamd_test.go | 258 ++++++++++++++++++ .../modules/rspamd/testdata/config.json | 20 ++ .../modules/rspamd/testdata/config.yaml | 17 ++ .../modules/rspamd/testdata/v3.4-stat.json | 66 +++++ 15 files changed, 1101 insertions(+), 10 deletions(-) create mode 100644 src/go/collectors/go.d.plugin/config/go.d/rspamd.conf create mode 100644 src/go/collectors/go.d.plugin/modules/rspamd/charts.go create mode 100644 src/go/collectors/go.d.plugin/modules/rspamd/collect.go create mode 100644 src/go/collectors/go.d.plugin/modules/rspamd/config_schema.json create mode 100644 src/go/collectors/go.d.plugin/modules/rspamd/metadata.yaml create mode 100644 src/go/collectors/go.d.plugin/modules/rspamd/rspamd.go create mode 100644 src/go/collectors/go.d.plugin/modules/rspamd/rspamd_test.go create mode 100644 src/go/collectors/go.d.plugin/modules/rspamd/testdata/config.json create mode 100644 src/go/collectors/go.d.plugin/modules/rspamd/testdata/config.yaml create mode 100644 src/go/collectors/go.d.plugin/modules/rspamd/testdata/v3.4-stat.json diff --git a/src/go/collectors/go.d.plugin/README.md b/src/go/collectors/go.d.plugin/README.md index 9414c217e26a19..ad7955a5c4412f 100644 --- a/src/go/collectors/go.d.plugin/README.md +++ b/src/go/collectors/go.d.plugin/README.md @@ -17,9 +17,8 @@ collection modules written in `go`. 1. It runs as an independent process (`ps fax` shows it). 2. It is started and stopped automatically by Netdata. 3. It communicates with Netdata via a unidirectional pipe (sending data to the Netdata daemon). -4. Supports any number of data collection [modules](https://github.com/netdata/netdata/tree/master/src/go/collectors/go.d.plugin/modules). -5. Allows each [module](https://github.com/netdata/netdata/tree/master/src/go/collectors/go.d.plugin/modules) to have any number of data - collection jobs. +4. Supports any number of data collection modules. +5. Allows each module to have any number of data collection jobs. ## Bug reports, feature requests, and questions @@ -115,6 +114,7 @@ see the appropriate collector readme. | [pulsar](https://github.com/netdata/netdata/tree/master/src/go/collectors/go.d.plugin/modules/portcheck) | Apache Pulsar | | [rabbitmq](https://github.com/netdata/netdata/tree/master/src/go/collectors/go.d.plugin/modules/rabbitmq) | RabbitMQ | | [redis](https://github.com/netdata/netdata/tree/master/src/go/collectors/go.d.plugin/modules/redis) | Redis | +| [rspamd](https://github.com/netdata/netdata/tree/master/src/go/collectors/go.d.plugin/modules/rspamd) | Rspamd | | [scaleio](https://github.com/netdata/netdata/tree/master/src/go/collectors/go.d.plugin/modules/scaleio) | Dell EMC ScaleIO | | [sensors](https://github.com/netdata/netdata/tree/master/src/go/collectors/go.d.plugin/modules) | Hardware Sensors | | [SNMP](https://github.com/netdata/netdata/blob/master/src/go/collectors/go.d.plugin/modules/snmp) | SNMP | @@ -141,8 +141,8 @@ see the appropriate collector readme. ## Configuration Edit the `go.d.conf` configuration file using `edit-config` from the -Netdata [config directory](https://github.com/netdata/netdata/blob/master/docs/netdata-agent/configuration/README.md#the-netdata-config-directory), which is typically -at `/etc/netdata`. +Netdata [config directory](https://github.com/netdata/netdata/blob/master/docs/netdata-agent/configuration/README.md#the-netdata-config-directory), +which is typically at `/etc/netdata`. ```bash cd /etc/netdata # Replace this path with your Netdata config directory @@ -173,8 +173,8 @@ modules: example: yes ``` -Then [restart netdata](https://github.com/netdata/netdata/blob/master/packaging/installer/README.md#maintaining-a-netdata-agent-installation) for the -change to take effect. +Then [restart netdata](https://github.com/netdata/netdata/blob/master/packaging/installer/README.md#maintaining-a-netdata-agent-installation) +for the change to take effect. ## Contributing diff --git a/src/go/collectors/go.d.plugin/agent/discovery/sd/discoverer/netlisteners/netlisteners.go b/src/go/collectors/go.d.plugin/agent/discovery/sd/discoverer/netlisteners/netlisteners.go index 9897405cd2ac7c..271a1d5dc9a1ea 100644 --- a/src/go/collectors/go.d.plugin/agent/discovery/sd/discoverer/netlisteners/netlisteners.go +++ b/src/go/collectors/go.d.plugin/agent/discovery/sd/discoverer/netlisteners/netlisteners.go @@ -310,10 +310,10 @@ func (e *localListenersExec) discover(ctx context.Context) ([]byte, error) { func extractComm(cmdLine string) string { i := strings.IndexByte(cmdLine, ' ') if i <= 0 { - return cmdLine + return strings.TrimSuffix(cmdLine, ":") } _, comm := filepath.Split(cmdLine[:i]) - return comm + return strings.TrimSuffix(comm, ":") } func calcHash(obj any) (uint64, error) { diff --git a/src/go/collectors/go.d.plugin/config/go.d.conf b/src/go/collectors/go.d.plugin/config/go.d.conf index 76b91b98bbafda..6ab1b87517d45d 100644 --- a/src/go/collectors/go.d.plugin/config/go.d.conf +++ b/src/go/collectors/go.d.plugin/config/go.d.conf @@ -78,6 +78,7 @@ modules: # pulsar: yes # rabbitmq: yes # redis: yes +# rspamd: yes # scaleio: yes # sensors: yes # snmp: yes diff --git a/src/go/collectors/go.d.plugin/config/go.d/rspamd.conf b/src/go/collectors/go.d.plugin/config/go.d/rspamd.conf new file mode 100644 index 00000000000000..fe4424768e54c6 --- /dev/null +++ b/src/go/collectors/go.d.plugin/config/go.d/rspamd.conf @@ -0,0 +1,6 @@ +## All available configuration options, their descriptions and default values: +## https://github.com/netdata/netdata/tree/master/src/go/collectors/go.d.plugin/modules/rspamd#readme + +#jobs: +# - name: local +# url: http://127.0.0.1:11334 diff --git a/src/go/collectors/go.d.plugin/config/go.d/sd/net_listeners.conf b/src/go/collectors/go.d.plugin/config/go.d/sd/net_listeners.conf index 6c2e22c71c9e05..257cf6b78ba851 100644 --- a/src/go/collectors/go.d.plugin/config/go.d/sd/net_listeners.conf +++ b/src/go/collectors/go.d.plugin/config/go.d/sd/net_listeners.conf @@ -69,7 +69,7 @@ classify: - tags: "mysql" expr: '{{ or (eq .Port "3306") (eq .Comm "mysqld" "mariadbd") }}' - tags: "nginx" - expr: '{{ and (eq .Port "80" "8080") (eq .Comm "nginx" "nginx:") }}' + expr: '{{ and (eq .Port "80" "8080") (eq .Comm "nginx") }}' - tags: "ntpd" expr: '{{ or (eq .Port "123") (eq .Comm "ntpd") }}' - tags: "openvpn" @@ -92,6 +92,8 @@ classify: expr: '{{ or (eq .Port "15672") (glob .Cmdline "*rabbitmq*") }}' - tags: "redis" expr: '{{ or (eq .Port "6379") (eq .Comm "redis-server") }}' + - tags: "rspamd" + expr: '{{ or (eq .Port "11334") (eq .Comm "rspamd") }}' - tags: "supervisord" expr: '{{ and (eq .Port "9001") (eq .Comm "supervisord") }}' - tags: "traefik" @@ -321,6 +323,11 @@ compose: module: pika name: local address: redis://@{{.IPAddress}}:{{.Port}} + - selector: "rspamd" + template: | + module: rspamd + name: local + url: http://{{.Address}} - selector: "postgres" template: | - module: postgres diff --git a/src/go/collectors/go.d.plugin/modules/init.go b/src/go/collectors/go.d.plugin/modules/init.go index 80927253214d92..32ffa99b5fe487 100644 --- a/src/go/collectors/go.d.plugin/modules/init.go +++ b/src/go/collectors/go.d.plugin/modules/init.go @@ -70,6 +70,7 @@ import ( _ "github.com/netdata/netdata/go/go.d.plugin/modules/pulsar" _ "github.com/netdata/netdata/go/go.d.plugin/modules/rabbitmq" _ "github.com/netdata/netdata/go/go.d.plugin/modules/redis" + _ "github.com/netdata/netdata/go/go.d.plugin/modules/rspamd" _ "github.com/netdata/netdata/go/go.d.plugin/modules/scaleio" _ "github.com/netdata/netdata/go/go.d.plugin/modules/sensors" _ "github.com/netdata/netdata/go/go.d.plugin/modules/smartctl" diff --git a/src/go/collectors/go.d.plugin/modules/rspamd/charts.go b/src/go/collectors/go.d.plugin/modules/rspamd/charts.go new file mode 100644 index 00000000000000..fcf21fb41f3a66 --- /dev/null +++ b/src/go/collectors/go.d.plugin/modules/rspamd/charts.go @@ -0,0 +1,110 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package rspamd + +import "github.com/netdata/netdata/go/go.d.plugin/agent/module" + +const ( + prioClassifications = module.Priority + iota + prioActions + prioScans + prioLearns + prioConnections + prioControlConnections +) + +var charts = module.Charts{ + classificationsChartTmpl.Copy(), + + actionsChart.Copy(), + + scanChartTmpl.Copy(), + learnChartTmpl.Copy(), + + connectionsChartTmpl.Copy(), + controlConnectionsChartTmpl.Copy(), +} + +var ( + classificationsChartTmpl = module.Chart{ + ID: "classifications", + Title: "Classifications", + Units: "messages/s", + Fam: "classification", + Ctx: "rspamd.classifications", + Type: module.Stacked, + Priority: prioClassifications, + Dims: module.Dims{ + {ID: "ham_count", Name: "ham", Algo: module.Incremental}, + {ID: "spam_count", Name: "spam", Algo: module.Incremental}, + }, + } + + actionsChart = module.Chart{ + ID: "actions", + Title: "Actions", + Units: "messages/s", + Fam: "actions", + Ctx: "rspamd.actions", + Type: module.Stacked, + Priority: prioActions, + Dims: module.Dims{ + {ID: "actions_reject", Name: "reject", Algo: module.Incremental}, + {ID: "actions_soft_reject", Name: "soft_reject", Algo: module.Incremental}, + {ID: "actions_rewrite_subject", Name: "rewrite_subject", Algo: module.Incremental}, + {ID: "actions_add_header", Name: "add_header", Algo: module.Incremental}, + {ID: "actions_greylist", Name: "greylist", Algo: module.Incremental}, + {ID: "actions_custom", Name: "custom", Algo: module.Incremental}, + {ID: "actions_discard", Name: "discard", Algo: module.Incremental}, + {ID: "actions_quarantine", Name: "quarantine", Algo: module.Incremental}, + {ID: "actions_no_action", Name: "no_action", Algo: module.Incremental}, + }, + } + + scanChartTmpl = module.Chart{ + ID: "scans", + Title: "Scanned messages", + Units: "messages/s", + Fam: "training", + Ctx: "rspamd.scans", + Priority: prioScans, + Dims: module.Dims{ + {ID: "scanned", Name: "scanned", Algo: module.Incremental}, + }, + } + + learnChartTmpl = module.Chart{ + ID: "learns", + Title: "Learned messages", + Units: "messages/s", + Fam: "training", + Ctx: "rspamd.learns", + Priority: prioLearns, + Dims: module.Dims{ + {ID: "learned", Name: "learned", Algo: module.Incremental}, + }, + } + + connectionsChartTmpl = module.Chart{ + ID: "connections", + Title: "Connections", + Units: "connections/s", + Fam: "connections", + Ctx: "rspamd.connections", + Priority: prioConnections, + Dims: module.Dims{ + {ID: "connections", Name: "connections", Algo: module.Incremental}, + }, + } + controlConnectionsChartTmpl = module.Chart{ + ID: "control_connections", + Title: "Control connections", + Units: "connections/s", + Fam: "connections", + Ctx: "rspamd.control_connections", + Priority: prioControlConnections, + Dims: module.Dims{ + {ID: "control_connections", Name: "control_connections", Algo: module.Incremental}, + }, + } +) diff --git a/src/go/collectors/go.d.plugin/modules/rspamd/collect.go b/src/go/collectors/go.d.plugin/modules/rspamd/collect.go new file mode 100644 index 00000000000000..6e204e9b55660d --- /dev/null +++ b/src/go/collectors/go.d.plugin/modules/rspamd/collect.go @@ -0,0 +1,94 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package rspamd + +import ( + "encoding/json" + "fmt" + "io" + "net/http" + + "github.com/netdata/netdata/go/go.d.plugin/pkg/stm" + "github.com/netdata/netdata/go/go.d.plugin/pkg/web" +) + +type rspamdStats struct { + Version string `json:"version"` + ConfigId string `json:"config_id"` + Scanned int64 `json:"scanned" stm:"scanned"` + Learned int64 `json:"learned" stm:"learned"` + Actions struct { + Reject int64 `json:"reject" stm:"reject"` + SoftReject int64 `json:"soft reject" stm:"soft_reject"` + RewriteSubject int64 `json:"rewrite subject" stm:"rewrite_subject"` + AddHeader int64 `json:"add header" stm:"add_header"` + Greylist int64 `json:"greylist" stm:"greylist"` + NoAction int64 `json:"no action" stm:"no_action"` + InvalidMaxAction int64 `json:"invalid max action" stm:"invalid_max_action"` + Custom int64 `json:"custom" stm:"custom"` + Discard int64 `json:"discard" stm:"discard"` + Quarantine int64 `json:"quarantine" stm:"quarantine"` + UnknownAction int64 `json:"unknown action" stm:"unknown_action"` + } `json:"actions" stm:"actions"` + ScanTimes []float64 `json:"scan_times"` + SpamCount int64 `json:"spam_count" stm:"spam_count"` + HamCount int64 `json:"ham_count" stm:"ham_count"` + Connections int64 `json:"connections" stm:"connections"` + ControlConnections int64 `json:"control_connections" stm:"control_connections"` + FuzzyHashes map[string]int64 `json:"fuzzy_hashes"` +} + +func (r *Rspamd) collect() (map[string]int64, error) { + stats, err := r.queryRspamdStats() + if err != nil { + return nil, err + } + + mx := stm.ToMap(stats) + + return mx, nil +} + +func (r *Rspamd) queryRspamdStats() (*rspamdStats, error) { + req, err := web.NewHTTPRequest(r.Request) + if err != nil { + return nil, err + } + + req.URL.Path = "/stat" + + var stats rspamdStats + if err := r.doOKDecode(req, &stats); err != nil { + return nil, err + } + + if stats.Version == "" || stats.ConfigId == "" || len(stats.FuzzyHashes) == 0 { + return nil, fmt.Errorf("unexpected response: not rspamd data") + } + + return &stats, nil +} + +func (r *Rspamd) doOKDecode(req *http.Request, in interface{}) error { + resp, err := r.httpClient.Do(req) + if err != nil { + return fmt.Errorf("error on HTTP request '%s': %v", req.URL, err) + } + defer closeBody(resp) + + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("'%s' returned HTTP status code: %d", req.URL, resp.StatusCode) + } + + if err := json.NewDecoder(resp.Body).Decode(in); err != nil { + return fmt.Errorf("error on decoding response from '%s': %v", req.URL, err) + } + return nil +} + +func closeBody(resp *http.Response) { + if resp != nil && resp.Body != nil { + _, _ = io.Copy(io.Discard, resp.Body) + _ = resp.Body.Close() + } +} diff --git a/src/go/collectors/go.d.plugin/modules/rspamd/config_schema.json b/src/go/collectors/go.d.plugin/modules/rspamd/config_schema.json new file mode 100644 index 00000000000000..8ed49475373d96 --- /dev/null +++ b/src/go/collectors/go.d.plugin/modules/rspamd/config_schema.json @@ -0,0 +1,177 @@ +{ + "jsonSchema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Rspamd collector configuration.", + "type": "object", + "properties": { + "update_every": { + "title": "Update every", + "description": "Data collection interval, measured in seconds.", + "type": "integer", + "minimum": 1, + "default": 1 + }, + "url": { + "title": "URL", + "description": "The base URL of the Rspamd [controller worker](https://rspamd.com/doc/workers/controller.html).", + "type": "string", + "default": "http://127.0.0.1:11334", + "format": "uri" + }, + "timeout": { + "title": "Timeout", + "description": "The timeout in seconds for the HTTP request.", + "type": "number", + "minimum": 0.5, + "default": 1 + }, + "not_follow_redirects": { + "title": "Not follow redirects", + "description": "If set, the client will not follow HTTP redirects automatically.", + "type": "boolean" + }, + "username": { + "title": "Username", + "description": "The username for basic authentication.", + "type": "string", + "sensitive": true + }, + "password": { + "title": "Password", + "description": "The password for basic authentication.", + "type": "string", + "sensitive": true + }, + "proxy_url": { + "title": "Proxy URL", + "description": "The URL of the proxy server.", + "type": "string" + }, + "proxy_username": { + "title": "Proxy username", + "description": "The username for proxy authentication.", + "type": "string", + "sensitive": true + }, + "proxy_password": { + "title": "Proxy password", + "description": "The password for proxy authentication.", + "type": "string", + "sensitive": true + }, + "headers": { + "title": "Headers", + "description": "Additional HTTP headers to include in the request.", + "type": [ + "object", + "null" + ], + "additionalProperties": { + "type": "string" + } + }, + "tls_skip_verify": { + "title": "Skip TLS verification", + "description": "If set, TLS certificate verification will be skipped.", + "type": "boolean" + }, + "tls_ca": { + "title": "TLS CA", + "description": "The path to the CA certificate file for TLS verification.", + "type": "string", + "pattern": "^$|^/" + }, + "tls_cert": { + "title": "TLS certificate", + "description": "The path to the client certificate file for TLS authentication.", + "type": "string", + "pattern": "^$|^/" + }, + "tls_key": { + "title": "TLS key", + "description": "The path to the client key file for TLS authentication.", + "type": "string", + "pattern": "^$|^/" + }, + "body": { + "title": "Body", + "type": "string" + }, + "method": { + "title": "Method", + "type": "string" + } + }, + "required": [ + "url" + ], + "additionalProperties": false, + "patternProperties": { + "^name$": {} + } + }, + "uiSchema": { + "uiOptions": { + "fullPage": true + }, + "body": { + "ui:widget": "hidden" + }, + "method": { + "ui:widget": "hidden" + }, + "timeout": { + "ui:help": "Accepts decimals for precise control (e.g., type 1.5 for 1.5 seconds)." + }, + "password": { + "ui:widget": "password" + }, + "proxy_password": { + "ui:widget": "password" + }, + "ui:flavour": "tabs", + "ui:options": { + "tabs": [ + { + "title": "Base", + "fields": [ + "update_every", + "url", + "timeout", + "not_follow_redirects" + ] + }, + { + "title": "Auth", + "fields": [ + "username", + "password" + ] + }, + { + "title": "TLS", + "fields": [ + "tls_skip_verify", + "tls_ca", + "tls_cert", + "tls_key" + ] + }, + { + "title": "Proxy", + "fields": [ + "proxy_url", + "proxy_username", + "proxy_password" + ] + }, + { + "title": "Headers", + "fields": [ + "headers" + ] + } + ] + } + } +} diff --git a/src/go/collectors/go.d.plugin/modules/rspamd/metadata.yaml b/src/go/collectors/go.d.plugin/modules/rspamd/metadata.yaml new file mode 100644 index 00000000000000..a8ab16b4910e72 --- /dev/null +++ b/src/go/collectors/go.d.plugin/modules/rspamd/metadata.yaml @@ -0,0 +1,221 @@ +plugin_name: go.d.plugin +modules: + - meta: + id: collector-go.d.plugin-rspamd + plugin_name: go.d.plugin + module_name: rspamd + monitored_instance: + name: Rspamd + link: https://rspamd.com/ + categories: + - data-collection.security-systems + icon_filename: globe.svg + related_resources: + integrations: + list: + - plugin_name: go.d.plugin + module_name: httpcheck + - plugin_name: apps.plugin + module_name: apps + alternative_monitored_instances: [] + info_provided_to_referring_integrations: + description: "" + keywords: + - spam + - rspamd + - email + most_popular: false + overview: + data_collection: + metrics_description: | + This collector monitors the activity and performance of Rspamd servers. It gathers various metrics including scanned emails, learned messages, spam/ham counts, and actions taken on emails (reject, rewrite, etc.). + method_description: | + It retrieves statistics from Rspamd's [built-in web server](https://rspamd.com/doc/workers/controller.html) by making HTTP requests to the `/stat` endpoint. + default_behavior: + auto_detection: + description: | + By default, it detects Rspamd instances running on localhost that are listening on port 11334. + limits: + description: "" + performance_impact: + description: "" + additional_permissions: + description: "" + multi_instance: true + supported_platforms: + include: [] + exclude: [] + setup: + prerequisites: + list: [] + configuration: + file: + name: go.d/rspamd.conf + options: + description: | + The following options can be defined globally: update_every, autodetection_retry. + folding: + title: Config options + enabled: true + list: + - name: update_every + description: Data collection frequency. + default_value: 1 + required: false + - name: autodetection_retry + description: Recheck interval in seconds. Zero means no recheck will be scheduled. + default_value: 0 + required: false + - name: url + description: Server URL. + default_value: http://127.0.0.1:11334 + required: true + - name: timeout + description: HTTP request timeout. + default_value: 1 + required: false + - name: username + description: Username for basic HTTP authentication. + default_value: "" + required: false + - name: password + description: Password for basic HTTP authentication. + default_value: "" + required: false + - name: proxy_url + description: Proxy URL. + default_value: "" + required: false + - name: proxy_username + description: Username for proxy basic HTTP authentication. + default_value: "" + required: false + - name: proxy_password + description: Password for proxy basic HTTP authentication. + default_value: "" + required: false + - name: method + description: HTTP request method. + default_value: GET + required: false + - name: body + description: HTTP request body. + default_value: "" + required: false + - name: headers + description: HTTP request headers. + default_value: "" + required: false + - name: not_follow_redirects + description: Redirect handling policy. Controls whether the client follows redirects. + default_value: false + required: false + - name: tls_skip_verify + description: Server certificate chain and hostname validation policy. Controls whether the client performs this check. + default_value: false + required: false + - name: tls_ca + description: Certification authority that the client uses when verifying the server's certificates. + default_value: "" + required: false + - name: tls_cert + description: Client TLS certificate. + default_value: "" + required: false + - name: tls_key + description: Client TLS key. + default_value: "" + required: false + examples: + folding: + title: Config + enabled: true + list: + - name: Basic + description: A basic example configuration. + folding: + enabled: false + config: | + jobs: + - name: local + url: http://127.0.0.1:11334 + - name: HTTP authentication + description: Basic HTTP authentication. + config: | + jobs: + - name: local + url: http://127.0.0.1:11334 + username: username + password: password + - name: Multi-instance + description: | + > **Note**: When you define multiple jobs, their names must be unique. + + Collecting metrics from local and remote instances. + config: | + jobs: + - name: local + url: http://127.0.0.1:11334 + + - name: remote + url: http://192.0.2.1:11334 + troubleshooting: + problems: + list: [] + alerts: [] + metrics: + folding: + title: Metrics + enabled: false + description: "" + availability: [] + scopes: + - name: global + description: These metrics refer to the entire monitored application. + labels: [] + metrics: + - name: rspamd.classifications + description: Classifications + unit: messages/s + chart_type: stacked + dimensions: + - name: ham + - name: spam + - name: rspamd.actions + description: Actions + unit: messages/s + chart_type: stacked + dimensions: + - name: reject + - name: soft_reject + - name: rewrite_subject + - name: add_header + - name: greylist + - name: custom + - name: discard + - name: quarantine + - name: no_action + - name: rspamd.scans + description: Scanned messages + unit: messages/s + chart_type: line + dimensions: + - name: scanned + - name: rspamd.learns + description: Learned messages + unit: messages/s + chart_type: line + dimensions: + - name: learned + - name: rspamd.connections + description: Connections + unit: connections/s + chart_type: line + dimensions: + - name: connections + - name: rspamd.control_connections + description: Control connections + unit: connections/s + chart_type: line + dimensions: + - name: control_connections diff --git a/src/go/collectors/go.d.plugin/modules/rspamd/rspamd.go b/src/go/collectors/go.d.plugin/modules/rspamd/rspamd.go new file mode 100644 index 00000000000000..635e5877417b16 --- /dev/null +++ b/src/go/collectors/go.d.plugin/modules/rspamd/rspamd.go @@ -0,0 +1,113 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package rspamd + +import ( + _ "embed" + "errors" + "net/http" + "time" + + "github.com/netdata/netdata/go/go.d.plugin/agent/module" + "github.com/netdata/netdata/go/go.d.plugin/pkg/web" +) + +//go:embed "config_schema.json" +var configSchema string + +func init() { + module.Register("rspamd", module.Creator{ + JobConfigSchema: configSchema, + Create: func() module.Module { return New() }, + }) +} + +func New() *Rspamd { + return &Rspamd{ + Config: Config{ + HTTP: web.HTTP{ + Request: web.Request{ + URL: "http://127.0.0.1:11334", + }, + Client: web.Client{ + Timeout: web.Duration(time.Second * 1), + }, + }, + }, + charts: charts.Copy(), + } +} + +type Config struct { + web.HTTP `yaml:",inline" json:""` + UpdateEvery int `yaml:"update_every" json:"update_every"` +} + +type Rspamd struct { + module.Base + Config `yaml:",inline" json:""` + + charts *module.Charts + + httpClient *http.Client +} + +func (r *Rspamd) Configuration() any { + return r.Config +} + +func (r *Rspamd) Init() error { + if r.URL == "" { + r.Error("URL not set") + return errors.New("url not set") + } + + client, err := web.NewHTTPClient(r.Client) + if err != nil { + r.Error(err) + return err + } + r.httpClient = client + + r.Debugf("using URL %s", r.URL) + r.Debugf("using timeout: %s", r.Timeout) + + return nil +} + +func (r *Rspamd) Check() error { + mx, err := r.collect() + if err != nil { + r.Error(err) + return err + } + + if len(mx) == 0 { + return errors.New("no metrics collected") + } + + return nil +} + +func (r *Rspamd) Charts() *module.Charts { + return r.charts +} + +func (r *Rspamd) Collect() map[string]int64 { + mx, err := r.collect() + if err != nil { + r.Error(err) + } + + if len(mx) == 0 { + return nil + } + + return mx +} + +func (r *Rspamd) Cleanup() { + if r.httpClient != nil { + r.httpClient.CloseIdleConnections() + } +} diff --git a/src/go/collectors/go.d.plugin/modules/rspamd/rspamd_test.go b/src/go/collectors/go.d.plugin/modules/rspamd/rspamd_test.go new file mode 100644 index 00000000000000..c5db42cec95f28 --- /dev/null +++ b/src/go/collectors/go.d.plugin/modules/rspamd/rspamd_test.go @@ -0,0 +1,258 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package rspamd + +import ( + "net/http" + "net/http/httptest" + "os" + "testing" + + "github.com/netdata/netdata/go/go.d.plugin/agent/module" + "github.com/netdata/netdata/go/go.d.plugin/pkg/web" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +var ( + dataConfigJSON, _ = os.ReadFile("testdata/config.json") + dataConfigYAML, _ = os.ReadFile("testdata/config.yaml") + + dataV34Stat, _ = os.ReadFile("testdata/v3.4-stat.json") +) + +func Test_testDataIsValid(t *testing.T) { + for name, data := range map[string][]byte{ + "dataConfigJSON": dataConfigJSON, + "dataConfigYAML": dataConfigYAML, + "dataV34Stat": dataV34Stat, + } { + require.NotNil(t, data, name) + } +} + +func TestRspamd_ConfigurationSerialize(t *testing.T) { + module.TestConfigurationSerialize(t, &Rspamd{}, dataConfigJSON, dataConfigYAML) +} + +func TestRspamd_Init(t *testing.T) { + tests := map[string]struct { + wantFail bool + config Config + }{ + "success with default": { + wantFail: false, + config: New().Config, + }, + "fail when URL not set": { + wantFail: true, + config: Config{ + HTTP: web.HTTP{ + Request: web.Request{URL: ""}, + }, + }, + }, + } + + for name, test := range tests { + t.Run(name, func(t *testing.T) { + rsp := New() + rsp.Config = test.config + + if test.wantFail { + assert.Error(t, rsp.Init()) + } else { + assert.NoError(t, rsp.Init()) + } + }) + } +} + +func TestRspamd_Charts(t *testing.T) { + assert.NotNil(t, New().Charts()) +} + +func TestRspamd_Check(t *testing.T) { + tests := map[string]struct { + wantFail bool + prepare func(t *testing.T) (*Rspamd, func()) + }{ + "success on valid response": { + wantFail: false, + prepare: prepareCaseOk, + }, + "fails on unexpected json response": { + wantFail: true, + prepare: prepareCaseUnexpectedJsonResponse, + }, + "fails on invalid format response": { + wantFail: true, + prepare: prepareCaseInvalidFormatResponse, + }, + "fails on connection refused": { + wantFail: true, + prepare: prepareCaseConnectionRefused, + }, + } + + for name, test := range tests { + t.Run(name, func(t *testing.T) { + rsp, cleanup := test.prepare(t) + defer cleanup() + + if test.wantFail { + assert.Error(t, rsp.Check()) + } else { + assert.NoError(t, rsp.Check()) + } + }) + } +} + +func TestRspamd_Collect(t *testing.T) { + tests := map[string]struct { + prepare func(t *testing.T) (*Rspamd, func()) + wantMetrics map[string]int64 + }{ + "success on valid response": { + prepare: prepareCaseOk, + wantMetrics: map[string]int64{ + "actions_add_header": 1, + "actions_custom": 0, + "actions_discard": 0, + "actions_greylist": 1, + "actions_invalid_max_action": 0, + "actions_no_action": 1, + "actions_quarantine": 0, + "actions_reject": 1, + "actions_rewrite_subject": 1, + "actions_soft_reject": 1, + "actions_unknown_action": 0, + "connections": 1, + "control_connections": 117, + "ham_count": 1, + "learned": 1, + "scanned": 1, + "spam_count": 1, + }, + }, + "fails on unexpected json response": { + prepare: prepareCaseUnexpectedJsonResponse, + }, + "fails on invalid format response": { + prepare: prepareCaseInvalidFormatResponse, + }, + "fails on connection refused": { + prepare: prepareCaseConnectionRefused, + }, + } + + for name, test := range tests { + t.Run(name, func(t *testing.T) { + rsp, cleanup := test.prepare(t) + defer cleanup() + + mx := rsp.Collect() + + require.Equal(t, test.wantMetrics, mx) + if len(test.wantMetrics) > 0 { + testMetricsHasAllChartsDims(t, rsp, mx) + } + }) + } +} + +func testMetricsHasAllChartsDims(t *testing.T, rsp *Rspamd, mx map[string]int64) { + for _, chart := range *rsp.Charts() { + if chart.Obsolete { + continue + } + for _, dim := range chart.Dims { + _, ok := mx[dim.ID] + assert.Truef(t, ok, "collected metrics has no data for dim '%s' chart '%s'", dim.ID, chart.ID) + } + for _, v := range chart.Vars { + _, ok := mx[v.ID] + assert.Truef(t, ok, "collected metrics has no data for var '%s' chart '%s'", v.ID, chart.ID) + } + } +} + +func prepareCaseOk(t *testing.T) (*Rspamd, func()) { + t.Helper() + srv := httptest.NewServer(http.HandlerFunc( + func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/stat": + _, _ = w.Write(dataV34Stat) + default: + w.WriteHeader(http.StatusNotFound) + } + })) + + rsp := New() + rsp.URL = srv.URL + require.NoError(t, rsp.Init()) + + return rsp, srv.Close +} + +func prepareCaseUnexpectedJsonResponse(t *testing.T) (*Rspamd, func()) { + t.Helper() + resp := ` +{ + "elephant": { + "burn": false, + "mountain": true, + "fog": false, + "skin": -1561907625, + "burst": "anyway", + "shadow": 1558616893 + }, + "start": "ever", + "base": 2093056027, + "mission": -2007590351, + "victory": 999053756, + "die": false +} +` + srv := httptest.NewServer(http.HandlerFunc( + func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/stat": + _, _ = w.Write([]byte(resp)) + default: + w.WriteHeader(http.StatusNotFound) + } + })) + + rsp := New() + rsp.URL = srv.URL + require.NoError(t, rsp.Init()) + + return rsp, srv.Close +} + +func prepareCaseInvalidFormatResponse(t *testing.T) (*Rspamd, func()) { + t.Helper() + srv := httptest.NewServer(http.HandlerFunc( + func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte("hello and\n goodbye")) + })) + + rsp := New() + rsp.URL = srv.URL + require.NoError(t, rsp.Init()) + + return rsp, srv.Close +} + +func prepareCaseConnectionRefused(t *testing.T) (*Rspamd, func()) { + t.Helper() + rsp := New() + rsp.URL = "http://127.0.0.1:65001/stat" + require.NoError(t, rsp.Init()) + + return rsp, func() {} +} diff --git a/src/go/collectors/go.d.plugin/modules/rspamd/testdata/config.json b/src/go/collectors/go.d.plugin/modules/rspamd/testdata/config.json new file mode 100644 index 00000000000000..984c3ed6e7a880 --- /dev/null +++ b/src/go/collectors/go.d.plugin/modules/rspamd/testdata/config.json @@ -0,0 +1,20 @@ +{ + "update_every": 123, + "url": "ok", + "body": "ok", + "method": "ok", + "headers": { + "ok": "ok" + }, + "username": "ok", + "password": "ok", + "proxy_url": "ok", + "proxy_username": "ok", + "proxy_password": "ok", + "timeout": 123.123, + "not_follow_redirects": true, + "tls_ca": "ok", + "tls_cert": "ok", + "tls_key": "ok", + "tls_skip_verify": true +} diff --git a/src/go/collectors/go.d.plugin/modules/rspamd/testdata/config.yaml b/src/go/collectors/go.d.plugin/modules/rspamd/testdata/config.yaml new file mode 100644 index 00000000000000..8558b61cc05cf8 --- /dev/null +++ b/src/go/collectors/go.d.plugin/modules/rspamd/testdata/config.yaml @@ -0,0 +1,17 @@ +update_every: 123 +url: "ok" +body: "ok" +method: "ok" +headers: + ok: "ok" +username: "ok" +password: "ok" +proxy_url: "ok" +proxy_username: "ok" +proxy_password: "ok" +timeout: 123.123 +not_follow_redirects: yes +tls_ca: "ok" +tls_cert: "ok" +tls_key: "ok" +tls_skip_verify: yes diff --git a/src/go/collectors/go.d.plugin/modules/rspamd/testdata/v3.4-stat.json b/src/go/collectors/go.d.plugin/modules/rspamd/testdata/v3.4-stat.json new file mode 100644 index 00000000000000..38145477eca941 --- /dev/null +++ b/src/go/collectors/go.d.plugin/modules/rspamd/testdata/v3.4-stat.json @@ -0,0 +1,66 @@ +{ + "version": "3.4", + "config_id": "gkwm3ysiqrx96kj1mwnfashx9hkypj833w1tgjaw4nysgwwxqthh7q78hyrezi9gzamke3n9ea7u8cjrzru7i5p4z7r9xhcoitjpjyy", + "uptime": 1774, + "read_only": false, + "scanned": 1, + "learned": 1, + "actions": { + "reject": 1, + "soft reject": 1, + "rewrite subject": 1, + "add header": 1, + "greylist": 1, + "no action": 1 + }, + "scan_times": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ], + "spam_count": 1, + "ham_count": 1, + "connections": 1, + "control_connections": 117, + "pools_allocated": 184, + "pools_freed": 147, + "bytes_allocated": 28807460, + "chunks_allocated": 282, + "shared_chunks_allocated": 4, + "chunks_freed": 0, + "chunks_oversized": 2, + "fragmented": 0, + "total_learns": 0, + "statfiles": [], + "fuzzy_hashes": { + "rspamd.com": 446607461 + } +}