Skip to content

Commit

Permalink
add go.d/rspamd (netdata#17679)
Browse files Browse the repository at this point in the history
  • Loading branch information
ilyam8 authored May 16, 2024
1 parent 8d9c464 commit d290766
Show file tree
Hide file tree
Showing 15 changed files with 1,101 additions and 10 deletions.
14 changes: 7 additions & 7 deletions src/go/collectors/go.d.plugin/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,8 @@ collection modules written in `go`.
1. It runs as an independent process (`ps fax` shows it).
2. It is started and stopped automatically by Netdata.
3. It communicates with Netdata via a unidirectional pipe (sending data to the Netdata daemon).
4. Supports any number of data collection [modules](https://github.com/netdata/netdata/tree/master/src/go/collectors/go.d.plugin/modules).
5. Allows each [module](https://github.com/netdata/netdata/tree/master/src/go/collectors/go.d.plugin/modules) to have any number of data
collection jobs.
4. Supports any number of data collection modules.
5. Allows each module to have any number of data collection jobs.

## Bug reports, feature requests, and questions

Expand Down Expand Up @@ -115,6 +114,7 @@ see the appropriate collector readme.
| [pulsar](https://github.com/netdata/netdata/tree/master/src/go/collectors/go.d.plugin/modules/portcheck) | Apache Pulsar |
| [rabbitmq](https://github.com/netdata/netdata/tree/master/src/go/collectors/go.d.plugin/modules/rabbitmq) | RabbitMQ |
| [redis](https://github.com/netdata/netdata/tree/master/src/go/collectors/go.d.plugin/modules/redis) | Redis |
| [rspamd](https://github.com/netdata/netdata/tree/master/src/go/collectors/go.d.plugin/modules/rspamd) | Rspamd |
| [scaleio](https://github.com/netdata/netdata/tree/master/src/go/collectors/go.d.plugin/modules/scaleio) | Dell EMC ScaleIO |
| [sensors](https://github.com/netdata/netdata/tree/master/src/go/collectors/go.d.plugin/modules) | Hardware Sensors |
| [SNMP](https://github.com/netdata/netdata/blob/master/src/go/collectors/go.d.plugin/modules/snmp) | SNMP |
Expand All @@ -141,8 +141,8 @@ see the appropriate collector readme.
## Configuration

Edit the `go.d.conf` configuration file using `edit-config` from the
Netdata [config directory](https://github.com/netdata/netdata/blob/master/docs/netdata-agent/configuration/README.md#the-netdata-config-directory), which is typically
at `/etc/netdata`.
Netdata [config directory](https://github.com/netdata/netdata/blob/master/docs/netdata-agent/configuration/README.md#the-netdata-config-directory),
which is typically at `/etc/netdata`.

```bash
cd /etc/netdata # Replace this path with your Netdata config directory
Expand Down Expand Up @@ -173,8 +173,8 @@ modules:
example: yes
```
Then [restart netdata](https://github.com/netdata/netdata/blob/master/packaging/installer/README.md#maintaining-a-netdata-agent-installation) for the
change to take effect.
Then [restart netdata](https://github.com/netdata/netdata/blob/master/packaging/installer/README.md#maintaining-a-netdata-agent-installation)
for the change to take effect.
## Contributing
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -310,10 +310,10 @@ func (e *localListenersExec) discover(ctx context.Context) ([]byte, error) {
func extractComm(cmdLine string) string {
i := strings.IndexByte(cmdLine, ' ')
if i <= 0 {
return cmdLine
return strings.TrimSuffix(cmdLine, ":")
}
_, comm := filepath.Split(cmdLine[:i])
return comm
return strings.TrimSuffix(comm, ":")
}

func calcHash(obj any) (uint64, error) {
Expand Down
1 change: 1 addition & 0 deletions src/go/collectors/go.d.plugin/config/go.d.conf
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,7 @@ modules:
# pulsar: yes
# rabbitmq: yes
# redis: yes
# rspamd: yes
# scaleio: yes
# sensors: yes
# snmp: yes
Expand Down
6 changes: 6 additions & 0 deletions src/go/collectors/go.d.plugin/config/go.d/rspamd.conf
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
## All available configuration options, their descriptions and default values:
## https://github.com/netdata/netdata/tree/master/src/go/collectors/go.d.plugin/modules/rspamd#readme

#jobs:
# - name: local
# url: http://127.0.0.1:11334
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@ classify:
- tags: "mysql"
expr: '{{ or (eq .Port "3306") (eq .Comm "mysqld" "mariadbd") }}'
- tags: "nginx"
expr: '{{ and (eq .Port "80" "8080") (eq .Comm "nginx" "nginx:") }}'
expr: '{{ and (eq .Port "80" "8080") (eq .Comm "nginx") }}'
- tags: "ntpd"
expr: '{{ or (eq .Port "123") (eq .Comm "ntpd") }}'
- tags: "openvpn"
Expand All @@ -92,6 +92,8 @@ classify:
expr: '{{ or (eq .Port "15672") (glob .Cmdline "*rabbitmq*") }}'
- tags: "redis"
expr: '{{ or (eq .Port "6379") (eq .Comm "redis-server") }}'
- tags: "rspamd"
expr: '{{ or (eq .Port "11334") (eq .Comm "rspamd") }}'
- tags: "supervisord"
expr: '{{ and (eq .Port "9001") (eq .Comm "supervisord") }}'
- tags: "traefik"
Expand Down Expand Up @@ -321,6 +323,11 @@ compose:
module: pika
name: local
address: redis://@{{.IPAddress}}:{{.Port}}
- selector: "rspamd"
template: |
module: rspamd
name: local
url: http://{{.Address}}
- selector: "postgres"
template: |
- module: postgres
Expand Down
1 change: 1 addition & 0 deletions src/go/collectors/go.d.plugin/modules/init.go
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,7 @@ import (
_ "github.com/netdata/netdata/go/go.d.plugin/modules/pulsar"
_ "github.com/netdata/netdata/go/go.d.plugin/modules/rabbitmq"
_ "github.com/netdata/netdata/go/go.d.plugin/modules/redis"
_ "github.com/netdata/netdata/go/go.d.plugin/modules/rspamd"
_ "github.com/netdata/netdata/go/go.d.plugin/modules/scaleio"
_ "github.com/netdata/netdata/go/go.d.plugin/modules/sensors"
_ "github.com/netdata/netdata/go/go.d.plugin/modules/smartctl"
Expand Down
110 changes: 110 additions & 0 deletions src/go/collectors/go.d.plugin/modules/rspamd/charts.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,110 @@
// SPDX-License-Identifier: GPL-3.0-or-later

package rspamd

import "github.com/netdata/netdata/go/go.d.plugin/agent/module"

const (
prioClassifications = module.Priority + iota
prioActions
prioScans
prioLearns
prioConnections
prioControlConnections
)

var charts = module.Charts{
classificationsChartTmpl.Copy(),

actionsChart.Copy(),

scanChartTmpl.Copy(),
learnChartTmpl.Copy(),

connectionsChartTmpl.Copy(),
controlConnectionsChartTmpl.Copy(),
}

var (
classificationsChartTmpl = module.Chart{
ID: "classifications",
Title: "Classifications",
Units: "messages/s",
Fam: "classification",
Ctx: "rspamd.classifications",
Type: module.Stacked,
Priority: prioClassifications,
Dims: module.Dims{
{ID: "ham_count", Name: "ham", Algo: module.Incremental},
{ID: "spam_count", Name: "spam", Algo: module.Incremental},
},
}

actionsChart = module.Chart{
ID: "actions",
Title: "Actions",
Units: "messages/s",
Fam: "actions",
Ctx: "rspamd.actions",
Type: module.Stacked,
Priority: prioActions,
Dims: module.Dims{
{ID: "actions_reject", Name: "reject", Algo: module.Incremental},
{ID: "actions_soft_reject", Name: "soft_reject", Algo: module.Incremental},
{ID: "actions_rewrite_subject", Name: "rewrite_subject", Algo: module.Incremental},
{ID: "actions_add_header", Name: "add_header", Algo: module.Incremental},
{ID: "actions_greylist", Name: "greylist", Algo: module.Incremental},
{ID: "actions_custom", Name: "custom", Algo: module.Incremental},
{ID: "actions_discard", Name: "discard", Algo: module.Incremental},
{ID: "actions_quarantine", Name: "quarantine", Algo: module.Incremental},
{ID: "actions_no_action", Name: "no_action", Algo: module.Incremental},
},
}

scanChartTmpl = module.Chart{
ID: "scans",
Title: "Scanned messages",
Units: "messages/s",
Fam: "training",
Ctx: "rspamd.scans",
Priority: prioScans,
Dims: module.Dims{
{ID: "scanned", Name: "scanned", Algo: module.Incremental},
},
}

learnChartTmpl = module.Chart{
ID: "learns",
Title: "Learned messages",
Units: "messages/s",
Fam: "training",
Ctx: "rspamd.learns",
Priority: prioLearns,
Dims: module.Dims{
{ID: "learned", Name: "learned", Algo: module.Incremental},
},
}

connectionsChartTmpl = module.Chart{
ID: "connections",
Title: "Connections",
Units: "connections/s",
Fam: "connections",
Ctx: "rspamd.connections",
Priority: prioConnections,
Dims: module.Dims{
{ID: "connections", Name: "connections", Algo: module.Incremental},
},
}
controlConnectionsChartTmpl = module.Chart{
ID: "control_connections",
Title: "Control connections",
Units: "connections/s",
Fam: "connections",
Ctx: "rspamd.control_connections",
Priority: prioControlConnections,
Dims: module.Dims{
{ID: "control_connections", Name: "control_connections", Algo: module.Incremental},
},
}
)
94 changes: 94 additions & 0 deletions src/go/collectors/go.d.plugin/modules/rspamd/collect.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,94 @@
// SPDX-License-Identifier: GPL-3.0-or-later

package rspamd

import (
"encoding/json"
"fmt"
"io"
"net/http"

"github.com/netdata/netdata/go/go.d.plugin/pkg/stm"
"github.com/netdata/netdata/go/go.d.plugin/pkg/web"
)

type rspamdStats struct {
Version string `json:"version"`
ConfigId string `json:"config_id"`
Scanned int64 `json:"scanned" stm:"scanned"`
Learned int64 `json:"learned" stm:"learned"`
Actions struct {
Reject int64 `json:"reject" stm:"reject"`
SoftReject int64 `json:"soft reject" stm:"soft_reject"`
RewriteSubject int64 `json:"rewrite subject" stm:"rewrite_subject"`
AddHeader int64 `json:"add header" stm:"add_header"`
Greylist int64 `json:"greylist" stm:"greylist"`
NoAction int64 `json:"no action" stm:"no_action"`
InvalidMaxAction int64 `json:"invalid max action" stm:"invalid_max_action"`
Custom int64 `json:"custom" stm:"custom"`
Discard int64 `json:"discard" stm:"discard"`
Quarantine int64 `json:"quarantine" stm:"quarantine"`
UnknownAction int64 `json:"unknown action" stm:"unknown_action"`
} `json:"actions" stm:"actions"`
ScanTimes []float64 `json:"scan_times"`
SpamCount int64 `json:"spam_count" stm:"spam_count"`
HamCount int64 `json:"ham_count" stm:"ham_count"`
Connections int64 `json:"connections" stm:"connections"`
ControlConnections int64 `json:"control_connections" stm:"control_connections"`
FuzzyHashes map[string]int64 `json:"fuzzy_hashes"`
}

func (r *Rspamd) collect() (map[string]int64, error) {
stats, err := r.queryRspamdStats()
if err != nil {
return nil, err
}

mx := stm.ToMap(stats)

return mx, nil
}

func (r *Rspamd) queryRspamdStats() (*rspamdStats, error) {
req, err := web.NewHTTPRequest(r.Request)
if err != nil {
return nil, err
}

req.URL.Path = "/stat"

var stats rspamdStats
if err := r.doOKDecode(req, &stats); err != nil {
return nil, err
}

if stats.Version == "" || stats.ConfigId == "" || len(stats.FuzzyHashes) == 0 {
return nil, fmt.Errorf("unexpected response: not rspamd data")
}

return &stats, nil
}

func (r *Rspamd) doOKDecode(req *http.Request, in interface{}) error {
resp, err := r.httpClient.Do(req)
if err != nil {
return fmt.Errorf("error on HTTP request '%s': %v", req.URL, err)
}
defer closeBody(resp)

if resp.StatusCode != http.StatusOK {
return fmt.Errorf("'%s' returned HTTP status code: %d", req.URL, resp.StatusCode)
}

if err := json.NewDecoder(resp.Body).Decode(in); err != nil {
return fmt.Errorf("error on decoding response from '%s': %v", req.URL, err)
}
return nil
}

func closeBody(resp *http.Response) {
if resp != nil && resp.Body != nil {
_, _ = io.Copy(io.Discard, resp.Body)
_ = resp.Body.Close()
}
}
Loading

0 comments on commit d290766

Please sign in to comment.