From 0c3dd2c1a4a3a1d9218c289e42b9d4a5ad222c5e Mon Sep 17 00:00:00 2001 From: Heiko Henning Date: Sun, 10 Dec 2023 10:53:44 +0100 Subject: [PATCH] #59 add semp v2 for queue stats --- README.md | 137 ++++++++++++++++++++++------------ exporter/config.struct.go | 5 ++ exporter/dataSource.struct.go | 14 ++-- exporter/exporter.collect.go | 22 +++++- go.sum | 13 +++- semp/getQueueStatsSemp2.go | 129 ++++++++++++++++++++++++++++++++ semp/helper.go | 45 +++++++++++ semp/{postHttp.go => http.go} | 26 +++++++ semp/metricDesc.go | 47 ++++++++---- semp/sempv2.desc.struct.go | 65 ++++++++++++++++ solace_prometheus_exporter.go | 17 +++-- 11 files changed, 441 insertions(+), 79 deletions(-) create mode 100644 semp/getQueueStatsSemp2.go create mode 100644 semp/helper.go rename semp/{postHttp.go => http.go} (53%) create mode 100644 semp/sempv2.desc.struct.go diff --git a/README.md b/README.md index 996cece..c9633f5 100644 --- a/README.md +++ b/README.md @@ -12,11 +12,11 @@ The exporter is written in go, based on the Solace Legacy SEMP protocol. -It grabs metrics via SEMP v1 and provide those as prometheus friendly http endpoints. +It grabs metrics via SEMP v1 and provides those as prometheus friendly http endpoints. -Video Intro available on youtube: [Integrating Prometheus and Grafana with Solace PubSub+ | Solace Community Lightning Talk +Video Intro available on YouTube: [Integrating Prometheus and Grafana with Solace PubSub+ | Solace Community Lightning Talk ](https://youtu.be/72Wz5rrStAU?t=35) ## Features @@ -36,19 +36,27 @@ http://:/solace The modular endpoint ### Modular endpoint explained -Configure the data you want ot receive, via [HTTP GET parameters](https://www.seobility.net/en/wiki/GET_Parameters). +Configure the data you want ot to receive via [HTTP GET parameters](https://www.seobility.net/en/wiki/GET_Parameters). The key is always the [scrape target](#scrape-targets) prefixed by a `m.`. -The value contains out of 2 parts, delimited by a pipe `|`. +The value contains out of 2–3 parts, delimited by a pipe `|`. - The first part is the VPN filter. -- The second part is the ITEM filter. +- The second part is the ITEM filter. +- The third part is the METRIC filter. -Not all scrape targets support both filter. Please see [scrape target](#scrape-targets) to find out what is supported where. -Both filter can contain multiple asterisk `*` as wildcard for N chars. +Not all scrape targets support both filters. Please see [scrape target](#scrape-targets) to find out what is supported where. +The first both filters can contain multiple asterisk `*` as wildcard for N chars. Each scrape target can be used multiple times, to implement or condition filters. +#### Endpoints using semp v1 + +Here are only the first two filters are supported. + +The VPN filter can be an asterix. +The ITME filter is using the semp v1 semantic (* is a wildcard for one or more chars). + #### Examples Get the same result as the legacy `solace-det` endpoint. @@ -69,41 +77,72 @@ Get all queue information, where the queue name starts with `BRAVO`or `ARBON` an Get the same result as the legacy `solace-det` endpoint, but from a specific broker. `http://your-exporter:9628/solace?m.ClientStats=*|*&m.VpnStats=*|*&m.BridgeStats=*|*&m.QueueRates=*|*&m.QueueDetails=*|*&scrapeURI=http://your-broker-url:8080` +#### Endpoints using semp v2 + +Here are only the first two filters are supported. + +The VPN filter may NOT be asterix. +You are advised to always provide a valid vpn name. +Wildcards are not supported. +In case you provide an asterix, the "DefaultVpn" from configuration will be used. + +The ITME filter is using the semp v2 semantic (* is a wildcard for one or more chars). +You can either provide only the filter string, in this case main field and == will be prepended. + +Or you provide full qualified solace sep [v2 filter](https://docs.solace.com/Admin/SEMP/SEMP-Features.htm#Filtering) like: + +`queueName!=internal*` All queues that are NOT internal. +`queueName==important*` Only important queues. + +The METRIC filter limits the metrics that are returned. +Please use the feature to save resources of the broker and your prometheus. +Some fields are more costly than others. +By only returning required metrics, you can speed up semp v2 query dramatically. +Provide a comma separated list of either semp v2 field names or metrics names. + + +#### Examples + +Get the metrics `solace_queue_msg_shutdown_discarded` and `solace_queue_msg_max_redelivered_discarded` for all queues not starting with the word "internal" +`http://your-exporter:9628/solace?m.QueueStatsV2=AaaBbbCcc|queueName!=internal*|solace_queue_msg_shutdown_discarded,solace_queue_msg_max_redelivered_discarded` + + #### Scrape targets -| scrape target | vpn filter supports | item filter supported | performance impact | corresponding cli cmd | supported by | -| :------------------------------------ | :------------------ | :-------------------- | :-------------------------------------------------- | :---------------------------------------------------------------------------- | :------------------ | -| Version | no | no | dont harm broker | show version | software, appliance | -| Health | no | no | dont harm broker | show system health | software | -| StorageElement | no | yes | dont harm broker | show storage-element storageElementFilter | software | -| Disk | no | no | dont harm broker | show disk detail | appliance | -| Memory | no | no | dont harm broker | show memory | software, appliance | -| Interface | no | yes | dont harm broker | show interface interfaceFilter | software, appliance | -| GlobalStats | no | no | dont harm broker | show stats client | software, appliance | -| Spool | no | no | dont harm broker | show message-spool | software, appliance | -| Redundancy (only for HA broker) | no | no | dont harm broker | show redundancy | software, appliance | -| ConfigSyncRouter (only for HA broker) | no | no | dont harm broker | show config-sync database router | software, appliance | -| Replication (only for DR broker) | no | no | dont harm broker | show replication stats | software, appliance | -| Vpn | yes | no | dont harm broker | show message-vpn vpnFilter | software, appliance | -| VpnReplication | yes | no | dont harm broker | show message-vpn vpnFilter replication | software, appliance | -| ConfigSyncVpn (only for HA broker) | yes | no | dont harm broker | show config-sync database message-vpn vpnFilter | software, appliance | -| Bridge | yes | yes | dont harm broker | show bridge itemFilter message-vpn vpnFilter | software, appliance | -| VpnSpool | yes | no | dont harm broker | show message-spool message-vpn vpnFilter | software, appliance | -| Client | yes | yes | may harm broker if many clients | show client itemFilter message-vpn vpnFilter connected | software, appliance | -| ClientSlowSubscriber | yes | yes | may harm broker if many clients | show client itemFilter message-vpn vpnFilter slow-subscriber | software, appliance | -| ClientStats | yes | no | may harm broker if many clients | show client itemFilter stats (paged) | software, appliance | -| ClientConnections | yes | no | may harm broker if many clients | show client itemFilter stats | software, appliance | -| ClientMessageSpoolStats | yes | no | may harm broker if many clients | show client itemFilter stats(paged) | software, appliance | -| VpnStats | yes | no | has a very small performance down site | show message-vpn vpnFilter stats | software, appliance | -| BridgeStats | yes | yes | has a very small performance down site | show bridge itemFilter message-vpn vpnFilter stats | software, appliance | -| QueueRates | yes | yes | DEPRECATED: may harm broker if many queues | show queue itemFilter message-vpn vpnFilter rates count 100 (paged) | software, appliance | -| QueueStats | yes | yes | may harm broker if many queues | show queue itemFilter message-vpn vpnFilter rates count 100 (paged) | software, appliance | -| QueueDetails | yes | yes | may harm broker if many queues | show queue itemFilter message-vpn vpnFilter detail count 100 (paged) | software, appliance | -| TopicEndpointRates | yes | yes | DEPRECATED: may harm broker if many topic-endpoints | show topic-endpoint itemFilter message-vpn vpnFilter rates count 100 (paged) | software, appliance | -| TopicEndpointStats | yes | yes | may harm broker if many topic-endpoint | show topic-endpoint itemFilter message-vpn vpnFilter rates count 100 (paged) | software, appliance | -| TopicEndpointDetails | yes | yes | may harm broker if many topic-endpoints | show topic-endpoint itemFilter message-vpn vpnFilter detail count 100 (paged) | software, appliance | -| ClusterLinks | yes | yes | dont harm broker | show the state of the cluster links. Filters are for clusterName and linkName | software, appliance | +| scrape target | vpn filter supports | item filter supported | metrics filter supported | performance impact | corresponding cli cmd | supported by | +|:--------------------------------------|:--------------------|:----------------------|--------------------------|:----------------------------------------------------|:------------------------------------------------------------------------------|:--------------------| +| Version | no | no | no | dont harm broker | show version | software, appliance | +| Health | no | no | no | dont harm broker | show system health | software | +| StorageElement | no | yes | no | dont harm broker | show storage-element storageElementFilter | software | +| Disk | no | no | no | dont harm broker | show disk detail | appliance | +| Memory | no | no | no | dont harm broker | show memory | software, appliance | +| Interface | no | yes | no | dont harm broker | show interface interfaceFilter | software, appliance | +| GlobalStats | no | no | no | dont harm broker | show stats client | software, appliance | +| Spool | no | no | no | dont harm broker | show message-spool | software, appliance | +| Redundancy (only for HA broker) | no | no | no | dont harm broker | show redundancy | software, appliance | +| ConfigSyncRouter (only for HA broker) | no | no | no | dont harm broker | show config-sync database router | software, appliance | +| Replication (only for DR broker) | no | no | no | dont harm broker | show replication stats | software, appliance | +| Vpn | yes | no | no | dont harm broker | show message-vpn vpnFilter | software, appliance | +| VpnReplication | yes | no | no | dont harm broker | show message-vpn vpnFilter replication | software, appliance | +| ConfigSyncVpn (only for HA broker) | yes | no | no | dont harm broker | show config-sync database message-vpn vpnFilter | software, appliance | +| Bridge | yes | yes | no | dont harm broker | show bridge itemFilter message-vpn vpnFilter | software, appliance | +| VpnSpool | yes | no | no | dont harm broker | show message-spool message-vpn vpnFilter | software, appliance | +| Client | yes | yes | no | may harm broker if many clients | show client itemFilter message-vpn vpnFilter connected | software, appliance | +| ClientSlowSubscriber | yes | yes | no | may harm broker if many clients | show client itemFilter message-vpn vpnFilter slow-subscriber | software, appliance | +| ClientStats | yes | no | no | may harm broker if many clients | show client itemFilter stats (paged) | software, appliance | +| ClientConnections | yes | no | no | may harm broker if many clients | show client itemFilter stats | software, appliance | +| ClientMessageSpoolStats | yes | no | no | may harm broker if many clients | show client itemFilter stats(paged) | software, appliance | +| VpnStats | yes | no | no | has a very small performance down site | show message-vpn vpnFilter stats | software, appliance | +| BridgeStats | yes | yes | no | has a very small performance down site | show bridge itemFilter message-vpn vpnFilter stats | software, appliance | +| QueueRates | yes | yes | no | DEPRECATED: may harm broker if many queues | show queue itemFilter message-vpn vpnFilter rates count 100 (paged) | software, appliance | +| QueueStats | yes | yes | no | may harm broker if many queues | show queue itemFilter message-vpn vpnFilter rates count 100 (paged) | software, appliance | +| QueueStatsV2 | yes | yes | yes | may harm broker if many queues | show queue itemFilter message-vpn vpnFilter rates count 100 (paged) | software, appliance | +| QueueDetails | yes | yes | no | may harm broker if many queues | SempV2 monitoring /queue/getMsgVpnQueues 100 (paged) | software, appliance | +| TopicEndpointRates | yes | yes | no | DEPRECATED: may harm broker if many topic-endpoints | show topic-endpoint itemFilter message-vpn vpnFilter rates count 100 (paged) | software, appliance | +| TopicEndpointStats | yes | yes | no | may harm broker if many topic-endpoint | show topic-endpoint itemFilter message-vpn vpnFilter rates count 100 (paged) | software, appliance | +| TopicEndpointDetails | yes | yes | no | may harm broker if many topic-endpoints | show topic-endpoint itemFilter message-vpn vpnFilter detail count 100 (paged) | software, appliance | +| ClusterLinks | yes | yes | no | dont harm broker | show the state of the cluster links. Filters are for clusterName and linkName | software, appliance | #### Broker Connectivity Metric @@ -142,7 +181,7 @@ http://your-exporter:9628/solace-det This will provide the same output as: http://your-exporter:9628/solace?m.ClientStats=*|*&?m.VpnStats=*|*&?m.BridgeStats=*|*&?m.QueueRates=*|*&?m.QueueDetails=*|* -If you want to use wildcards to only have a subset but needs more then one wildcard, +If you want to use wildcards to only have a subset but need more than one wildcard, you have to add a dot and an incrementing number. Like this: ```ini @@ -172,7 +211,7 @@ Flags: ``` The configuration parameters can be placed into a config file or into a set of environment variables or can be given via URL. If you use docker, you should prefer the environment variable configuration method (see below). -If the exporter is started with a config file argument then the config file entries have precedence over the environment variables. If a parameter is neither found in the URL, nor the config file nor in the environment the exporter exits with an error. +If the exporter is started with a config file argument, then the config file entries have precedence over the environment variables. If a parameter is neither found in the URL, nor the config file nor in the environment, the exporter exits with an error. ### Config File @@ -187,15 +226,15 @@ Sample config file: listenAddr=0.0.0.0:9628 # Enable TLS on listenAddr endpoint. Make sure to provide certificate and private key files. -# can be overridden via env varibale SOLACE_LISTEN_TLS +# can be overridden via env variable SOLACE_LISTEN_TLS enableTLS=true # Path to the server certificate (including intermediates and CA's certificate) -# can be overridden via env varibale SOLACE_SERVER_CERT +# can be overridden via env variable SOLACE_SERVER_CERT certificate=cert.pem # Path to the private key pem file -# can be overridden via env varibale SOLACE_PRIVATE_KEY +# can be overridden via env variable SOLACE_PRIVATE_KEY privateKey=key.pem # Base URI on which to scrape Solace broker. @@ -238,14 +277,14 @@ SOLACE_SSL_VERIFY=false You can call: `https://your-exporter:9628/solace?m.ClientStats=*|*&m.VpnStats=*|*&scrapeURI=https%3A%2F%2Fyour-broker%3A943&username=monitoring&password=monitoring&timeout=10s` -This service grabs metrics via SEMP v1 and provide those as prometheus friendly http endpoints. +This service grabs metrics via SEMP v1 and provides those as prometheus friendly http endpoints. This allows you to overwrite the parameters, which are in the ini-config file / environment variables: - scrapeURI - username - password - timeout -This provides you a single exporter for all your on prem broker. +This provides you a single exporter for all your OnPrem brokers. Security: Only use this feature with HTTPS. @@ -287,7 +326,7 @@ This is used to automatically build and push the latest image to the Dockerhub r ### Run Docker Image Environment variables are recommended to parameterize the exporter in Docker.
-Put the following parameters, adapted to your situation, into a file on the local host, e.g. env.txt:
+Put the following parameters, adapted to your situation, into a file on the local host, e.g., env.txt:
```bash SOLACE_LISTEN_ADDR=0.0.0.0:9628 SOLACE_SCRAPE_URI=http://your-broker:8080 @@ -320,7 +359,7 @@ port or to run this application within kubernetes/openshift or similar to add an ### How to enable TLS By default, the endpoint configured via `listenAddr=0.0.0.0:9628` is unencrypted and served via HTTP only. -To enable encryption make sure to set `enableTLS=true` or use the environment varibale `export SOLACE_LISTEN_TLS=true` +To enable encryption make sure to set `enableTLS=true` or use the environment variable `export SOLACE_LISTEN_TLS=true` or cli flag `--enable-tls` respectively. TLS encryption requires you to provide two files in PEM (base64) format. You can define the path to those files in @@ -359,7 +398,7 @@ SOLACE_PRIVATE_KEY=/etc/solace/key.pem ## Resources -For more information try these resources: +For more information, try these resources: - The Solace Developer Portal website at: https://solace.dev - Ask the [Solace Community](https://solace.community) diff --git a/exporter/config.struct.go b/exporter/config.struct.go index 5531465..cbff280 100644 --- a/exporter/config.struct.go +++ b/exporter/config.struct.go @@ -23,6 +23,7 @@ type Config struct { ScrapeURI string Username string Password string + DefaultVpn string SslVerify bool useSystemProxy bool Timeout time.Duration @@ -82,6 +83,10 @@ func ParseConfig(configFile string) (map[string][]DataSource, *Config, error) { if err != nil { return nil, nil, err } + conf.DefaultVpn, err = parseConfigString(cfg, "solace", "defaultVpn", "SOLACE_DEFAULT_VPN") + if err != nil { + return nil, nil, err + } conf.Timeout, err = parseConfigDuration(cfg, "solace", "timeout", "SOLACE_TIMEOUT") if err != nil { return nil, nil, err diff --git a/exporter/dataSource.struct.go b/exporter/dataSource.struct.go index 0c6c77e..f17ce5b 100644 --- a/exporter/dataSource.struct.go +++ b/exporter/dataSource.struct.go @@ -1,13 +1,17 @@ package exporter -import "fmt" +import ( + "fmt" + "strings" +) type DataSource struct { - Name string - VpnFilter string - ItemFilter string + Name string + VpnFilter string + ItemFilter string + MetricFilter []string } func (dataSource DataSource) String() string { - return fmt.Sprintf("%s=%s|%s", dataSource.Name, dataSource.VpnFilter, dataSource.ItemFilter) + return fmt.Sprintf("%s=%s|%s|%s", dataSource.Name, dataSource.VpnFilter, dataSource.ItemFilter, strings.Join(dataSource.MetricFilter, ",")) } diff --git a/exporter/exporter.collect.go b/exporter/exporter.collect.go index 1b04309..c02bbc3 100644 --- a/exporter/exporter.collect.go +++ b/exporter/exporter.collect.go @@ -1,9 +1,10 @@ package exporter import ( - "solace_exporter/semp" - + "errors" "github.com/prometheus/client_golang/prometheus" + "solace_exporter/semp" + "strings" ) // Collect fetches the stats from configured Solace location and delivers them @@ -11,6 +12,7 @@ import ( func (e *Exporter) Collect(ch chan<- prometheus.Metric) { var up float64 = 1 var err error = nil + var vpnName = "" for _, dataSource := range *e.dataSource { if up < 1 { @@ -76,6 +78,11 @@ func (e *Exporter) Collect(ch chan<- prometheus.Metric) { up, err = e.semp.GetQueueRatesSemp1(ch, dataSource.VpnFilter, dataSource.ItemFilter) case "QueueStats": up, err = e.semp.GetQueueStatsSemp1(ch, dataSource.VpnFilter, dataSource.ItemFilter) + case "QueueStatsV2": + vpnName, err = e.getVpnName(dataSource.VpnFilter) + if err == nil { + up, err = e.semp.GetQueueStatsSemp2(ch, vpnName, dataSource.ItemFilter, dataSource.MetricFilter) + } case "QueueDetails": up, err = e.semp.GetQueueDetailsSemp1(ch, dataSource.VpnFilter, dataSource.ItemFilter) case "TopicEndpointRates": @@ -88,3 +95,14 @@ func (e *Exporter) Collect(ch chan<- prometheus.Metric) { } ch <- prometheus.MustNewConstMetric(semp.MetricDesc["Global"]["up"], prometheus.GaugeValue, 1, "") } + +func (e *Exporter) getVpnName(vpnFilter string) (vpnName string, err error) { + if vpnFilter == "*" { + if len(strings.TrimSpace(e.config.DefaultVpn)) == 0 { + return "", errors.New("Can't scrape Semp2 As vpnFilter was an * given and the defaultVpn is not set in configuration") + } + return e.config.DefaultVpn, nil + } + + return vpnFilter, nil +} diff --git a/go.sum b/go.sum index 8a8e05e..690e564 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,5 @@ -github.com/alecthomas/kingpin/v2 v2.4.0 h1:f48lwail6p8zpO1bC4TxtqACaGqHYA22qkHjHpqDjYY= -github.com/alecthomas/kingpin/v2 v2.4.0/go.mod h1:0gyi0zQnjuFk8xrkNKamJoyUo382HRL7ATRpFZCw6tE= +github.com/alecthomas/kingpin/v2 v2.3.2 h1:H0aULhgmSzN8xQ3nX1uxtdlTHYoPLu5AhHxWrKI6ocU= +github.com/alecthomas/kingpin/v2 v2.3.2/go.mod h1:0gyi0zQnjuFk8xrkNKamJoyUo382HRL7ATRpFZCw6tE= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20231202071711-9a357b53e9c9 h1:ez/4by2iGztzR4L0zgAOR8lTQK9VlyBVVd7G4omaOQs= @@ -10,6 +10,7 @@ github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/go-kit/kit v0.13.0 h1:OoneCcHKHQ03LfBpoQCUfCluwd2Vt3ohz+kvbJneZAU= github.com/go-kit/kit v0.13.0/go.mod h1:phqEHMMUbyrCFCTgH48JueqrM3md2HcAZ8N3XE4FKDg= github.com/go-kit/log v0.2.1 h1:MRVx0/zhvdseW+Gza6N9rVzU/IVzaeE1SFI4raAhmBU= @@ -19,8 +20,15 @@ github.com/go-logfmt/logfmt v0.6.0/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KE github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 h1:jWpvCLoY8Z/e3VKvlsiIGKtc+UG6U5vzxaoagmhXfyg= github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0/go.mod h1:QUyp042oQthUoa9bqDv0ER0wrtXnBruoNd7aNjkbP+k= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v1.17.0 h1:rl2sfwZMtSthVU752MqfjQozy7blglC+1SOtjMAMh+Q= @@ -49,4 +57,5 @@ gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/semp/getQueueStatsSemp2.go b/semp/getQueueStatsSemp2.go new file mode 100644 index 0000000..e39a7c2 --- /dev/null +++ b/semp/getQueueStatsSemp2.go @@ -0,0 +1,129 @@ +package semp + +import ( + "encoding/json" + "errors" + "github.com/go-kit/kit/log/level" + "github.com/prometheus/client_golang/prometheus" + "strings" +) + +// Get rates for each individual queue of all vpn's +// This can result in heavy system load for lots of queues +func (e *Semp) GetQueueStatsSemp2(ch chan<- prometheus.Metric, vpnName string, itemFilter string, metricFilter []string) (ok float64, err error) { + type Response struct { + Queue []struct { + QueueName string `json:"queueName"` + MsgVpnName string `json:""` + TotalByteSpooled float64 `json:"spooledByteCount"` + TotalMsgSpooled float64 `json:"spooledMsgCount"` + MsgRedelivered float64 `json:"redeliveredMsgCount"` + MsgRetransmit float64 `json:"transportRetransmitMsgCount"` + SpoolUsageExceeded float64 `json:"maxMsgSpoolUsageExceededDiscardedMsgCount"` + MsgSizeExceeded float64 `json:"maxMsgSizeExceededDiscardedMsgCount"` + SpoolShutdownDiscard float64 `json:"disabledDiscardedMsgCount"` + DestinationGroupError float64 `json:"destinationGroupErrorDiscardedMsgCount"` + LowPrioMsgDiscard float64 `json:"lowPriorityMsgCongestionDiscardedMsgCount"` + Deleted float64 `json:"deletedMsgCount"` + TtlDiscarded float64 `json:"maxTtlExpiredDiscardedMsgCount"` + TtlDmq float64 `json:"maxTtlExpiredToDmqMsgCount"` + TtlDmqFailed float64 `json:"maxTtlExpiredToDmqFailedMsgCount"` + MaxRedeliveryDiscarded float64 `json:"maxRedeliveryExceededDiscardedMsgCount"` + MaxRedeliveryDmq float64 `json:"maxRedeliveryExceededToDmqMsgCount"` + MaxRedeliveryDmqFailed float64 `json:"maxRedeliveryExceededToDmqFailedMsgCount"` + TxUnackedMsg float64 `json:"txUnackedMsgCount"` + TransactionNotSupportedDiscardedMsg float64 `json:"xaTransactionNotSupportedDiscardedMsgCount"` + } `json:"data"` + Meta struct { + Count int64 `json:"count"` + ResponseCode int `json:"responseCode"` + Paging struct { + CursorQuery string `json:"cursorQuery"` + NextPageUri string `json:"nextPageUri"` + } `json:",paging"` + Error struct { + Code int `json:"code"` + Description string `json:"description"` + Status string `json:"status"` + } `json:",error"` + } `json:"meta"` + } + + var getParameter = "count=100" + if len(strings.TrimSpace(itemFilter)) > 0 && itemFilter != "*" { + if strings.Contains(itemFilter, "=") { + getParameter += "&where=" + itemFilter + } else { + getParameter += "&where=queueName==" + itemFilter + } + } + + var fieldsToSelect []string + if len(metricFilter) > 0 { + fieldsToSelect, err = getSempV2FieldsToSelect( + metricFilter, + []string{"queueName", "msgVpnName"}, + QueueStatsSempV2, + ) + + if err != nil { + _ = level.Error(e.logger).Log("msg", "Unable to map metric filter", "err", err, "broker", e.brokerURI) + return 0, err + } + getParameter += "&select=" + strings.Join(fieldsToSelect, ",") + } + + var lastQueueName = "" + for nextUrl := e.brokerURI + "/SEMP/v2/monitor/msgVpns/" + vpnName + "/queues?" + getParameter; nextUrl != ""; { + body, err := e.getHTTPbytes(nextUrl, "application/json ") + if err != nil { + _ = level.Error(e.logger).Log("msg", "Can't scrape QueueStatsSemp2", "command", nextUrl, "err", err, "broker", e.brokerURI) + return 0, err + } + + var response Response + err = json.Unmarshal(body, &response) + if err != nil { + _ = level.Error(e.logger).Log("msg", "Can't decode QueueStatsSemp2", "err", err, "broker", e.brokerURI) + return 0, err + } + if response.Meta.ResponseCode != 200 { + _ = level.Error(e.logger).Log("msg", "unexpected result", "command", nextUrl, "remoteError", response.Meta.Error.Description, "broker", e.brokerURI) + return 0, errors.New("unexpected result: see log") + } + + //fmt.Printf("Next request: %v\n", response.Meta.Paging.NextPageUri) + nextUrl = response.Meta.Paging.NextPageUri + for _, queue := range response.Queue { + queueKey := queue.MsgVpnName + "___" + queue.QueueName + if queueKey == lastQueueName { + continue + } + lastQueueName = queueKey + + var values = []SempV2Result{ + {v2Desc: QueueStatsSempV2["total_bytes_spooled"], value: queue.TotalByteSpooled}, + {v2Desc: QueueStatsSempV2["messages_redelivered"], value: queue.MsgRedelivered}, + {v2Desc: QueueStatsSempV2["messages_transport_retransmited"], value: queue.MsgRetransmit}, + {v2Desc: QueueStatsSempV2["spool_usage_exceeded"], value: queue.SpoolUsageExceeded}, + {v2Desc: QueueStatsSempV2["max_message_size_exceeded"], value: queue.MsgSizeExceeded}, + {v2Desc: QueueStatsSempV2["total_deleted_messages"], value: queue.Deleted}, + {v2Desc: QueueStatsSempV2["messages_shutdown_discarded"], value: queue.SpoolShutdownDiscard}, + {v2Desc: QueueStatsSempV2["messages_ttl_discarded"], value: queue.TtlDiscarded}, + {v2Desc: QueueStatsSempV2["messages_ttl_dmq"], value: queue.TtlDmq}, + {v2Desc: QueueStatsSempV2["messages_ttl_dmq_failed"], value: queue.TtlDmqFailed}, + {v2Desc: QueueStatsSempV2["messages_max_redelivered_discarded"], value: queue.MaxRedeliveryDiscarded}, + {v2Desc: QueueStatsSempV2["messages_max_redelivered_dmq"], value: queue.MaxRedeliveryDmq}, + {v2Desc: QueueStatsSempV2["messages_max_redelivered_dmq_failed"], value: queue.MaxRedeliveryDmqFailed}, + } + + for _, v := range values { + if v.v2Desc.isSelected(fieldsToSelect) { + ch <- prometheus.MustNewConstMetric(v.v2Desc.NewPrometheusDesc(), prometheus.GaugeValue, v.value, queue.MsgVpnName, queue.QueueName) + } + } + } + } + + return 1, nil +} diff --git a/semp/helper.go b/semp/helper.go new file mode 100644 index 0000000..c60e252 --- /dev/null +++ b/semp/helper.go @@ -0,0 +1,45 @@ +package semp + +import ( + "fmt" + "strings" +) + +func mapItems(items []string, translateMap map[string]string) ([]string, error) { + validRawItems := make(map[string]bool, len(translateMap)) + translated := make([]string, 0, len(items)) + validItems := make([]string, 0, len(translateMap)*2) + + for key, validRawItem := range translateMap { + validRawItems[validRawItem] = true + + validItems = append(validItems, key) + validItems = append(validItems, validRawItem) + } + + for _, item := range items { + if translatedItem, ok := translateMap[item]; ok { + translated = append(translated, translatedItem) + } else if _, ok := validRawItems[item]; ok { + translated = append(translated, item) + } else { + return nil, fmt.Errorf( + "Item \"%s\" is not valid. Pleaee choose from: %s", + item, + strings.Join(validItems, ","), + ) + } + } + + return translated, nil +} + +func sliceContains(slice []string, lookUp string) bool { + for _, selectedField := range slice { + if selectedField == lookUp { + return true + } + } + + return false +} diff --git a/semp/postHttp.go b/semp/http.go similarity index 53% rename from semp/postHttp.go rename to semp/http.go index 599a24a..5e9ea1e 100644 --- a/semp/postHttp.go +++ b/semp/http.go @@ -27,3 +27,29 @@ func (s *Semp) postHTTP(uri string, _ string, body string) (io.ReadCloser, error } return resp.Body, nil } + +func (s *Semp) getHTTPbytes(uri string, _ string) ([]byte, error) { + req, err := http.NewRequest("GET", uri, nil) + if err != nil { + return nil, err + } + + s.httpRequestVisitor(req) + + resp, err := s.httpClient.Do(req) + if err != nil { + return nil, err + } + + if !(resp.StatusCode >= 200 && resp.StatusCode < 500) { + _ = resp.Body.Close() + return nil, fmt.Errorf("HTTP status %d (%s)", resp.StatusCode, http.StatusText(resp.StatusCode)) + } + + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + + return body, nil +} diff --git a/semp/metricDesc.go b/semp/metricDesc.go index b7812c1..fc295e1 100644 --- a/semp/metricDesc.go +++ b/semp/metricDesc.go @@ -29,6 +29,25 @@ var ( type Metrics map[string]*prometheus.Desc +var QueueStatsSempV2 = SempV2Descs{ + "total_bytes_spooled": NewSempV2Desc(namespace+"_"+"queue_byte_spooled", "spooledByteCount", "Queue spool total of all spooled messages in bytes.", variableLabelsVpnQueue), + "total_messages_spooled": NewSempV2Desc(namespace+"_"+"queue_msg_spooled", "spooledMsgCount", "Queue spool total of all spooled messages.", variableLabelsVpnQueue), + "messages_redelivered": NewSempV2Desc(namespace+"_"+"queue_msg_redelivered", "redeliveredMsgCount", "Queue total msg redeliveries.", variableLabelsVpnQueue), + "messages_transport_retransmited": NewSempV2Desc(namespace+"_"+"queue_msg_retransmited", "transportRetransmitMsgCount", "Queue total msg retransmitted on transport.", variableLabelsVpnQueue), + "spool_usage_exceeded": NewSempV2Desc(namespace+"_"+"queue_msg_spool_usage_exceeded", "maxMsgSpoolUsageExceededDiscardedMsgCount", "Queue total number of messages exceeded the spool usage.", variableLabelsVpnQueue), + "max_message_size_exceeded": NewSempV2Desc(namespace+"_"+"queue_msg_max_msg_size_exceeded", "maxMsgSizeExceededDiscardedMsgCount", "Queue total number of messages exceeded the max message size.", variableLabelsVpnQueue), + "total_deleted_messages": NewSempV2Desc(namespace+"_"+"queue_msg_total_deleted", "deletedMsgCount", "Queue total number that was deleted.", variableLabelsVpnQueue), + "messages_shutdown_discarded": NewSempV2Desc(namespace+"_"+"queue_msg_shutdown_discarded", "disabledDiscardedMsgCount", "Queue total number of messages discarded due to spool shutdown.", variableLabelsVpnQueue), + "messages_ttl_discarded": NewSempV2Desc(namespace+"_"+"queue_msg_ttl_discarded", "maxTtlExpiredDiscardedMsgCount", "Queue total number of messages discarded due to ttl expiry.", variableLabelsVpnQueue), + "messages_ttl_dmq": NewSempV2Desc(namespace+"_"+"queue_msg_ttl_dmq", "maxTtlExpiredToDmqMsgCount", "Queue total number of messages delivered to dmq due to ttl expiry.", variableLabelsVpnQueue), + "messages_ttl_dmq_failed": NewSempV2Desc(namespace+"_"+"queue_msg_ttl_dmq_failed", "maxTtlExpiredToDmqFailedMsgCount", "Queue total number of messages that failed delivery to dmq due to ttl expiry.", variableLabelsVpnQueue), + "messages_max_redelivered_discarded": NewSempV2Desc(namespace+"_"+"queue_msg_max_redelivered_discarded", "maxRedeliveryExceededDiscardedMsgCount", "Queue total number of messages discarded due to exceeded max redelivery.", variableLabelsVpnQueue), + "messages_max_redelivered_dmq": NewSempV2Desc(namespace+"_"+"queue_msg_max_redelivered_dmq", "maxRedeliveryExceededToDmqMsgCount", "Queue total number of messages delivered to dmq due to exceeded max redelivery.", variableLabelsVpnQueue), + "messages_max_redelivered_dmq_failed": NewSempV2Desc(namespace+"_"+"queue_msg_max_redelivered_dmq_failed", "maxRedeliveryExceededToDmqFailedMsgCount", "Queue total number of messages failed delivery to dmq due to exceeded max redelivery.", variableLabelsVpnQueue), +} + +var QueueStats = toMetrics(QueueStatsSempV2) + var MetricDesc = map[string]Metrics{ "Global": { "up": prometheus.NewDesc(namespace+"_up", "Was the last scrape of Solace broker successful.", variableLabelsUp, nil), @@ -301,22 +320,8 @@ var MetricDesc = map[string]Metrics{ "queue_spool_usage_msgs": prometheus.NewDesc(namespace+"_"+"queue_spool_usage_msgs", "Queue spooled number of messages.", variableLabelsVpnQueue, nil), "queue_binds": prometheus.NewDesc(namespace+"_"+"queue_binds", "Number of clients bound to queue.", variableLabelsVpnQueue, nil), }, - "QueueStats": { - "total_bytes_spooled": prometheus.NewDesc(namespace+"_"+"queue_byte_spooled", "Queue spool total of all spooled messages in bytes.", variableLabelsVpnQueue, nil), - "total_messages_spooled": prometheus.NewDesc(namespace+"_"+"queue_msg_spooled", "Queue spool total of all spooled messages.", variableLabelsVpnQueue, nil), - "messages_redelivered": prometheus.NewDesc(namespace+"_"+"queue_msg_redelivered", "Queue total msg redeliveries.", variableLabelsVpnQueue, nil), - "messages_transport_retransmited": prometheus.NewDesc(namespace+"_"+"queue_msg_retransmited", "Queue total msg retransmitted on transport.", variableLabelsVpnQueue, nil), - "spool_usage_exceeded": prometheus.NewDesc(namespace+"_"+"queue_msg_spool_usage_exceeded", "Queue total number of messages exceeded the spool usage.", variableLabelsVpnQueue, nil), - "max_message_size_exceeded": prometheus.NewDesc(namespace+"_"+"queue_msg_max_msg_size_exceeded", "Queue total number of messages exceeded the max message size.", variableLabelsVpnQueue, nil), - "total_deleted_messages": prometheus.NewDesc(namespace+"_"+"queue_msg_total_deleted", "Queue total number that was deleted.", variableLabelsVpnQueue, nil), - "messages_shutdown_discarded": prometheus.NewDesc(namespace+"_"+"queue_msg_shutdown_discarded", "Queue total number of messages discarded due to spool shutdown.", variableLabelsVpnQueue, nil), - "messages_ttl_discarded": prometheus.NewDesc(namespace+"_"+"queue_msg_ttl_discarded", "Queue total number of messages discarded due to ttl expiry.", variableLabelsVpnQueue, nil), - "messages_ttl_dmq": prometheus.NewDesc(namespace+"_"+"queue_msg_ttl_dmq", "Queue total number of messages delivered to dmq due to ttl expiry.", variableLabelsVpnQueue, nil), - "messages_ttl_dmq_failed": prometheus.NewDesc(namespace+"_"+"queue_msg_ttl_dmq_failed", "Queue total number of messages that failed delivery to dmq due to ttl expiry.", variableLabelsVpnQueue, nil), - "messages_max_redelivered_discarded": prometheus.NewDesc(namespace+"_"+"queue_msg_max_redelivered_discarded", "Queue total number of messages discarded due to exceeded max redelivery.", variableLabelsVpnQueue, nil), - "messages_max_redelivered_dmq": prometheus.NewDesc(namespace+"_"+"queue_msg_max_redelivered_dmq", "Queue total number of messages delivered to dmq due to exceeded max redelivery.", variableLabelsVpnQueue, nil), - "messages_max_redelivered_dmq_failed": prometheus.NewDesc(namespace+"_"+"queue_msg_max_redelivered_dmq_failed", "Queue total number of messages failed delivery to dmq due to exceeded max redelivery.", variableLabelsVpnQueue, nil), - }, + "QueueStats": QueueStats, + "QueueStatsV2": QueueStats, "TopicEndpointRates": { "rx_msg_rate": prometheus.NewDesc(namespace+"_"+"topic_endpoint_rx_msg_rate", "Rate of received messages.", variableLabelsVpnTopicEndpoint, nil), "tx_msg_rate": prometheus.NewDesc(namespace+"_"+"topic_endpoint_tx_msg_rate", "Rate of transmitted messages.", variableLabelsVpnTopicEndpoint, nil), @@ -373,3 +378,13 @@ var MetricDesc = map[string]Metrics{ "connection_timed_retransmit": prometheus.NewDesc(namespace+"_"+"connection_timed_retransmit", "The number of TCP segments re-transmitted due to timeout awaiting an ACK. See RFC 793 for further details.", variableLabelsVpnClient, nil), }, } + +func toMetrics(v2Desc SempV2Descs) Metrics { + metrics := make(map[string]*prometheus.Desc, len(v2Desc)) + + for key, v2Description := range v2Desc { + metrics[key] = v2Description.NewPrometheusDesc() + } + + return metrics +} diff --git a/semp/sempv2.desc.struct.go b/semp/sempv2.desc.struct.go new file mode 100644 index 0000000..56314d5 --- /dev/null +++ b/semp/sempv2.desc.struct.go @@ -0,0 +1,65 @@ +package semp + +import ( + "github.com/prometheus/client_golang/prometheus" +) + +type SempV2Desc struct { + fqName string + sempV2field string + help string + variableLabels []string + constLabels prometheus.Labels +} + +func NewSempV2Desc(fqName string, sempV2field string, help string, variableLabels []string) *SempV2Desc { + return &SempV2Desc{ + fqName: fqName, + sempV2field: sempV2field, + help: help, + variableLabels: variableLabels, + constLabels: nil, + } +} + +func (v2Desc *SempV2Desc) NewPrometheusDesc() *prometheus.Desc { + return prometheus.NewDesc(v2Desc.fqName, v2Desc.help, v2Desc.variableLabels, v2Desc.constLabels) +} +func (v2Desc *SempV2Desc) isSelected(selectedFields []string) bool { + if len(selectedFields) < 1 { + return true + } + + return sliceContains(selectedFields, v2Desc.sempV2field) +} + +func getSempV2FieldMapList(descs SempV2Descs) map[string]string { + mapList := make(map[string]string, len(descs)) + + for _, desc := range descs { + mapList[desc.fqName] = desc.sempV2field + } + return mapList +} + +func getSempV2FieldsToSelect(metricFilter []string, mandatoryFields []string, descs SempV2Descs) ([]string, error) { + var fields, err = mapItems(metricFilter, getSempV2FieldMapList(descs)) + if err != nil { + return []string{}, err + } + + for _, mandatoryField := range mandatoryFields { + if !sliceContains(fields, mandatoryField) { + fields = append(fields, mandatoryField) + } + } + + return fields, nil +} + +type SempV2Descs map[string]*SempV2Desc + +type SempV2Result struct { + v2Desc *SempV2Desc + value float64 +} diff --git a/solace_prometheus_exporter.go b/solace_prometheus_exporter.go index 223bc81..2939dd2 100644 --- a/solace_prometheus_exporter.go +++ b/solace_prometheus_exporter.go @@ -129,13 +129,19 @@ func main() { if strings.HasPrefix(key, "m.") { for _, value := range values { parts := strings.Split(value, "|") - if len(parts) != 2 { - level.Error(logger).Log("msg", "Exactly one | expected. Use VPN wildcard. |. Item wildcard.", "key", key, "value", value) + if len(parts) < 2 { + level.Error(logger).Log("msg", "One or two | expected. Use VPN wildcard. | Item wildcard. | Optional metric filter for v2 apis", "key", key, "value", value) } else { + var metricFilter []string + if len(parts) == 3 && len(strings.TrimSpace(parts[2])) > 0 { + metricFilter = strings.Split(parts[2], ",") + } + dataSource = append(dataSource, exporter.DataSource{ - Name: strings.TrimPrefix(key, "m."), - VpnFilter: parts[0], - ItemFilter: parts[1], + Name: strings.TrimPrefix(key, "m."), + VpnFilter: parts[0], + ItemFilter: parts[1], + MetricFilter: metricFilter, }) } } @@ -194,6 +200,7 @@ func main() { BridgeStatsyesyeshas a very small performance down site QueueRatesyesyesDEPRECATED: may harm broker if many queues QueueStatsyesyesmay harm broker if many queues + QueueStatsV2yesyesmay harm broker if many queues QueueDetailsyesyesmay harm broker if many queues TopicEndpointRatesyesyesDEPRECATED: may harm broker if many topic-endpoints TopicEndpointStatsyesyesmay harm broker if many topic-endpoints