From 2de4b397b69ce007043e4161cd3153dd260c76e9 Mon Sep 17 00:00:00 2001 From: Ken Rowland Date: Tue, 29 Oct 2024 14:04:12 -0400 Subject: [PATCH] HPCC-32734 Add ElasticSearch metric sink configuration Added configuration options for the ElasticSearch sink Updated metrics readme with information on adding the sink to the cluster Added example ElasticSearch configuraion yaml file Signed-Off-By: Kenneth Rowland kenneth.rowland@lexisnexisrisk.com --- helm/examples/metrics/README.md | 111 ++++++++++++++++++ .../metrics/elasticsearch_metrics.yaml | 36 ++++++ system/metrics/sinks/elastic/elasticSink.cpp | 65 +++++++++- system/metrics/sinks/elastic/elasticSink.hpp | 6 +- 4 files changed, 213 insertions(+), 5 deletions(-) create mode 100644 helm/examples/metrics/elasticsearch_metrics.yaml diff --git a/helm/examples/metrics/README.md b/helm/examples/metrics/README.md index 5c2d42001f5..e29fd25cc1c 100644 --- a/helm/examples/metrics/README.md +++ b/helm/examples/metrics/README.md @@ -177,3 +177,114 @@ HPCC metrics can be easily routed to Azure Insights for application level monito 11/5/2021, 9:02:00.000 PM prometheus esp_requests_active 0 {"app":"eclservices","namespace":"default","pod_name":"eclservices-778477d679-vgpj2"} 11/5/2021, 9:02:00.000 PM prometheus esp_requests_active 3 {"app":"eclservices","namespace":"default","pod_name":"eclservices-778477d679-vgpj2"} ``` +### ElasticSearch Support +HPCC metrics can be routed to ElasticSearch for advanced metrics processing and +alerting. This process involves two requirements: enabling the ElasticSearch metric +sink, and configuring the index in ElasticSearch to receive the metrics. + +Since the metrics configuration is common across all HPCC components, the ElasticSearch +sink will report metrics from all components to the same index. Therefore, the +index must be configured to receive metrics from all components. + +#### Index Configuration +The index must be created in ElasticSearch before metrics can be reported to it. The name is passed +to the sink as a configuration setting. The index must be created with the following settings: + +##### Dynamic Mapping +The index must be created with dynamic mapping enabled. Dynamic mapping allows framework metric +data types to be stored in the index in their native types. Without dynamic mapping, the ElasticSearch +default mapping does not properly map value to unsigned 64-bit integers. + +To create an index with dynamic mapping, use the following object when creating the index: +```code json +{ + "mappings": { + "dynamic_templates": [ + { + "hpcc_metric_count_to_unsigned_long": { + "match": "*_count", + "mapping": { + "type": "unsigned_long" + } + } + }, + { + "hpcc_metric_gauge_to_unsigned_long": { + "match": "*_gauge", + "mapping": { + "type": "unsigned_long" + } + } + }, + { + "hpcc_metric_histogram_to_histogram": { + "match": "*_histogram", + "mapping": { + "type": "histogram" + } + } + } + ] + } +} +``` + +Note that there may be other means for adding the required dynamic mapping to the index. + +The _match_ values above are representative of typical values. The actual values may +vary depending on the cluster configuration and shared use of the index across multiple clusters. +In all cases, the configuration of the sink must match that of the dynamic mappings in the index. + +#### Enabling ElasticSearch Metrics Sink for Kubernetes +To enable reporting of metrics to ElasticSearch, add the metric configuration settings to +the helm chart. + +The provided HPCC helm chart provides all global settings from its values.yml file to all components. +To enable metrics reporting, either include the metrics configuration as a permanent part of +your HPCC helm chart values.yml file, or add it as command line settings at chart installation. +To enable the ElasticSearch sink on the command line, use the following to add the ElasticSearch +settings: + +```code +helm install mycluster ./hpcc -f /elasticsearch_metrics.yml +``` +An example _yml_ file can be found in the repository at helm/examples/metrics/elasticsearch_metrics.yml. +Make a copy and modify as needed for your installation. + +##### Configuration Settings +The ElasticSearch sink defines the following settings: + +* hostProtocol - The protocol used to connect to the ElasticSearch server. (default: https) +* hostName - The host name or IP address of the ElasticSearch server. (required) +* hostPort - The port number of the ElasticSearch server. (default: 9200) +* indexName - The name of the index to which metrics are reported. (required) +* countMetricSuffix - The suffix used to identify count metrics. (default: count) +* gaugeMetricSuffix - The suffix used to identify gauge metrics. (default: gauge) +* histogramMetricSuffix - The suffix used to identify histogram metrics. (default: histogram) + +Standard periodic metric sink settings are also available. + +#### Enabling ElasticSearch Metrics Sink for Bare Metal +To enable reporting of metrics to ElasticSearch, add the metric configuration settings to +the environment configuration file (enviroment.xml). These settings must be added manually +since there is no support in the config manager. + +Add the following to the environment configuration file (note only the required +settings are shown): + +```code xml + + + + + + + + + + + + + +``` +See section above for additional settings that can be added to the ElasticSearch sink. \ No newline at end of file diff --git a/helm/examples/metrics/elasticsearch_metrics.yaml b/helm/examples/metrics/elasticsearch_metrics.yaml new file mode 100644 index 00000000000..0b44d232b59 --- /dev/null +++ b/helm/examples/metrics/elasticsearch_metrics.yaml @@ -0,0 +1,36 @@ +# +# Defines an elastic sink for reporting metrics to an ElasticSearch instance +# Settings: +# type - sink type (must be elastic for ElasticSearch support) +# name - name for the sink instance +# settings.countMetricSuffix - suffix for count metrics (default: count) +# settings.gaugeMetricSuffix - suffix for gauge metrics (default: gauge) +# settings.histogramMetricSuffix - suffix for histogram metrics (default: histogram) +# settings.host - ElasticSearch host settings +# settings.host.protocol - protocol to use, http or https (default) +# settings.host.name - host name +# settings.host.port - port number (default 9200) +# settings.index - ElasticSearch index settings +# settings.index.name - index name +# +# If not overridden, the following suffixes are used by default: +# countMetricSuffix: count +# gaugeMetricSuffix: gauge +# histogramMetricSuffix: histogram + +global: + metrics: + sinks: + - type: elastic + name: myelasticsink + settings: + countMetricSuffix: count + gaugeMetricSuffix: gauge + histogramMetricSuffix: histogram + host: + protocol: https + name: hostname + port: 9200 + index: + name: hpccmetrics + diff --git a/system/metrics/sinks/elastic/elasticSink.cpp b/system/metrics/sinks/elastic/elasticSink.cpp index cfe03b35ca3..ceb0ff9fb87 100644 --- a/system/metrics/sinks/elastic/elasticSink.cpp +++ b/system/metrics/sinks/elastic/elasticSink.cpp @@ -12,7 +12,6 @@ ############################################################################## */ #include "elasticSink.hpp" - #include "nlohmann/json.hpp" //including cpp-httplib single header file REST client @@ -43,8 +42,66 @@ ElasticMetricSink::ElasticMetricSink(const char *name, const IPropertyTree *pSet PeriodicMetricSink(name, "elastic", pSettingsTree) { ignoreZeroMetrics = pSettingsTree->getPropBool("@ignoreZeroMetrics", true); - pSettingsTree->getProp("@elasticHost", elasticHost); - pSettingsTree->getProp("@indexName", indexName); + + StringBuffer hostName; + StringBuffer hostProtocol; + StringBuffer hostPort; + + Owned pHostConfigTree = pSettingsTree->getPropTree("host"); + if (pHostConfigTree) + { + pHostConfigTree->getProp("@name", hostName); + + if (!pHostConfigTree->getProp("@protocol", hostProtocol)) + { + hostProtocol.append("https"); + } + + if (!pHostConfigTree->getProp("@port", hostPort)) + { + hostPort.append("9200"); + } + } + + if (!hostName.isEmpty() && !hostPort.isEmpty() && !hostProtocol.isEmpty()) + { + elasticHostUrl.append(hostProtocol).append("://").append(hostName).append(":").append(hostPort); + } + else + { + WARNLOG("ElasticMetricSink: Host configuration missing or invalid"); + } + + Owned pIndexConfigTree = pSettingsTree->getPropTree("index"); + if (pIndexConfigTree) + { + pSettingsTree->getProp("@name", indexName); + } + + if (indexName.isEmpty()) + { + WARNLOG("ElasticMetricSink: Index configuration missing or invalid"); + } + + + // Both a host url and an index name are required + configurationValid = !elasticHostUrl.isEmpty() && !indexName.isEmpty(); + + // Initialize standard suffixes + if (!pSettingsTree->getProp("@countMetricSuffix", countMetricSuffix)) + { + countMetricSuffix.append("count"); + } + + if (!pSettingsTree->getProp("@gaugeMetricSuffix", gaugeMetricSuffix)) + { + gaugeMetricSuffix.append("gauge"); + } + + if (!pSettingsTree->getProp("@histogramMetricSuffix", histogramMetricSuffix)) + { + histogramMetricSuffix.append("histogram"); + } } @@ -62,6 +119,6 @@ void ElasticMetricSink::doCollection() void ElasticMetricSink::collectingHasStopped() { - ; + } diff --git a/system/metrics/sinks/elastic/elasticSink.hpp b/system/metrics/sinks/elastic/elasticSink.hpp index 337f446d9b8..c435ed04abb 100644 --- a/system/metrics/sinks/elastic/elasticSink.hpp +++ b/system/metrics/sinks/elastic/elasticSink.hpp @@ -38,5 +38,9 @@ class ELASTICSINK_API ElasticMetricSink : public hpccMetrics::PeriodicMetricSink protected: StringBuffer indexName; bool ignoreZeroMetrics = false; - StringBuffer elasticHost; + StringBuffer elasticHostUrl; + StringBuffer countMetricSuffix; + StringBuffer gaugeMetricSuffix; + StringBuffer histogramMetricSuffix; + bool configurationValid = false; };