diff --git a/helm/examples/metrics/README.md b/helm/examples/metrics/README.md index 5c2d42001f5..e0833f192c5 100644 --- a/helm/examples/metrics/README.md +++ b/helm/examples/metrics/README.md @@ -177,3 +177,108 @@ HPCC metrics can be easily routed to Azure Insights for application level monito 11/5/2021, 9:02:00.000 PM prometheus esp_requests_active 0 {"app":"eclservices","namespace":"default","pod_name":"eclservices-778477d679-vgpj2"} 11/5/2021, 9:02:00.000 PM prometheus esp_requests_active 3 {"app":"eclservices","namespace":"default","pod_name":"eclservices-778477d679-vgpj2"} ``` +### ElasticSearch Support +HPCC metrics can be routed to ElasticSearch for advanced metrics processing and +alerting. This process involves two requirements: enabling the ElasticSearch metric +sink, and configuring the index in ElasticSearch to receive the metrics. + +Since the metrics configuration is common across all HPCC components, the ElasticSearch +sink will report metrics from all components to the same index. Therefore, the +index must be configured to receive metrics from all components. + +#### Index Configuration +The index must be created in ElasticSearch before metrics can be reported to it. The name is passed +to the sink as a configuration setting. The index must be created with the following settings: + +##### Dynamic Mapping +The index must be created with dynamic mapping enabled. Dynamic mapping allows framework metric +data types to be stored in the index in their native types. Without dynamic mapping, the ElasticSearch +default mapping does not properly map value to unsigned 64-bit integers. + +To create an index with dynamic mapping, use the following object when creating the index: +```code json +{ + "mappings": { + "dynamic_templates": [ + { + "hpcc_metric_count_to_unsigned_long": { + "match": "*_count", + "mapping": { + "type": "unsigned_long" + } + } + }, + { + "hpcc_metric_gauge_to_unsigned_long": { + "match": "*_gauge", + "mapping": { + "type": "unsigned_long" + } + } + }, + { + "hpcc_metric_histogram_to_histogram": { + "match": "*_histogram", + "mapping": { + "type": "histogram" + } + } + } + ] + } +} +``` + +Note that there may be other means for adding the required dynamic mapping to the index. + +#### Enabling ElasticSearch Metrics Sink for Kubernetes +To enable reporting of metrics to ElasticSearch, add the metric configuration settings to +the helm chart. + +The provided HPCC helm chart provides all global settings from its values.yml file to all components. +To enable metrics reporting, either include the metrics configuration as a permanent part of +your HPCC helm chart values.yml file, or add it as command line settings at chart installation. +To enable the ElasticSearch sink on the command line, use the following to add the ElasticSearch +settings: + +```code +helm install mycluster ./hpcc -f /elasticsearch_metrics.yml +``` +An example _yml_ file can be found in the repository at helm/examples/metrics/elasticsearch_metrics.yml. +Make a copy and modify as needed for your installation. + +##### Configuration Settings +The ElasticSearch sink defines the following settings: + +* elasticHost - The host name or IP address of the ElasticSearch server. (required) +* indexName - The name of the index to which metrics are reported. (required) +* countMetricSuffix - The suffix used to identify count metrics. (default: count) +* gaugeMetricSuffix - The suffix used to identify gauge metrics. (default: gauge) +* histogramMetricSuffix - The suffix used to identify histogram metrics. (default: histogram) + +Standard periodic metric sink settings are also available. + +#### Enabling ElasticSearch Metrics Sink for Bare Metal +To enable reporting of metrics to ElasticSearch, add the metric configuration settings to +the environment configuration file (enviroment.xml). These settings must be added manually +since there is no support in the config manager. + +Add the following to the environment configuration file (note only the required +settings are shown): + +```code xml + + + + + + + + + + +``` +See section above for additional settings that can be added to the ElasticSearch sink. \ No newline at end of file diff --git a/helm/examples/metrics/elasticsearch_metrics.yaml b/helm/examples/metrics/elasticsearch_metrics.yaml new file mode 100644 index 00000000000..8a7bd6af19b --- /dev/null +++ b/helm/examples/metrics/elasticsearch_metrics.yaml @@ -0,0 +1,27 @@ +# +# Defines a elastic sink for reporting metrics to an ElasticSearch instance +# Settings: +# type - sink type (must be elastic for ElasticSearch support) +# name - name for the sink instance +# settings.elasticHost - url for the ElasticSearch instance +# settings.indexName - ElasticSearch index name where metrics are indexed +# settings.countMetricSuffix - suffix for count metrics +# settings.gaugeMetricSuffix - suffix for gauge metrics +# settings.histogramMetricSuffix - suffix for histogram metrics +# +# If not overridden, the following suffixes are used by default: +# countMetricSuffix: count +# gaugeMetricSuffix: gauge +# histogramMetricSuffix: histogram + +global: + metrics: + sinks: + - type: elastic + name: elasticsink + settings: + elasticHost: http://localhost:9200, + indexName: hpccmetrics, + countMetricSuffix: count, + gaugeMetricSuffix: gauge, + histogramMetricSuffix: histogram diff --git a/system/metrics/sinks/elastic/elasticSink.cpp b/system/metrics/sinks/elastic/elasticSink.cpp index cfe03b35ca3..d3978907b09 100644 --- a/system/metrics/sinks/elastic/elasticSink.cpp +++ b/system/metrics/sinks/elastic/elasticSink.cpp @@ -45,11 +45,26 @@ ElasticMetricSink::ElasticMetricSink(const char *name, const IPropertyTree *pSet ignoreZeroMetrics = pSettingsTree->getPropBool("@ignoreZeroMetrics", true); pSettingsTree->getProp("@elasticHost", elasticHost); pSettingsTree->getProp("@indexName", indexName); + + // Initialize standard suffixes + countMetricSuffix.append("_count"); + gaugeMetricSuffix.append("_gauge"); + histogramMetricSuffix.append("_histogram"); + + // See if any are overridden + pSettingsTree->getProp("@countMetricSuffix", countMetricSuffix); + pSettingsTree->getProp("@gaugeMetricSuffix", gaugeMetricSuffix); + pSettingsTree->getProp("@histogramMetricSuffix", histogramMetricSuffix); } void ElasticMetricSink::prepareToStartCollecting() { + if (elasticHost.isEmpty()) + throw MakeStringException(-1, "ElasticMetricSink: ElasticSearch host missing"); + + if (indexName.isEmpty()) + throw MakeStringException(-1, "ElasticMetricSink: Index name missing"); } diff --git a/system/metrics/sinks/elastic/elasticSink.hpp b/system/metrics/sinks/elastic/elasticSink.hpp index 337f446d9b8..c1120d8228f 100644 --- a/system/metrics/sinks/elastic/elasticSink.hpp +++ b/system/metrics/sinks/elastic/elasticSink.hpp @@ -39,4 +39,7 @@ class ELASTICSINK_API ElasticMetricSink : public hpccMetrics::PeriodicMetricSink StringBuffer indexName; bool ignoreZeroMetrics = false; StringBuffer elasticHost; + StringBuffer countMetricSuffix; + StringBuffer gaugeMetricSuffix; + StringBuffer histogramMetricSuffix; };