Merge pull request #19310 from kenrowland/HPCC-32968

HPCC-32968 Add ElasticSearch server security configuration values Reviewed-By: Rodrigo Pastrana <[email protected]> Reviewed-by: Gavin Halliday <[email protected]> Merged-by: Gavin Halliday <[email protected]>
hpcc-systems · Dec 18, 2024 · aeeb2f8 · aeeb2f8
2 parents 61189d3 + 400d590
commit aeeb2f8
Show file tree

Hide file tree

Showing 4 changed files with 165 additions and 46 deletions.
diff --git a/helm/examples/metrics/README.md b/helm/examples/metrics/README.md
@@ -252,12 +252,41 @@ An example _yml_ file can be found in the repository at helm/examples/metrics/el
 Make a copy and modify as needed for your installation.
 
 ##### Configuration Settings
-The ElasticSearch sink defines the following settings:
+The ElasticSearch sink defines the following settings
 
-* hostProtocol - The protocol used to connect to the ElasticSearch server. (default: https)
-* hostName - The host name or IP address of the ElasticSearch server. (required)
-* hostPort - The port number of the ElasticSearch server. (default: 9200)
-* indexName - The name of the index to which metrics are reported. (required)
+**Host**
+The host settings define the ElasticSearch server to which metrics are reported. The settings are:
+
+* domain - The domain or IP address of the ElasticSearch server. (required)
+* protocol - The protocol used to connect to the ElasticSearch server. (default: https)
+* port - The port number of the ElasticSearch server. (default: 9200)
+* certificateFilePath - Path to the file containing the certificate used to connect to the ElasticSearch server. (optional)
+
+
+**Authentication**
+
+Optional child of the host configuration where authentication settings are defined. If missing, 
+no authentication is used. If defined, the settings are:
+
+* type - Required Authentication type used to connect to the ElasticSearch server. Value defines the 
+remaining settings. The allowed values are:
+  * basic - Basic authentication is used.
+* credentialsSecret - The name of the secret containing the credentials used to authenticate to 
+the ElasticSearch server. (optional, valid for Kubernetes only)
+* credentialsVaultId - The vault ID containing the credentials used to authenticate to the 
+ElasticSearch server. (optional, valid for Vault only)
+
+For **basic** authentication, the following settings are required, regardless if the credentials are stored 
+as a secret or in the environment.xml file.
+* username - The username used to authenticate to the ElasticSearch server. 
+* password - The password used to authenticate to the ElasticSearch server. When stored in the 
+environment.xml file, it shall be encrypted using standard environment.xml encryption.
+
+**Index**
+
+The index settings define the index where metrics are indexed. The settings are:
+
+* name - The name of the index to which metrics are reported. (required)
 * countMetricSuffix - The suffix used to identify count metrics. (default: count)
 * gaugeMetricSuffix - The suffix used to identify gauge metrics. (default: gauge)
 * histogramMetricSuffix - The suffix used to identify histogram metrics. (default: histogram)
@@ -269,17 +298,17 @@ To enable reporting of metrics to ElasticSearch, add the metric configuration se
 the environment configuration file (enviroment.xml). These settings must be added manually
 since there is no support in the config manager.
 
-Add the following to the environment configuration file (note only the required
-settings are shown):
+Add the following to the environment.xml configuration file (note that some values may not be required): 
 
 ```code xml
-
 <Environment>
     <Software>
         <metrics name="mymetricsconfig">
             <sinks name="myelasticsink" type="elastic">
                 <settings period="30" ignoreZeroMetrics="1">
-                    <host name="<hostname>" port="<port>" protocol="http|htps"/>
+                    <host domain="<domainname>" port="<port>" protocol="http|https">
+                        <authentication type="basic" username="<username>" password="<password>"/>
+                    </host>
                     <index name="<index>"/> 
                 <settings/>
             </sinks>

diff --git a/helm/examples/metrics/elasticsearch_metrics.yaml b/helm/examples/metrics/elasticsearch_metrics.yaml
@@ -1,17 +1,24 @@
 #
 # Defines an elastic sink for reporting metrics to an ElasticSearch instance
 # Settings:
-#   type                                - sink type (must be elastic for ElasticSearch support)
-#   name                                - name for the sink instance
-#   settings.countMetricSuffix          - suffix for count metrics (default: count)
-#   settings.gaugeMetricSuffix          - suffix for gauge metrics (default: gauge)
-#   settings.histogramMetricSuffix      - suffix for histogram metrics (default: histogram)
-#   settings.host                       - ElasticSearch host settings
-#   settings.host.protocol               - protocol to use, http or https (default)
-#   settings.host.name                   - host name
-#   settings.host.port                   - port number (default 9200)
-#   settings.index                       - ElasticSearch index settings
-#   settings.index.name                  - index name
+#   type                                  - sink type (must be elastic for ElasticSearch support)
+#   name                                  - name for the sink instance
+#   settings.countMetricSuffix            - suffix for count metrics (default: count)
+#   settings.gaugeMetricSuffix            - suffix for gauge metrics (default: gauge)
+#   settings.histogramMetricSuffix        - suffix for histogram metrics (default: histogram)
+#   settings.host                         - ElasticSearch host settings
+#   settings.host.protocol                - protocol to use, http or https (default)
+#   settings.host.domain                  - host domain
+#   settings.host.certificateFilePath     - path to certificate file (optional)
+#   settings.host.port                    - port number (default 9200)
+#   settings.host.authentication          - authentication settings if authentication is enabled (optional)
+#   settings.host.authentication.type     - authentication type (determines remaining settings) (only 'basic' is supported)
+#   settings.host.authentication.username - username for basic authentication (if not stored in a secret)
+#   settings.host.authentication.password - encrypted password for basic authentication (if not stored in a secret)
+#   settings.host.authentication.credentialsSecret  - name of secret containing username and password for basic authentication
+#   settings.host.authentication.credentialsVaultId - optional vault id for secret containing username password for basic authentication
+#   settings.index                        - ElasticSearch index settings
+#   settings.index.name                   - index name
 #
 # If not overridden, the following suffixes are used by default:
 #   countMetricSuffix: count
@@ -29,8 +36,14 @@ global:
         histogramMetricSuffix: histogram
         host:
           protocol: https
-          name: hostname
+          domain: domain
           port: 9200
+          certificateFilePath: "path/to/cert"
+          authentication:
+            type: basic
+            username: username
+            password: password
+            credentialsSecret: secretName
+            credentialsVaultId: vaultId
         index:
           name: hpccmetrics
-
diff --git a/system/metrics/sinks/elastic/elasticSink.cpp b/system/metrics/sinks/elastic/elasticSink.cpp
@@ -13,6 +13,8 @@
 
 #include "elasticSink.hpp"
 #include "nlohmann/json.hpp"
+#include "jsecrets.hpp"
+#include "jencrypt.hpp"
 
 //including cpp-httplib single header file REST client
 //  doesn't work with format-nonliteral as an error
@@ -41,67 +43,136 @@ extern "C" MetricSink* getSinkInstance(const char *name, const IPropertyTree *pS
 ElasticMetricSink::ElasticMetricSink(const char *name, const IPropertyTree *pSettingsTree) :
     PeriodicMetricSink(name, "elastic", pSettingsTree)
 {
+    // Standard sink settings
     ignoreZeroMetrics = pSettingsTree->getPropBool("@ignoreZeroMetrics", true);
 
-    StringBuffer hostName;
+    // Get the host and index configuration
+    if (getHostConfig(pSettingsTree) && getIndexConfig(pSettingsTree))
+    {
+        configurationValid = true;
+        PROGLOG("ElasticMetricSink: Loaded and configured");
+    }
+}
+
+
+bool ElasticMetricSink::getHostConfig(const IPropertyTree *pSettingsTree)
+{
+    StringBuffer hostDomain;
     StringBuffer hostProtocol;
     StringBuffer hostPort;
 
     Owned<IPropertyTree> pHostConfigTree = pSettingsTree->getPropTree("host");
     if (pHostConfigTree)
     {
-        pHostConfigTree->getProp("@name", hostName);
+        pHostConfigTree->getProp("@domain", hostDomain);
 
         if (!pHostConfigTree->getProp("@protocol", hostProtocol))
-        {
             hostProtocol.append("https");
-        }
 
         if (!pHostConfigTree->getProp("@port", hostPort))
-        {
             hostPort.append("9200");
-        }
     }
 
-    if (!hostName.isEmpty() && !hostPort.isEmpty() && !hostProtocol.isEmpty())
+    // Validate the host configuration minimal settings are present
+    if (hostDomain.isEmpty() || hostProtocol.isEmpty())
     {
-        elasticHostUrl.append(hostProtocol).append("://").append(hostName).append(":").append(hostPort);
+        WARNLOG("ElasticMetricSink: Host configuration missing domain and/or protocol");
+        return false;
+    }
+
+    // build url for use with httplib Client
+    elasticHostUrl.append(hostProtocol).append("://").append(hostDomain);
+    if (!hostPort.isEmpty())
+        elasticHostUrl.append(":").append(hostPort);
+
+    // Read optional certificate file path
+    pHostConfigTree->getProp("@certificateFilePath", certificateFilePath);
+
+    // Get authentication settings, if present
+    Owned<IPropertyTree> pAuthConfigTree = pSettingsTree->getPropTree("authentication");
+    if (!pAuthConfigTree)
+        return true;
+
+    // Retrieve the authentication type and validate (only basic is supported)
+    if (!pAuthConfigTree->getProp("@type", authenticationType) || !streq(authenticationType, "basic"))
+    {
+        WARNLOG("ElasticMetricSink: Only basic authentication is supported");
+        return false;
+    }
+
+    StringBuffer credentialsSecretKey;
+    pAuthConfigTree->getProp("@credentialsSecret", credentialsSecretKey);  // vault/secrets key
+    if (!credentialsSecretKey.isEmpty())
+    {
+        StringBuffer credentialsVaultId;
+        pAuthConfigTree->getProp("@credentialsVaultId", credentialsVaultId);//optional HashiCorp vault ID
+
+        PROGLOG("Retrieving ElasticSearch host authentication username/password from secrets tree '%s', from vault '%s'",
+               credentialsSecretKey.str(), !credentialsVaultId.isEmpty() ? credentialsVaultId.str() : "");
+
+        Owned<const IPropertyTree> secretTree(getSecret("authn", credentialsSecretKey.str(), credentialsVaultId, nullptr));
+        if (secretTree == nullptr)
+        {
+            WARNLOG("ElasticMetricSink: Unable to load secret tree '%s', from vault '%s'", credentialsSecretKey.str(),
+                    !credentialsVaultId.isEmpty() ? credentialsVaultId.str() : "n/a");
+            return false;
+        }
+
+        // authentication type defines the secret key name/value pairs to retrieve
+        if (streq(authenticationType, "basic"))
+        {
+            if (!getSecretKeyValue(username, secretTree, "username") || !getSecretKeyValue(password, secretTree, "password"))
+            {
+                WARNLOG("ElasticMetricSink: Missing username and/or password from secrets tree '%s', vault '%s'",
+                        credentialsSecretKey.str(), !credentialsVaultId.isEmpty() ? credentialsVaultId.str() : "n/a");
+                return false;
+            }
+        }
     }
     else
     {
-        WARNLOG("ElasticMetricSink: Host configuration missing or invalid");
+        // if basic auth, username and password are stored directly in the configuration
+        if (streq(authenticationType, "basic"))
+        {
+            StringBuffer encryptedPassword;
+            if (!pAuthConfigTree->getProp("@username", username) || !pAuthConfigTree->getProp("@password", encryptedPassword))
+            {
+                WARNLOG("ElasticMetricSink: Missing username and/or password from configuration");
+                return false;
+            }
+            decrypt(password, encryptedPassword.str()); //MD5 encrypted in config
+        }
     }
+    return true;
+}
+
 
+bool ElasticMetricSink::getIndexConfig(const IPropertyTree *pSettingsTree)
+{
     Owned<IPropertyTree> pIndexConfigTree = pSettingsTree->getPropTree("index");
-    if (pIndexConfigTree)
+    if (!pIndexConfigTree)
     {
-        pSettingsTree->getProp("@name", indexName);
+        WARNLOG("ElasticMetricSink: Index configuration missing");
+        return false;
     }
 
-    if (indexName.isEmpty())
+    if (!pIndexConfigTree->getProp("@name", indexName))
     {
-        WARNLOG("ElasticMetricSink: Index configuration missing or invalid");
+        WARNLOG("ElasticMetricSink: Index configuration missing name");
+        return false;
     }
 
-
-    // Both a host url and an index name are required
-    configurationValid = !elasticHostUrl.isEmpty() && !indexName.isEmpty();
-
     // Initialize standard suffixes
-    if (!pSettingsTree->getProp("@countMetricSuffix", countMetricSuffix))
-    {
+    if (!pIndexConfigTree->getProp("@countMetricSuffix", countMetricSuffix))
         countMetricSuffix.append("count");
-    }
 
     if (!pSettingsTree->getProp("@gaugeMetricSuffix", gaugeMetricSuffix))
-    {
         gaugeMetricSuffix.append("gauge");
-    }
 
     if (!pSettingsTree->getProp("@histogramMetricSuffix", histogramMetricSuffix))
-    {
         histogramMetricSuffix.append("histogram");
-    }
+
+    return true;
 }
 
 

diff --git a/system/metrics/sinks/elastic/elasticSink.hpp b/system/metrics/sinks/elastic/elasticSink.hpp
@@ -34,11 +34,17 @@ class ELASTICSINK_API ElasticMetricSink : public hpccMetrics::PeriodicMetricSink
     virtual bool prepareToStartCollecting() override;
     virtual void collectingHasStopped() override;
     virtual void doCollection() override;
+    bool getHostConfig(const IPropertyTree *pSettingsTree);
+    bool getIndexConfig(const IPropertyTree *pSettingsTree);
 
 protected:
     StringBuffer indexName;
     bool ignoreZeroMetrics = false;
     StringBuffer elasticHostUrl;
+    StringBuffer certificateFilePath;
+    StringBuffer authenticationType;
+    StringBuffer username;
+    StringBuffer password;
     StringBuffer countMetricSuffix;
     StringBuffer gaugeMetricSuffix;
     StringBuffer histogramMetricSuffix;