Introduce indices filter

Signed-off-by: nhuttran <[email protected]>

README.md

@@ -139,6 +139,16 @@ For example to get stats for all cluster nodes from any node use settings:
 prometheus.nodes.filter: "_all"
 ```
 
+#### Indices filter
+
+Prometheus exporter can be configured to filter indices statistics with indices starting with prefixes.
+Default value: `""`.
+
+For example to filter indices statistics with indices starting with prefixes such as logs, alarms:
+```
+prometheus.indices_filter: "logs-*,alarms*"
+```
+
 ## Usage
 
 Metrics are directly available at:

src/main/java/org/compuscene/metrics/prometheus/PrometheusSettings.java

@@ -34,6 +34,7 @@ public class PrometheusSettings {
     static String PROMETHEUS_CLUSTER_SETTINGS_KEY = "prometheus.cluster.settings";
     static String PROMETHEUS_INDICES_KEY = "prometheus.indices";
     static String PROMETHEUS_NODES_FILTER_KEY = "prometheus.nodes.filter";
+    static String PROMETHEUS_INDICES_FILTER_KEY = "prometheus.indices_filter";
 
     /**
      * This setting is used configure weather to expose cluster settings metrics or not. The default value is true.
@@ -59,9 +60,18 @@ public class PrometheusSettings {
             Setting.simpleString(PROMETHEUS_NODES_FILTER_KEY, "_local",
                     Setting.Property.Dynamic, Setting.Property.NodeScope);
 
+    /**
+     * This setting is used configure to filter indices statistics with indices starting with prefixes. The default value is "".
+     * Can be configured in opensearch.yml file or update dynamically under key {@link #PROMETHEUS_INDICES_FILTER_KEY}.
+     */
+    public static final Setting<String> PROMETHEUS_INDICES_FILTER =
+            Setting.simpleString(PROMETHEUS_INDICES_FILTER_KEY, "",
+                    Setting.Property.Dynamic, Setting.Property.NodeScope);
+
     private volatile boolean clusterSettings;
     private volatile boolean indices;
     private volatile String nodesFilter;
+    private volatile String indicesFilter;
 
     /**
      * A constructor.
@@ -72,9 +82,11 @@ public PrometheusSettings(Settings settings, ClusterSettings clusterSettings) {
         setPrometheusClusterSettings(PROMETHEUS_CLUSTER_SETTINGS.get(settings));
         setPrometheusIndices(PROMETHEUS_INDICES.get(settings));
         setPrometheusNodesFilter(PROMETHEUS_NODES_FILTER.get(settings));
+        setPrometheusIndicesFilter(PROMETHEUS_INDICES_FILTER.get(settings));
         clusterSettings.addSettingsUpdateConsumer(PROMETHEUS_CLUSTER_SETTINGS, this::setPrometheusClusterSettings);
         clusterSettings.addSettingsUpdateConsumer(PROMETHEUS_INDICES, this::setPrometheusIndices);
         clusterSettings.addSettingsUpdateConsumer(PROMETHEUS_NODES_FILTER, this::setPrometheusNodesFilter);
+        clusterSettings.addSettingsUpdateConsumer(PROMETHEUS_INDICES_FILTER, this::setPrometheusIndicesFilter);
     }
 
     private void setPrometheusClusterSettings(boolean flag) {
@@ -87,6 +99,10 @@ private void setPrometheusIndices(boolean flag) {
 
     private void setPrometheusNodesFilter(String filter) { this.nodesFilter = filter; }
 
+    private void setPrometheusIndicesFilter(String indicesFilter) {
+        this.indicesFilter = indicesFilter;
+    }
+
     /**
      * Get value of settings key {@link #PROMETHEUS_CLUSTER_SETTINGS_KEY}.
      * @return boolean value of the key
@@ -108,4 +124,12 @@ public boolean getPrometheusIndices() {
      * @return boolean value of the key
      */
     public String getNodesFilter() { return this.nodesFilter; }
+
+    /**
+     * Get value of settings key {@link #PROMETHEUS_INDICES_FILTER_KEY}.
+     * @return string value of the key
+     */
+    public String getPrometheusIndicesFilter() {
+        return this.indicesFilter;
+    }
 }

src/main/java/org/opensearch/action/TransportNodePrometheusMetricsAction.java

@@ -37,6 +37,7 @@
 import org.opensearch.client.Client;
 import org.opensearch.client.Requests;
 import org.opensearch.common.Nullable;
+import org.opensearch.common.Strings;
 import org.opensearch.common.inject.Inject;
 import org.opensearch.common.settings.ClusterSettings;
 import org.opensearch.common.settings.Settings;
@@ -132,7 +133,7 @@ private AsyncAction(ActionListener<NodePrometheusMetricsResponse> listener) {
 
             // Indices stats request is not "node-specific", it does not support any "_local" notion
             // it is broad-casted to all cluster nodes.
-            this.indicesStatsRequest = isPrometheusIndices ? new IndicesStatsRequest() : null;
+            this.indicesStatsRequest = isPrometheusIndices ? (new IndicesStatsRequest()).indices(Strings.splitStringByCommaToArray(prometheusSettings.getPrometheusIndicesFilter())) : null;
 
             // Cluster settings are get via ClusterStateRequest (see elasticsearch RestClusterGetSettingsAction for details)
             // We prefer to send it to master node (hence local=false; it should be set by default but we want to be sure).

src/main/java/org/opensearch/plugin/prometheus/PrometheusExporterPlugin.java

@@ -76,6 +76,7 @@ public List<Setting<?>> getSettings() {
                 PrometheusSettings.PROMETHEUS_CLUSTER_SETTINGS,
                 PrometheusSettings.PROMETHEUS_INDICES,
                 PrometheusSettings.PROMETHEUS_NODES_FILTER,
+                PrometheusSettings.PROMETHEUS_INDICES_FILTER,
                 RestPrometheusMetricsAction.METRIC_PREFIX
         );
         return Collections.unmodifiableList(settings);

src/yamlRestTest/resources/rest-api-spec/test/50_10_nodes_filter.yml

@@ -17,6 +17,7 @@
   - match: {defaults.prometheus.cluster.settings: "true"}
   - match: {defaults.prometheus.indices: "true"}
   - match: {defaults.prometheus.nodes.filter: "_local"}
+  - match: {defaults.prometheus.indices_filter: ""}
   - match: {defaults.prometheus.metric_name.prefix: "opensearch_"}
 
 
@@ -44,6 +45,16 @@
 
   - match: {persistent: {prometheus.nodes.filter: "_all"}}
 
+  # Change indices filter to "logs*"
+  - do:
+      cluster.put_settings:
+        body:
+          persistent:
+            prometheus.indices_filter: "logs*"
+        flat_settings: true
+
+  - match: {persistent: {prometheus.indices_filter: "logs*"}}
+
   # Verify in Prometheus metrics:
   - do:
       prometheus.metrics: {}
@@ -64,3 +75,27 @@
           persistent:
             prometheus.nodes.filter: null
         flat_settings: true
+
+  # Remove persistent settings
+  - do:
+      cluster.put_settings:
+        body:
+          persistent:
+            prometheus.indices_filter: null
+        flat_settings: true
+
+  # Remove persistent settings
+  - do:
+      cluster.put_settings:
+        body:
+          persistent:
+            prometheus.indices_filter: null
+        flat_settings: true
+
+  # Remove persistent settings
+  - do:
+      cluster.put_settings:
+        body:
+          persistent:
+            prometheus.indices_filter: null
+        flat_settings: true