diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 052db28..887b209 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -19,10 +19,11 @@ jobs: contents: read steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v1 + uses: actions/setup-java@v4 with: + distribution: "adopt" java-version: ${{ matrix.java }} - name: Build and Run Tests diff --git a/.github/workflows/dco.yml b/.github/workflows/dco.yml index 3100de7..fc2044e 100644 --- a/.github/workflows/dco.yml +++ b/.github/workflows/dco.yml @@ -12,7 +12,7 @@ jobs: steps: - name: Get PR Commits id: 'get-pr-commits' - uses: tim-actions/get-pr-commits@v1.1.0 + uses: tim-actions/get-pr-commits@v1.3.1 with: token: ${{ secrets.GITHUB_TOKEN }} - name: DCO Check diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 4f92835..ab5ebf6 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -84,7 +84,7 @@ Each commit must include a DCO which looks like this Signed-off-by: Jane Smith ``` -You may type this line on your own when writing your commit messages. However, if your user.name and user.email are set in your git configs, you can use `-s` or `– – signoff` to add the `Signed-off-by` line to the end of the commit message. +You may type this line on your own when writing your commit messages. However, if your user.name and user.email are set in your git configs, you can use `-s` or `--signoff` to add the `Signed-off-by` line to the end of the commit message. ## Finding contributions to work on Looking at the existing issues is a great way to find something to contribute on. As our projects, by default, use the default GitHub issue labels (enhancement/bug/duplicate/help wanted/invalid/question/wontfix), looking at any 'help wanted' issues is a great place to start. diff --git a/README.md b/README.md index 5aaab3b..3187c75 100644 --- a/README.md +++ b/README.md @@ -38,12 +38,25 @@ This plugin was started as a fork of [Prometheus exporter for Elasticsearch®](h ## Compatibility Matrix -| OpenSearch | Plugin | Release date | -|-----------:|--------:|-------------:| -| 1.3.2 | 1.3.2.0 | May 10, 2022 | -| 1.3.1 | 1.3.1.0 | Apr 08, 2022 | -| 1.3.0 | 1.3.0.0 | Mar 22, 2022 | -| <= 1.2.4 | (*) | (*) | +| OpenSearch | Plugin | Release date | +|-----------:|---------:|-------------:| +| 1.3.15 | 1.3.15.0 | Apr 05, 2024 | +| 1.3.14 | 1.3.14.0 | Dec 20, 2023 | +| 1.3.13 | 1.3.13.0 | Sep 27, 2023 | +| 1.3.12 | 1.3.12.0 | Aug 31, 2023 | +| 1.3.11 | 1.3.11.0 | Jul 06, 2023 | +| 1.3.10 | 1.3.10.0 | May 23, 2023 | +| 1.3.9 | 1.3.9.0 | Mar 20, 2023 | +| 1.3.8 | 1.3.8.0 | Mar 03, 2023 | +| 1.3.7 | 1.3.7.0 | Dec 19, 2022 | +| 1.3.6 | 1.3.6.0 | Oct 10, 2022 | +| 1.3.5 | 1.3.5.0 | Sep 05, 2022 | +| 1.3.4 | 1.3.4.0 | Jul 15, 2022 | +| 1.3.3 | 1.3.3.0 | Jun 15, 2022 | +| 1.3.2 | 1.3.2.0 | May 10, 2022 | +| 1.3.1 | 1.3.1.0 | Apr 08, 2022 | +| 1.3.0 | 1.3.0.0 | Mar 22, 2022 | +| <= 1.2.4 | (*) | (*) | _(*) If you are looking for plugin releases supporting earlier (`<= 1.2.4`) OpenSearch versions please visit ._ @@ -54,7 +67,7 @@ You need to install the plugin on every OpenSearch node that will be scraped by To **install** the plugin: -`./bin/opensearch-plugin install https://github.com/aiven/prometheus-exporter-plugin-for-opensearch/releases/download/1.3.2.0/prometheus-exporter-1.3.2.0.zip` +`./bin/opensearch-plugin install https://github.com/aiven/prometheus-exporter-plugin-for-opensearch/releases/download/1.3.15.0/prometheus-exporter-1.3.15.0.zip` To **remove** the plugin. @@ -105,6 +118,7 @@ Metrics include statistics about individual OpenSearch nodes. By default, only statistics from the node that received the request are included. Prometheus exporter can be configured to include statistics from other nodes as well. +This filter is directly utilizing OpenSearch [Node filters](https://opensearch.org/docs/latest/opensearch/rest-api/nodes-apis/index/#node-filters) feature. Default value: `"_local"`. For example to get stats for all cluster nodes from any node use settings: @@ -112,6 +126,40 @@ For example to get stats for all cluster nodes from any node use settings: prometheus.nodes.filter: "_all" ``` +#### Indices filter + +Prometheus exporter can be configured to filter indices statistics from selected indices. +To target all indices, use "" or * or _all or null. +Default value: `""`. + +For example to filter indices statistics: +``` +prometheus.indices_filter.selected_indices: "log-*,*log,*log*,log*-test" +``` + +Users can select which option for filtering indices statistics. +Default value: `"STRICT_EXPAND_OPEN_FORBID_CLOSED"`. + +For example to select an option: +``` +prometheus.indices_filter.selected_option: "LENIENT_EXPAND_OPEN" +``` +Here is a list of indices options available for selection +``` +STRICT_EXPAND_OPEN: indices options that requires every specified index to exist, expands wildcards only to open indices and allows that no indices are resolved from wildcard expressions (not returning an error). +STRICT_EXPAND_OPEN_HIDDEN: indices options that requires every specified index to exist, expands wildcards only to open indices, includes hidden indices, and allows that no indices are resolved from wildcard expressions (not returning an error). +STRICT_EXPAND_OPEN_FORBID_CLOSED: indices options that requires every specified index to exist, expands wildcards only to open indices, allows that no indices are resolved from wildcard expressions (not returning an error) and forbids the use of closed indices by throwing an error. +STRICT_EXPAND_OPEN_HIDDEN_FORBID_CLOSED: indices options that requires every specified index to exist, expands wildcards only to open indices, includes hidden indices, and allows that no indices are resolved from wildcard expressions (not returning an error) and forbids the use of closed indices by throwing an error. +STRICT_EXPAND_OPEN_FORBID_CLOSED_IGNORE_THROTTLED: indices options that requires every specified index to exist, expands wildcards only to open indices, allows that no indices are resolved from wildcard expressions (not returning an error), forbids the use of closed indices by throwing an error and ignores indices that are throttled. +STRICT_EXPAND_OPEN_CLOSED: indices option that requires every specified index to exist, expands wildcards to both open and closed indices and allows that no indices are resolved from wildcard expressions (not returning an error). +STRICT_EXPAND_OPEN_CLOSED_HIDDEN: indices option that requires every specified index to exist, expands wildcards to both open and closed indices, includes hidden indices, and allows that no indices are resolved from wildcard expressions (not returning an error). +STRICT_SINGLE_INDEX_NO_EXPAND_FORBID_CLOSED: indices option that requires each specified index or alias to exist, doesn't expand wildcards and throws error if any of the aliases resolves to multiple indices. +LENIENT_EXPAND_OPEN: indices options that ignores unavailable indices, expands wildcards only to open indices and allows that no indices are resolved from wildcard expressions (not returning an error). +LENIENT_EXPAND_OPEN_HIDDEN: indices options that ignores unavailable indices, expands wildcards to open and hidden indices, and allows that no indices are resolved from wildcard expressions (not returning an error). +LENIENT_EXPAND_OPEN_CLOSED: indices options that ignores unavailable indices, expands wildcards to both open and closed indices and allows that no indices are resolved from wildcard expressions (not returning an error). +LENIENT_EXPAND_OPEN_CLOSED_HIDDEN: indices options that ignores unavailable indices, expands wildcards to all open and closed indices and allows that no indices are resolved from wildcard expressions (not returning an error). +``` + ## Usage Metrics are directly available at: diff --git a/build.gradle b/build.gradle index dda94f0..481cca0 100644 --- a/build.gradle +++ b/build.gradle @@ -3,49 +3,8 @@ import org.opensearch.gradle.test.RestIntegTestTask import java.util.regex.Matcher import java.util.regex.Pattern -apply plugin: 'java' -apply plugin: 'idea' -apply plugin: 'opensearch.opensearchplugin' -apply plugin: 'opensearch.yaml-rest-test' - -def pluginName = pluginName -def pluginDescription = pluginDescription -//def projectPath = 'org.opensearch' -//def pathToPlugin = 'path.to.plugin' -def pluginClassName = pluginClassname - -//tasks.register("preparePluginPathDirs") { -// mustRunAfter clean -// doLast { -// def newPath = pathToPlugin.replace(".", "/") -// mkdir "src/main/java/org/opensearch/$newPath" -// mkdir "src/test/java/org/opensearch/$newPath" -// mkdir "src/yamlRestTest/java/org/opensearch/$newPath" -// } -//} - -opensearchplugin { - name pluginName - description pluginDescription -// classname "${projectPath}.${pathToPlugin}.${pluginClassName}" - classname pluginClassName - licenseFile rootProject.file('LICENSE.txt') - noticeFile rootProject.file('NOTICE.txt') -} - -// thirdparty audit needs can be enabled -thirdPartyAudit.enabled = false - -// This requires an additional Jar not published as part of build-tools -loggerUsageCheck.enabled = false - -// No need to validate pom, as we do not upload to maven/sonatype -validateNebulaPom.enabled = false - buildscript { - ext { - // Four {digit(s)} concatenated with a "." followed by -rc{digit(s)} // Examples: 1.0.0.1-rc2, 2.10.4.0-rc5 // Group (3) refers to the last .{digit{(s)} pattern (this is what we want to remove for OpS version) @@ -75,16 +34,16 @@ buildscript { versions = [ "opensearch": opensearch_version, - "prometheus": "0.15.0" + "prometheus": "0.16.0" ] } repositories { mavenLocal() + maven { url "https://aws.oss.sonatype.org/content/repositories/releases" } maven { url "https://aws.oss.sonatype.org/content/repositories/snapshots" } mavenCentral() maven { url "https://plugins.gradle.org/m2/" } - jcenter() } dependencies { @@ -92,8 +51,39 @@ buildscript { } } +apply plugin: 'java' +apply plugin: 'idea' +apply plugin: 'opensearch.opensearchplugin' +apply plugin: 'opensearch.yaml-rest-test' +//apply plugin: 'checkstyle' + +def pluginName = pluginName +def pluginDescription = pluginDescription +//def projectPath = 'org.opensearch' +//def pathToPlugin = 'path.to.plugin' +def pluginClassName = pluginClassname + +opensearchplugin { + name pluginName + description pluginDescription +// classname "${projectPath}.${pathToPlugin}.${pluginClassName}" + classname pluginClassName + licenseFile rootProject.file('LICENSE.txt') + noticeFile rootProject.file('NOTICE.txt') +} + +// thirdparty audit needs can be enabled +thirdPartyAudit.enabled = false + +// This requires an additional Jar not published as part of build-tools +loggerUsageCheck.enabled = false + +// No need to validate pom, as we do not upload to maven/sonatype +validateNebulaPom.enabled = false + repositories { mavenLocal() + maven { url "https://aws.oss.sonatype.org/content/repositories/releases" } maven { url "https://aws.oss.sonatype.org/content/repositories/snapshots" } mavenCentral() maven { url "https://plugins.gradle.org/m2/" } @@ -105,8 +95,14 @@ dependencies { // See https://docs.gradle.org/6.8.1/userguide/java_plugin.html#sec:java_plugin_and_dependency_management // compile "io.prometheus:simpleclient:${versions.prometheus}" // compile "io.prometheus:simpleclient_common:${versions.prometheus}" - implementation group: 'io.prometheus', name: 'simpleclient', version: '0.15.0' - implementation group: 'io.prometheus', name: 'simpleclient_common', version: '0.15.0' + implementation group: 'io.prometheus', name: 'simpleclient', version: '0.16.0' + implementation group: 'io.prometheus', name: 'simpleclient_common', version: '0.16.0' +} + +restResources { + restApi { + includeCore '_common', 'cat', 'cluster', 'nodes', 'indices', 'index' + } } test { diff --git a/config/checkstyle/checkstyle.xml b/config/checkstyle/checkstyle.xml new file mode 100644 index 0000000..188c119 --- /dev/null +++ b/config/checkstyle/checkstyle.xml @@ -0,0 +1,298 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/gradle.properties b/gradle.properties index 8c8d987..4d52188 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,6 +1,6 @@ #group = org.opensearch.plugin.prometheus -version = 1.3.2.0 +version = 1.3.15.0 pluginName = prometheus-exporter pluginClassname = org.opensearch.plugin.prometheus.PrometheusExporterPlugin diff --git a/licenses/simpleclient-0.15.0.jar.sha1 b/licenses/simpleclient-0.15.0.jar.sha1 deleted file mode 100644 index 80582d1..0000000 --- a/licenses/simpleclient-0.15.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -144aaf1ac9361a497d98079e0db8757a95e22fc4 \ No newline at end of file diff --git a/licenses/simpleclient-0.16.0.jar.sha1 b/licenses/simpleclient-0.16.0.jar.sha1 new file mode 100644 index 0000000..36a11e4 --- /dev/null +++ b/licenses/simpleclient-0.16.0.jar.sha1 @@ -0,0 +1 @@ +28b0eaf7c500c506976da8d0fc9cad6c278e8d87 \ No newline at end of file diff --git a/licenses/simpleclient_common-0.15.0.jar.sha1 b/licenses/simpleclient_common-0.15.0.jar.sha1 deleted file mode 100644 index f10a8d9..0000000 --- a/licenses/simpleclient_common-0.15.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -57bd1d8be9f4d965a38c6b1b35ee60358cc679fc \ No newline at end of file diff --git a/licenses/simpleclient_common-0.16.0.jar.sha1 b/licenses/simpleclient_common-0.16.0.jar.sha1 new file mode 100644 index 0000000..6359603 --- /dev/null +++ b/licenses/simpleclient_common-0.16.0.jar.sha1 @@ -0,0 +1 @@ +a09a8c790a20309b942a9fdbfe77da22407096e6 \ No newline at end of file diff --git a/src/main/java/org/compuscene/metrics/prometheus/PrometheusSettings.java b/src/main/java/org/compuscene/metrics/prometheus/PrometheusSettings.java index 9505e5b..bd1d820 100644 --- a/src/main/java/org/compuscene/metrics/prometheus/PrometheusSettings.java +++ b/src/main/java/org/compuscene/metrics/prometheus/PrometheusSettings.java @@ -17,6 +17,8 @@ package org.compuscene.metrics.prometheus; +import org.opensearch.action.support.IndicesOptions; +import org.opensearch.common.Strings; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; @@ -30,28 +32,91 @@ * } */ public class PrometheusSettings { + public enum INDEX_FILTER_OPTIONS { + STRICT_EXPAND_OPEN, + STRICT_EXPAND_OPEN_HIDDEN, + STRICT_EXPAND_OPEN_FORBID_CLOSED, + STRICT_EXPAND_OPEN_HIDDEN_FORBID_CLOSED, + STRICT_EXPAND_OPEN_FORBID_CLOSED_IGNORE_THROTTLED, + STRICT_EXPAND_OPEN_CLOSED, + STRICT_EXPAND_OPEN_CLOSED_HIDDEN, + STRICT_SINGLE_INDEX_NO_EXPAND_FORBID_CLOSED, + LENIENT_EXPAND_OPEN, + LENIENT_EXPAND_OPEN_HIDDEN, + LENIENT_EXPAND_OPEN_CLOSED, + LENIENT_EXPAND_OPEN_CLOSED_HIDDEN + } + + static String PROMETHEUS_CLUSTER_SETTINGS_KEY = "prometheus.cluster.settings"; + static String PROMETHEUS_INDICES_KEY = "prometheus.indices"; + static String PROMETHEUS_NODES_FILTER_KEY = "prometheus.nodes.filter"; + static String PROMETHEUS_SELECTED_INDICES_KEY = "prometheus.indices_filter.selected_indices"; + static String PROMETHEUS_SELECTED_OPTION_KEY = "prometheus.indices_filter.selected_option"; + /** + * This setting is used configure weather to expose cluster settings metrics or not. The default value is true. + * Can be configured in opensearch.yml file or update dynamically under key {@link #PROMETHEUS_CLUSTER_SETTINGS_KEY}. + */ public static final Setting PROMETHEUS_CLUSTER_SETTINGS = - Setting.boolSetting("prometheus.cluster.settings", true, + Setting.boolSetting(PROMETHEUS_CLUSTER_SETTINGS_KEY, true, Setting.Property.Dynamic, Setting.Property.NodeScope); + + /** + * This setting is used configure weather to expose low level index metrics or not. The default value is true. + * Can be configured in opensearch.yml file or update dynamically under key {@link #PROMETHEUS_INDICES_KEY}. + */ public static final Setting PROMETHEUS_INDICES = - Setting.boolSetting("prometheus.indices", true, + Setting.boolSetting(PROMETHEUS_INDICES_KEY, true, Setting.Property.Dynamic, Setting.Property.NodeScope); + + /** + * This setting is used configure which cluster nodes to gather metrics from. The default value is _local. + * Can be configured in opensearch.yml file or update dynamically under key {@link #PROMETHEUS_NODES_FILTER_KEY}. + */ public static final Setting PROMETHEUS_NODES_FILTER = - Setting.simpleString("prometheus.nodes.filter", "_local", + Setting.simpleString(PROMETHEUS_NODES_FILTER_KEY, "_local", Setting.Property.Dynamic, Setting.Property.NodeScope); + /** + * This setting is used configure to filter indices statistics from selected indices. The default value is "". + * Can be configured in opensearch.yml file or update dynamically under key {@link #PROMETHEUS_SELECTED_INDICES_KEY}. + */ + public static final Setting PROMETHEUS_SELECTED_INDICES = + Setting.simpleString(PROMETHEUS_SELECTED_INDICES_KEY, "", + Setting.Property.Dynamic, Setting.Property.NodeScope); + + /** + * This setting is used configure which option for filtering indices statistics. + * The default value is STRICT_EXPAND_OPEN_FORBID_CLOSED. + * Can be configured in opensearch.yml file or update dynamically under key {@link #PROMETHEUS_SELECTED_OPTION_KEY}. + */ + public static final Setting PROMETHEUS_SELECTED_OPTION = + new Setting<>(PROMETHEUS_SELECTED_OPTION_KEY, + String.valueOf(INDEX_FILTER_OPTIONS.STRICT_EXPAND_OPEN_FORBID_CLOSED), + INDEX_FILTER_OPTIONS::valueOf, Setting.Property.Dynamic, Setting.Property.NodeScope); + private volatile boolean clusterSettings; private volatile boolean indices; private volatile String nodesFilter; + private volatile String selectedIndices; + private volatile INDEX_FILTER_OPTIONS selectedOption; + /** + * A constructor. + * @param settings Settings + * @param clusterSettings Cluster settings + */ public PrometheusSettings(Settings settings, ClusterSettings clusterSettings) { setPrometheusClusterSettings(PROMETHEUS_CLUSTER_SETTINGS.get(settings)); setPrometheusIndices(PROMETHEUS_INDICES.get(settings)); setPrometheusNodesFilter(PROMETHEUS_NODES_FILTER.get(settings)); + setPrometheusSelectedIndices(PROMETHEUS_SELECTED_INDICES.get(settings)); + setPrometheusSelectedOption(PROMETHEUS_SELECTED_OPTION.get(settings)); clusterSettings.addSettingsUpdateConsumer(PROMETHEUS_CLUSTER_SETTINGS, this::setPrometheusClusterSettings); clusterSettings.addSettingsUpdateConsumer(PROMETHEUS_INDICES, this::setPrometheusIndices); clusterSettings.addSettingsUpdateConsumer(PROMETHEUS_NODES_FILTER, this::setPrometheusNodesFilter); + clusterSettings.addSettingsUpdateConsumer(PROMETHEUS_SELECTED_INDICES, this::setPrometheusSelectedIndices); + clusterSettings.addSettingsUpdateConsumer(PROMETHEUS_SELECTED_OPTION, this::setPrometheusSelectedOption); } private void setPrometheusClusterSettings(boolean flag) { @@ -64,13 +129,88 @@ private void setPrometheusIndices(boolean flag) { private void setPrometheusNodesFilter(String filter) { this.nodesFilter = filter; } + private void setPrometheusSelectedIndices(String selectedIndices) { + this.selectedIndices = selectedIndices; + } + + private void setPrometheusSelectedOption(INDEX_FILTER_OPTIONS selectedOption) { + this.selectedOption = selectedOption; + } + + /** + * Get value of settings key {@link #PROMETHEUS_CLUSTER_SETTINGS_KEY}. + * @return boolean value of the key + */ public boolean getPrometheusClusterSettings() { return this.clusterSettings; } + /** + * Get value of settings key {@link #PROMETHEUS_INDICES_KEY}. + * @return boolean value of the key + */ public boolean getPrometheusIndices() { return this.indices; } + /** + * Get value of settings key {@link #PROMETHEUS_NODES_FILTER_KEY}. + * @return boolean value of the key + */ public String getNodesFilter() { return this.nodesFilter; } + + /** + * Get value of settings key {@link #PROMETHEUS_SELECTED_INDICES_KEY}. + * @return string value of the key + */ + public String[] getPrometheusSelectedIndices() { + return Strings.splitStringByCommaToArray(this.selectedIndices); + } + + /** + * Get IndicesOptions of settings key {@link #PROMETHEUS_SELECTED_OPTION_KEY}. + * @return IndicesOptions of the key + */ + public IndicesOptions getIndicesOptions() { + IndicesOptions indicesOptions = null; + switch (this.selectedOption) { + case STRICT_EXPAND_OPEN: + indicesOptions = IndicesOptions.strictExpandOpen(); + break; + case STRICT_EXPAND_OPEN_HIDDEN: + indicesOptions = IndicesOptions.strictExpandOpenHidden(); + break; + case STRICT_EXPAND_OPEN_FORBID_CLOSED: + indicesOptions = IndicesOptions.strictExpandOpenAndForbidClosed(); + break; + case STRICT_EXPAND_OPEN_HIDDEN_FORBID_CLOSED: + indicesOptions = IndicesOptions.STRICT_EXPAND_OPEN_HIDDEN_FORBID_CLOSED; + break; + case STRICT_EXPAND_OPEN_FORBID_CLOSED_IGNORE_THROTTLED: + indicesOptions = IndicesOptions.strictExpandOpenAndForbidClosedIgnoreThrottled(); + break; + case STRICT_EXPAND_OPEN_CLOSED: + indicesOptions = IndicesOptions.strictExpand(); + break; + case STRICT_EXPAND_OPEN_CLOSED_HIDDEN: + indicesOptions = IndicesOptions.strictExpandHidden(); + break; + case STRICT_SINGLE_INDEX_NO_EXPAND_FORBID_CLOSED: + indicesOptions = IndicesOptions.strictSingleIndexNoExpandForbidClosed(); + break; + case LENIENT_EXPAND_OPEN: + indicesOptions = IndicesOptions.lenientExpandOpen(); + break; + case LENIENT_EXPAND_OPEN_HIDDEN: + indicesOptions = IndicesOptions.lenientExpandOpenHidden(); + break; + case LENIENT_EXPAND_OPEN_CLOSED: + indicesOptions = IndicesOptions.lenientExpand(); + break; + case LENIENT_EXPAND_OPEN_CLOSED_HIDDEN: + indicesOptions = IndicesOptions.lenientExpandHidden(); + break; + } + return indicesOptions; + } } diff --git a/src/main/java/org/opensearch/action/TransportNodePrometheusMetricsAction.java b/src/main/java/org/opensearch/action/TransportNodePrometheusMetricsAction.java index 531112d..d9fc7c7 100644 --- a/src/main/java/org/opensearch/action/TransportNodePrometheusMetricsAction.java +++ b/src/main/java/org/opensearch/action/TransportNodePrometheusMetricsAction.java @@ -124,7 +124,14 @@ private AsyncAction(ActionListener listener) { // Indices stats request is not "node-specific", it does not support any "_local" notion // it is broad-casted to all cluster nodes. - this.indicesStatsRequest = isPrometheusIndices ? new IndicesStatsRequest() : null; + if (isPrometheusIndices) { + IndicesStatsRequest indicesStatsRequest = new IndicesStatsRequest(); + indicesStatsRequest.indices(prometheusSettings.getPrometheusSelectedIndices()); + indicesStatsRequest.indicesOptions(prometheusSettings.getIndicesOptions()); + this.indicesStatsRequest = indicesStatsRequest; + } else { + this.indicesStatsRequest = null; + } // Cluster settings are get via ClusterStateRequest (see elasticsearch RestClusterGetSettingsAction for details) // We prefer to send it to master node (hence local=false; it should be set by default but we want to be sure). diff --git a/src/main/java/org/opensearch/plugin/prometheus/PrometheusExporterPlugin.java b/src/main/java/org/opensearch/plugin/prometheus/PrometheusExporterPlugin.java index e3f1c74..ea804a1 100644 --- a/src/main/java/org/opensearch/plugin/prometheus/PrometheusExporterPlugin.java +++ b/src/main/java/org/opensearch/plugin/prometheus/PrometheusExporterPlugin.java @@ -73,6 +73,8 @@ public List> getSettings() { PrometheusSettings.PROMETHEUS_CLUSTER_SETTINGS, PrometheusSettings.PROMETHEUS_INDICES, PrometheusSettings.PROMETHEUS_NODES_FILTER, + PrometheusSettings.PROMETHEUS_SELECTED_INDICES, + PrometheusSettings.PROMETHEUS_SELECTED_OPTION, RestPrometheusMetricsAction.METRIC_PREFIX ); return Collections.unmodifiableList(settings); diff --git a/src/yamlRestTest/resources/rest-api-spec/test/60_10_indices_filter.yml b/src/yamlRestTest/resources/rest-api-spec/test/60_10_indices_filter.yml new file mode 100644 index 0000000..097f6aa --- /dev/null +++ b/src/yamlRestTest/resources/rest-api-spec/test/60_10_indices_filter.yml @@ -0,0 +1,259 @@ +--- +"Indices filter": + + # Make sure indices filter setting is in the default state + - do: + cluster.get_settings: + include_defaults: false + + - match: {persistent: {}} + - match: {transient: {}} + + - do: + cluster.get_settings: + include_defaults: true + filter_path: defaults.prometheus + + - match: {defaults.prometheus.indices_filter.selected_indices: ""} + - match: {defaults.prometheus.indices_filter.selected_option: "STRICT_EXPAND_OPEN_FORBID_CLOSED"} + + # Create 3 indices + - do: + index: + index: log-a-test + id: 1 + body: { foo: bar } + + - do: + index: + index: log-b-test + id: 2 + body: { foo: bar } + + - do: + index: + index: log-c-test + id: 3 + body: { foo: bar } + + # Should have metrics for all indices + - do: + prometheus.metrics: {} + + - match: + $body: | + /.* + (opensearch_index_status\{ + cluster="yamlRestTest", + index="(log-a-test|log-b-test|log-c-test)", + \} \s+ \d+\.\d+ \n?){3} + .*/ + + # Change selected_indices to "log-a-*" + - do: + cluster.put_settings: + body: + persistent: + prometheus.indices_filter.selected_indices: "log-a-*" + flat_settings: true + + - match: {persistent: {prometheus.indices_filter.selected_indices: "log-a-*"}} + + # Should have metrics for index log-a-test only, no metrics for others + - do: + prometheus.metrics: {} + + - match: + $body: | + /.* + opensearch_index_status\{ + cluster="yamlRestTest", + index="(?=log-a-test)(?!(log-b-test|log-c-test))(.*)", + \} \s+ \d+\.\d+ \n? + .*/ + + # Change selected_indices to "*b-*,log*c-test" + - do: + cluster.put_settings: + body: + persistent: + prometheus.indices_filter.selected_indices: "*b-*,log*c-test" + flat_settings: true + + - match: {persistent: {prometheus.indices_filter.selected_indices: "*b-*,log*c-test"}} + + # Should have metrics for indices log-b-test and log-c-test, no metrics for index log-a-test + - do: + prometheus.metrics: {} + + - match: + $body: | + /.* + (opensearch_index_status\{ + cluster="yamlRestTest", + index="(?=(log-b-test|log-c-test))(?!log-a-test)(.*)", + \} \s+ \d+\.\d+ \n?){2} + .*/ + + # Change selected_indices to "*" + - do: + cluster.put_settings: + body: + persistent: + prometheus.indices_filter.selected_indices: "*" + flat_settings: true + + - match: {persistent: {prometheus.indices_filter.selected_indices: "*"}} + + # Should have metrics for all indices + - do: + prometheus.metrics: {} + + - match: + $body: | + /.* + (opensearch_index_status\{ + cluster="yamlRestTest", + index="(log-a-test|log-b-test|log-c-test)", + \} \s+ \d+\.\d+ \n?){3} + .*/ + + # Change selected_indices to "_all" + - do: + cluster.put_settings: + body: + persistent: + prometheus.indices_filter.selected_indices: "_all" + flat_settings: true + + - match: {persistent: {prometheus.indices_filter.selected_indices: "_all"}} + + # Should have metrics for all indices + - do: + prometheus.metrics: {} + + - match: + $body: | + /.* + (opensearch_index_status\{ + cluster="yamlRestTest", + index="(log-a-test|log-b-test|log-c-test)", + \} \s+ \d+\.\d+ \n?){3} + .*/ + + # Change selected_indices to null + - do: + cluster.put_settings: + body: + persistent: + prometheus.indices_filter.selected_indices: null + flat_settings: true + + - match: {persistent: {}} + + # Should have metrics for all indices + - do: + prometheus.metrics: {} + + - match: + $body: | + /.* + (opensearch_index_status\{ + cluster="yamlRestTest", + index="(log-a-test|log-b-test|log-c-test)", + \} \s+ \d+\.\d+ \n?){3} + .*/ + + # Change selected_indices to "log*,test" + - do: + cluster.put_settings: + body: + persistent: + prometheus.indices_filter.selected_indices: "log*,test" + flat_settings: true + + - match: {persistent: {prometheus.indices_filter.selected_indices: "log*,test"}} + + # Should have errors + - do: + catch: request + prometheus.metrics: {} + + - match: {status: 500} + - match: {error.type: "exception"} + - match: {error.reason: "Indices stats request failed"} + - match: {error.caused_by.type: "index_not_found_exception"} + - match: {error.caused_by.reason: "no such index [test]"} + + # Change selected_option to "LENIENT_EXPAND_OPEN" + - do: + cluster.put_settings: + body: + persistent: + prometheus.indices_filter.selected_option: "LENIENT_EXPAND_OPEN" + flat_settings: true + + - match: {persistent: {prometheus.indices_filter.selected_option: "LENIENT_EXPAND_OPEN"}} + + # Should have metrics for indices log* + - do: + prometheus.metrics: {} + + - match: + $body: | + /.* + (opensearch_index_status\{ + cluster="yamlRestTest", + index="(log-a-test|log-b-test|log-c-test)", + \} \s+ \d+\.\d+ \n?){3} + .*/ + + # Change selected_indices to "test" + - do: + cluster.put_settings: + body: + persistent: + prometheus.indices_filter.selected_indices: "test" + flat_settings: true + + - match: {persistent: {prometheus.indices_filter.selected_indices: "test"}} + + # Should no metrics for index test + - do: + prometheus.metrics: {} + + - match: + $body: | + /.* + \# \s HELP \s opensearch_index_status (\s|\w|\d)+ \n + \# \s TYPE \s opensearch_index_status \s gauge \n + (?!opensearch_index_status) + .*/ + + # Change selected_option to "TEST" + - do: + catch: bad_request + cluster.put_settings: + body: + persistent: + prometheus.indices_filter.selected_option: "TEST" + flat_settings: true + + - match: {status: 400} + - match: {error.type: "illegal_argument_exception"} + - match: {error.reason: "No enum constant org.compuscene.metrics.prometheus.PrometheusSettings.INDEX_FILTER_OPTIONS.TEST"} + + # Unexpected input values + - do: + catch: request + cluster.put_settings: + body: + persistent: + prometheus.indices_filter.selected_indices: [ "", null, 10 ] + flat_settings: true + + - match: {status: 500} + - match: {error.type: "settings_exception"} + - match: {error.reason: "Failed to load settings from [{\"prometheus.indices_filter.selected_indices\":[\"\",null,10]}]"} + - match: {error.caused_by.type: "illegal_state_exception"} + - match: {error.caused_by.reason: "only value lists are allowed in serialized settings"}