From ff7a809642099d6aaf9425ed69c4d2e2ae8046e5 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna <93581129+salvatore-campagna@users.noreply.github.com> Date: Thu, 29 Sep 2022 10:57:30 +0200 Subject: [PATCH] Downsampling: remove null value setting (#90025) (#90385) Do not copy the total fields limit setting into the downsampled index if the value is null in the source index. (cherry picked from commit d430bff9c5936c52bfa62e543849da6b37cb3400) Co-authored-by: weizijun --- .../downsample/TransportRollupAction.java | 45 +++++----- .../DownsampleActionSingleNodeTests.java | 83 ++++++++++--------- 2 files changed, 66 insertions(+), 62 deletions(-) diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/downsample/TransportRollupAction.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/downsample/TransportRollupAction.java index a6014c7e9cd25..f9192772ea1e5 100644 --- a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/downsample/TransportRollupAction.java +++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/downsample/TransportRollupAction.java @@ -594,32 +594,33 @@ private void createRollupIndex( DownsampleAction.Request request, ActionListener listener ) { + /* + * When creating the rollup index, we copy the index.number_of_shards from source index, + * and we set the index.number_of_replicas to 0, to avoid replicating the index being built. + * Also, we set the index.refresh_interval to -1. + * We will set the correct number of replicas and refresh the index later. + * + * We should note that there is a risk of losing a node during the rollup process. In this + * case rollup will fail. + */ + Settings.Builder builder = Settings.builder() + .put(IndexMetadata.SETTING_INDEX_HIDDEN, true) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, sourceIndexMetadata.getNumberOfShards()) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + .put(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(), "-1") + .put(IndexMetadata.INDEX_DOWNSAMPLE_STATUS.getKey(), IndexMetadata.DownsampleTaskStatus.STARTED); + if (sourceIndexMetadata.getSettings().hasValue(MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING.getKey())) { + builder.put( + MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING.getKey(), + sourceIndexMetadata.getSettings().get(MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING.getKey()) + ); + } + CreateIndexClusterStateUpdateRequest createIndexClusterStateUpdateRequest = new CreateIndexClusterStateUpdateRequest( "rollup", rollupIndexName, rollupIndexName - ).settings( - /* - * When creating the rollup index, we copy the index.number_of_shards from source index, - * and we set the index.number_of_replicas to 0, to avoid replicating the index being built. - * Also, we set the index.refresh_interval to -1. - * We will set the correct number of replicas and refresh the index later. - * - * We should note that there is a risk of losing a node during the rollup process. In this - * case rollup will fail. - */ - Settings.builder() - .put(IndexMetadata.SETTING_INDEX_HIDDEN, true) - .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, sourceIndexMetadata.getNumberOfShards()) - .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) - .put(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(), "-1") - .put(IndexMetadata.INDEX_DOWNSAMPLE_STATUS.getKey(), IndexMetadata.DownsampleTaskStatus.STARTED) - .put( - MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING.getKey(), - sourceIndexMetadata.getSettings().get(MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING.getKey()) - ) - .build() - ).mappings(mapping); + ).settings(builder.build()).mappings(mapping); clusterService.submitStateUpdateTask("create-rollup-index [" + rollupIndexName + "]", new RollupClusterStateUpdateTask(listener) { @Override public ClusterState execute(ClusterState currentState) throws Exception { diff --git a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/downsample/DownsampleActionSingleNodeTests.java b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/downsample/DownsampleActionSingleNodeTests.java index f075a986334ee..f7078c5fba367 100644 --- a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/downsample/DownsampleActionSingleNodeTests.java +++ b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/downsample/DownsampleActionSingleNodeTests.java @@ -840,70 +840,73 @@ private void assertFieldMappings( } private void assertRollupIndexSettings(String sourceIndex, String rollupIndex, GetIndexResponse indexSettingsResp) { + Settings sourceSettings = indexSettingsResp.settings().get(sourceIndex); + Settings rollupSettings = indexSettingsResp.settings().get(rollupIndex); + // Assert rollup metadata are set in index settings - assertEquals("success", indexSettingsResp.getSetting(rollupIndex, IndexMetadata.INDEX_DOWNSAMPLE_STATUS_KEY)); + assertEquals("success", rollupSettings.get(IndexMetadata.INDEX_DOWNSAMPLE_STATUS_KEY)); - assertNotNull(indexSettingsResp.getSetting(sourceIndex, IndexMetadata.SETTING_INDEX_UUID)); - assertNotNull(indexSettingsResp.getSetting(rollupIndex, IndexMetadata.INDEX_DOWNSAMPLE_SOURCE_UUID_KEY)); + assertNotNull(sourceSettings.get(IndexMetadata.SETTING_INDEX_UUID)); + assertNotNull(rollupSettings.get(IndexMetadata.INDEX_DOWNSAMPLE_SOURCE_UUID_KEY)); assertEquals( - indexSettingsResp.getSetting(sourceIndex, IndexMetadata.SETTING_INDEX_UUID), - indexSettingsResp.getSetting(rollupIndex, IndexMetadata.INDEX_DOWNSAMPLE_SOURCE_UUID_KEY) + sourceSettings.get(IndexMetadata.SETTING_INDEX_UUID), + rollupSettings.get(IndexMetadata.INDEX_DOWNSAMPLE_SOURCE_UUID_KEY) ); - assertEquals(sourceIndex, indexSettingsResp.getSetting(rollupIndex, IndexMetadata.INDEX_DOWNSAMPLE_SOURCE_NAME_KEY)); - assertEquals( - indexSettingsResp.getSetting(sourceIndex, IndexSettings.MODE.getKey()), - indexSettingsResp.getSetting(rollupIndex, IndexSettings.MODE.getKey()) - ); + assertEquals(sourceIndex, rollupSettings.get(IndexMetadata.INDEX_DOWNSAMPLE_SOURCE_NAME_KEY)); + assertEquals(sourceSettings.get(IndexSettings.MODE.getKey()), rollupSettings.get(IndexSettings.MODE.getKey())); - assertNotNull(indexSettingsResp.getSetting(sourceIndex, IndexSettings.TIME_SERIES_START_TIME.getKey())); - assertNotNull(indexSettingsResp.getSetting(rollupIndex, IndexSettings.TIME_SERIES_START_TIME.getKey())); + assertNotNull(sourceSettings.get(IndexSettings.TIME_SERIES_START_TIME.getKey())); + assertNotNull(rollupSettings.get(IndexSettings.TIME_SERIES_START_TIME.getKey())); assertEquals( - indexSettingsResp.getSetting(sourceIndex, IndexSettings.TIME_SERIES_START_TIME.getKey()), - indexSettingsResp.getSetting(rollupIndex, IndexSettings.TIME_SERIES_START_TIME.getKey()) + sourceSettings.get(IndexSettings.TIME_SERIES_START_TIME.getKey()), + rollupSettings.get(IndexSettings.TIME_SERIES_START_TIME.getKey()) ); - assertNotNull(indexSettingsResp.getSetting(sourceIndex, IndexSettings.TIME_SERIES_END_TIME.getKey())); - assertNotNull(indexSettingsResp.getSetting(rollupIndex, IndexSettings.TIME_SERIES_END_TIME.getKey())); + assertNotNull(sourceSettings.get(IndexSettings.TIME_SERIES_END_TIME.getKey())); + assertNotNull(rollupSettings.get(IndexSettings.TIME_SERIES_END_TIME.getKey())); assertEquals( - indexSettingsResp.getSetting(sourceIndex, IndexSettings.TIME_SERIES_END_TIME.getKey()), - indexSettingsResp.getSetting(rollupIndex, IndexSettings.TIME_SERIES_END_TIME.getKey()) + sourceSettings.get(IndexSettings.TIME_SERIES_END_TIME.getKey()), + rollupSettings.get(IndexSettings.TIME_SERIES_END_TIME.getKey()) ); - assertNotNull(indexSettingsResp.getSetting(sourceIndex, IndexMetadata.INDEX_ROUTING_PATH.getKey())); - assertNotNull(indexSettingsResp.getSetting(rollupIndex, IndexMetadata.INDEX_ROUTING_PATH.getKey())); + assertNotNull(sourceSettings.get(IndexMetadata.INDEX_ROUTING_PATH.getKey())); + assertNotNull(rollupSettings.get(IndexMetadata.INDEX_ROUTING_PATH.getKey())); assertEquals( - indexSettingsResp.getSetting(sourceIndex, IndexMetadata.INDEX_ROUTING_PATH.getKey()), - indexSettingsResp.getSetting(rollupIndex, IndexMetadata.INDEX_ROUTING_PATH.getKey()) + sourceSettings.get(IndexMetadata.INDEX_ROUTING_PATH.getKey()), + rollupSettings.get(IndexMetadata.INDEX_ROUTING_PATH.getKey()) ); - assertNotNull(indexSettingsResp.getSetting(sourceIndex, IndexMetadata.SETTING_NUMBER_OF_SHARDS)); - assertNotNull(indexSettingsResp.getSetting(rollupIndex, IndexMetadata.SETTING_NUMBER_OF_SHARDS)); + assertNotNull(sourceSettings.get(IndexMetadata.SETTING_NUMBER_OF_SHARDS)); + assertNotNull(rollupSettings.get(IndexMetadata.SETTING_NUMBER_OF_SHARDS)); assertEquals( - indexSettingsResp.getSetting(sourceIndex, IndexMetadata.SETTING_NUMBER_OF_SHARDS), - indexSettingsResp.getSetting(rollupIndex, IndexMetadata.SETTING_NUMBER_OF_SHARDS) + sourceSettings.get(IndexMetadata.SETTING_NUMBER_OF_SHARDS), + rollupSettings.get(IndexMetadata.SETTING_NUMBER_OF_SHARDS) ); - assertNotNull(indexSettingsResp.getSetting(sourceIndex, IndexMetadata.SETTING_NUMBER_OF_REPLICAS)); - assertNotNull(indexSettingsResp.getSetting(rollupIndex, IndexMetadata.SETTING_NUMBER_OF_REPLICAS)); - assertEquals( - indexSettingsResp.getSetting(sourceIndex, IndexMetadata.SETTING_NUMBER_OF_REPLICAS), - indexSettingsResp.getSetting(rollupIndex, IndexMetadata.SETTING_NUMBER_OF_REPLICAS) - ); - assertEquals("true", indexSettingsResp.getSetting(rollupIndex, IndexMetadata.SETTING_BLOCKS_WRITE)); + assertNotNull(sourceSettings.get(IndexMetadata.SETTING_NUMBER_OF_REPLICAS)); + assertNotNull(rollupSettings.get(IndexMetadata.SETTING_NUMBER_OF_REPLICAS)); assertEquals( - indexSettingsResp.getSetting(sourceIndex, IndexMetadata.SETTING_INDEX_HIDDEN), - indexSettingsResp.getSetting(rollupIndex, IndexMetadata.SETTING_INDEX_HIDDEN) + sourceSettings.get(IndexMetadata.SETTING_NUMBER_OF_REPLICAS), + rollupSettings.get(IndexMetadata.SETTING_NUMBER_OF_REPLICAS) ); - if (indexSettingsResp.getSetting(sourceIndex, MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING.getKey()) != null) { + assertEquals("true", rollupSettings.get(IndexMetadata.SETTING_BLOCKS_WRITE)); + assertEquals(sourceSettings.get(IndexMetadata.SETTING_INDEX_HIDDEN), rollupSettings.get(IndexMetadata.SETTING_INDEX_HIDDEN)); + + if (sourceSettings.keySet().contains(MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING.getKey())) { assertNotNull(indexSettingsResp.getSetting(rollupIndex, MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING.getKey())); } - if (indexSettingsResp.getSetting(sourceIndex, MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING.getKey()) != null - && indexSettingsResp.getSetting(rollupIndex, MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING.getKey()) != null) + + if (sourceSettings.keySet().contains(MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING.getKey()) + && rollupSettings.keySet().contains(MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING.getKey())) { assertEquals( - indexSettingsResp.getSetting(sourceIndex, MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING.getKey()), - indexSettingsResp.getSetting(rollupIndex, MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING.getKey()) + sourceSettings.get(MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING.getKey()), + rollupSettings.get(MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING.getKey()) ); + } else { + assertFalse(sourceSettings.keySet().contains(MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING.getKey())); + assertFalse(rollupSettings.keySet().contains(MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING.getKey())); + } } private AggregationBuilder buildAggregations(