Skip to content

Commit

Permalink
Downsampling: remove null value setting (#90025) (#90385)
Browse files Browse the repository at this point in the history 
Do not copy the total fields limit setting into the downsampled
index if the value is null in the source index.

(cherry picked from commit d430bff)

Co-authored-by: weizijun <[email protected]>
  • Loading branch information
@salvatore-campagna @weizijun
salvatore-campagna and weizijun authored Sep 29, 2022
1 parent e26a827 commit ff7a809
Show file tree
Hide file tree
Showing 2 changed files with 66 additions and 62 deletions.
45 changes: 23 additions & 22 deletions ...plugin/rollup/src/main/java/org/elasticsearch/xpack/downsample/TransportRollupAction.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -594,32 +594,33 @@ private void createRollupIndex(
DownsampleAction.Request request,
ActionListener<AcknowledgedResponse> listener
) {
/*
* When creating the rollup index, we copy the index.number_of_shards from source index,
* and we set the index.number_of_replicas to 0, to avoid replicating the index being built.
* Also, we set the index.refresh_interval to -1.
* We will set the correct number of replicas and refresh the index later.
*
* We should note that there is a risk of losing a node during the rollup process. In this
* case rollup will fail.
*/
Settings.Builder builder = Settings.builder()
.put(IndexMetadata.SETTING_INDEX_HIDDEN, true)
.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, sourceIndexMetadata.getNumberOfShards())
.put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
.put(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(), "-1")
.put(IndexMetadata.INDEX_DOWNSAMPLE_STATUS.getKey(), IndexMetadata.DownsampleTaskStatus.STARTED);
if (sourceIndexMetadata.getSettings().hasValue(MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING.getKey())) {
builder.put(
MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING.getKey(),
sourceIndexMetadata.getSettings().get(MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING.getKey())
);
}

CreateIndexClusterStateUpdateRequest createIndexClusterStateUpdateRequest = new CreateIndexClusterStateUpdateRequest(
"rollup",
rollupIndexName,
rollupIndexName
).settings(
/*
* When creating the rollup index, we copy the index.number_of_shards from source index,
* and we set the index.number_of_replicas to 0, to avoid replicating the index being built.
* Also, we set the index.refresh_interval to -1.
* We will set the correct number of replicas and refresh the index later.
*
* We should note that there is a risk of losing a node during the rollup process. In this
* case rollup will fail.
*/
Settings.builder()
.put(IndexMetadata.SETTING_INDEX_HIDDEN, true)
.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, sourceIndexMetadata.getNumberOfShards())
.put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
.put(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(), "-1")
.put(IndexMetadata.INDEX_DOWNSAMPLE_STATUS.getKey(), IndexMetadata.DownsampleTaskStatus.STARTED)
.put(
MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING.getKey(),
sourceIndexMetadata.getSettings().get(MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING.getKey())
)
.build()
).mappings(mapping);
).settings(builder.build()).mappings(mapping);
clusterService.submitStateUpdateTask("create-rollup-index [" + rollupIndexName + "]", new RollupClusterStateUpdateTask(listener) {
@Override
public ClusterState execute(ClusterState currentState) throws Exception {
Expand Down
83 changes: 43 additions & 40 deletions ...lup/src/test/java/org/elasticsearch/xpack/downsample/DownsampleActionSingleNodeTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -840,70 +840,73 @@ private void assertFieldMappings(
}

private void assertRollupIndexSettings(String sourceIndex, String rollupIndex, GetIndexResponse indexSettingsResp) {
Settings sourceSettings = indexSettingsResp.settings().get(sourceIndex);
Settings rollupSettings = indexSettingsResp.settings().get(rollupIndex);

// Assert rollup metadata are set in index settings
assertEquals("success", indexSettingsResp.getSetting(rollupIndex, IndexMetadata.INDEX_DOWNSAMPLE_STATUS_KEY));
assertEquals("success", rollupSettings.get(IndexMetadata.INDEX_DOWNSAMPLE_STATUS_KEY));

assertNotNull(indexSettingsResp.getSetting(sourceIndex, IndexMetadata.SETTING_INDEX_UUID));
assertNotNull(indexSettingsResp.getSetting(rollupIndex, IndexMetadata.INDEX_DOWNSAMPLE_SOURCE_UUID_KEY));
assertNotNull(sourceSettings.get(IndexMetadata.SETTING_INDEX_UUID));
assertNotNull(rollupSettings.get(IndexMetadata.INDEX_DOWNSAMPLE_SOURCE_UUID_KEY));
assertEquals(
indexSettingsResp.getSetting(sourceIndex, IndexMetadata.SETTING_INDEX_UUID),
indexSettingsResp.getSetting(rollupIndex, IndexMetadata.INDEX_DOWNSAMPLE_SOURCE_UUID_KEY)
sourceSettings.get(IndexMetadata.SETTING_INDEX_UUID),
rollupSettings.get(IndexMetadata.INDEX_DOWNSAMPLE_SOURCE_UUID_KEY)
);

assertEquals(sourceIndex, indexSettingsResp.getSetting(rollupIndex, IndexMetadata.INDEX_DOWNSAMPLE_SOURCE_NAME_KEY));
assertEquals(
indexSettingsResp.getSetting(sourceIndex, IndexSettings.MODE.getKey()),
indexSettingsResp.getSetting(rollupIndex, IndexSettings.MODE.getKey())
);
assertEquals(sourceIndex, rollupSettings.get(IndexMetadata.INDEX_DOWNSAMPLE_SOURCE_NAME_KEY));
assertEquals(sourceSettings.get(IndexSettings.MODE.getKey()), rollupSettings.get(IndexSettings.MODE.getKey()));

assertNotNull(indexSettingsResp.getSetting(sourceIndex, IndexSettings.TIME_SERIES_START_TIME.getKey()));
assertNotNull(indexSettingsResp.getSetting(rollupIndex, IndexSettings.TIME_SERIES_START_TIME.getKey()));
assertNotNull(sourceSettings.get(IndexSettings.TIME_SERIES_START_TIME.getKey()));
assertNotNull(rollupSettings.get(IndexSettings.TIME_SERIES_START_TIME.getKey()));
assertEquals(
indexSettingsResp.getSetting(sourceIndex, IndexSettings.TIME_SERIES_START_TIME.getKey()),
indexSettingsResp.getSetting(rollupIndex, IndexSettings.TIME_SERIES_START_TIME.getKey())
sourceSettings.get(IndexSettings.TIME_SERIES_START_TIME.getKey()),
rollupSettings.get(IndexSettings.TIME_SERIES_START_TIME.getKey())
);

assertNotNull(indexSettingsResp.getSetting(sourceIndex, IndexSettings.TIME_SERIES_END_TIME.getKey()));
assertNotNull(indexSettingsResp.getSetting(rollupIndex, IndexSettings.TIME_SERIES_END_TIME.getKey()));
assertNotNull(sourceSettings.get(IndexSettings.TIME_SERIES_END_TIME.getKey()));
assertNotNull(rollupSettings.get(IndexSettings.TIME_SERIES_END_TIME.getKey()));
assertEquals(
indexSettingsResp.getSetting(sourceIndex, IndexSettings.TIME_SERIES_END_TIME.getKey()),
indexSettingsResp.getSetting(rollupIndex, IndexSettings.TIME_SERIES_END_TIME.getKey())
sourceSettings.get(IndexSettings.TIME_SERIES_END_TIME.getKey()),
rollupSettings.get(IndexSettings.TIME_SERIES_END_TIME.getKey())
);
assertNotNull(indexSettingsResp.getSetting(sourceIndex, IndexMetadata.INDEX_ROUTING_PATH.getKey()));
assertNotNull(indexSettingsResp.getSetting(rollupIndex, IndexMetadata.INDEX_ROUTING_PATH.getKey()));
assertNotNull(sourceSettings.get(IndexMetadata.INDEX_ROUTING_PATH.getKey()));
assertNotNull(rollupSettings.get(IndexMetadata.INDEX_ROUTING_PATH.getKey()));
assertEquals(
indexSettingsResp.getSetting(sourceIndex, IndexMetadata.INDEX_ROUTING_PATH.getKey()),
indexSettingsResp.getSetting(rollupIndex, IndexMetadata.INDEX_ROUTING_PATH.getKey())
sourceSettings.get(IndexMetadata.INDEX_ROUTING_PATH.getKey()),
rollupSettings.get(IndexMetadata.INDEX_ROUTING_PATH.getKey())
);

assertNotNull(indexSettingsResp.getSetting(sourceIndex, IndexMetadata.SETTING_NUMBER_OF_SHARDS));
assertNotNull(indexSettingsResp.getSetting(rollupIndex, IndexMetadata.SETTING_NUMBER_OF_SHARDS));
assertNotNull(sourceSettings.get(IndexMetadata.SETTING_NUMBER_OF_SHARDS));
assertNotNull(rollupSettings.get(IndexMetadata.SETTING_NUMBER_OF_SHARDS));
assertEquals(
indexSettingsResp.getSetting(sourceIndex, IndexMetadata.SETTING_NUMBER_OF_SHARDS),
indexSettingsResp.getSetting(rollupIndex, IndexMetadata.SETTING_NUMBER_OF_SHARDS)
sourceSettings.get(IndexMetadata.SETTING_NUMBER_OF_SHARDS),
rollupSettings.get(IndexMetadata.SETTING_NUMBER_OF_SHARDS)
);

assertNotNull(indexSettingsResp.getSetting(sourceIndex, IndexMetadata.SETTING_NUMBER_OF_REPLICAS));
assertNotNull(indexSettingsResp.getSetting(rollupIndex, IndexMetadata.SETTING_NUMBER_OF_REPLICAS));
assertEquals(
indexSettingsResp.getSetting(sourceIndex, IndexMetadata.SETTING_NUMBER_OF_REPLICAS),
indexSettingsResp.getSetting(rollupIndex, IndexMetadata.SETTING_NUMBER_OF_REPLICAS)
);
assertEquals("true", indexSettingsResp.getSetting(rollupIndex, IndexMetadata.SETTING_BLOCKS_WRITE));
assertNotNull(sourceSettings.get(IndexMetadata.SETTING_NUMBER_OF_REPLICAS));
assertNotNull(rollupSettings.get(IndexMetadata.SETTING_NUMBER_OF_REPLICAS));
assertEquals(
indexSettingsResp.getSetting(sourceIndex, IndexMetadata.SETTING_INDEX_HIDDEN),
indexSettingsResp.getSetting(rollupIndex, IndexMetadata.SETTING_INDEX_HIDDEN)
sourceSettings.get(IndexMetadata.SETTING_NUMBER_OF_REPLICAS),
rollupSettings.get(IndexMetadata.SETTING_NUMBER_OF_REPLICAS)
);

if (indexSettingsResp.getSetting(sourceIndex, MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING.getKey()) != null) {
assertEquals("true", rollupSettings.get(IndexMetadata.SETTING_BLOCKS_WRITE));
assertEquals(sourceSettings.get(IndexMetadata.SETTING_INDEX_HIDDEN), rollupSettings.get(IndexMetadata.SETTING_INDEX_HIDDEN));

if (sourceSettings.keySet().contains(MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING.getKey())) {
assertNotNull(indexSettingsResp.getSetting(rollupIndex, MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING.getKey()));
}
if (indexSettingsResp.getSetting(sourceIndex, MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING.getKey()) != null
&& indexSettingsResp.getSetting(rollupIndex, MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING.getKey()) != null)

if (sourceSettings.keySet().contains(MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING.getKey())
&& rollupSettings.keySet().contains(MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING.getKey())) {
assertEquals(
indexSettingsResp.getSetting(sourceIndex, MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING.getKey()),
indexSettingsResp.getSetting(rollupIndex, MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING.getKey())
sourceSettings.get(MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING.getKey()),
rollupSettings.get(MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING.getKey())
);
} else {
assertFalse(sourceSettings.keySet().contains(MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING.getKey()));
assertFalse(rollupSettings.keySet().contains(MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING.getKey()));
}
}

private AggregationBuilder buildAggregations(
Expand Down

0 comments on commit ff7a809

Please sign in to comment.