diff --git a/docs/maps/maps-aggregations.asciidoc b/docs/maps/maps-aggregations.asciidoc index 98aa21f6a07a3..627fd49dafa51 100644 --- a/docs/maps/maps-aggregations.asciidoc +++ b/docs/maps/maps-aggregations.asciidoc @@ -47,6 +47,7 @@ To enable top hits: . Set *Entity* to the field that identifies entities in your documents. This field will be used in the terms aggregation to group your documents into entity buckets. . Set *Documents per entity* to configure the maximum number of documents accumulated per entity. +This setting is limited to the `index.max_inner_result_window` index setting, which defaults to 100. [role="screenshot"] image::maps/images/top_hits.png[] diff --git a/docs/maps/vector-layer.asciidoc b/docs/maps/vector-layer.asciidoc index 1b9d0e6556f54..1d4ba9912529a 100644 --- a/docs/maps/vector-layer.asciidoc +++ b/docs/maps/vector-layer.asciidoc @@ -15,7 +15,7 @@ See map.regionmap.* in <> for details. *Documents*:: Vector data from a Kibana index pattern. The index must contain at least one field mapped as {ref}/geo-point.html[geo_point] or {ref}/geo-shape.html[geo_shape]. -NOTE: Document results are limited to the first 10000 matching documents. +NOTE: Document results are limited to the `index.max_result_window` index setting, which defaults to 10000. Use <> to plot large data sets. *Grid aggregation*:: Geospatial data grouped in grids with metrics for each gridded cell. diff --git a/x-pack/legacy/plugins/maps/common/constants.js b/x-pack/legacy/plugins/maps/common/constants.js index a6bdee3d2ed12..b97845a458d51 100644 --- a/x-pack/legacy/plugins/maps/common/constants.js +++ b/x-pack/legacy/plugins/maps/common/constants.js @@ -26,6 +26,7 @@ export const APP_ICON = 'gisApp'; export const MAP_APP_PATH = `app/${APP_ID}`; export const GIS_API_PATH = `api/${APP_ID}`; +export const INDEX_SETTINGS_API_PATH = `${GIS_API_PATH}/indexSettings`; export const MAP_BASE_URL = `/${MAP_APP_PATH}#/${MAP_SAVED_OBJECT_TYPE}`; @@ -69,7 +70,9 @@ export const MAX_ZOOM = 24; export const DECIMAL_DEGREES_PRECISION = 5; // meters precision export const ZOOM_PRECISION = 2; -export const ES_SIZE_LIMIT = 10000; +export const DEFAULT_MAX_RESULT_WINDOW = 10000; +export const DEFAULT_MAX_INNER_RESULT_WINDOW = 100; +export const DEFAULT_MAX_BUCKETS_LIMIT = 10000; export const FEATURE_ID_PROPERTY_NAME = '__kbn__feature_id__'; export const FEATURE_VISIBLE_PROPERTY_NAME = '__kbn_isvisibleduetojoin__'; diff --git a/x-pack/legacy/plugins/maps/public/layers/sources/client_file_source/geojson_file_source.js b/x-pack/legacy/plugins/maps/public/layers/sources/client_file_source/geojson_file_source.js index bd7f5f634ad20..a38669fcd1d1a 100644 --- a/x-pack/legacy/plugins/maps/public/layers/sources/client_file_source/geojson_file_source.js +++ b/x-pack/legacy/plugins/maps/public/layers/sources/client_file_source/geojson_file_source.js @@ -6,7 +6,11 @@ import { AbstractVectorSource } from '../vector_source'; import React from 'react'; -import { ES_GEO_FIELD_TYPE, GEOJSON_FILE, ES_SIZE_LIMIT } from '../../../../common/constants'; +import { + ES_GEO_FIELD_TYPE, + GEOJSON_FILE, + DEFAULT_MAX_RESULT_WINDOW, +} from '../../../../common/constants'; import { ClientFileCreateSourceEditor } from './create_client_file_source_editor'; import { ESSearchSource } from '../es_search_source'; import uuid from 'uuid/v4'; @@ -82,7 +86,7 @@ export class GeojsonFileSource extends AbstractVectorSource { addAndViewSource(null); } else { // Only turn on bounds filter for large doc counts - const filterByMapBounds = indexDataResp.docCount > ES_SIZE_LIMIT; + const filterByMapBounds = indexDataResp.docCount > DEFAULT_MAX_RESULT_WINDOW; const source = new ESSearchSource( { id: uuid(), diff --git a/x-pack/legacy/plugins/maps/public/layers/sources/es_search_source/create_source_editor.js b/x-pack/legacy/plugins/maps/public/layers/sources/es_search_source/create_source_editor.js index 11506d74505ba..28045eeb5e9b5 100644 --- a/x-pack/legacy/plugins/maps/public/layers/sources/es_search_source/create_source_editor.js +++ b/x-pack/legacy/plugins/maps/public/layers/sources/es_search_source/create_source_editor.js @@ -15,7 +15,11 @@ import { NoIndexPatternCallout } from '../../../components/no_index_pattern_call import { FormattedMessage } from '@kbn/i18n/react'; import { i18n } from '@kbn/i18n'; import { kfetch } from 'ui/kfetch'; -import { ES_GEO_FIELD_TYPE, GIS_API_PATH, ES_SIZE_LIMIT } from '../../../../common/constants'; +import { + ES_GEO_FIELD_TYPE, + GIS_API_PATH, + DEFAULT_MAX_RESULT_WINDOW, +} from '../../../../common/constants'; import { DEFAULT_FILTER_BY_MAP_BOUNDS } from './constants'; import { npStart } from 'ui/new_platform'; @@ -96,7 +100,7 @@ export class CreateSourceEditor extends Component { let indexHasSmallDocCount = false; try { const indexDocCount = await this.loadIndexDocCount(indexPattern.title); - indexHasSmallDocCount = indexDocCount <= ES_SIZE_LIMIT; + indexHasSmallDocCount = indexDocCount <= DEFAULT_MAX_RESULT_WINDOW; } catch (error) { // retrieving index count is a nice to have and is not essential // do not interrupt user flow if unable to retrieve count diff --git a/x-pack/legacy/plugins/maps/public/layers/sources/es_search_source/es_search_source.js b/x-pack/legacy/plugins/maps/public/layers/sources/es_search_source/es_search_source.js index d852332ac2f84..8ef4966e03c1b 100644 --- a/x-pack/legacy/plugins/maps/public/layers/sources/es_search_source/es_search_source.js +++ b/x-pack/legacy/plugins/maps/public/layers/sources/es_search_source/es_search_source.js @@ -17,12 +17,13 @@ import { UpdateSourceEditor } from './update_source_editor'; import { ES_SEARCH, ES_GEO_FIELD_TYPE, - ES_SIZE_LIMIT, + DEFAULT_MAX_BUCKETS_LIMIT, SORT_ORDER, } from '../../../../common/constants'; import { i18n } from '@kbn/i18n'; import { getDataSourceLabel } from '../../../../common/i18n_getters'; import { getSourceFields } from '../../../index_pattern_util'; +import { loadIndexSettings } from './load_index_settings'; import { DEFAULT_FILTER_BY_MAP_BOUNDS } from './constants'; import { ESDocField } from '../../fields/es_doc_field'; @@ -267,8 +268,8 @@ export class ESSearchSource extends AbstractESSource { entitySplit: { terms: { field: topHitsSplitField, - size: ES_SIZE_LIMIT, - shard_size: ES_SIZE_LIMIT, + size: DEFAULT_MAX_BUCKETS_LIMIT, + shard_size: DEFAULT_MAX_BUCKETS_LIMIT, }, aggs: { entityHits: { @@ -290,7 +291,7 @@ export class ESSearchSource extends AbstractESSource { const entityBuckets = _.get(resp, 'aggregations.entitySplit.buckets', []); const totalEntities = _.get(resp, 'aggregations.totalEntities.value', 0); // can not compare entityBuckets.length to totalEntities because totalEntities is an approximate - const areEntitiesTrimmed = entityBuckets.length >= ES_SIZE_LIMIT; + const areEntitiesTrimmed = entityBuckets.length >= DEFAULT_MAX_BUCKETS_LIMIT; let areTopHitsTrimmed = false; entityBuckets.forEach(entityBucket => { const total = _.get(entityBucket, 'entityHits.hits.total', 0); @@ -315,7 +316,7 @@ export class ESSearchSource extends AbstractESSource { // searchFilters.fieldNames contains geo field and any fields needed for styling features // Performs Elasticsearch search request being careful to pull back only required fields to minimize response size - async _getSearchHits(layerName, searchFilters, registerCancelCallback) { + async _getSearchHits(layerName, searchFilters, maxResultWindow, registerCancelCallback) { const initialSearchContext = { docvalue_fields: await this._getDateDocvalueFields(searchFilters.fieldNames), }; @@ -331,7 +332,7 @@ export class ESSearchSource extends AbstractESSource { ); searchSource = await this._makeSearchSource( searchFilters, - ES_SIZE_LIMIT, + maxResultWindow, initialSearchContext ); searchSource.setField('source', false); // do not need anything from _source @@ -340,7 +341,7 @@ export class ESSearchSource extends AbstractESSource { // geo_shape fields do not support docvalue_fields yet, so still have to be pulled from _source searchSource = await this._makeSearchSource( searchFilters, - ES_SIZE_LIMIT, + maxResultWindow, initialSearchContext ); // Setting "fields" instead of "source: { includes: []}" @@ -382,11 +383,19 @@ export class ESSearchSource extends AbstractESSource { } async getGeoJsonWithMeta(layerName, searchFilters, registerCancelCallback) { + const indexPattern = await this.getIndexPattern(); + + const indexSettings = await loadIndexSettings(indexPattern.title); + const { hits, meta } = this._isTopHits() ? await this._getTopHits(layerName, searchFilters, registerCancelCallback) - : await this._getSearchHits(layerName, searchFilters, registerCancelCallback); + : await this._getSearchHits( + layerName, + searchFilters, + indexSettings.maxResultWindow, + registerCancelCallback + ); - const indexPattern = await this.getIndexPattern(); const unusedMetaFields = indexPattern.metaFields.filter(metaField => { return !['_id', '_index'].includes(metaField); }); diff --git a/x-pack/legacy/plugins/maps/public/layers/sources/es_search_source/load_index_settings.js b/x-pack/legacy/plugins/maps/public/layers/sources/es_search_source/load_index_settings.js new file mode 100644 index 0000000000000..1a58b5b073b08 --- /dev/null +++ b/x-pack/legacy/plugins/maps/public/layers/sources/es_search_source/load_index_settings.js @@ -0,0 +1,58 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { + DEFAULT_MAX_RESULT_WINDOW, + DEFAULT_MAX_INNER_RESULT_WINDOW, + INDEX_SETTINGS_API_PATH, +} from '../../../../common/constants'; +import { kfetch } from 'ui/kfetch'; +import { toastNotifications } from 'ui/notify'; +import { i18n } from '@kbn/i18n'; + +let toastDisplayed = false; +const indexSettings = new Map(); + +export async function loadIndexSettings(indexPatternTitle) { + if (indexSettings.has(indexPatternTitle)) { + return indexSettings.get(indexPatternTitle); + } + + const fetchPromise = fetchIndexSettings(indexPatternTitle); + indexSettings.set(indexPatternTitle, fetchPromise); + return fetchPromise; +} + +async function fetchIndexSettings(indexPatternTitle) { + try { + const indexSettings = await kfetch({ + pathname: `../${INDEX_SETTINGS_API_PATH}`, + query: { + indexPatternTitle, + }, + }); + return indexSettings; + } catch (err) { + const warningMsg = i18n.translate('xpack.maps.indexSettings.fetchErrorMsg', { + defaultMessage: `Unable to fetch index settings for index pattern '{indexPatternTitle}'. + Ensure you have '{viewIndexMetaRole}' role.`, + values: { + indexPatternTitle, + viewIndexMetaRole: 'view_index_metadata', + }, + }); + if (!toastDisplayed) { + // Only show toast for first failure to avoid flooding user with warnings + toastDisplayed = true; + toastNotifications.addWarning(warningMsg); + } + console.warn(warningMsg); + return { + maxResultWindow: DEFAULT_MAX_RESULT_WINDOW, + maxInnerResultWindow: DEFAULT_MAX_INNER_RESULT_WINDOW, + }; + } +} diff --git a/x-pack/legacy/plugins/maps/public/layers/sources/es_search_source/update_source_editor.js b/x-pack/legacy/plugins/maps/public/layers/sources/es_search_source/update_source_editor.js index 595e1d0c755cf..4503856829ef2 100644 --- a/x-pack/legacy/plugins/maps/public/layers/sources/es_search_source/update_source_editor.js +++ b/x-pack/legacy/plugins/maps/public/layers/sources/es_search_source/update_source_editor.js @@ -22,9 +22,10 @@ import { indexPatternService } from '../../../kibana_services'; import { i18n } from '@kbn/i18n'; import { getTermsFields, getSourceFields } from '../../../index_pattern_util'; import { ValidatedRange } from '../../../components/validated_range'; -import { SORT_ORDER } from '../../../../common/constants'; +import { DEFAULT_MAX_INNER_RESULT_WINDOW, SORT_ORDER } from '../../../../common/constants'; import { ESDocField } from '../../fields/es_doc_field'; import { FormattedMessage } from '@kbn/i18n/react'; +import { loadIndexSettings } from './load_index_settings'; export class UpdateSourceEditor extends Component { static propTypes = { @@ -43,17 +44,31 @@ export class UpdateSourceEditor extends Component { sourceFields: null, termFields: null, sortFields: null, + maxInnerResultWindow: DEFAULT_MAX_INNER_RESULT_WINDOW, }; componentDidMount() { this._isMounted = true; this.loadFields(); + this.loadIndexSettings(); } componentWillUnmount() { this._isMounted = false; } + async loadIndexSettings() { + try { + const indexPattern = await indexPatternService.get(this.props.indexPatternId); + const { maxInnerResultWindow } = await loadIndexSettings(indexPattern.title); + if (this._isMounted) { + this.setState({ maxInnerResultWindow }); + } + } catch (err) { + return; + } + } + async loadFields() { let indexPattern; try { @@ -149,7 +164,7 @@ export class UpdateSourceEditor extends Component { > ({})); +jest.mock('./load_index_settings', () => ({ + loadIndexSettings: async () => { + return { maxInnerResultWindow: 100 }; + }, +})); + import React from 'react'; import { shallow } from 'enzyme'; diff --git a/x-pack/legacy/plugins/maps/public/layers/sources/es_term_source.js b/x-pack/legacy/plugins/maps/public/layers/sources/es_term_source.js index 42d07b8d45acd..6e8b2e1e7c6f5 100644 --- a/x-pack/legacy/plugins/maps/public/layers/sources/es_term_source.js +++ b/x-pack/legacy/plugins/maps/public/layers/sources/es_term_source.js @@ -9,7 +9,7 @@ import _ from 'lodash'; import { Schemas } from 'ui/vis/editors/default/schemas'; import { AggConfigs } from 'ui/agg_types'; import { i18n } from '@kbn/i18n'; -import { ES_SIZE_LIMIT, FIELD_ORIGIN, METRIC_TYPE } from '../../../common/constants'; +import { DEFAULT_MAX_BUCKETS_LIMIT, FIELD_ORIGIN, METRIC_TYPE } from '../../../common/constants'; import { ESDocField } from '../fields/es_doc_field'; import { AbstractESAggSource } from './es_agg_source'; @@ -170,7 +170,7 @@ export class ESTermSource extends AbstractESAggSource { schema: 'segment', params: { field: this._termField.getName(), - size: ES_SIZE_LIMIT, + size: DEFAULT_MAX_BUCKETS_LIMIT, }, }, ]; diff --git a/x-pack/legacy/plugins/maps/public/layers/vector_layer.js b/x-pack/legacy/plugins/maps/public/layers/vector_layer.js index 6ebc1b3d95250..dd9a1b7a14c10 100644 --- a/x-pack/legacy/plugins/maps/public/layers/vector_layer.js +++ b/x-pack/legacy/plugins/maps/public/layers/vector_layer.js @@ -373,7 +373,6 @@ export class VectorLayer extends AbstractLayer { const requestToken = Symbol(`layer-${this.getId()}-${SOURCE_DATA_ID_ORIGIN}`); const searchFilters = this._getSearchFilters(dataFilters); const prevDataRequest = this.getSourceDataRequest(); - const canSkipFetch = await canSkipSourceUpdate({ source: this._source, prevDataRequest, diff --git a/x-pack/legacy/plugins/maps/server/lib/get_index_pattern_settings.js b/x-pack/legacy/plugins/maps/server/lib/get_index_pattern_settings.js new file mode 100644 index 0000000000000..c5522b7ba21c5 --- /dev/null +++ b/x-pack/legacy/plugins/maps/server/lib/get_index_pattern_settings.js @@ -0,0 +1,30 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import _ from 'lodash'; +import { DEFAULT_MAX_RESULT_WINDOW, DEFAULT_MAX_INNER_RESULT_WINDOW } from '../../common/constants'; + +export function getIndexPatternSettings(indicesSettingsResp) { + let maxResultWindow = Infinity; + let maxInnerResultWindow = Infinity; + Object.values(indicesSettingsResp).forEach(indexSettings => { + const indexMaxResultWindow = _.get( + indexSettings, + 'settings.index.max_result_window', + DEFAULT_MAX_RESULT_WINDOW + ); + maxResultWindow = Math.min(maxResultWindow, indexMaxResultWindow); + + const indexMaxInnerResultWindow = _.get( + indexSettings, + 'settings.index.max_inner_result_window', + DEFAULT_MAX_INNER_RESULT_WINDOW + ); + maxInnerResultWindow = Math.min(indexMaxInnerResultWindow, indexMaxResultWindow); + }); + + return { maxResultWindow, maxInnerResultWindow }; +} diff --git a/x-pack/legacy/plugins/maps/server/lib/get_index_pattern_settings.test.js b/x-pack/legacy/plugins/maps/server/lib/get_index_pattern_settings.test.js new file mode 100644 index 0000000000000..01a1ba2703cba --- /dev/null +++ b/x-pack/legacy/plugins/maps/server/lib/get_index_pattern_settings.test.js @@ -0,0 +1,86 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { getIndexPatternSettings } from './get_index_pattern_settings'; +import { DEFAULT_MAX_RESULT_WINDOW, DEFAULT_MAX_INNER_RESULT_WINDOW } from '../../common/constants'; + +describe('max_result_window and max_inner_result_window are not set', () => { + test('Should provide default values when values not set', () => { + const indicesSettingsResp = { + kibana_sample_data_logs: { + settings: { + index: {}, + }, + }, + }; + const { maxResultWindow, maxInnerResultWindow } = getIndexPatternSettings(indicesSettingsResp); + expect(maxResultWindow).toBe(DEFAULT_MAX_RESULT_WINDOW); + expect(maxInnerResultWindow).toBe(DEFAULT_MAX_INNER_RESULT_WINDOW); + }); + + test('Should include default values when providing minimum values for indices in index pattern', () => { + const indicesSettingsResp = { + kibana_sample_data_logs: { + settings: { + index: { + max_result_window: '15000', + max_inner_result_window: '200', + }, + }, + }, + kibana_sample_data_flights: { + settings: { + index: {}, + }, + }, + }; + const { maxResultWindow, maxInnerResultWindow } = getIndexPatternSettings(indicesSettingsResp); + expect(maxResultWindow).toBe(DEFAULT_MAX_RESULT_WINDOW); + expect(maxInnerResultWindow).toBe(DEFAULT_MAX_INNER_RESULT_WINDOW); + }); +}); + +describe('max_result_window and max_inner_result_window are set', () => { + test('Should provide values from settings', () => { + const indicesSettingsResp = { + kibana_sample_data_logs: { + settings: { + index: { + max_result_window: '15000', // value is returned as string API + max_inner_result_window: '200', + }, + }, + }, + }; + const { maxResultWindow, maxInnerResultWindow } = getIndexPatternSettings(indicesSettingsResp); + expect(maxResultWindow).toBe(15000); + expect(maxInnerResultWindow).toBe(200); + }); + + test('Should provide minimum values for indices in index pattern', () => { + const indicesSettingsResp = { + kibana_sample_data_logs: { + settings: { + index: { + max_result_window: '15000', + max_inner_result_window: '200', + }, + }, + }, + kibana_sample_data_flights: { + settings: { + index: { + max_result_window: '7000', + max_inner_result_window: '75', + }, + }, + }, + }; + const { maxResultWindow, maxInnerResultWindow } = getIndexPatternSettings(indicesSettingsResp); + expect(maxResultWindow).toBe(7000); + expect(maxInnerResultWindow).toBe(75); + }); +}); diff --git a/x-pack/legacy/plugins/maps/server/routes.js b/x-pack/legacy/plugins/maps/server/routes.js index e8c8c23ae2ca1..5e9cd3cfa87bd 100644 --- a/x-pack/legacy/plugins/maps/server/routes.js +++ b/x-pack/legacy/plugins/maps/server/routes.js @@ -17,10 +17,12 @@ import { EMS_TILES_VECTOR_TILE_PATH, GIS_API_PATH, EMS_SPRITES_PATH, + INDEX_SETTINGS_API_PATH, } from '../common/constants'; import { EMSClient } from '@elastic/ems-client'; import fetch from 'node-fetch'; import { i18n } from '@kbn/i18n'; +import { getIndexPatternSettings } from './lib/get_index_pattern_settings'; import Boom from 'boom'; @@ -414,6 +416,33 @@ export function initRoutes(server, licenseUid) { }, }); + server.route({ + method: 'GET', + path: `/${INDEX_SETTINGS_API_PATH}`, + handler: async (request, h) => { + const { server, query } = request; + + if (!query.indexPatternTitle) { + server.log('warning', `Required query parameter 'indexPatternTitle' not provided.`); + return h.response().code(400); + } + + const { callWithRequest } = server.plugins.elasticsearch.getCluster('data'); + try { + const resp = await callWithRequest(request, 'indices.getSettings', { + index: query.indexPatternTitle, + }); + return getIndexPatternSettings(resp); + } catch (error) { + server.log( + 'warning', + `Cannot load index settings for index pattern '${query.indexPatternTitle}', error: ${error.message}.` + ); + return h.response().code(400); + } + }, + }); + function checkEMSProxyConfig() { if (!mapConfig.proxyElasticMapsServiceInMaps) { server.log( diff --git a/x-pack/test/api_integration/apis/maps/index.js b/x-pack/test/api_integration/apis/maps/index.js index d2fe97abbe5c5..cba042e3a9c29 100644 --- a/x-pack/test/api_integration/apis/maps/index.js +++ b/x-pack/test/api_integration/apis/maps/index.js @@ -4,8 +4,17 @@ * you may not use this file except in compliance with the Elastic License. */ -export default function({ loadTestFile }) { +export default function({ loadTestFile, getService }) { + const esArchiver = getService('esArchiver'); + describe('Maps endpoints', () => { - loadTestFile(require.resolve('./migrations')); + before(async () => { + await esArchiver.loadIfNeeded('logstash_functional'); + }); + + describe('', () => { + loadTestFile(require.resolve('./index_settings')); + loadTestFile(require.resolve('./migrations')); + }); }); } diff --git a/x-pack/test/api_integration/apis/maps/index_settings.js b/x-pack/test/api_integration/apis/maps/index_settings.js new file mode 100644 index 0000000000000..3ec4a4d9f5deb --- /dev/null +++ b/x-pack/test/api_integration/apis/maps/index_settings.js @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import expect from '@kbn/expect'; + +export default function({ getService }) { + const supertest = getService('supertest'); + + describe('index settings', () => { + it('should return index settings', async () => { + const resp = await supertest + .get(`/api/maps/indexSettings?indexPatternTitle=logstash*`) + .set('kbn-xsrf', 'kibana') + .expect(200); + + expect(resp.body.maxResultWindow).to.be(10000); + expect(resp.body.maxInnerResultWindow).to.be(100); + }); + }); +}