diff --git a/.buildkite/ftr_oblt_serverless_configs.yml b/.buildkite/ftr_oblt_serverless_configs.yml index fbf0406f37be4..75909e7c21c46 100644 --- a/.buildkite/ftr_oblt_serverless_configs.yml +++ b/.buildkite/ftr_oblt_serverless_configs.yml @@ -6,6 +6,10 @@ disabled: - x-pack/test_serverless/functional/test_suites/observability/cypress/config_headless.ts - x-pack/test_serverless/functional/test_suites/observability/cypress/config_runner.ts + # serverless config files that run deployment-agnostic tests + # Failing https://github.com/elastic/kibana/issues/195811 + - x-pack/test/api_integration/deployment_agnostic/configs/serverless/oblt.serverless.config.ts + defaultQueue: 'n2-4-spot' enabled: - x-pack/test_serverless/api_integration/test_suites/observability/config.ts @@ -25,5 +29,3 @@ enabled: - x-pack/test_serverless/functional/test_suites/observability/common_configs/config.group5.ts - x-pack/test_serverless/functional/test_suites/observability/common_configs/config.group6.ts - x-pack/test_serverless/functional/test_suites/observability/config.screenshots.ts - # serverless config files that run deployment-agnostic tests - - x-pack/test/api_integration/deployment_agnostic/configs/serverless/oblt.serverless.config.ts diff --git a/.buildkite/ftr_search_serverless_configs.yml b/.buildkite/ftr_search_serverless_configs.yml index e6efee5860806..413558bffa0fe 100644 --- a/.buildkite/ftr_search_serverless_configs.yml +++ b/.buildkite/ftr_search_serverless_configs.yml @@ -1,6 +1,10 @@ disabled: # Base config files, only necessary to inform config finding script + # serverless config files that run deployment-agnostic tests + # Failing https://github.com/elastic/kibana/issues/195811 + - x-pack/test/api_integration/deployment_agnostic/configs/serverless/search.serverless.config.ts + defaultQueue: 'n2-4-spot' enabled: - x-pack/test_serverless/api_integration/test_suites/search/config.ts @@ -18,5 +22,3 @@ enabled: - x-pack/test_serverless/functional/test_suites/search/common_configs/config.group4.ts - x-pack/test_serverless/functional/test_suites/search/common_configs/config.group5.ts - x-pack/test_serverless/functional/test_suites/search/common_configs/config.group6.ts - # serverless config files that run deployment-agnostic tests - - x-pack/test/api_integration/deployment_agnostic/configs/serverless/search.serverless.config.ts diff --git a/.buildkite/ftr_security_serverless_configs.yml b/.buildkite/ftr_security_serverless_configs.yml index 6d42c030b2d4f..caf9fcc5ac92a 100644 --- a/.buildkite/ftr_security_serverless_configs.yml +++ b/.buildkite/ftr_security_serverless_configs.yml @@ -20,6 +20,10 @@ disabled: - x-pack/test_serverless/functional/config.base.ts - x-pack/test_serverless/shared/config.base.ts + # serverless config files that run deployment-agnostic tests + # Failing https://github.com/elastic/kibana/issues/195811 + - x-pack/test/api_integration/deployment_agnostic/configs/serverless/security.serverless.config.ts + defaultQueue: 'n2-4-spot' enabled: - x-pack/test_serverless/api_integration/test_suites/security/config.ts @@ -100,5 +104,3 @@ enabled: - x-pack/test/security_solution_api_integration/test_suites/edr_workflows/response_actions/trial_license_complete_tier/configs/serverless.config.ts - x-pack/test/security_solution_endpoint/configs/serverless.endpoint.config.ts - x-pack/test/security_solution_endpoint/configs/serverless.integrations.config.ts - # serverless config files that run deployment-agnostic tests - - x-pack/test/api_integration/deployment_agnostic/configs/serverless/security.serverless.config.ts diff --git a/packages/kbn-esql-utils/index.ts b/packages/kbn-esql-utils/index.ts index 223181f2bd154..333557964d873 100644 --- a/packages/kbn-esql-utils/index.ts +++ b/packages/kbn-esql-utils/index.ts @@ -29,6 +29,7 @@ export { isQueryWrappedByPipes, retrieveMetadataColumns, getQueryColumnsFromESQLQuery, + isESQLColumnSortable, TextBasedLanguages, } from './src'; diff --git a/packages/kbn-esql-utils/src/index.ts b/packages/kbn-esql-utils/src/index.ts index e36283c7a9238..3b3228e7a2a4a 100644 --- a/packages/kbn-esql-utils/src/index.ts +++ b/packages/kbn-esql-utils/src/index.ts @@ -31,3 +31,4 @@ export { getStartEndParams, hasStartEndParams, } from './utils/run_query'; +export { isESQLColumnSortable } from './utils/esql_fields_utils'; diff --git a/packages/kbn-esql-utils/src/utils/esql_fields_utils.test.ts b/packages/kbn-esql-utils/src/utils/esql_fields_utils.test.ts new file mode 100644 index 0000000000000..ef8a24e686bd6 --- /dev/null +++ b/packages/kbn-esql-utils/src/utils/esql_fields_utils.test.ts @@ -0,0 +1,66 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ +import type { DatatableColumn } from '@kbn/expressions-plugin/common'; +import { isESQLColumnSortable } from './esql_fields_utils'; + +describe('esql fields helpers', () => { + describe('isESQLColumnSortable', () => { + it('returns false for geo fields', () => { + const geoField = { + id: 'geo.coordinates', + name: 'geo.coordinates', + meta: { + type: 'geo_point', + esType: 'geo_point', + }, + isNull: false, + } as DatatableColumn; + expect(isESQLColumnSortable(geoField)).toBeFalsy(); + }); + + it('returns false for source fields', () => { + const sourceField = { + id: '_source', + name: '_source', + meta: { + type: '_source', + esType: '_source', + }, + isNull: false, + } as DatatableColumn; + expect(isESQLColumnSortable(sourceField)).toBeFalsy(); + }); + + it('returns false for counter fields', () => { + const tsdbField = { + id: 'tsbd_counter', + name: 'tsbd_counter', + meta: { + type: 'number', + esType: 'counter_long', + }, + isNull: false, + } as DatatableColumn; + expect(isESQLColumnSortable(tsdbField)).toBeFalsy(); + }); + + it('returns true for everything else', () => { + const keywordField = { + id: 'sortable', + name: 'sortable', + meta: { + type: 'string', + esType: 'keyword', + }, + isNull: false, + } as DatatableColumn; + expect(isESQLColumnSortable(keywordField)).toBeTruthy(); + }); + }); +}); diff --git a/packages/kbn-esql-utils/src/utils/esql_fields_utils.ts b/packages/kbn-esql-utils/src/utils/esql_fields_utils.ts new file mode 100644 index 0000000000000..f5a0fe7b81340 --- /dev/null +++ b/packages/kbn-esql-utils/src/utils/esql_fields_utils.ts @@ -0,0 +1,40 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +import type { DatatableColumn } from '@kbn/expressions-plugin/common'; + +const SPATIAL_FIELDS = ['geo_point', 'geo_shape', 'point', 'shape']; +const SOURCE_FIELD = '_source'; +const TSDB_COUNTER_FIELDS_PREFIX = 'counter_'; + +/** + * Check if a column is sortable. + * + * @param column The DatatableColumn of the field. + * @returns True if the column is sortable, false otherwise. + */ + +export const isESQLColumnSortable = (column: DatatableColumn): boolean => { + // We don't allow sorting on spatial fields + if (SPATIAL_FIELDS.includes(column.meta?.type)) { + return false; + } + + // we don't allow sorting on the _source field + if (column.meta?.type === SOURCE_FIELD) { + return false; + } + + // we don't allow sorting on tsdb counter fields + if (column.meta?.esType && column.meta?.esType?.indexOf(TSDB_COUNTER_FIELDS_PREFIX) !== -1) { + return false; + } + + return true; +}; diff --git a/src/plugins/unified_histogram/public/services/lens_vis_service.suggestions.test.ts b/src/plugins/unified_histogram/public/services/lens_vis_service.suggestions.test.ts index 28819f7a5c54b..1719adebe7a49 100644 --- a/src/plugins/unified_histogram/public/services/lens_vis_service.suggestions.test.ts +++ b/src/plugins/unified_histogram/public/services/lens_vis_service.suggestions.test.ts @@ -12,6 +12,7 @@ import { DataViewField } from '@kbn/data-views-plugin/common'; import { deepMockedFields, buildDataViewMock } from '@kbn/discover-utils/src/__mocks__'; import { allSuggestionsMock } from '../__mocks__/suggestions'; import { getLensVisMock } from '../__mocks__/lens_vis'; +import { convertDatatableColumnToDataViewFieldSpec } from '@kbn/data-view-utils'; import { UnifiedHistogramSuggestionType } from '../types'; describe('LensVisService suggestions', () => { @@ -198,6 +199,11 @@ describe('LensVisService suggestions', () => { }); test('should return histogramSuggestion if no suggestions returned by the api with the breakdown field if it is given', async () => { + const breakdown = convertDatatableColumnToDataViewFieldSpec({ + name: 'var0', + id: 'var0', + meta: { type: 'number' }, + }); const lensVis = await getLensVisMock({ filters: [], query: { esql: 'from the-data-view | limit 100' }, @@ -207,7 +213,7 @@ describe('LensVisService suggestions', () => { from: '2023-09-03T08:00:00.000Z', to: '2023-09-04T08:56:28.274Z', }, - breakdownField: { name: 'var0' } as DataViewField, + breakdownField: breakdown as DataViewField, columns: [ { id: 'var0', @@ -247,4 +253,54 @@ describe('LensVisService suggestions', () => { expect(lensVis.visContext?.attributes.state.query).toStrictEqual(histogramQuery); }); + + test('should return histogramSuggestion if no suggestions returned by the api with a geo point breakdown field correctly', async () => { + const lensVis = await getLensVisMock({ + filters: [], + query: { esql: 'from the-data-view | limit 100' }, + dataView: dataViewMock, + timeInterval: 'auto', + timeRange: { + from: '2023-09-03T08:00:00.000Z', + to: '2023-09-04T08:56:28.274Z', + }, + breakdownField: { name: 'coordinates' } as DataViewField, + columns: [ + { + id: 'coordinates', + name: 'coordinates', + meta: { + type: 'geo_point', + }, + }, + ], + isPlainRecord: true, + allSuggestions: [], + hasHistogramSuggestionForESQL: true, + }); + + expect(lensVis.currentSuggestionContext?.type).toBe( + UnifiedHistogramSuggestionType.histogramForESQL + ); + expect(lensVis.currentSuggestionContext?.suggestion).toBeDefined(); + expect(lensVis.currentSuggestionContext?.suggestion?.visualizationState).toHaveProperty( + 'layers', + [ + { + layerId: '662552df-2cdc-4539-bf3b-73b9f827252c', + seriesType: 'bar_stacked', + xAccessor: '@timestamp every 30 second', + accessors: ['results'], + layerType: 'data', + }, + ] + ); + + const histogramQuery = { + esql: `from the-data-view | limit 100 +| EVAL timestamp=DATE_TRUNC(30 minute, @timestamp) | stats results = count(*) by timestamp, \`coordinates\` | rename timestamp as \`@timestamp every 30 minute\``, + }; + + expect(lensVis.visContext?.attributes.state.query).toStrictEqual(histogramQuery); + }); }); diff --git a/src/plugins/unified_histogram/public/services/lens_vis_service.ts b/src/plugins/unified_histogram/public/services/lens_vis_service.ts index eccfd663b2557..25bb8be6f6242 100644 --- a/src/plugins/unified_histogram/public/services/lens_vis_service.ts +++ b/src/plugins/unified_histogram/public/services/lens_vis_service.ts @@ -9,7 +9,11 @@ import { BehaviorSubject, distinctUntilChanged, map, Observable } from 'rxjs'; import { isEqual } from 'lodash'; -import { removeDropCommandsFromESQLQuery, appendToESQLQuery } from '@kbn/esql-utils'; +import { + removeDropCommandsFromESQLQuery, + appendToESQLQuery, + isESQLColumnSortable, +} from '@kbn/esql-utils'; import type { DataView, DataViewField } from '@kbn/data-views-plugin/common'; import type { CountIndexPatternColumn, @@ -553,12 +557,17 @@ export class LensVisService { const queryInterval = interval ?? computeInterval(timeRange, this.services.data); const language = getAggregateQueryMode(query); const safeQuery = removeDropCommandsFromESQLQuery(query[language]); - const breakdown = breakdownColumn - ? `, \`${breakdownColumn.name}\` | sort \`${breakdownColumn.name}\` asc` - : ''; + const breakdown = breakdownColumn ? `, \`${breakdownColumn.name}\`` : ''; + + // sort by breakdown column if it's sortable + const sortBy = + breakdownColumn && isESQLColumnSortable(breakdownColumn) + ? ` | sort \`${breakdownColumn.name}\` asc` + : ''; + return appendToESQLQuery( safeQuery, - `| EVAL timestamp=DATE_TRUNC(${queryInterval}, ${dataView.timeFieldName}) | stats results = count(*) by timestamp${breakdown} | rename timestamp as \`${dataView.timeFieldName} every ${queryInterval}\`` + `| EVAL timestamp=DATE_TRUNC(${queryInterval}, ${dataView.timeFieldName}) | stats results = count(*) by timestamp${breakdown}${sortBy} | rename timestamp as \`${dataView.timeFieldName} every ${queryInterval}\`` ); }; diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_creation/esql_rule.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_creation/esql_rule.cy.ts index 0045a79ff4394..64423a921e595 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_creation/esql_rule.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_creation/esql_rule.cy.ts @@ -67,7 +67,8 @@ const workaroundForResizeObserver = () => } }); -describe( +// Failing: See https://github.com/elastic/kibana/issues/184558 +describe.skip( 'Detection ES|QL rules, creation', { tags: ['@ess', '@serverless', '@skipInServerlessMKI'], diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/investigations/threat_intelligence/indicators.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/investigations/threat_intelligence/indicators.cy.ts index ebfc5d4e9a0cb..f485ead495949 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/investigations/threat_intelligence/indicators.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/investigations/threat_intelligence/indicators.cy.ts @@ -67,7 +67,8 @@ const URL = '/app/security/threat_intelligence/indicators'; const URL_WITH_CONTRADICTORY_FILTERS = '/app/security/threat_intelligence/indicators?indicators=(filterQuery:(language:kuery,query:%27%27),filters:!((%27$state%27:(store:appState),meta:(alias:!n,disabled:!f,index:%27%27,key:threat.indicator.type,negate:!f,params:(query:file),type:phrase),query:(match_phrase:(threat.indicator.type:file))),(%27$state%27:(store:appState),meta:(alias:!n,disabled:!f,index:%27%27,key:threat.indicator.type,negate:!f,params:(query:url),type:phrase),query:(match_phrase:(threat.indicator.type:url)))),timeRange:(from:now/d,to:now/d))'; -describe('Single indicator', { tags: ['@ess'] }, () => { +// Failing: See https://github.com/elastic/kibana/issues/195804 +describe.skip('Single indicator', { tags: ['@ess'] }, () => { before(() => cy.task('esArchiverLoad', { archiveName: 'ti_indicators_data_single' })); after(() => cy.task('esArchiverUnload', { archiveName: 'ti_indicators_data_single' }));