Skip to content

Commit

Permalink
Merge branch 'main' into rule-footer-link
Browse files Browse the repository at this point in the history
  • Loading branch information
christineweng authored Oct 10, 2024
2 parents 0744a96 + 3ec1908 commit a87c687
Show file tree
Hide file tree
Showing 24 changed files with 385 additions and 211 deletions.
6 changes: 4 additions & 2 deletions .buildkite/ftr_oblt_serverless_configs.yml
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,10 @@ disabled:
- x-pack/test_serverless/functional/test_suites/observability/cypress/config_headless.ts
- x-pack/test_serverless/functional/test_suites/observability/cypress/config_runner.ts

# serverless config files that run deployment-agnostic tests
# Failing https://github.com/elastic/kibana/issues/195811
- x-pack/test/api_integration/deployment_agnostic/configs/serverless/oblt.serverless.config.ts

defaultQueue: 'n2-4-spot'
enabled:
- x-pack/test_serverless/api_integration/test_suites/observability/config.ts
Expand All @@ -25,5 +29,3 @@ enabled:
- x-pack/test_serverless/functional/test_suites/observability/common_configs/config.group5.ts
- x-pack/test_serverless/functional/test_suites/observability/common_configs/config.group6.ts
- x-pack/test_serverless/functional/test_suites/observability/config.screenshots.ts
# serverless config files that run deployment-agnostic tests
- x-pack/test/api_integration/deployment_agnostic/configs/serverless/oblt.serverless.config.ts
6 changes: 4 additions & 2 deletions .buildkite/ftr_search_serverless_configs.yml
Original file line number Diff line number Diff line change
@@ -1,6 +1,10 @@
disabled:
# Base config files, only necessary to inform config finding script

# serverless config files that run deployment-agnostic tests
# Failing https://github.com/elastic/kibana/issues/195811
- x-pack/test/api_integration/deployment_agnostic/configs/serverless/search.serverless.config.ts

defaultQueue: 'n2-4-spot'
enabled:
- x-pack/test_serverless/api_integration/test_suites/search/config.ts
Expand All @@ -18,5 +22,3 @@ enabled:
- x-pack/test_serverless/functional/test_suites/search/common_configs/config.group4.ts
- x-pack/test_serverless/functional/test_suites/search/common_configs/config.group5.ts
- x-pack/test_serverless/functional/test_suites/search/common_configs/config.group6.ts
# serverless config files that run deployment-agnostic tests
- x-pack/test/api_integration/deployment_agnostic/configs/serverless/search.serverless.config.ts
6 changes: 4 additions & 2 deletions .buildkite/ftr_security_serverless_configs.yml
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,10 @@ disabled:
- x-pack/test_serverless/functional/config.base.ts
- x-pack/test_serverless/shared/config.base.ts

# serverless config files that run deployment-agnostic tests
# Failing https://github.com/elastic/kibana/issues/195811
- x-pack/test/api_integration/deployment_agnostic/configs/serverless/security.serverless.config.ts

defaultQueue: 'n2-4-spot'
enabled:
- x-pack/test_serverless/api_integration/test_suites/security/config.ts
Expand Down Expand Up @@ -100,5 +104,3 @@ enabled:
- x-pack/test/security_solution_api_integration/test_suites/edr_workflows/response_actions/trial_license_complete_tier/configs/serverless.config.ts
- x-pack/test/security_solution_endpoint/configs/serverless.endpoint.config.ts
- x-pack/test/security_solution_endpoint/configs/serverless.integrations.config.ts
# serverless config files that run deployment-agnostic tests
- x-pack/test/api_integration/deployment_agnostic/configs/serverless/security.serverless.config.ts
45 changes: 31 additions & 14 deletions docs/maps/connect-to-ems.asciidoc
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
:ems: Elastic Maps Service
:ems-docker-repo: docker.elastic.co/elastic-maps-service/elastic-maps-server
:ems-docker-image: {ems-docker-repo}:{version}-amd64
:ems-docker-image: {ems-docker-repo}:{version}
:ems-headers-url: https://deployment-host

[[maps-connect-to-ems]]
Expand Down Expand Up @@ -81,34 +81,53 @@ If you cannot connect to {ems} from the {kib} server or browser clients, and you

{hosted-ems} is a self-managed version of {ems} offered as a Docker image that provides both the EMS basemaps and EMS boundaries. The image is bundled with basemaps up to zoom level 8. After connecting it to your {es} cluster for license validation, you have the option to download and configure a more detailed basemaps database.

You can use +docker pull+ to download the {hosted-ems} image from the Elastic Docker registry.

. Pull the {hosted-ems} Docker image.
+
ifeval::["{release-state}"=="unreleased"]
Version {version} of {hosted-ems} has not yet been released, so no Docker image is currently available for this version.
WARNING: Version {version} of {hosted-ems} has not yet been released.
No Docker image is currently available for this version.
endif::[]

ifeval::["{release-state}"!="unreleased"]

+
["source","bash",subs="attributes"]
----------------------------------
docker pull {ems-docker-image}
----------------------------------

Start {hosted-ems} and expose the default port `8080`:
. Optional: Install
https://docs.sigstore.dev/system_config/installation/[Cosign] for your
environment. Then use Cosign to verify the {es} image's signature.
+
[source,sh,subs="attributes"]
----
wget https://artifacts.elastic.co/cosign.pub
cosign verify --key cosign.pub {ems-docker-image}
----
+
The `cosign` command prints the check results and the signature payload in JSON format:
+
[source,sh,subs="attributes"]
--------------------------------------------
Verification for {ems-docker-image} --
The following checks were performed on each of these signatures:
- The cosign claims were validated
- Existence of the claims in the transparency log was verified offline
- The signatures were verified against the specified public key
--------------------------------------------


. Start {hosted-ems} and expose the default port `8080`:
+
["source","bash",subs="attributes"]
----------------------------------
docker run --rm --init --publish 8080:8080 \
{ems-docker-image}
----------------------------------

+
Once {hosted-ems} is running, follow instructions from the webpage at `localhost:8080` to define a configuration file and optionally download a more detailed basemaps database.

+
[role="screenshot"]
image::images/elastic-maps-server-instructions.png[Set-up instructions]

endif::[]

[float]
[[elastic-maps-server-configuration]]
==== Configuration
Expand Down Expand Up @@ -193,7 +212,6 @@ One way to configure {hosted-ems} is to provide `elastic-maps-server.yml` via bi

["source","yaml",subs="attributes"]
--------------------------------------------
version: '2'
services:
ems-server:
image: {ems-docker-image}
Expand All @@ -212,7 +230,6 @@ These variables can be set with +docker-compose+ like this:

["source","yaml",subs="attributes"]
----------------------------------------------------------
version: '2'
services:
ems-server:
image: {ems-docker-image}
Expand Down
Binary file modified docs/maps/images/elastic-maps-server-instructions.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
1 change: 1 addition & 0 deletions packages/kbn-esql-utils/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ export {
isQueryWrappedByPipes,
retrieveMetadataColumns,
getQueryColumnsFromESQLQuery,
isESQLColumnSortable,
TextBasedLanguages,
} from './src';

Expand Down
1 change: 1 addition & 0 deletions packages/kbn-esql-utils/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -31,3 +31,4 @@ export {
getStartEndParams,
hasStartEndParams,
} from './utils/run_query';
export { isESQLColumnSortable } from './utils/esql_fields_utils';
66 changes: 66 additions & 0 deletions packages/kbn-esql-utils/src/utils/esql_fields_utils.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the "Elastic License
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
* Public License v 1"; you may not use this file except in compliance with, at
* your election, the "Elastic License 2.0", the "GNU Affero General Public
* License v3.0 only", or the "Server Side Public License, v 1".
*/
import type { DatatableColumn } from '@kbn/expressions-plugin/common';
import { isESQLColumnSortable } from './esql_fields_utils';

describe('esql fields helpers', () => {
describe('isESQLColumnSortable', () => {
it('returns false for geo fields', () => {
const geoField = {
id: 'geo.coordinates',
name: 'geo.coordinates',
meta: {
type: 'geo_point',
esType: 'geo_point',
},
isNull: false,
} as DatatableColumn;
expect(isESQLColumnSortable(geoField)).toBeFalsy();
});

it('returns false for source fields', () => {
const sourceField = {
id: '_source',
name: '_source',
meta: {
type: '_source',
esType: '_source',
},
isNull: false,
} as DatatableColumn;
expect(isESQLColumnSortable(sourceField)).toBeFalsy();
});

it('returns false for counter fields', () => {
const tsdbField = {
id: 'tsbd_counter',
name: 'tsbd_counter',
meta: {
type: 'number',
esType: 'counter_long',
},
isNull: false,
} as DatatableColumn;
expect(isESQLColumnSortable(tsdbField)).toBeFalsy();
});

it('returns true for everything else', () => {
const keywordField = {
id: 'sortable',
name: 'sortable',
meta: {
type: 'string',
esType: 'keyword',
},
isNull: false,
} as DatatableColumn;
expect(isESQLColumnSortable(keywordField)).toBeTruthy();
});
});
});
40 changes: 40 additions & 0 deletions packages/kbn-esql-utils/src/utils/esql_fields_utils.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the "Elastic License
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
* Public License v 1"; you may not use this file except in compliance with, at
* your election, the "Elastic License 2.0", the "GNU Affero General Public
* License v3.0 only", or the "Server Side Public License, v 1".
*/

import type { DatatableColumn } from '@kbn/expressions-plugin/common';

const SPATIAL_FIELDS = ['geo_point', 'geo_shape', 'point', 'shape'];
const SOURCE_FIELD = '_source';
const TSDB_COUNTER_FIELDS_PREFIX = 'counter_';

/**
* Check if a column is sortable.
*
* @param column The DatatableColumn of the field.
* @returns True if the column is sortable, false otherwise.
*/

export const isESQLColumnSortable = (column: DatatableColumn): boolean => {
// We don't allow sorting on spatial fields
if (SPATIAL_FIELDS.includes(column.meta?.type)) {
return false;
}

// we don't allow sorting on the _source field
if (column.meta?.type === SOURCE_FIELD) {
return false;
}

// we don't allow sorting on tsdb counter fields
if (column.meta?.esType && column.meta?.esType?.indexOf(TSDB_COUNTER_FIELDS_PREFIX) !== -1) {
return false;
}

return true;
};
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import { DataViewField } from '@kbn/data-views-plugin/common';
import { deepMockedFields, buildDataViewMock } from '@kbn/discover-utils/src/__mocks__';
import { allSuggestionsMock } from '../__mocks__/suggestions';
import { getLensVisMock } from '../__mocks__/lens_vis';
import { convertDatatableColumnToDataViewFieldSpec } from '@kbn/data-view-utils';
import { UnifiedHistogramSuggestionType } from '../types';

describe('LensVisService suggestions', () => {
Expand Down Expand Up @@ -198,6 +199,11 @@ describe('LensVisService suggestions', () => {
});

test('should return histogramSuggestion if no suggestions returned by the api with the breakdown field if it is given', async () => {
const breakdown = convertDatatableColumnToDataViewFieldSpec({
name: 'var0',
id: 'var0',
meta: { type: 'number' },
});
const lensVis = await getLensVisMock({
filters: [],
query: { esql: 'from the-data-view | limit 100' },
Expand All @@ -207,7 +213,7 @@ describe('LensVisService suggestions', () => {
from: '2023-09-03T08:00:00.000Z',
to: '2023-09-04T08:56:28.274Z',
},
breakdownField: { name: 'var0' } as DataViewField,
breakdownField: breakdown as DataViewField,
columns: [
{
id: 'var0',
Expand Down Expand Up @@ -247,4 +253,54 @@ describe('LensVisService suggestions', () => {

expect(lensVis.visContext?.attributes.state.query).toStrictEqual(histogramQuery);
});

test('should return histogramSuggestion if no suggestions returned by the api with a geo point breakdown field correctly', async () => {
const lensVis = await getLensVisMock({
filters: [],
query: { esql: 'from the-data-view | limit 100' },
dataView: dataViewMock,
timeInterval: 'auto',
timeRange: {
from: '2023-09-03T08:00:00.000Z',
to: '2023-09-04T08:56:28.274Z',
},
breakdownField: { name: 'coordinates' } as DataViewField,
columns: [
{
id: 'coordinates',
name: 'coordinates',
meta: {
type: 'geo_point',
},
},
],
isPlainRecord: true,
allSuggestions: [],
hasHistogramSuggestionForESQL: true,
});

expect(lensVis.currentSuggestionContext?.type).toBe(
UnifiedHistogramSuggestionType.histogramForESQL
);
expect(lensVis.currentSuggestionContext?.suggestion).toBeDefined();
expect(lensVis.currentSuggestionContext?.suggestion?.visualizationState).toHaveProperty(
'layers',
[
{
layerId: '662552df-2cdc-4539-bf3b-73b9f827252c',
seriesType: 'bar_stacked',
xAccessor: '@timestamp every 30 second',
accessors: ['results'],
layerType: 'data',
},
]
);

const histogramQuery = {
esql: `from the-data-view | limit 100
| EVAL timestamp=DATE_TRUNC(30 minute, @timestamp) | stats results = count(*) by timestamp, \`coordinates\` | rename timestamp as \`@timestamp every 30 minute\``,
};

expect(lensVis.visContext?.attributes.state.query).toStrictEqual(histogramQuery);
});
});
19 changes: 14 additions & 5 deletions src/plugins/unified_histogram/public/services/lens_vis_service.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,11 @@

import { BehaviorSubject, distinctUntilChanged, map, Observable } from 'rxjs';
import { isEqual } from 'lodash';
import { removeDropCommandsFromESQLQuery, appendToESQLQuery } from '@kbn/esql-utils';
import {
removeDropCommandsFromESQLQuery,
appendToESQLQuery,
isESQLColumnSortable,
} from '@kbn/esql-utils';
import type { DataView, DataViewField } from '@kbn/data-views-plugin/common';
import type {
CountIndexPatternColumn,
Expand Down Expand Up @@ -553,12 +557,17 @@ export class LensVisService {
const queryInterval = interval ?? computeInterval(timeRange, this.services.data);
const language = getAggregateQueryMode(query);
const safeQuery = removeDropCommandsFromESQLQuery(query[language]);
const breakdown = breakdownColumn
? `, \`${breakdownColumn.name}\` | sort \`${breakdownColumn.name}\` asc`
: '';
const breakdown = breakdownColumn ? `, \`${breakdownColumn.name}\`` : '';

// sort by breakdown column if it's sortable
const sortBy =
breakdownColumn && isESQLColumnSortable(breakdownColumn)
? ` | sort \`${breakdownColumn.name}\` asc`
: '';

return appendToESQLQuery(
safeQuery,
`| EVAL timestamp=DATE_TRUNC(${queryInterval}, ${dataView.timeFieldName}) | stats results = count(*) by timestamp${breakdown} | rename timestamp as \`${dataView.timeFieldName} every ${queryInterval}\``
`| EVAL timestamp=DATE_TRUNC(${queryInterval}, ${dataView.timeFieldName}) | stats results = count(*) by timestamp${breakdown}${sortBy} | rename timestamp as \`${dataView.timeFieldName} every ${queryInterval}\``
);
};

Expand Down
Loading

0 comments on commit a87c687

Please sign in to comment.