From 39b36b471e3304a19839cd03e31f6b1d00548905 Mon Sep 17 00:00:00 2001 From: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> Date: Tue, 26 Sep 2023 10:42:57 -0400 Subject: [PATCH 01/12] [8.10] [Fleet] Increase package install max timeout + add concurrency control to rollovers (#166775) (#167184) # Backport This will backport the following commits from `main` to `8.10`: - [[Fleet] Increase package install max timeout + add concurrency control to rollovers (#166775)](https://github.com/elastic/kibana/pull/166775) ### Questions ? Please refer to the [Backport tool documentation](https://github.com/sqren/backport) --------- Co-authored-by: Kyle Pollich Co-authored-by: Julia Bardi <90178898+juliaElastic@users.noreply.github.com> --- x-pack/plugins/fleet/common/constants/epm.ts | 2 +- .../epm/elasticsearch/template/template.ts | 25 +++- .../epm/packages/_install_package.test.ts | 141 +++++++++++++++++- .../services/epm/packages/_install_package.ts | 32 ++-- 4 files changed, 176 insertions(+), 24 deletions(-) diff --git a/x-pack/plugins/fleet/common/constants/epm.ts b/x-pack/plugins/fleet/common/constants/epm.ts index 52cb24271afa5..3548fee93fbf2 100644 --- a/x-pack/plugins/fleet/common/constants/epm.ts +++ b/x-pack/plugins/fleet/common/constants/epm.ts @@ -9,7 +9,7 @@ import { ElasticsearchAssetType, KibanaAssetType } from '../types/models'; export const PACKAGES_SAVED_OBJECT_TYPE = 'epm-packages'; export const ASSETS_SAVED_OBJECT_TYPE = 'epm-packages-assets'; -export const MAX_TIME_COMPLETE_INSTALL = 60000; +export const MAX_TIME_COMPLETE_INSTALL = 30 * 60 * 1000; // 30 minutes export const FLEET_SYSTEM_PACKAGE = 'system'; export const FLEET_ELASTIC_AGENT_PACKAGE = 'elastic_agent'; diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts index 3e9363fa9828f..5682749d7e381 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts @@ -11,6 +11,8 @@ import type { MappingTypeMapping, } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; +import pMap from 'p-map'; + import type { Field, Fields } from '../../fields/field'; import type { RegistryDataStream, @@ -729,15 +731,22 @@ const updateAllDataStreams = async ( esClient: ElasticsearchClient, logger: Logger ): Promise => { - const updatedataStreamPromises = indexNameWithTemplates.map((templateEntry) => { - return updateExistingDataStream({ - esClient, - logger, - dataStreamName: templateEntry.dataStreamName, - }); - }); - await Promise.all(updatedataStreamPromises); + await pMap( + indexNameWithTemplates, + (templateEntry) => { + return updateExistingDataStream({ + esClient, + logger, + dataStreamName: templateEntry.dataStreamName, + }); + }, + { + // Limit concurrent putMapping/rollover requests to avoid overhwhelming ES cluster + concurrency: 20, + } + ); }; + const updateExistingDataStream = async ({ dataStreamName, esClient, diff --git a/x-pack/plugins/fleet/server/services/epm/packages/_install_package.test.ts b/x-pack/plugins/fleet/server/services/epm/packages/_install_package.test.ts index 96582ff554190..850164f51787d 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/_install_package.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/_install_package.test.ts @@ -5,11 +5,21 @@ * 2.0. */ -import type { SavedObjectsClientContract, ElasticsearchClient } from '@kbn/core/server'; +import type { + SavedObjectsClientContract, + ElasticsearchClient, + SavedObject, +} from '@kbn/core/server'; import { savedObjectsClientMock, elasticsearchServiceMock } from '@kbn/core/server/mocks'; import { loggerMock } from '@kbn/logging-mocks'; import { DEFAULT_SPACE_ID } from '@kbn/spaces-plugin/common/constants'; +import { ConcurrentInstallOperationError } from '../../../errors'; + +import type { Installation } from '../../../../common'; + +import { PACKAGES_SAVED_OBJECT_TYPE } from '../../../../common'; + import { appContextService } from '../../app_context'; import { createAppContextStartContractMock } from '../../../mocks'; import { saveArchiveEntries } from '../archive/storage'; @@ -29,7 +39,9 @@ jest.mock('../elasticsearch/datastream_ilm/install'); import { updateCurrentWriteIndices } from '../elasticsearch/template/template'; import { installKibanaAssetsAndReferences } from '../kibana/assets/install'; -import { installIndexTemplatesAndPipelines } from './install'; +import { MAX_TIME_COMPLETE_INSTALL } from '../../../../common/constants'; + +import { installIndexTemplatesAndPipelines, restartInstallation } from './install'; import { _installPackage } from './_install_package'; @@ -69,9 +81,7 @@ describe('_installPackage', () => { jest.mocked(saveArchiveEntries).mockResolvedValue({ saved_objects: [], }); - }); - afterEach(async () => { - appContextService.stop(); + jest.mocked(restartInstallation).mockReset(); }); it('handles errors from installKibanaAssets', async () => { // force errors from this function @@ -220,4 +230,125 @@ describe('_installPackage', () => { expect(installILMPolicy).toBeCalled(); expect(installIlmForDataStream).toBeCalled(); }); + + describe('when package is stuck in `installing`', () => { + afterEach(() => {}); + const mockInstalledPackageSo: SavedObject = { + id: 'mocked-package', + attributes: { + name: 'test-package', + version: '1.0.0', + install_status: 'installing', + install_version: '1.0.0', + install_started_at: new Date().toISOString(), + install_source: 'registry', + verification_status: 'verified', + installed_kibana: [] as any, + installed_es: [] as any, + es_index_patterns: {}, + }, + type: PACKAGES_SAVED_OBJECT_TYPE, + references: [], + }; + + beforeEach(() => { + appContextService.start( + createAppContextStartContractMock({ + internal: { + disableILMPolicies: true, + disableProxies: false, + fleetServerStandalone: false, + onlyAllowAgentUpgradeToKnownVersions: false, + capabilities: [], + }, + }) + ); + }); + + describe('timeout reached', () => { + it('restarts installation', async () => { + await _installPackage({ + savedObjectsClient: soClient, + // @ts-ignore + savedObjectsImporter: jest.fn(), + esClient, + logger: loggerMock.create(), + paths: [], + packageInfo: { + name: mockInstalledPackageSo.attributes.name, + version: mockInstalledPackageSo.attributes.version, + title: mockInstalledPackageSo.attributes.name, + } as any, + installedPkg: { + ...mockInstalledPackageSo, + attributes: { + ...mockInstalledPackageSo.attributes, + install_started_at: new Date( + Date.now() - MAX_TIME_COMPLETE_INSTALL * 2 + ).toISOString(), + }, + }, + }); + + expect(restartInstallation).toBeCalled(); + }); + }); + + describe('timeout not reached', () => { + describe('force flag not provided', () => { + it('throws concurrent installation error if force flag is not provided', async () => { + expect( + _installPackage({ + savedObjectsClient: soClient, + // @ts-ignore + savedObjectsImporter: jest.fn(), + esClient, + logger: loggerMock.create(), + paths: [], + packageInfo: { + name: mockInstalledPackageSo.attributes.name, + version: mockInstalledPackageSo.attributes.version, + title: mockInstalledPackageSo.attributes.name, + } as any, + installedPkg: { + ...mockInstalledPackageSo, + attributes: { + ...mockInstalledPackageSo.attributes, + install_started_at: new Date(Date.now() - 1000).toISOString(), + }, + }, + }) + ).rejects.toThrowError(ConcurrentInstallOperationError); + }); + }); + + describe('force flag provided', () => { + it('restarts installation', async () => { + await _installPackage({ + savedObjectsClient: soClient, + // @ts-ignore + savedObjectsImporter: jest.fn(), + esClient, + logger: loggerMock.create(), + paths: [], + packageInfo: { + name: mockInstalledPackageSo.attributes.name, + version: mockInstalledPackageSo.attributes.version, + title: mockInstalledPackageSo.attributes.name, + } as any, + installedPkg: { + ...mockInstalledPackageSo, + attributes: { + ...mockInstalledPackageSo.attributes, + install_started_at: new Date(Date.now() - 1000).toISOString(), + }, + }, + force: true, + }); + + expect(restartInstallation).toBeCalled(); + }); + }); + }); + }); }); diff --git a/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts b/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts index 337cf59bbd613..3bfc74bf68968 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts @@ -99,18 +99,30 @@ export async function _installPackage({ try { // if some installation already exists if (installedPkg) { + const isStatusInstalling = installedPkg.attributes.install_status === 'installing'; + const hasExceededTimeout = + Date.now() - Date.parse(installedPkg.attributes.install_started_at) < + MAX_TIME_COMPLETE_INSTALL; + // if the installation is currently running, don't try to install // instead, only return already installed assets - if ( - installedPkg.attributes.install_status === 'installing' && - Date.now() - Date.parse(installedPkg.attributes.install_started_at) < - MAX_TIME_COMPLETE_INSTALL - ) { - throw new ConcurrentInstallOperationError( - `Concurrent installation or upgrade of ${pkgName || 'unknown'}-${ - pkgVersion || 'unknown' - } detected, aborting.` - ); + if (isStatusInstalling && hasExceededTimeout) { + // If this is a forced installation, ignore the timeout and restart the installation anyway + if (force) { + await restartInstallation({ + savedObjectsClient, + pkgName, + pkgVersion, + installSource, + verificationResult, + }); + } else { + throw new ConcurrentInstallOperationError( + `Concurrent installation or upgrade of ${pkgName || 'unknown'}-${ + pkgVersion || 'unknown' + } detected, aborting.` + ); + } } else { // if no installation is running, or the installation has been running longer than MAX_TIME_COMPLETE_INSTALL // (it might be stuck) update the saved object and proceed From 7cf6622556cb5ba70d037fb8d90c7932fab5a27f Mon Sep 17 00:00:00 2001 From: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> Date: Tue, 26 Sep 2023 13:34:54 -0400 Subject: [PATCH 02/12] [8.10] Gracefully handle 404s for documents and mappings (#167101) (#167311) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit # Backport This will backport the following commits from `main` to `8.10`: - [Gracefully handle 404s for documents and mappings (#167101)](https://github.com/elastic/kibana/pull/167101) ### Questions ? Please refer to the [Backport tool documentation](https://github.com/sqren/backport) Co-authored-by: Efe Gürkan YALAMAN --- .../components/search_index/documents.tsx | 27 +++++++++++++++-- .../search_index/index_mappings.tsx | 30 ++++++++++++++++--- 2 files changed, 50 insertions(+), 7 deletions(-) diff --git a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/documents.tsx b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/documents.tsx index b82305b4c4d04..e197d0551253e 100644 --- a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/documents.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/documents.tsx @@ -10,6 +10,7 @@ import React, { useEffect, useState, ChangeEvent } from 'react'; import { useActions, useValues } from 'kea'; import { + EuiCallOut, EuiFieldSearch, EuiFlexGroup, EuiFlexItem, @@ -73,7 +74,7 @@ export const SearchIndexDocuments: React.FC = () => { const { makeRequest: getDocuments } = useActions(documentLogic); const { makeRequest: getMappings } = useActions(mappingLogic); - const { data, status } = useValues(documentLogic); + const { data, status, error } = useValues(documentLogic); const { data: mappingData, status: mappingStatus } = useValues(mappingLogic); const docs = data?.results?.hits.hits ?? []; @@ -83,6 +84,8 @@ export const SearchIndexDocuments: React.FC = () => { const shouldShowAccessControlSwitcher = hasDocumentLevelSecurityFeature && productFeatures.hasDocumentLevelSecurityEnabled; + const isAccessControlIndexNotFound = + shouldShowAccessControlSwitcher && error?.body?.statusCode === 404; useEffect(() => { getDocuments({ @@ -140,11 +143,29 @@ export const SearchIndexDocuments: React.FC = () => { - {docs.length === 0 && + {isAccessControlIndexNotFound && ( + +

+ {i18n.translate('xpack.enterpriseSearch.content.searchIndex.documents.noIndex', { + defaultMessage: + "An Access Control Index won't be created until you enable document-level security and run your first access control sync.", + })} +

+ + )} + {!isAccessControlIndexNotFound && + docs.length === 0 && i18n.translate('xpack.enterpriseSearch.content.searchIndex.documents.noMappings', { defaultMessage: 'No documents found for index', })} - {docs.length > 0 && ( + {!isAccessControlIndexNotFound && docs.length > 0 && ( { ? indexName : indexName.replace('search-', CONNECTORS_ACCESS_CONTROL_INDEX_PREFIX); const { makeRequest: makeMappingRequest } = useActions(mappingsWithPropsApiLogic(indexToShow)); - const { data: mappingData } = useValues(mappingsWithPropsApiLogic(indexToShow)); + const { data: mappingData, error } = useValues(mappingsWithPropsApiLogic(indexToShow)); const shouldShowAccessControlSwitch = hasDocumentLevelSecurityFeature && productFeatures.hasDocumentLevelSecurityEnabled; + const isAccessControlIndexNotFound = + shouldShowAccessControlSwitch && error?.body?.statusCode === 404; + useEffect(() => { makeMappingRequest({ indexName: indexToShow }); }, [indexToShow, indexName]); @@ -76,9 +80,27 @@ export const SearchIndexIndexMappings: React.FC = () => { )} - - {JSON.stringify(mappingData, null, 2)} - + {isAccessControlIndexNotFound ? ( + +

+ {i18n.translate('xpack.enterpriseSearch.content.searchIndex.mappings.noIndex', { + defaultMessage: + "An Access Control Index won't be created until you enable document-level security and run your first access control sync.", + })} +

+ + ) : ( + + {JSON.stringify(mappingData, null, 2)} + + )} From a3a681accead38dbd3de4dd5fa7a7b6bf2b94d79 Mon Sep 17 00:00:00 2001 From: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> Date: Tue, 26 Sep 2023 15:45:19 -0400 Subject: [PATCH 03/12] [8.10] Upgrade execa from v4.1.0 to v5.1.1 (#167211) (#167329) # Backport This will backport the following commits from `main` to `8.10`: - [Upgrade execa from v4.1.0 to v5.1.1 (#167211)](https://github.com/elastic/kibana/pull/167211) ### Questions ? Please refer to the [Backport tool documentation](https://github.com/sqren/backport) Co-authored-by: Thomas Watson --- package.json | 2 +- yarn.lock | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index 807da90338d78..c449259651787 100644 --- a/package.json +++ b/package.json @@ -849,7 +849,7 @@ "del": "^6.1.0", "elastic-apm-node": "^3.49.1", "email-addresses": "^5.0.0", - "execa": "^4.0.2", + "execa": "^5.1.1", "expiry-js": "0.1.7", "extract-zip": "^2.0.1", "fast-deep-equal": "^3.1.1", diff --git a/yarn.lock b/yarn.lock index 38ca85722f935..19317d14c6716 100644 --- a/yarn.lock +++ b/yarn.lock @@ -15987,7 +15987,7 @@ exec-sh@^0.3.2: resolved "https://registry.yarnpkg.com/exec-sh/-/exec-sh-0.3.2.tgz#6738de2eb7c8e671d0366aea0b0db8c6f7d7391b" integrity sha512-9sLAvzhI5nc8TpuQUh4ahMdCrWT00wPWz7j47/emR5+2qEfoZP5zzUXvx+vdx+H6ohhnsYC31iX04QLYJK8zTg== -execa@4.1.0, execa@^4.0.2: +execa@4.1.0: version "4.1.0" resolved "https://registry.yarnpkg.com/execa/-/execa-4.1.0.tgz#4e5491ad1572f2f17a77d388c6c857135b22847a" integrity sha512-j5W0//W7f8UxAn8hXVnwG8tLwdiUy4FJLcSupCg6maBYZDpyBvTApK7KyuI4bKj8KOh1r2YH+6ucuYtJv1bTZA== From ad070ef2d3ccf6f28ce7cf69a4689b87b47a6ccc Mon Sep 17 00:00:00 2001 From: Kevin Delemme Date: Tue, 26 Sep 2023 17:22:23 -0400 Subject: [PATCH 04/12] [8.10] fix(slo): date range filter format (#166989) (#167178) # Backport This will backport the following commits from `main` to `8.10`: - [fix(slo): date range filter format (#166989)](https://github.com/elastic/kibana/pull/166989) ### Questions ? Please refer to the [Backport tool documentation](https://github.com/sqren/backport) --- .../apm_transaction_duration.test.ts.snap | 16 ++++++++-------- .../apm_transaction_error_rate.test.ts.snap | 16 ++++++++-------- .../__snapshots__/histogram.test.ts.snap | 6 +++--- .../__snapshots__/kql_custom.test.ts.snap | 6 +++--- .../__snapshots__/metric_custom.test.ts.snap | 6 +++--- .../apm_transaction_duration.ts | 2 +- .../apm_transaction_error_rate.ts | 2 +- .../slo/transform_generators/histogram.ts | 2 +- .../slo/transform_generators/kql_custom.ts | 2 +- .../slo/transform_generators/metric_custom.ts | 2 +- 10 files changed, 30 insertions(+), 30 deletions(-) diff --git a/x-pack/plugins/observability/server/services/slo/transform_generators/__snapshots__/apm_transaction_duration.test.ts.snap b/x-pack/plugins/observability/server/services/slo/transform_generators/__snapshots__/apm_transaction_duration.test.ts.snap index 5f828ee51b625..aea9ba75f2863 100644 --- a/x-pack/plugins/observability/server/services/slo/transform_generators/__snapshots__/apm_transaction_duration.test.ts.snap +++ b/x-pack/plugins/observability/server/services/slo/transform_generators/__snapshots__/apm_transaction_duration.test.ts.snap @@ -24,7 +24,7 @@ Object { Object { "range": Object { "@timestamp": Object { - "gte": "now-7d", + "gte": "now-7d/d", }, }, }, @@ -130,7 +130,7 @@ Object { Object { "range": Object { "@timestamp": Object { - "gte": "now-7d", + "gte": "now-7d/d", }, }, }, @@ -163,7 +163,7 @@ Object { Object { "range": Object { "@timestamp": Object { - "gte": "now-7d", + "gte": "now-7d/d", }, }, }, @@ -277,7 +277,7 @@ Object { Object { "range": Object { "@timestamp": Object { - "gte": "now-7d", + "gte": "now-7d/d", }, }, }, @@ -391,7 +391,7 @@ Object { Object { "range": Object { "@timestamp": Object { - "gte": "now-7d", + "gte": "now-7d/d", }, }, }, @@ -505,7 +505,7 @@ Object { Object { "range": Object { "@timestamp": Object { - "gte": "now-7d", + "gte": "now-7d/d", }, }, }, @@ -765,7 +765,7 @@ Object { Object { "range": Object { "@timestamp": Object { - "gte": "now-7d", + "gte": "now-7d/d", }, }, }, @@ -1039,7 +1039,7 @@ Object { Object { "range": Object { "@timestamp": Object { - "gte": "now-7d", + "gte": "now-7d/d", }, }, }, diff --git a/x-pack/plugins/observability/server/services/slo/transform_generators/__snapshots__/apm_transaction_error_rate.test.ts.snap b/x-pack/plugins/observability/server/services/slo/transform_generators/__snapshots__/apm_transaction_error_rate.test.ts.snap index dc9278511a864..b7c4ebf0f6b8c 100644 --- a/x-pack/plugins/observability/server/services/slo/transform_generators/__snapshots__/apm_transaction_error_rate.test.ts.snap +++ b/x-pack/plugins/observability/server/services/slo/transform_generators/__snapshots__/apm_transaction_error_rate.test.ts.snap @@ -20,7 +20,7 @@ Object { Object { "range": Object { "@timestamp": Object { - "gte": "now-7d", + "gte": "now-7d/d", }, }, }, @@ -122,7 +122,7 @@ Object { Object { "range": Object { "@timestamp": Object { - "gte": "now-7d", + "gte": "now-7d/d", }, }, }, @@ -151,7 +151,7 @@ Object { Object { "range": Object { "@timestamp": Object { - "gte": "now-7d", + "gte": "now-7d/d", }, }, }, @@ -261,7 +261,7 @@ Object { Object { "range": Object { "@timestamp": Object { - "gte": "now-7d", + "gte": "now-7d/d", }, }, }, @@ -371,7 +371,7 @@ Object { Object { "range": Object { "@timestamp": Object { - "gte": "now-7d", + "gte": "now-7d/d", }, }, }, @@ -481,7 +481,7 @@ Object { Object { "range": Object { "@timestamp": Object { - "gte": "now-7d", + "gte": "now-7d/d", }, }, }, @@ -730,7 +730,7 @@ Object { Object { "range": Object { "@timestamp": Object { - "gte": "now-7d", + "gte": "now-7d/d", }, }, }, @@ -993,7 +993,7 @@ Object { Object { "range": Object { "@timestamp": Object { - "gte": "now-7d", + "gte": "now-7d/d", }, }, }, diff --git a/x-pack/plugins/observability/server/services/slo/transform_generators/__snapshots__/histogram.test.ts.snap b/x-pack/plugins/observability/server/services/slo/transform_generators/__snapshots__/histogram.test.ts.snap index ee4001303fec5..ad100c4662fe9 100644 --- a/x-pack/plugins/observability/server/services/slo/transform_generators/__snapshots__/histogram.test.ts.snap +++ b/x-pack/plugins/observability/server/services/slo/transform_generators/__snapshots__/histogram.test.ts.snap @@ -51,7 +51,7 @@ Object { Object { "range": Object { "log_timestamp": Object { - "gte": "now-7d", + "gte": "now-7d/d", }, }, }, @@ -232,7 +232,7 @@ Object { Object { "range": Object { "log_timestamp": Object { - "gte": "now-7d", + "gte": "now-7d/d", }, }, }, @@ -488,7 +488,7 @@ Object { Object { "range": Object { "log_timestamp": Object { - "gte": "now-7d", + "gte": "now-7d/d", }, }, }, diff --git a/x-pack/plugins/observability/server/services/slo/transform_generators/__snapshots__/kql_custom.test.ts.snap b/x-pack/plugins/observability/server/services/slo/transform_generators/__snapshots__/kql_custom.test.ts.snap index 126437173f84a..3297a8c513821 100644 --- a/x-pack/plugins/observability/server/services/slo/transform_generators/__snapshots__/kql_custom.test.ts.snap +++ b/x-pack/plugins/observability/server/services/slo/transform_generators/__snapshots__/kql_custom.test.ts.snap @@ -92,7 +92,7 @@ Object { Object { "range": Object { "log_timestamp": Object { - "gte": "now-7d", + "gte": "now-7d/d", }, }, }, @@ -247,7 +247,7 @@ Object { Object { "range": Object { "log_timestamp": Object { - "gte": "now-7d", + "gte": "now-7d/d", }, }, }, @@ -477,7 +477,7 @@ Object { Object { "range": Object { "log_timestamp": Object { - "gte": "now-7d", + "gte": "now-7d/d", }, }, }, diff --git a/x-pack/plugins/observability/server/services/slo/transform_generators/__snapshots__/metric_custom.test.ts.snap b/x-pack/plugins/observability/server/services/slo/transform_generators/__snapshots__/metric_custom.test.ts.snap index ced0801d859d4..362b1d4a9e01e 100644 --- a/x-pack/plugins/observability/server/services/slo/transform_generators/__snapshots__/metric_custom.test.ts.snap +++ b/x-pack/plugins/observability/server/services/slo/transform_generators/__snapshots__/metric_custom.test.ts.snap @@ -63,7 +63,7 @@ Object { Object { "range": Object { "log_timestamp": Object { - "gte": "now-7d", + "gte": "now-7d/d", }, }, }, @@ -256,7 +256,7 @@ Object { Object { "range": Object { "log_timestamp": Object { - "gte": "now-7d", + "gte": "now-7d/d", }, }, }, @@ -524,7 +524,7 @@ Object { Object { "range": Object { "log_timestamp": Object { - "gte": "now-7d", + "gte": "now-7d/d", }, }, }, diff --git a/x-pack/plugins/observability/server/services/slo/transform_generators/apm_transaction_duration.ts b/x-pack/plugins/observability/server/services/slo/transform_generators/apm_transaction_duration.ts index 3dd6469a7f2c5..171a5ca15e2de 100644 --- a/x-pack/plugins/observability/server/services/slo/transform_generators/apm_transaction_duration.ts +++ b/x-pack/plugins/observability/server/services/slo/transform_generators/apm_transaction_duration.ts @@ -72,7 +72,7 @@ export class ApmTransactionDurationTransformGenerator extends TransformGenerator { range: { '@timestamp': { - gte: `now-${slo.timeWindow.duration.format()}`, + gte: `now-${slo.timeWindow.duration.format()}/d`, }, }, }, diff --git a/x-pack/plugins/observability/server/services/slo/transform_generators/apm_transaction_error_rate.ts b/x-pack/plugins/observability/server/services/slo/transform_generators/apm_transaction_error_rate.ts index 3818111c70df2..d038876f5ee9d 100644 --- a/x-pack/plugins/observability/server/services/slo/transform_generators/apm_transaction_error_rate.ts +++ b/x-pack/plugins/observability/server/services/slo/transform_generators/apm_transaction_error_rate.ts @@ -72,7 +72,7 @@ export class ApmTransactionErrorRateTransformGenerator extends TransformGenerato { range: { '@timestamp': { - gte: `now-${slo.timeWindow.duration.format()}`, + gte: `now-${slo.timeWindow.duration.format()}/d`, }, }, }, diff --git a/x-pack/plugins/observability/server/services/slo/transform_generators/histogram.ts b/x-pack/plugins/observability/server/services/slo/transform_generators/histogram.ts index 654f8d67a3673..844703003257a 100644 --- a/x-pack/plugins/observability/server/services/slo/transform_generators/histogram.ts +++ b/x-pack/plugins/observability/server/services/slo/transform_generators/histogram.ts @@ -54,7 +54,7 @@ export class HistogramTransformGenerator extends TransformGenerator { { range: { [indicator.params.timestampField]: { - gte: `now-${slo.timeWindow.duration.format()}`, + gte: `now-${slo.timeWindow.duration.format()}/d`, }, }, }, diff --git a/x-pack/plugins/observability/server/services/slo/transform_generators/kql_custom.ts b/x-pack/plugins/observability/server/services/slo/transform_generators/kql_custom.ts index 8321a0cb7172e..02c7757ec1362 100644 --- a/x-pack/plugins/observability/server/services/slo/transform_generators/kql_custom.ts +++ b/x-pack/plugins/observability/server/services/slo/transform_generators/kql_custom.ts @@ -49,7 +49,7 @@ export class KQLCustomTransformGenerator extends TransformGenerator { { range: { [indicator.params.timestampField]: { - gte: `now-${slo.timeWindow.duration.format()}`, + gte: `now-${slo.timeWindow.duration.format()}/d`, }, }, }, diff --git a/x-pack/plugins/observability/server/services/slo/transform_generators/metric_custom.ts b/x-pack/plugins/observability/server/services/slo/transform_generators/metric_custom.ts index 52209533f828c..063e6fbe1e3dc 100644 --- a/x-pack/plugins/observability/server/services/slo/transform_generators/metric_custom.ts +++ b/x-pack/plugins/observability/server/services/slo/transform_generators/metric_custom.ts @@ -52,7 +52,7 @@ export class MetricCustomTransformGenerator extends TransformGenerator { { range: { [indicator.params.timestampField]: { - gte: `now-${slo.timeWindow.duration.format()}`, + gte: `now-${slo.timeWindow.duration.format()}/d`, }, }, }, From 5e0d4274bd8d02576d547a20a76dd53c88eca873 Mon Sep 17 00:00:00 2001 From: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> Date: Wed, 27 Sep 2023 03:04:11 -0400 Subject: [PATCH 05/12] [8.10.3] [Security Solutions] Fix The 'Show top N' action inside the timeline (#165109) (#165231) # Backport This will backport the following commits from `main` to `8.10`: - [[Security Solutions] Fix The 'Show top N' action inside the timeline (#165109)](https://github.com/elastic/kibana/pull/165109) ### Questions ? Please refer to the [Backport tool documentation](https://github.com/sqren/backport) Co-authored-by: Pablo Machado --- .../components/matrix_histogram/index.tsx | 19 +++++++++++++++++-- .../public/common/components/top_n/top_n.tsx | 1 + .../lens_attributes/common/events.ts | 5 +++-- .../components/events_by_dataset/index.tsx | 2 ++ 4 files changed, 23 insertions(+), 4 deletions(-) diff --git a/x-pack/plugins/security_solution/public/common/components/matrix_histogram/index.tsx b/x-pack/plugins/security_solution/public/common/components/matrix_histogram/index.tsx index c2d1e6dc59ff8..e14a0c7f4c224 100644 --- a/x-pack/plugins/security_solution/public/common/components/matrix_histogram/index.tsx +++ b/x-pack/plugins/security_solution/public/common/components/matrix_histogram/index.tsx @@ -12,6 +12,7 @@ import styled from 'styled-components'; import { EuiFlexGroup, EuiFlexItem, EuiProgress, EuiSelect, EuiSpacer } from '@elastic/eui'; import { useDispatch } from 'react-redux'; import type { AggregationsTermsAggregateBase } from '@elastic/elasticsearch/lib/api/types'; +import { isString } from 'lodash/fp'; import * as i18n from './translations'; import { HeaderSection } from '../header_section'; import { Panel } from '../panel'; @@ -66,6 +67,7 @@ export type MatrixHistogramComponentProps = MatrixHistogramProps & scopeId?: string; title: string | GetTitle; hideQueryToggle?: boolean; + applyGlobalQueriesAndFilters?: boolean; }; const DEFAULT_PANEL_HEIGHT = 300; @@ -117,6 +119,7 @@ export const MatrixHistogramComponent: React.FC = yTickFormatter, skip, hideQueryToggle = false, + applyGlobalQueriesAndFilters = true, }) => { const visualizationId = `${id}-embeddable`; const dispatch = useDispatch(); @@ -258,9 +261,20 @@ export const MatrixHistogramComponent: React.FC = const timerange = useMemo(() => ({ from: startDate, to: endDate }), [startDate, endDate]); const extraVisualizationOptions = useMemo( - () => ({ dnsIsPtrIncluded: isPtrIncluded ?? false }), - [isPtrIncluded] + () => ({ + dnsIsPtrIncluded: isPtrIncluded ?? false, + filters: filterQuery + ? [ + { + query: isString(filterQuery) ? JSON.parse(filterQuery) : filterQuery, + meta: {}, + }, + ] + : undefined, + }), + [isPtrIncluded, filterQuery] ); + if (hideHistogram) { return null; } @@ -329,6 +343,7 @@ export const MatrixHistogramComponent: React.FC = {toggleStatus ? ( isChartEmbeddablesEnabled ? ( = ({ {view === 'raw' || view === 'all' ? ( { return { title: 'Events', @@ -55,7 +56,7 @@ export const getEventsHistogramLensAttributes: GetLensAttributes = ( query: '', language: 'kuery', }, - filters: [], + filters: extraOptions.filters ?? [], datasourceStates: { formBased: { layers: { diff --git a/x-pack/plugins/security_solution/public/overview/components/events_by_dataset/index.tsx b/x-pack/plugins/security_solution/public/overview/components/events_by_dataset/index.tsx index 360cfbbce9825..360c95ec94019 100644 --- a/x-pack/plugins/security_solution/public/overview/components/events_by_dataset/index.tsx +++ b/x-pack/plugins/security_solution/public/overview/components/events_by_dataset/index.tsx @@ -60,6 +60,7 @@ interface Props extends Pick void; hideQueryToggle?: boolean; + applyGlobalQueriesAndFilters?: boolean; } const getHistogramOption = (fieldName: string): MatrixHistogramOption => ({ @@ -95,6 +96,7 @@ const EventsByDatasetComponent: React.FC = ({ to, toggleTopN, hideQueryToggle = false, + applyGlobalQueriesAndFilters = true, }) => { const uniqueQueryId = useMemo(() => `${ID}-${queryType}`, [queryType]); From 7742bf2215f7f5d5dc7b4e6818df5f06ce6aefb6 Mon Sep 17 00:00:00 2001 From: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> Date: Wed, 27 Sep 2023 06:01:03 -0400 Subject: [PATCH 06/12] [8.10] [Saved Objects] Fix document migrator to prevent losing namespaces on import (#164848) (#167372) # Backport This will backport the following commits from `main` to `8.10`: - [[Saved Objects] Fix document migrator to prevent losing namespaces on import (#164848)](https://github.com/elastic/kibana/pull/164848) ### Questions ? Please refer to the [Backport tool documentation](https://github.com/sqren/backport) Co-authored-by: Michael Dokolin --- .../document_migrator.test.ts | 30 +++++++++++++++++++ .../document_migrator/internal_transforms.ts | 7 +++++ 2 files changed, 37 insertions(+) diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/document_migrator/document_migrator.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/document_migrator/document_migrator.test.ts index 2e3a99b4b463e..43cf127a9ab19 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/document_migrator/document_migrator.test.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/document_migrator/document_migrator.test.ts @@ -713,6 +713,36 @@ describe('DocumentMigrator', () => { ]); }); + it('does not lose namespaces in documents with undefined namespace and defined namespaces property', () => { + const migrator = new DocumentMigrator({ + ...testOpts(), + typeRegistry: createRegistry( + { name: 'dog', namespaceType: 'multiple', convertToMultiNamespaceTypeVersion: '1.0.0' } + // no migration transforms are defined, the typeMigrationVersion will be derived from 'convertToMultiNamespaceTypeVersion' + ), + }); + migrator.prepareMigrations(); + const obj = { + id: 'mischievous', + type: 'dog', + attributes: { name: 'Ann' }, + coreMigrationVersion: kibanaVersion, + typeMigrationVersion: '0.1.0', + namespaces: ['something'], + } as SavedObjectUnsanitizedDoc; + const actual = migrator.migrateAndConvert(obj); + expect(actual).toEqual([ + { + id: 'mischievous', + type: 'dog', + attributes: { name: 'Ann' }, + coreMigrationVersion: kibanaVersion, + typeMigrationVersion: '1.0.0', + namespaces: ['something'], + }, + ]); + }); + it('does not fail when encountering documents with coreMigrationVersion higher than the latest known', () => { const migrator = new DocumentMigrator({ ...testOpts(), diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/document_migrator/internal_transforms.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/document_migrator/internal_transforms.ts index 762fa20342c49..11090badd9173 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/document_migrator/internal_transforms.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/document_migrator/internal_transforms.ts @@ -105,6 +105,13 @@ function convertNamespaceType(doc: SavedObjectUnsanitizedDoc) { const { namespace, ...otherAttrs } = doc; const additionalDocs: SavedObjectUnsanitizedDoc[] = []; + if (namespace == null && otherAttrs.namespaces) { + return { + additionalDocs, + transformedDoc: otherAttrs, + }; + } + // If this object exists in the default namespace, return it with the appropriate `namespaces` field without changing its ID. if (namespace === undefined) { return { From 6e1ef68c2dff28d5c751d525e4bec684033b094f Mon Sep 17 00:00:00 2001 From: Jill Guyonnet Date: Wed, 27 Sep 2023 12:40:03 +0200 Subject: [PATCH 07/12] [8.10] [Fleet] Fix bulk action dropdown (#166475) (#167368) # Backport This will backport the following commits from `main` to `8.10`: - [[Fleet] Fix bulk action dropdown (#166475)](https://github.com/elastic/kibana/pull/166475) ### Questions ? Please refer to the [Backport tool documentation](https://github.com/sqren/backport) --- .../agents_selection_status.test.tsx | 100 +++++++ .../components/agents_selection_status.tsx | 20 +- .../components/bulk_actions.test.tsx | 282 +++--------------- .../components/bulk_actions.tsx | 85 ++---- .../components/search_and_filter_bar.test.tsx | 12 +- .../components/search_and_filter_bar.tsx | 15 +- .../components/table_header.tsx | 3 + .../agents/agent_list_page/index.test.tsx | 1 + .../sections/agents/agent_list_page/index.tsx | 121 +++++--- 9 files changed, 286 insertions(+), 353 deletions(-) create mode 100644 x-pack/plugins/fleet/public/applications/fleet/sections/agents/agent_list_page/components/agents_selection_status.test.tsx diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/agents/agent_list_page/components/agents_selection_status.test.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/agents/agent_list_page/components/agents_selection_status.test.tsx new file mode 100644 index 0000000000000..07f42b65fdb0c --- /dev/null +++ b/x-pack/plugins/fleet/public/applications/fleet/sections/agents/agent_list_page/components/agents_selection_status.test.tsx @@ -0,0 +1,100 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React from 'react'; + +import { createFleetTestRendererMock } from '../../../../../../mock'; + +import { AgentsSelectionStatus } from './agents_selection_status'; + +function render(props: any) { + const renderer = createFleetTestRendererMock(); + + return renderer.render(); +} + +const defaultProps = { + totalAgents: 30, + selectableAgents: 20, + managedAgentsOnCurrentPage: 0, + selectionMode: 'manual', + setSelectionMode: jest.fn(), + selectedAgents: [], + setSelectedAgents: jest.fn(), +}; + +function generateAgents(n: number) { + return [...Array(n).keys()].map((i) => ({ + id: `agent${i}`, + active: true, + })); +} + +describe('AgentsSelectionStatus', () => { + describe('when selection mode is manual', () => { + describe('when there are no selected agents', () => { + it('should not show any selection options', () => { + const res = render(defaultProps); + expect(res.queryByTestId('selectedAgentCountLabel')).toBeNull(); + expect(res.queryByTestId('clearAgentSelectionButton')).toBeNull(); + expect(res.queryByTestId('selectedEverythingOnAllPagesButton')).toBeNull(); + }); + }); + + describe('when there are selected agents', () => { + it('should show the number of selected agents and the Clear selection button', () => { + const res = render({ ...defaultProps, selectedAgents: generateAgents(2) }); + expect(res.queryByTestId('selectedAgentCountLabel')).not.toBeNull(); + expect(res.queryByTestId('clearAgentSelectionButton')).not.toBeNull(); + }); + + it('should not show the Select everything on all pages button if not all agents are selected', () => { + const res = render({ ...defaultProps, selectedAgents: generateAgents(2) }); + expect(res.queryByTestId('selectedEverythingOnAllPagesButton')).toBeNull(); + }); + + it('should not show the Select everything on all pages button if all agents are selected but there are no more selectable agents', () => { + const res = render({ + ...defaultProps, + totalAgents: 20, + selectableAgents: 19, + managedAgentsOnCurrentPage: 1, + selectedAgents: generateAgents(19), + }); + expect(res.queryByTestId('selectedEverythingOnAllPagesButton')).toBeNull(); + }); + + it('should show the Select everything on all pages button if all agents are selected and there are more selectable agents', () => { + const res = render({ ...defaultProps, selectedAgents: generateAgents(20) }); + expect(res.queryByTestId('selectedEverythingOnAllPagesButton')).not.toBeNull(); + }); + }); + }); + + describe('when selection mode is query', () => { + describe('when there are agents', () => { + it('should show the number of selected agents and the Clear selection button', () => { + const res = render({ + ...defaultProps, + selectionMode: 'query', + selectedAgents: generateAgents(2), + }); + expect(res.queryByTestId('selectedAgentCountLabel')).not.toBeNull(); + expect(res.queryByTestId('clearAgentSelectionButton')).not.toBeNull(); + }); + + it('should not show the Select everything on all pages button', () => { + const res = render({ + ...defaultProps, + selectionMode: 'query', + selectedAgents: generateAgents(20), + }); + expect(res.queryByTestId('selectedEverythingOnAllPagesButton')).toBeNull(); + }); + }); + }); +}); diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/agents/agent_list_page/components/agents_selection_status.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/agents/agent_list_page/components/agents_selection_status.tsx index 57728f275ccb5..0e1b6e43db46e 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/sections/agents/agent_list_page/components/agents_selection_status.tsx +++ b/x-pack/plugins/fleet/public/applications/fleet/sections/agents/agent_list_page/components/agents_selection_status.tsx @@ -34,6 +34,7 @@ const Button = styled(EuiButtonEmpty)` export const AgentsSelectionStatus: React.FunctionComponent<{ totalAgents: number; selectableAgents: number; + managedAgentsOnCurrentPage: number; selectionMode: SelectionMode; setSelectionMode: (mode: SelectionMode) => void; selectedAgents: Agent[]; @@ -41,15 +42,19 @@ export const AgentsSelectionStatus: React.FunctionComponent<{ }> = ({ totalAgents, selectableAgents, + managedAgentsOnCurrentPage, selectionMode, setSelectionMode, selectedAgents, setSelectedAgents, }) => { + const showSelectionInfoAndOptions = + (selectionMode === 'manual' && selectedAgents.length > 0) || + (selectionMode === 'query' && totalAgents > 0); const showSelectEverything = selectionMode === 'manual' && selectedAgents.length === selectableAgents && - selectableAgents < totalAgents; + selectableAgents < totalAgents - managedAgentsOnCurrentPage; return ( <> @@ -74,14 +79,13 @@ export const AgentsSelectionStatus: React.FunctionComponent<{ )} - {(selectionMode === 'manual' && selectedAgents.length) || - (selectionMode === 'query' && totalAgents > 0) ? ( + {showSelectionInfoAndOptions ? ( <> - + -