From 969ee98a9343a21e4eb674cc38c896e9e267d579 Mon Sep 17 00:00:00 2001 From: Kevin Lacabane Date: Wed, 16 Oct 2024 21:03:48 +0200 Subject: [PATCH] [eem] use current user to delete indices (#195886) We were trying to cleanup `.entities` indices with the system user that does not have necessary privileges. This failed silently because of `ignore_unavailable: true` (cherry picked from commit 9f291dc55ec4c82380b81aa2fb9d7a8d84a1ff22) # Conflicts: # x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_store_data_client.ts --- .../server/lib/entities/delete_index.ts | 13 ++--- .../entities/uninstall_entity_definition.ts | 27 +++------- .../server/lib/entity_client.ts | 50 +++++++++++++------ .../plugins/entity_manager/server/plugin.ts | 4 +- .../server/routes/enablement/disable.ts | 8 ++- .../entity_store_data_client.test.ts | 5 +- .../entity_store/entity_store_data_client.ts | 38 +++++++------- .../server/request_context_factory.ts | 4 +- .../apis/entity_manager/definitions.ts | 8 ++- 9 files changed, 82 insertions(+), 75 deletions(-) diff --git a/x-pack/plugins/entity_manager/server/lib/entities/delete_index.ts b/x-pack/plugins/entity_manager/server/lib/entities/delete_index.ts index 433b6e392c27e..d40d9975a8820 100644 --- a/x-pack/plugins/entity_manager/server/lib/entities/delete_index.ts +++ b/x-pack/plugins/entity_manager/server/lib/entities/delete_index.ts @@ -7,7 +7,7 @@ import { ElasticsearchClient, Logger } from '@kbn/core/server'; import { EntityDefinition } from '@kbn/entities-schema'; -import { generateHistoryIndexName, generateLatestIndexName } from './helpers/generate_component_id'; +import { generateLatestIndexName } from './helpers/generate_component_id'; export async function deleteIndices( esClient: ElasticsearchClient, @@ -15,15 +15,8 @@ export async function deleteIndices( logger: Logger ) { try { - const { indices: historyIndices } = await esClient.indices.resolveIndex({ - name: `${generateHistoryIndexName(definition)}.*`, - expand_wildcards: 'all', - }); - const indices = [ - ...historyIndices.map(({ name }) => name), - generateLatestIndexName(definition), - ]; - await esClient.indices.delete({ index: indices, ignore_unavailable: true }); + const index = generateLatestIndexName(definition); + await esClient.indices.delete({ index, ignore_unavailable: true }); } catch (e) { logger.error(`Unable to remove entity definition index [${definition.id}}]`); throw e; diff --git a/x-pack/plugins/entity_manager/server/lib/entities/uninstall_entity_definition.ts b/x-pack/plugins/entity_manager/server/lib/entities/uninstall_entity_definition.ts index d0e0410b6e422..f8e27353082d0 100644 --- a/x-pack/plugins/entity_manager/server/lib/entities/uninstall_entity_definition.ts +++ b/x-pack/plugins/entity_manager/server/lib/entities/uninstall_entity_definition.ts @@ -10,63 +10,48 @@ import { SavedObjectsClientContract } from '@kbn/core-saved-objects-api-server'; import { EntityDefinition } from '@kbn/entities-schema'; import { Logger } from '@kbn/logging'; import { deleteEntityDefinition } from './delete_entity_definition'; -import { deleteIndices } from './delete_index'; import { deleteIngestPipelines } from './delete_ingest_pipeline'; -import { findEntityDefinitions } from './find_entity_definition'; import { deleteTemplates } from '../manage_index_templates'; import { stopTransforms } from './stop_transforms'; import { deleteTransforms } from './delete_transforms'; +import { EntityClient } from '../entity_client'; export async function uninstallEntityDefinition({ definition, esClient, soClient, logger, - deleteData = false, }: { definition: EntityDefinition; esClient: ElasticsearchClient; soClient: SavedObjectsClientContract; logger: Logger; - deleteData?: boolean; }) { await stopTransforms(esClient, definition, logger); await deleteTransforms(esClient, definition, logger); await deleteIngestPipelines(esClient, definition, logger); - if (deleteData) { - await deleteIndices(esClient, definition, logger); - } - await deleteTemplates(esClient, definition, logger); await deleteEntityDefinition(soClient, definition); } export async function uninstallBuiltInEntityDefinitions({ - esClient, - soClient, - logger, + entityClient, deleteData = false, }: { - esClient: ElasticsearchClient; - soClient: SavedObjectsClientContract; - logger: Logger; + entityClient: EntityClient; deleteData?: boolean; }): Promise { - const definitions = await findEntityDefinitions({ - soClient, - esClient, - builtIn: true, - }); + const { definitions } = await entityClient.getEntityDefinitions({ builtIn: true }); await Promise.all( - definitions.map(async (definition) => { - await uninstallEntityDefinition({ definition, esClient, soClient, logger, deleteData }); + definitions.map(async ({ id }) => { + await entityClient.deleteEntityDefinition({ id, deleteData }); }) ); diff --git a/x-pack/plugins/entity_manager/server/lib/entity_client.ts b/x-pack/plugins/entity_manager/server/lib/entity_client.ts index 3b4413cd0497a..7e930d29bba73 100644 --- a/x-pack/plugins/entity_manager/server/lib/entity_client.ts +++ b/x-pack/plugins/entity_manager/server/lib/entity_client.ts @@ -7,7 +7,7 @@ import { EntityDefinition, EntityDefinitionUpdate } from '@kbn/entities-schema'; import { SavedObjectsClientContract } from '@kbn/core-saved-objects-api-server'; -import { ElasticsearchClient } from '@kbn/core-elasticsearch-server'; +import { IScopedClusterClient } from '@kbn/core-elasticsearch-server'; import { Logger } from '@kbn/logging'; import { installEntityDefinition, @@ -20,13 +20,14 @@ import { uninstallEntityDefinition } from './entities/uninstall_entity_definitio import { EntityDefinitionNotFound } from './entities/errors/entity_not_found'; import { stopTransforms } from './entities/stop_transforms'; +import { deleteIndices } from './entities/delete_index'; import { EntityDefinitionWithState } from './entities/types'; import { EntityDefinitionUpdateConflict } from './entities/errors/entity_definition_update_conflict'; export class EntityClient { constructor( private options: { - esClient: ElasticsearchClient; + clusterClient: IScopedClusterClient; soClient: SavedObjectsClientContract; logger: Logger; } @@ -39,15 +40,16 @@ export class EntityClient { definition: EntityDefinition; installOnly?: boolean; }) { + const secondaryAuthClient = this.options.clusterClient.asSecondaryAuthUser; const installedDefinition = await installEntityDefinition({ definition, + esClient: secondaryAuthClient, soClient: this.options.soClient, - esClient: this.options.esClient, logger: this.options.logger, }); if (!installOnly) { - await startTransforms(this.options.esClient, installedDefinition, this.options.logger); + await startTransforms(secondaryAuthClient, installedDefinition, this.options.logger); } return installedDefinition; @@ -60,10 +62,11 @@ export class EntityClient { id: string; definitionUpdate: EntityDefinitionUpdate; }) { + const secondaryAuthClient = this.options.clusterClient.asSecondaryAuthUser; const definition = await findEntityDefinitionById({ id, soClient: this.options.soClient, - esClient: this.options.esClient, + esClient: secondaryAuthClient, includeState: true, }); @@ -87,22 +90,22 @@ export class EntityClient { definition, definitionUpdate, soClient: this.options.soClient, - esClient: this.options.esClient, + esClient: secondaryAuthClient, logger: this.options.logger, }); if (shouldRestartTransforms) { - await startTransforms(this.options.esClient, updatedDefinition, this.options.logger); + await startTransforms(secondaryAuthClient, updatedDefinition, this.options.logger); } return updatedDefinition; } async deleteEntityDefinition({ id, deleteData = false }: { id: string; deleteData?: boolean }) { - const [definition] = await findEntityDefinitions({ + const secondaryAuthClient = this.options.clusterClient.asSecondaryAuthUser; + const definition = await findEntityDefinitionById({ id, - perPage: 1, + esClient: secondaryAuthClient, soClient: this.options.soClient, - esClient: this.options.esClient, }); if (!definition) { @@ -111,11 +114,20 @@ export class EntityClient { await uninstallEntityDefinition({ definition, - deleteData, + esClient: secondaryAuthClient, soClient: this.options.soClient, - esClient: this.options.esClient, logger: this.options.logger, }); + + if (deleteData) { + // delete data with current user as system user does not have + // .entities privileges + await deleteIndices( + this.options.clusterClient.asCurrentUser, + definition, + this.options.logger + ); + } } async getEntityDefinitions({ @@ -134,7 +146,7 @@ export class EntityClient { builtIn?: boolean; }) { const definitions = await findEntityDefinitions({ - esClient: this.options.esClient, + esClient: this.options.clusterClient.asSecondaryAuthUser, soClient: this.options.soClient, page, perPage, @@ -148,10 +160,18 @@ export class EntityClient { } async startEntityDefinition(definition: EntityDefinition) { - return startTransforms(this.options.esClient, definition, this.options.logger); + return startTransforms( + this.options.clusterClient.asSecondaryAuthUser, + definition, + this.options.logger + ); } async stopEntityDefinition(definition: EntityDefinition) { - return stopTransforms(this.options.esClient, definition, this.options.logger); + return stopTransforms( + this.options.clusterClient.asSecondaryAuthUser, + definition, + this.options.logger + ); } } diff --git a/x-pack/plugins/entity_manager/server/plugin.ts b/x-pack/plugins/entity_manager/server/plugin.ts index 2677b78042620..152b1b59b3107 100644 --- a/x-pack/plugins/entity_manager/server/plugin.ts +++ b/x-pack/plugins/entity_manager/server/plugin.ts @@ -99,9 +99,9 @@ export class EntityManagerServerPlugin request: KibanaRequest; coreStart: CoreStart; }) { - const esClient = coreStart.elasticsearch.client.asScoped(request).asSecondaryAuthUser; + const clusterClient = coreStart.elasticsearch.client.asScoped(request); const soClient = coreStart.savedObjects.getScopedClient(request); - return new EntityClient({ esClient, soClient, logger: this.logger }); + return new EntityClient({ clusterClient, soClient, logger: this.logger }); } public start( diff --git a/x-pack/plugins/entity_manager/server/routes/enablement/disable.ts b/x-pack/plugins/entity_manager/server/routes/enablement/disable.ts index 9c1c4f403636b..f8629fe46497b 100644 --- a/x-pack/plugins/entity_manager/server/routes/enablement/disable.ts +++ b/x-pack/plugins/entity_manager/server/routes/enablement/disable.ts @@ -49,7 +49,7 @@ export const disableEntityDiscoveryRoute = createEntityManagerServerRoute({ deleteData: z.optional(BooleanFromString).default(false), }), }), - handler: async ({ context, response, params, logger, server }) => { + handler: async ({ context, request, response, params, logger, server, getScopedClient }) => { try { const esClientAsCurrentUser = (await context.core).elasticsearch.client.asCurrentUser; const canDisable = await canDisableEntityDiscovery(esClientAsCurrentUser); @@ -62,15 +62,13 @@ export const disableEntityDiscoveryRoute = createEntityManagerServerRoute({ }); } - const esClient = (await context.core).elasticsearch.client.asSecondaryAuthUser; + const entityClient = await getScopedClient({ request }); const soClient = (await context.core).savedObjects.getClient({ includedHiddenTypes: [EntityDiscoveryApiKeyType.name], }); await uninstallBuiltInEntityDefinitions({ - soClient, - esClient, - logger, + entityClient, deleteData: params.query.deleteData, }); diff --git a/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_store_data_client.test.ts b/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_store_data_client.test.ts index 8bc4a7c3832d7..733e85fd6ed55 100644 --- a/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_store_data_client.test.ts +++ b/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_store_data_client.test.ts @@ -19,10 +19,11 @@ import type { EntityStoreConfig } from './types'; describe('EntityStoreDataClient', () => { const mockSavedObjectClient = savedObjectsClientMock.create(); - const esClientMock = elasticsearchServiceMock.createScopedClusterClient().asInternalUser; + const clusterClientMock = elasticsearchServiceMock.createScopedClusterClient(); + const esClientMock = clusterClientMock.asCurrentUser; const loggerMock = loggingSystemMock.createLogger(); const dataClient = new EntityStoreDataClient({ - esClient: esClientMock, + clusterClient: clusterClientMock, logger: loggerMock, namespace: 'default', soClient: mockSavedObjectClient, diff --git a/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_store_data_client.ts b/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_store_data_client.ts index bb47e38b6c541..879ddc52641f1 100644 --- a/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_store_data_client.ts +++ b/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_store_data_client.ts @@ -11,6 +11,7 @@ import type { SavedObjectsClientContract, AuditLogger, AnalyticsServiceSetup, + IScopedClusterClient, } from '@kbn/core/server'; import { EntityClient } from '@kbn/entityManager-plugin/server/lib/entity_client'; import type { SortOrder } from '@elastic/elasticsearch/lib/api/types'; @@ -65,7 +66,7 @@ import { CRITICALITY_VALUES } from '../asset_criticality/constants'; interface EntityStoreClientOpts { logger: Logger; - esClient: ElasticsearchClient; + clusterClient: IScopedClusterClient; namespace: string; soClient: SavedObjectsClientContract; taskManager?: TaskManagerStartContract; @@ -91,12 +92,14 @@ export class EntityStoreDataClient { private assetCriticalityMigrationClient: AssetCriticalityEcsMigrationClient; private entityClient: EntityClient; private riskScoreDataClient: RiskScoreDataClient; + private esClient: ElasticsearchClient; constructor(private readonly options: EntityStoreClientOpts) { - const { esClient, logger, soClient, auditLogger, kibanaVersion, namespace } = options; + const { clusterClient, logger, soClient, auditLogger, kibanaVersion, namespace } = options; + this.esClient = clusterClient.asCurrentUser; this.entityClient = new EntityClient({ - esClient, + clusterClient, soClient, logger, }); @@ -107,14 +110,14 @@ export class EntityStoreDataClient { }); this.assetCriticalityMigrationClient = new AssetCriticalityEcsMigrationClient({ - esClient, + esClient: this.esClient, logger, auditLogger, }); this.riskScoreDataClient = new RiskScoreDataClient({ soClient, - esClient, + esClient: this.esClient, logger, namespace, kibanaVersion, @@ -181,8 +184,8 @@ export class EntityStoreDataClient { config: EntityStoreConfig, pipelineDebugMode: boolean ) { - const { esClient, logger, namespace, appClient, dataViewsService } = this.options; const setupStartTime = moment().utc().toISOString(); + const { logger, namespace, appClient, dataViewsService } = this.options; const indexPatterns = await buildIndexPatterns(namespace, appClient, dataViewsService); const unitedDefinition = getUnitedEntityDefinition({ @@ -219,12 +222,12 @@ export class EntityStoreDataClient { // this is because the enrich policy will fail if the index does not exist with the correct fields await createEntityIndexComponentTemplate({ unitedDefinition, - esClient, + esClient: this.esClient, }); debugLog(`Created entity index component template`); await createEntityIndex({ entityType, - esClient, + esClient: this.esClient, namespace, logger, }); @@ -234,12 +237,12 @@ export class EntityStoreDataClient { // this is because the pipeline will fail if the enrich index does not exist await createFieldRetentionEnrichPolicy({ unitedDefinition, - esClient, + esClient: this.esClient, }); debugLog(`Created field retention enrich policy`); await executeFieldRetentionEnrichPolicy({ unitedDefinition, - esClient, + esClient: this.esClient, logger, }); debugLog(`Executed field retention enrich policy`); @@ -247,7 +250,7 @@ export class EntityStoreDataClient { debugMode: pipelineDebugMode, unitedDefinition, logger, - esClient, + esClient: this.esClient, }); debugLog(`Created @platform pipeline`); @@ -354,8 +357,9 @@ export class EntityStoreDataClient { taskManager: TaskManagerStartContract, options = { deleteData: false, deleteEngine: true } ) { - const { namespace, logger, esClient, appClient, dataViewsService, config } = this.options; + const { namespace, logger, appClient, dataViewsService, config } = this.options; const { deleteData, deleteEngine } = options; + const descriptor = await this.engineClient.maybeGet(entityType); const indexPatterns = await buildIndexPatterns(namespace, appClient, dataViewsService); @@ -383,23 +387,23 @@ export class EntityStoreDataClient { } await deleteEntityIndexComponentTemplate({ unitedDefinition, - esClient, + esClient: this.esClient, }); await deletePlatformPipeline({ unitedDefinition, logger, - esClient, + esClient: this.esClient, }); await deleteFieldRetentionEnrichPolicy({ unitedDefinition, - esClient, logger, + esClient: this.esClient, }); if (deleteData) { await deleteEntityIndex({ entityType, - esClient, + esClient: this.esClient, namespace, logger, }); @@ -439,7 +443,7 @@ export class EntityStoreDataClient { const sort = sortField ? [{ [sortField]: sortOrder }] : undefined; const query = filterQuery ? JSON.parse(filterQuery) : undefined; - const response = await this.options.esClient.search({ + const response = await this.esClient.search({ index, query, size: Math.min(perPage, MAX_SEARCH_RESPONSE_SIZE), diff --git a/x-pack/plugins/security_solution/server/request_context_factory.ts b/x-pack/plugins/security_solution/server/request_context_factory.ts index 09e7f36f3f987..aebf0fb6dfd8f 100644 --- a/x-pack/plugins/security_solution/server/request_context_factory.ts +++ b/x-pack/plugins/security_solution/server/request_context_factory.ts @@ -199,14 +199,14 @@ export class RequestContextFactory implements IRequestContextFactory { }) ), getEntityStoreDataClient: memoize(() => { - const esClient = coreContext.elasticsearch.client.asCurrentUser; + const clusterClient = coreContext.elasticsearch.client; const logger = options.logger; const soClient = coreContext.savedObjects.client; return new EntityStoreDataClient({ namespace: getSpaceId(), + clusterClient, dataViewsService, appClient: getAppClient(), - esClient, logger, soClient, taskManager: startPlugins.taskManager, diff --git a/x-pack/test/api_integration/apis/entity_manager/definitions.ts b/x-pack/test/api_integration/apis/entity_manager/definitions.ts index b51a26ad7b5ad..a1fdab08ff42a 100644 --- a/x-pack/test/api_integration/apis/entity_manager/definitions.ts +++ b/x-pack/test/api_integration/apis/entity_manager/definitions.ts @@ -154,7 +154,6 @@ export default function ({ getService }: FtrProviderContext) { after(async () => { await esDeleteAllIndices(dataForgeIndices); - await uninstallDefinition(supertest, { id: mockDefinition.id, deleteData: true }); await cleanup({ client: esClient, config: dataForgeConfig, logger }); }); @@ -171,6 +170,13 @@ export default function ({ getService }: FtrProviderContext) { const parsedSample = entityLatestSchema.safeParse(sample.hits.hits[0]._source); expect(parsedSample.success).to.be(true); }); + + it('should delete entities data when specified', async () => { + const index = generateLatestIndexName(mockDefinition); + expect(await esClient.indices.exists({ index })).to.be(true); + await uninstallDefinition(supertest, { id: mockDefinition.id, deleteData: true }); + expect(await esClient.indices.exists({ index })).to.be(false); + }); }); }); }