diff --git a/x-pack/plugins/entity_manager/server/lib/entities/delete_index.ts b/x-pack/plugins/entity_manager/server/lib/entities/delete_index.ts index 433b6e392c27e..d40d9975a8820 100644 --- a/x-pack/plugins/entity_manager/server/lib/entities/delete_index.ts +++ b/x-pack/plugins/entity_manager/server/lib/entities/delete_index.ts @@ -7,7 +7,7 @@ import { ElasticsearchClient, Logger } from '@kbn/core/server'; import { EntityDefinition } from '@kbn/entities-schema'; -import { generateHistoryIndexName, generateLatestIndexName } from './helpers/generate_component_id'; +import { generateLatestIndexName } from './helpers/generate_component_id'; export async function deleteIndices( esClient: ElasticsearchClient, @@ -15,15 +15,8 @@ export async function deleteIndices( logger: Logger ) { try { - const { indices: historyIndices } = await esClient.indices.resolveIndex({ - name: `${generateHistoryIndexName(definition)}.*`, - expand_wildcards: 'all', - }); - const indices = [ - ...historyIndices.map(({ name }) => name), - generateLatestIndexName(definition), - ]; - await esClient.indices.delete({ index: indices, ignore_unavailable: true }); + const index = generateLatestIndexName(definition); + await esClient.indices.delete({ index, ignore_unavailable: true }); } catch (e) { logger.error(`Unable to remove entity definition index [${definition.id}}]`); throw e; diff --git a/x-pack/plugins/entity_manager/server/lib/entities/uninstall_entity_definition.ts b/x-pack/plugins/entity_manager/server/lib/entities/uninstall_entity_definition.ts index d0e0410b6e422..f8e27353082d0 100644 --- a/x-pack/plugins/entity_manager/server/lib/entities/uninstall_entity_definition.ts +++ b/x-pack/plugins/entity_manager/server/lib/entities/uninstall_entity_definition.ts @@ -10,63 +10,48 @@ import { SavedObjectsClientContract } from '@kbn/core-saved-objects-api-server'; import { EntityDefinition } from '@kbn/entities-schema'; import { Logger } from '@kbn/logging'; import { deleteEntityDefinition } from './delete_entity_definition'; -import { deleteIndices } from './delete_index'; import { deleteIngestPipelines } from './delete_ingest_pipeline'; -import { findEntityDefinitions } from './find_entity_definition'; import { deleteTemplates } from '../manage_index_templates'; import { stopTransforms } from './stop_transforms'; import { deleteTransforms } from './delete_transforms'; +import { EntityClient } from '../entity_client'; export async function uninstallEntityDefinition({ definition, esClient, soClient, logger, - deleteData = false, }: { definition: EntityDefinition; esClient: ElasticsearchClient; soClient: SavedObjectsClientContract; logger: Logger; - deleteData?: boolean; }) { await stopTransforms(esClient, definition, logger); await deleteTransforms(esClient, definition, logger); await deleteIngestPipelines(esClient, definition, logger); - if (deleteData) { - await deleteIndices(esClient, definition, logger); - } - await deleteTemplates(esClient, definition, logger); await deleteEntityDefinition(soClient, definition); } export async function uninstallBuiltInEntityDefinitions({ - esClient, - soClient, - logger, + entityClient, deleteData = false, }: { - esClient: ElasticsearchClient; - soClient: SavedObjectsClientContract; - logger: Logger; + entityClient: EntityClient; deleteData?: boolean; }): Promise { - const definitions = await findEntityDefinitions({ - soClient, - esClient, - builtIn: true, - }); + const { definitions } = await entityClient.getEntityDefinitions({ builtIn: true }); await Promise.all( - definitions.map(async (definition) => { - await uninstallEntityDefinition({ definition, esClient, soClient, logger, deleteData }); + definitions.map(async ({ id }) => { + await entityClient.deleteEntityDefinition({ id, deleteData }); }) ); diff --git a/x-pack/plugins/entity_manager/server/lib/entity_client.ts b/x-pack/plugins/entity_manager/server/lib/entity_client.ts index 1bb1322be356f..dcb3dfab8f328 100644 --- a/x-pack/plugins/entity_manager/server/lib/entity_client.ts +++ b/x-pack/plugins/entity_manager/server/lib/entity_client.ts @@ -7,7 +7,7 @@ import { EntityDefinition, EntityDefinitionUpdate } from '@kbn/entities-schema'; import { SavedObjectsClientContract } from '@kbn/core-saved-objects-api-server'; -import { ElasticsearchClient } from '@kbn/core-elasticsearch-server'; +import { IScopedClusterClient } from '@kbn/core-elasticsearch-server'; import { Logger } from '@kbn/logging'; import { installEntityDefinition, @@ -20,13 +20,14 @@ import { uninstallEntityDefinition } from './entities/uninstall_entity_definitio import { EntityDefinitionNotFound } from './entities/errors/entity_not_found'; import { stopTransforms } from './entities/stop_transforms'; +import { deleteIndices } from './entities/delete_index'; import { EntityDefinitionWithState } from './entities/types'; import { EntityDefinitionUpdateConflict } from './entities/errors/entity_definition_update_conflict'; export class EntityClient { constructor( private options: { - esClient: ElasticsearchClient; + clusterClient: IScopedClusterClient; soClient: SavedObjectsClientContract; logger: Logger; } @@ -39,15 +40,16 @@ export class EntityClient { definition: EntityDefinition; installOnly?: boolean; }) { + const secondaryAuthClient = this.options.clusterClient.asSecondaryAuthUser; const installedDefinition = await installEntityDefinition({ definition, + esClient: secondaryAuthClient, soClient: this.options.soClient, - esClient: this.options.esClient, logger: this.options.logger, }); if (!installOnly) { - await startTransforms(this.options.esClient, installedDefinition, this.options.logger); + await startTransforms(secondaryAuthClient, installedDefinition, this.options.logger); } return installedDefinition; @@ -60,10 +62,11 @@ export class EntityClient { id: string; definitionUpdate: EntityDefinitionUpdate; }) { + const secondaryAuthClient = this.options.clusterClient.asSecondaryAuthUser; const definition = await findEntityDefinitionById({ id, soClient: this.options.soClient, - esClient: this.options.esClient, + esClient: secondaryAuthClient, includeState: true, }); @@ -87,22 +90,22 @@ export class EntityClient { definition, definitionUpdate, soClient: this.options.soClient, - esClient: this.options.esClient, + esClient: secondaryAuthClient, logger: this.options.logger, }); if (shouldRestartTransforms) { - await startTransforms(this.options.esClient, updatedDefinition, this.options.logger); + await startTransforms(secondaryAuthClient, updatedDefinition, this.options.logger); } return updatedDefinition; } async deleteEntityDefinition({ id, deleteData = false }: { id: string; deleteData?: boolean }) { - const [definition] = await findEntityDefinitions({ + const secondaryAuthClient = this.options.clusterClient.asSecondaryAuthUser; + const definition = await findEntityDefinitionById({ id, - perPage: 1, + esClient: secondaryAuthClient, soClient: this.options.soClient, - esClient: this.options.esClient, }); if (!definition) { @@ -113,11 +116,20 @@ export class EntityClient { await uninstallEntityDefinition({ definition, - deleteData, + esClient: secondaryAuthClient, soClient: this.options.soClient, - esClient: this.options.esClient, logger: this.options.logger, }); + + if (deleteData) { + // delete data with current user as system user does not have + // .entities privileges + await deleteIndices( + this.options.clusterClient.asCurrentUser, + definition, + this.options.logger + ); + } } async getEntityDefinitions({ @@ -136,7 +148,7 @@ export class EntityClient { builtIn?: boolean; }) { const definitions = await findEntityDefinitions({ - esClient: this.options.esClient, + esClient: this.options.clusterClient.asSecondaryAuthUser, soClient: this.options.soClient, page, perPage, @@ -150,10 +162,18 @@ export class EntityClient { } async startEntityDefinition(definition: EntityDefinition) { - return startTransforms(this.options.esClient, definition, this.options.logger); + return startTransforms( + this.options.clusterClient.asSecondaryAuthUser, + definition, + this.options.logger + ); } async stopEntityDefinition(definition: EntityDefinition) { - return stopTransforms(this.options.esClient, definition, this.options.logger); + return stopTransforms( + this.options.clusterClient.asSecondaryAuthUser, + definition, + this.options.logger + ); } } diff --git a/x-pack/plugins/entity_manager/server/plugin.ts b/x-pack/plugins/entity_manager/server/plugin.ts index 2677b78042620..152b1b59b3107 100644 --- a/x-pack/plugins/entity_manager/server/plugin.ts +++ b/x-pack/plugins/entity_manager/server/plugin.ts @@ -99,9 +99,9 @@ export class EntityManagerServerPlugin request: KibanaRequest; coreStart: CoreStart; }) { - const esClient = coreStart.elasticsearch.client.asScoped(request).asSecondaryAuthUser; + const clusterClient = coreStart.elasticsearch.client.asScoped(request); const soClient = coreStart.savedObjects.getScopedClient(request); - return new EntityClient({ esClient, soClient, logger: this.logger }); + return new EntityClient({ clusterClient, soClient, logger: this.logger }); } public start( diff --git a/x-pack/plugins/entity_manager/server/routes/enablement/disable.ts b/x-pack/plugins/entity_manager/server/routes/enablement/disable.ts index 9c1c4f403636b..f8629fe46497b 100644 --- a/x-pack/plugins/entity_manager/server/routes/enablement/disable.ts +++ b/x-pack/plugins/entity_manager/server/routes/enablement/disable.ts @@ -49,7 +49,7 @@ export const disableEntityDiscoveryRoute = createEntityManagerServerRoute({ deleteData: z.optional(BooleanFromString).default(false), }), }), - handler: async ({ context, response, params, logger, server }) => { + handler: async ({ context, request, response, params, logger, server, getScopedClient }) => { try { const esClientAsCurrentUser = (await context.core).elasticsearch.client.asCurrentUser; const canDisable = await canDisableEntityDiscovery(esClientAsCurrentUser); @@ -62,15 +62,13 @@ export const disableEntityDiscoveryRoute = createEntityManagerServerRoute({ }); } - const esClient = (await context.core).elasticsearch.client.asSecondaryAuthUser; + const entityClient = await getScopedClient({ request }); const soClient = (await context.core).savedObjects.getClient({ includedHiddenTypes: [EntityDiscoveryApiKeyType.name], }); await uninstallBuiltInEntityDefinitions({ - soClient, - esClient, - logger, + entityClient, deleteData: params.query.deleteData, }); diff --git a/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_store_data_client.test.ts b/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_store_data_client.test.ts index 4156ea1dbd4ea..8079e54ac9ba6 100644 --- a/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_store_data_client.test.ts +++ b/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_store_data_client.test.ts @@ -18,10 +18,11 @@ import type { AppClient } from '../../..'; describe('EntityStoreDataClient', () => { const mockSavedObjectClient = savedObjectsClientMock.create(); - const esClientMock = elasticsearchServiceMock.createScopedClusterClient().asInternalUser; + const clusterClientMock = elasticsearchServiceMock.createScopedClusterClient(); + const esClientMock = clusterClientMock.asCurrentUser; const loggerMock = loggingSystemMock.createLogger(); const dataClient = new EntityStoreDataClient({ - esClient: esClientMock, + clusterClient: clusterClientMock, logger: loggerMock, namespace: 'default', soClient: mockSavedObjectClient, diff --git a/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_store_data_client.ts b/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_store_data_client.ts index d2e21a1d10903..5b1acaa433cd0 100644 --- a/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_store_data_client.ts +++ b/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_store_data_client.ts @@ -10,6 +10,7 @@ import type { ElasticsearchClient, SavedObjectsClientContract, AuditLogger, + IScopedClusterClient, } from '@kbn/core/server'; import { EntityClient } from '@kbn/entityManager-plugin/server/lib/entity_client'; import type { SortOrder } from '@elastic/elasticsearch/lib/api/types'; @@ -55,7 +56,7 @@ import { interface EntityStoreClientOpts { logger: Logger; - esClient: ElasticsearchClient; + clusterClient: IScopedClusterClient; namespace: string; soClient: SavedObjectsClientContract; taskManager?: TaskManagerStartContract; @@ -79,12 +80,14 @@ export class EntityStoreDataClient { private assetCriticalityMigrationClient: AssetCriticalityEcsMigrationClient; private entityClient: EntityClient; private riskScoreDataClient: RiskScoreDataClient; + private esClient: ElasticsearchClient; constructor(private readonly options: EntityStoreClientOpts) { - const { esClient, logger, soClient, auditLogger, kibanaVersion, namespace } = options; + const { clusterClient, logger, soClient, auditLogger, kibanaVersion, namespace } = options; + this.esClient = clusterClient.asCurrentUser; this.entityClient = new EntityClient({ - esClient, + clusterClient, soClient, logger, }); @@ -95,14 +98,14 @@ export class EntityStoreDataClient { }); this.assetCriticalityMigrationClient = new AssetCriticalityEcsMigrationClient({ - esClient, + esClient: this.esClient, logger, auditLogger, }); this.riskScoreDataClient = new RiskScoreDataClient({ soClient, - esClient, + esClient: this.esClient, logger, namespace, kibanaVersion, @@ -165,7 +168,7 @@ export class EntityStoreDataClient { filter: string, pipelineDebugMode: boolean ) { - const { esClient, logger, namespace, appClient, dataViewsService } = this.options; + const { logger, namespace, appClient, dataViewsService } = this.options; const indexPatterns = await buildIndexPatterns(namespace, appClient, dataViewsService); const unitedDefinition = getUnitedEntityDefinition({ @@ -200,12 +203,12 @@ export class EntityStoreDataClient { // this is because the enrich policy will fail if the index does not exist with the correct fields await createEntityIndexComponentTemplate({ unitedDefinition, - esClient, + esClient: this.esClient, }); debugLog(`Created entity index component template`); await createEntityIndex({ entityType, - esClient, + esClient: this.esClient, namespace, logger, }); @@ -215,12 +218,12 @@ export class EntityStoreDataClient { // this is because the pipeline will fail if the enrich index does not exist await createFieldRetentionEnrichPolicy({ unitedDefinition, - esClient, + esClient: this.esClient, }); debugLog(`Created field retention enrich policy`); await executeFieldRetentionEnrichPolicy({ unitedDefinition, - esClient, + esClient: this.esClient, logger, }); debugLog(`Executed field retention enrich policy`); @@ -228,7 +231,7 @@ export class EntityStoreDataClient { debugMode: pipelineDebugMode, unitedDefinition, logger, - esClient, + esClient: this.esClient, }); debugLog(`Created @platform pipeline`); @@ -325,8 +328,9 @@ export class EntityStoreDataClient { taskManager: TaskManagerStartContract, options = { deleteData: false, deleteEngine: true } ) { - const { namespace, logger, esClient, appClient, dataViewsService } = this.options; + const { namespace, logger, appClient, dataViewsService } = this.options; const { deleteData, deleteEngine } = options; + const descriptor = await this.engineClient.maybeGet(entityType); const indexPatterns = await buildIndexPatterns(namespace, appClient, dataViewsService); const unitedDefinition = getUnitedEntityDefinition({ @@ -348,22 +352,22 @@ export class EntityStoreDataClient { } await deleteEntityIndexComponentTemplate({ unitedDefinition, - esClient, + esClient: this.esClient, }); await deletePlatformPipeline({ unitedDefinition, logger, - esClient, + esClient: this.esClient, }); await deleteFieldRetentionEnrichPolicy({ unitedDefinition, - esClient, + esClient: this.esClient, }); if (deleteData) { await deleteEntityIndex({ entityType, - esClient, + esClient: this.esClient, namespace, logger, }); @@ -402,7 +406,7 @@ export class EntityStoreDataClient { const sort = sortField ? [{ [sortField]: sortOrder }] : undefined; const query = filterQuery ? JSON.parse(filterQuery) : undefined; - const response = await this.options.esClient.search({ + const response = await this.esClient.search({ index, query, size: Math.min(perPage, MAX_SEARCH_RESPONSE_SIZE), diff --git a/x-pack/plugins/security_solution/server/request_context_factory.ts b/x-pack/plugins/security_solution/server/request_context_factory.ts index 0782fa25c71eb..d2bd579dc6b03 100644 --- a/x-pack/plugins/security_solution/server/request_context_factory.ts +++ b/x-pack/plugins/security_solution/server/request_context_factory.ts @@ -199,14 +199,14 @@ export class RequestContextFactory implements IRequestContextFactory { }) ), getEntityStoreDataClient: memoize(() => { - const esClient = coreContext.elasticsearch.client.asCurrentUser; + const clusterClient = coreContext.elasticsearch.client; const logger = options.logger; const soClient = coreContext.savedObjects.client; return new EntityStoreDataClient({ namespace: getSpaceId(), + clusterClient, dataViewsService, appClient: getAppClient(), - esClient, logger, soClient, taskManager: startPlugins.taskManager, diff --git a/x-pack/test/api_integration/apis/entity_manager/definitions.ts b/x-pack/test/api_integration/apis/entity_manager/definitions.ts index b51a26ad7b5ad..a1fdab08ff42a 100644 --- a/x-pack/test/api_integration/apis/entity_manager/definitions.ts +++ b/x-pack/test/api_integration/apis/entity_manager/definitions.ts @@ -154,7 +154,6 @@ export default function ({ getService }: FtrProviderContext) { after(async () => { await esDeleteAllIndices(dataForgeIndices); - await uninstallDefinition(supertest, { id: mockDefinition.id, deleteData: true }); await cleanup({ client: esClient, config: dataForgeConfig, logger }); }); @@ -171,6 +170,13 @@ export default function ({ getService }: FtrProviderContext) { const parsedSample = entityLatestSchema.safeParse(sample.hits.hits[0]._source); expect(parsedSample.success).to.be(true); }); + + it('should delete entities data when specified', async () => { + const index = generateLatestIndexName(mockDefinition); + expect(await esClient.indices.exists({ index })).to.be(true); + await uninstallDefinition(supertest, { id: mockDefinition.id, deleteData: true }); + expect(await esClient.indices.exists({ index })).to.be(false); + }); }); }); }