[eem] update builtin definitions (elastic#188351)

## Summary

Update built in definitions on plugin start. The update overwrites index
templates and ingest pipelines with the latest versions but has to
delete the transforms since we can only update a subset of settings in
the [update
api](https://www.elastic.co/guide/en/elasticsearch/reference/current/update-transform.html)
which does not include the aggregations.

## Testing
### api tests
Since the upgrade logic happens in plugin startup method we cannot
directly trigger it from api tests without some tweaks. I've added a
[fixture
plugin](https://github.com/elastic/kibana/blob/a87ae8b2109bbbbe24e99638782fd73b165289f2/x-pack/test/api_integration/apis/entity_manager/fixture_plugin/server/plugin.ts)
that is launched in the entity manager test server, this plugin creates
a test route exposing the upgrade api which can then be called in api
tests.

### manual
- install builtin definitions `PUT
kbn:/internal/api/entities/managed/enablement`
- bump builtin [service
definition](https://github.com/elastic/kibana/blob/main/x-pack/plugins/observability_solution/entity_manager/server/lib/entities/built_in/services.ts#L23)
version
- restart kibana server
- logs should output `[INFO ][plugins.entityManager] Updating built-in
entity definition [builtin_services] from v0.1.0 to v<new version>`
- `GET kbn:/internal/api/entities/definition` should output the new
definition
- verify latest version of definition components are installed

---------

Co-authored-by: kibanamachine <[email protected]>

.github/CODEOWNERS

@@ -395,6 +395,7 @@ x-pack/plugins/encrypted_saved_objects @elastic/kibana-security
 x-pack/plugins/enterprise_search @elastic/search-kibana
 x-pack/plugins/observability_solution/entities_data_access @elastic/obs-entities
 x-pack/packages/kbn-entities-schema @elastic/obs-entities
+x-pack/test/api_integration/apis/entity_manager/fixture_plugin @elastic/obs-entities
 x-pack/plugins/observability_solution/entity_manager @elastic/obs-entities
 examples/error_boundary @elastic/appex-sharedux
 packages/kbn-es @elastic/kibana-operations

package.json

@@ -457,6 +457,7 @@
     "@kbn/enterprise-search-plugin": "link:x-pack/plugins/enterprise_search",
     "@kbn/entities-data-access-plugin": "link:x-pack/plugins/observability_solution/entities_data_access",
     "@kbn/entities-schema": "link:x-pack/packages/kbn-entities-schema",
+    "@kbn/entity-manager-fixture-plugin": "link:x-pack/test/api_integration/apis/entity_manager/fixture_plugin",
     "@kbn/entityManager-plugin": "link:x-pack/plugins/observability_solution/entity_manager",
     "@kbn/error-boundary-example-plugin": "link:examples/error_boundary",
     "@kbn/es-errors": "link:packages/kbn-es-errors",

src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts

@@ -90,7 +90,7 @@ describe('checking migration metadata changes on all registered SO types', () =>
         "endpoint:unified-user-artifact-manifest": "71c7fcb52c658b21ea2800a6b6a76972ae1c776e",
         "endpoint:user-artifact-manifest": "1c3533161811a58772e30cdc77bac4631da3ef2b",
         "enterprise_search_telemetry": "9ac912e1417fc8681e0cd383775382117c9e3d3d",
-        "entity-definition": "331a2ba0ee9f24936ef049683549c8af7e46f03a",
+        "entity-definition": "61be3e95966045122b55e181bb39658b1dc9bbe9",
         "entity-discovery-api-key": "c267a65c69171d1804362155c1378365f5acef88",
         "epm-packages": "8042d4a1522f6c4e6f5486e791b3ffe3a22f88fd",
         "epm-packages-assets": "7a3e58efd9a14191d0d1a00b8aaed30a145fd0b1",

tsconfig.base.json

@@ -784,6 +784,8 @@
       "@kbn/entities-data-access-plugin/*": ["x-pack/plugins/observability_solution/entities_data_access/*"],
       "@kbn/entities-schema": ["x-pack/packages/kbn-entities-schema"],
       "@kbn/entities-schema/*": ["x-pack/packages/kbn-entities-schema/*"],
+      "@kbn/entity-manager-fixture-plugin": ["x-pack/test/api_integration/apis/entity_manager/fixture_plugin"],
+      "@kbn/entity-manager-fixture-plugin/*": ["x-pack/test/api_integration/apis/entity_manager/fixture_plugin/*"],
       "@kbn/entityManager-plugin": ["x-pack/plugins/observability_solution/entity_manager"],
       "@kbn/entityManager-plugin/*": ["x-pack/plugins/observability_solution/entity_manager/*"],
       "@kbn/error-boundary-example-plugin": ["examples/error_boundary"],

x-pack/packages/kbn-entities-schema/src/schema/entity_definition.ts

@@ -48,6 +48,15 @@ export const entityDefinitionSchema = z.object({
       ),
     })
   ),
+  installStatus: z.optional(
+    z.union([
+      z.literal('installing'),
+      z.literal('upgrading'),
+      z.literal('installed'),
+      z.literal('failed'),
+    ])
+  ),
+  installStartedAt: z.optional(z.string()),
 });
 
 export type EntityDefinition = z.infer<typeof entityDefinitionSchema>;

x-pack/plugins/observability_solution/entity_manager/common/errors.ts

@@ -10,3 +10,4 @@ export const ERROR_API_KEY_NOT_VALID = 'api_key_not_valid';
 export const ERROR_API_KEY_SERVICE_DISABLED = 'api_key_service_disabled';
 export const ERROR_PARTIAL_BUILTIN_INSTALLATION = 'partial_builtin_installation';
 export const ERROR_DEFINITION_STOPPED = 'error_definition_stopped';
+export const ERROR_BUILTIN_UPGRADE_REQUIRED = 'builtin_upgrade_required';

x-pack/plugins/observability_solution/entity_manager/kibana.jsonc

@@ -6,7 +6,7 @@
   "plugin": {
     "id": "entityManager",
     "configPath": ["xpack", "entityManager"],
-    "requiredPlugins": ["security", "encryptedSavedObjects"],
+    "requiredPlugins": ["security", "encryptedSavedObjects", "licensing"],
     "browser": true,
     "server": true,
     "requiredBundles": []

x-pack/plugins/observability_solution/entity_manager/server/lib/entities/delete_entity_definition.ts

@@ -5,27 +5,22 @@
  * 2.0.
  */
 
-import { Logger, SavedObjectsClientContract } from '@kbn/core/server';
+import { SavedObjectsClientContract, SavedObjectsErrorHelpers } from '@kbn/core/server';
 import { EntityDefinition } from '@kbn/entities-schema';
 import { SO_ENTITY_DEFINITION_TYPE } from '../../saved_objects';
 import { EntityDefinitionNotFound } from './errors/entity_not_found';
 
 export async function deleteEntityDefinition(
   soClient: SavedObjectsClientContract,
-  definition: EntityDefinition,
-  logger: Logger
+  definition: EntityDefinition
 ) {
-  const response = await soClient.find<EntityDefinition>({
-    type: SO_ENTITY_DEFINITION_TYPE,
-    page: 1,
-    perPage: 1,
-    filter: `${SO_ENTITY_DEFINITION_TYPE}.attributes.id:(${definition.id})`,
-  });
+  try {
+    await soClient.delete(SO_ENTITY_DEFINITION_TYPE, definition.id);
+  } catch (err) {
+    if (SavedObjectsErrorHelpers.isNotFoundError(err)) {
+      throw new EntityDefinitionNotFound(`Entity definition with [${definition.id}] not found.`);
+    }
 
-  if (response.total === 0) {
-    logger.error(`Unable to delete entity definition [${definition.id}] because it doesn't exist.`);
-    throw new EntityDefinitionNotFound(`Entity definition with [${definition.id}] not found.`);
+    throw err;
   }
-
-  await soClient.delete(SO_ENTITY_DEFINITION_TYPE, response.saved_objects[0].id);
 }

x-pack/plugins/observability_solution/entity_manager/server/lib/entities/find_entity_definition.ts

@@ -11,12 +11,16 @@ import { EntityDefinition } from '@kbn/entities-schema';
 import { SO_ENTITY_DEFINITION_TYPE } from '../../saved_objects';
 import {
   generateHistoryTransformId,
+  generateHistoryBackfillTransformId,
   generateHistoryIngestPipelineId,
+  generateHistoryIndexTemplateId,
   generateLatestTransformId,
   generateLatestIngestPipelineId,
+  generateLatestIndexTemplateId,
 } from './helpers/generate_component_id';
 import { BUILT_IN_ID_PREFIX } from './built_in';
 import { EntityDefinitionWithState } from './types';
+import { isBackfillEnabled } from './helpers/is_backfill_enabled';
 
 export async function findEntityDefinitions({
   soClient,
@@ -60,27 +64,40 @@ async function getEntityDefinitionState(
 ) {
   const historyIngestPipelineId = generateHistoryIngestPipelineId(definition);
   const latestIngestPipelineId = generateLatestIngestPipelineId(definition);
-  const [ingestPipelines, transforms] = await Promise.all([
-    esClient.ingest.getPipeline({
-      id: `${historyIngestPipelineId},${latestIngestPipelineId}`,
+  const transformIds = [
+    generateHistoryTransformId(definition),
+    generateLatestTransformId(definition),
+    ...(isBackfillEnabled(definition) ? [generateHistoryBackfillTransformId(definition)] : []),
+  ];
+  const [ingestPipelines, indexTemplatesInstalled, transforms] = await Promise.all([
+    esClient.ingest.getPipeline(
+      {
+        id: `${historyIngestPipelineId},${latestIngestPipelineId}`,
+      },
+      { ignore: [404] }
+    ),
+    esClient.indices.existsIndexTemplate({
+      name: `${
+        (generateLatestIndexTemplateId(definition), generateHistoryIndexTemplateId(definition))
+      }`,
     }),
     esClient.transform.getTransformStats({
-      transform_id: [generateHistoryTransformId(definition), generateLatestTransformId(definition)],
+      transform_id: transformIds,
     }),
   ]);
 
   const ingestPipelinesInstalled = !!(
     ingestPipelines[historyIngestPipelineId] && ingestPipelines[latestIngestPipelineId]
   );
-  const transformsInstalled = transforms.count === 2;
+  const transformsInstalled = transforms.count === transformIds.length;
   const transformsRunning =
     transformsInstalled &&
     transforms.transforms.every(
       (transform) => transform.state === 'started' || transform.state === 'indexing'
     );
 
   return {
-    installed: ingestPipelinesInstalled && transformsInstalled,
+    installed: ingestPipelinesInstalled && transformsInstalled && indexTemplatesInstalled,
     running: transformsRunning,
   };
 }

x-pack/plugins/observability_solution/entity_manager/server/lib/entities/helpers/fixtures/builtin_entity_definition.ts

@@ -0,0 +1,24 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { entityDefinitionSchema } from '@kbn/entities-schema';
+export const builtInEntityDefinition = entityDefinitionSchema.parse({
+  id: 'builtin_mock_entity_definition',
+  version: '1.0.0',
+  name: 'Mock builtin definition',
+  type: 'service',
+  indexPatterns: ['kbn-data-forge-fake_stack.*'],
+  managed: true,
+  history: {
+    timestampField: '@timestamp',
+    interval: '1m',
+  },
+  identityFields: ['log.logger', { field: 'event.category', optional: true }],
+  displayNameTemplate: '{{log.logger}}{{#event.category}}:{{.}}{{/event.category}}',
+  metadata: ['tags', 'host.name', 'host.os.name', { source: '_index', destination: 'sourceIndex' }],
+  metrics: [],
+});

x-pack/plugins/observability_solution/entity_manager/server/lib/entities/helpers/fixtures/entity_definition.ts

@@ -8,7 +8,7 @@
 import { entityDefinitionSchema } from '@kbn/entities-schema';
 export const rawEntityDefinition = {
   id: 'admin-console-services',
-  version: '999.999.999',
+  version: '1.0.0',
   name: 'Services for Admin Console',
   type: 'service',
   indexPatterns: ['kbn-data-forge-fake_stack.*'],

x-pack/plugins/observability_solution/entity_manager/server/lib/entities/helpers/fixtures/index.ts

@@ -7,3 +7,4 @@
 
 export { entityDefinition } from './entity_definition';
 export { entityDefinitionWithBackfill } from './entity_definition_with_backfill';
+export { builtInEntityDefinition } from './builtin_entity_definition';

x-pack/plugins/observability_solution/entity_manager/server/lib/entities/helpers/generate_component_id.ts

@@ -6,6 +6,7 @@
  */
 
 import {
+  ENTITY_BASE_PREFIX,
   ENTITY_HISTORY,
   ENTITY_LATEST,
   ENTITY_SCHEMA_VERSION_V1,
@@ -38,6 +39,10 @@ export function generateHistoryIndexName(definition: EntityDefinition) {
   });
 }
 
+export function generateHistoryIndexTemplateId(definition: EntityDefinition) {
+  return `${ENTITY_BASE_PREFIX}_${ENTITY_SCHEMA_VERSION_V1}_${ENTITY_HISTORY}_${definition.id}_index_template` as const;
+}
+
 // Latest
 function generateLatestId(definition: EntityDefinition) {
   return `${ENTITY_LATEST_PREFIX_V1}-${definition.id}` as const;
@@ -53,3 +58,6 @@ export function generateLatestIndexName(definition: EntityDefinition) {
     definitionId: definition.id,
   });
 }
+
+export const generateLatestIndexTemplateId = (definition: EntityDefinition) =>
+  `${ENTITY_BASE_PREFIX}_${ENTITY_SCHEMA_VERSION_V1}_${ENTITY_LATEST}_${definition.id}_index_template` as const;