[EEM] Entity Manager UI POC

packages/deeplinks/observability/constants.ts

@@ -26,6 +26,8 @@ export const OBSERVABILITY_ONBOARDING_APP_ID = 'observabilityOnboarding';
 
 export const SLO_APP_ID = 'slo';
 
+export const ENTITY_MANAGER_APP_ID = 'entityManager';
+
 export const AI_ASSISTANT_APP_ID = 'observabilityAIAssistant';
 
 export const INVESTIGATE_APP_ID = 'investigate';

packages/deeplinks/observability/deep_links.ts

@@ -21,6 +21,7 @@ import {
   OBLT_UX_APP_ID,
   OBLT_PROFILING_APP_ID,
   INVENTORY_APP_ID,
+  ENTITY_MANAGER_APP_ID,
 } from './constants';
 
 type LogsApp = typeof LOGS_APP_ID;
@@ -32,6 +33,7 @@ type ApmApp = typeof APM_APP_ID;
 type SyntheticsApp = typeof SYNTHETICS_APP_ID;
 type ObservabilityOnboardingApp = typeof OBSERVABILITY_ONBOARDING_APP_ID;
 type SloApp = typeof SLO_APP_ID;
+type EntityManagerApp = typeof ENTITY_MANAGER_APP_ID;
 type AiAssistantApp = typeof AI_ASSISTANT_APP_ID;
 type ObltUxApp = typeof OBLT_UX_APP_ID;
 type ObltProfilingApp = typeof OBLT_PROFILING_APP_ID;
@@ -47,6 +49,7 @@ export type AppId =
   | MetricsApp
   | SyntheticsApp
   | SloApp
+  | EntityManagerApp
   | AiAssistantApp
   | ObltUxApp
   | ObltProfilingApp

x-pack/.i18nrc.json

@@ -146,6 +146,7 @@
     "xpack.securitySolutionEss": "plugins/security_solution_ess",
     "xpack.securitySolutionServerless": "plugins/security_solution_serverless",
     "xpack.sessionView": "plugins/session_view",
+    "xpack.entityManager": "plugins/observability_solution/entity_manager",
     "xpack.slo": "plugins/observability_solution/slo",
     "xpack.snapshotRestore": "plugins/snapshot_restore",
     "xpack.spaces": "plugins/spaces",

x-pack/packages/kbn-data-forge/src/constants.ts

@@ -9,6 +9,7 @@ export const FAKE_HOSTS = 'fake_hosts';
 export const FAKE_LOGS = 'fake_logs';
 export const FAKE_STACK = 'fake_stack';
 export const SERVICE_LOGS = 'service.logs';
+export const FAKE_STACK_DS = 'fake_stack_ds';
 
 export const INDEX_PREFIX = 'kbn-data-forge';
 

x-pack/packages/kbn-data-forge/src/data_sources/fake_stack_ds/index.ts

@@ -0,0 +1,69 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { isArray, omit } from 'lodash';
+import { generateEvent as generateAdminConsole } from '../fake_stack/admin_console';
+import { generateEvent as generateMongoDB } from '../fake_stack/mongodb';
+import { generateEvent as generateMessageProcessor } from '../fake_stack/message_processor';
+import { generateEvent as generateNginxProxy } from '../fake_stack/nginx_proxy';
+import { generateEvent as generateHeartbeat } from '../fake_stack/heartbeat';
+import { Doc, GeneratorFunction } from '../../types';
+import { ADMIN_CONSOLE } from '../fake_stack/common/constants';
+
+const convertToMongoDBRawLog = (event: any) => {
+  return {
+    '@timestamp': event['@timestamp'],
+    message: `${event['@timestamp']} ${event.log.level === 'ERROR' ? 'E' : 'I'} ${
+      event.mongodb.component
+    } [${event.mongodb.context}] ${event.message}`,
+  } as unknown as Doc & { message: string };
+};
+
+const convertToNginxRawLog = (event: any) => {
+  return { '@timestamp': event['@timestamp'], message: event.message } as unknown as Doc & {
+    message: string;
+  };
+};
+
+export const generateEvent: GeneratorFunction = (config, schedule, index, timestamp) => {
+  const scenario = config.indexing.scenario || 'fake_stack';
+  const adminConsoleEvents = generateAdminConsole(config, schedule, index, timestamp);
+  const mongodbEvents = generateMongoDB(config, schedule, index, timestamp).map(
+    convertToMongoDBRawLog
+  );
+  const messageProcessorEvents = generateMessageProcessor(config, schedule, index, timestamp);
+  const nginxProxyEvents = generateNginxProxy(config, schedule, index, timestamp).map(
+    convertToNginxRawLog
+  );
+  const heartbeatEvents = generateHeartbeat(config, schedule, index, timestamp);
+  return [
+    ...(isArray(adminConsoleEvents) ? adminConsoleEvents : [adminConsoleEvents]),
+    ...(isArray(mongodbEvents) ? mongodbEvents : [mongodbEvents]),
+    ...(isArray(messageProcessorEvents) ? messageProcessorEvents : [messageProcessorEvents]),
+    ...(isArray(nginxProxyEvents) ? nginxProxyEvents : [nginxProxyEvents]),
+    ...(isArray(heartbeatEvents) ? heartbeatEvents : [heartbeatEvents]),
+  ].map((event: Doc) => {
+    const labels = event.labels ?? {};
+    const message = JSON.stringify(
+      omit(
+        {
+          ...event,
+          labels: { ...labels, scenario },
+        },
+        'namespace'
+      )
+    );
+    const finalDoc = {
+      namespace: ADMIN_CONSOLE,
+      '@timestamp': event['@timestamp'],
+      message,
+      data_stream: { dataset: ADMIN_CONSOLE, namespace: 'default' },
+    };
+
+    return finalDoc;
+  });
+};

x-pack/packages/kbn-data-forge/src/data_sources/index.ts

@@ -6,30 +6,34 @@
  */
 
 import { GeneratorFunction, Dataset, IndexTemplateDef } from '../types';
-import { FAKE_HOSTS, FAKE_LOGS, FAKE_STACK, SERVICE_LOGS } from '../constants';
+import { FAKE_HOSTS, FAKE_LOGS, FAKE_STACK, FAKE_STACK_DS, SERVICE_LOGS } from '../constants';
 
 import * as fakeLogs from './fake_logs';
 import * as fakeHosts from './fake_hosts';
 import * as fakeStack from './fake_stack';
 import * as serviceLogs from './service_logs';
+import * as fakeStackDs from './fake_stack_ds';
 
 export const indexTemplates: Record<Dataset, IndexTemplateDef[]> = {
   [FAKE_HOSTS]: [fakeHosts.indexTemplate],
   [FAKE_LOGS]: [fakeLogs.indexTemplate],
   [FAKE_STACK]: fakeStack.indexTemplate,
   [SERVICE_LOGS]: [], // uses logs-*-* index templates
+  [FAKE_STACK_DS]: [], // uses logs-*-* index templates
 };
 
 export const generateEvents: Record<Dataset, GeneratorFunction> = {
   [FAKE_HOSTS]: fakeHosts.generateEvent,
   [FAKE_LOGS]: fakeLogs.generateEvent,
   [FAKE_STACK]: fakeStack.generteEvent,
   [SERVICE_LOGS]: serviceLogs.generateEvent,
+  [FAKE_STACK_DS]: fakeStackDs.generateEvent,
 };
 
 export const kibanaAssets: Record<Dataset, string[]> = {
   [FAKE_HOSTS]: [],
   [FAKE_LOGS]: [],
   [FAKE_STACK]: fakeStack.kibanaAssets,
   [SERVICE_LOGS]: [],
+  [FAKE_STACK_DS]: [],
 };

x-pack/packages/kbn-data-forge/src/types/index.ts

@@ -8,7 +8,7 @@
 import type { Moment } from 'moment';
 import { Client } from '@elastic/elasticsearch';
 import * as rt from 'io-ts';
-import { FAKE_HOSTS, FAKE_LOGS, FAKE_STACK, SERVICE_LOGS } from '../constants';
+import { FAKE_HOSTS, FAKE_LOGS, FAKE_STACK, FAKE_STACK_DS, SERVICE_LOGS } from '../constants';
 
 export interface Doc {
   namespace: string;
@@ -27,6 +27,7 @@ export const DatasetRT = rt.keyof({
   [FAKE_LOGS]: null,
   [FAKE_STACK]: null,
   [SERVICE_LOGS]: null,
+  [FAKE_STACK_DS]: null,
 });
 
 export type Dataset = rt.TypeOf<typeof DatasetRT>;

x-pack/packages/kbn-entities-schema/index.ts

@@ -9,7 +9,10 @@ export * from './src/schema/entity_definition';
 export * from './src/schema/entity';
 export * from './src/schema/common';
 export * from './src/schema/patterns';
-export * from './src/rest_spec/create';
-export * from './src/rest_spec/delete';
-export * from './src/rest_spec/reset';
-export * from './src/rest_spec/get';
+export * from './src/rest_spec/definition/delete';
+export * from './src/rest_spec/definition/reset';
+export * from './src/rest_spec/definition/get';
+export * from './src/rest_spec/definition/find';
+export * from './src/rest_spec/definition/create';
+export * from './src/rest_spec/entities/find';
+export * from './src/rest_spec/enablement/check';

x-pack/packages/kbn-entities-schema/src/rest_spec/definition/find.ts

@@ -0,0 +1,74 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { z } from '@kbn/zod';
+import {
+  IndicesGetIndexTemplateIndexTemplateItem,
+  TransformGetTransformStatsTransformStats,
+  TransformGetTransformTransformSummary,
+} from '@elastic/elasticsearch/lib/api/types';
+import { BooleanFromString } from '@kbn/zod-helpers';
+import { entityDefinitionSchema } from '../../schema/entity_definition';
+
+export const findEntityDefinitionQuerySchema = z.object({
+  page: z.optional(z.coerce.number()),
+  perPage: z.optional(z.coerce.number()),
+  includeState: z.optional(BooleanFromString).default(false),
+});
+
+export type FindEntityDefinitionQuery = z.infer<typeof findEntityDefinitionQuerySchema>;
+
+export const entitiyDefinitionWithStateSchema = entityDefinitionSchema.extend({
+  state: z.object({
+    installed: z.boolean(),
+    running: z.boolean(),
+  }),
+  stats: z.object({
+    entityCount: z.number(),
+    totalDocs: z.number(),
+    lastSeenTimestamp: z.string().or(z.null()),
+  }),
+});
+
+interface IngestPipelineState {
+  id: string;
+  installed: boolean;
+  stats: {
+    count: number;
+    failed: number;
+  };
+}
+
+interface IndexTemplateState {
+  id: string;
+  installed: boolean;
+  template: IndicesGetIndexTemplateIndexTemplateItem;
+}
+
+interface TransformState {
+  id: string;
+  installed: boolean;
+  running: boolean;
+  summary: TransformGetTransformTransformSummary;
+  stats: TransformGetTransformStatsTransformStats;
+}
+
+export type EntityDefinitionWithState = z.infer<typeof entitiyDefinitionWithStateSchema> & {
+  resources: {
+    ingestPipelines: IngestPipelineState[];
+    indexTemplates: IndexTemplateState[];
+    transforms: TransformState[];
+  };
+};
+
+export const entityDefintionResponseSchema = z.object({
+  definitions: z.array(entitiyDefinitionWithStateSchema),
+});
+
+export interface EntityDefintionResponse {
+  definitions: EntityDefinitionWithState[];
+}

x-pack/packages/kbn-entities-schema/src/rest_spec/get.ts → x-pack/packages/kbn-entities-schema/src/rest_spec/definition/get.ts

@@ -4,7 +4,6 @@
  * 2.0; you may not use this file except in compliance with the Elastic License
  * 2.0.
  */
-
 import { z } from '@kbn/zod';
 import { BooleanFromString } from '@kbn/zod-helpers';
 
@@ -13,5 +12,9 @@ export const getEntityDefinitionQuerySchema = z.object({
   perPage: z.optional(z.coerce.number()),
   includeState: z.optional(BooleanFromString).default(false),
 });
-
 export type GetEntityDefinitionQuerySchema = z.infer<typeof getEntityDefinitionQuerySchema>;
+
+export const getEntityDefinitionParamsSchema = z.object({
+  id: z.string(),
+});
+export type GetEntityDefinitionParams = z.infer<typeof getEntityDefinitionParamsSchema>;

x-pack/packages/kbn-entities-schema/src/rest_spec/enablement/check.ts

@@ -0,0 +1,14 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+import { z } from 'zod';
+
+export const entityManagerEnablementResponseSchema = z.object({
+  enabled: z.boolean(),
+  reason: z.optional(z.string()),
+});
+
+export type EntityManagerEnablementResponse = z.infer<typeof entityManagerEnablementResponseSchema>;

x-pack/packages/kbn-entities-schema/src/rest_spec/entities/find.test.ts

@@ -0,0 +1,26 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { searchAfterSchema } from './find';
+import rison from '@kbn/rison';
+
+describe('FindEntitiesResponse Schema', () => {
+  describe('searchAfterSchema', () => {
+    it('should parse from rison.encode to schema', () => {
+      const input = rison.encode(['example', 123]);
+      const result = searchAfterSchema.safeParse(input);
+      expect(result).toHaveProperty('success', true);
+      expect(result.data).toEqual(['example', 123]);
+    });
+    it('should work with regular arrays', () => {
+      const input = ['example', 123];
+      const result = searchAfterSchema.safeParse(input);
+      expect(result).toHaveProperty('success', true);
+      expect(result.data).toEqual(['example', 123]);
+    });
+  });
+});

x-pack/packages/kbn-entities-schema/src/rest_spec/entities/find.ts

@@ -0,0 +1,43 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { z } from '@kbn/zod';
+import rison from '@kbn/rison';
+import { entityLatestSchema } from '../../schema/entity';
+
+const searchAfterArraySchema = z.array(z.string().or(z.number()));
+
+export const searchAfterSchema = z
+  .string()
+  .transform((value: string, ctx: z.RefinementCtx) => {
+    try {
+      const result = rison.decode(value);
+      return searchAfterArraySchema.parse(result);
+    } catch (e) {
+      ctx.addIssue({ code: z.ZodIssueCode.custom, message: e.message });
+    }
+    return z.NEVER;
+  })
+  .or(searchAfterArraySchema);
+
+export const findEntitiesQuerySchema = z.object({
+  perPage: z.coerce.number().default(10),
+  query: z.optional(z.string()),
+  searchAfter: z.optional(searchAfterSchema),
+  sortField: z.string().default('entity.displayName.keyword'),
+  sortDirection: z.enum(['asc', 'desc']).default('asc'),
+});
+
+export type FindEntitiesQuery = z.infer<typeof findEntitiesQuerySchema>;
+
+export const findEntitiesResponseSchema = z.object({
+  entities: z.array(entityLatestSchema),
+  total: z.number(),
+  searchAfter: z.optional(z.string()),
+});
+
+export type FindEntitiesResponse = z.infer<typeof findEntitiesResponseSchema>;

x-pack/packages/kbn-entities-schema/src/schema/entity.ts

@@ -40,9 +40,13 @@ export const entityLatestSchema = z
   })
   .and(entityMetadataSchema);
 
+export type EntityLatestDoc = z.infer<typeof entityLatestSchema>;
+
 export const entityHistorySchema = z
   .object({
     '@timestamp': z.string(),
     entity: entityBaseSchema,
   })
   .and(entityMetadataSchema);
+
+export type EntityHistoryDoc = z.infer<typeof entityHistorySchema>;

x-pack/packages/kbn-entities-schema/tsconfig.json

@@ -17,5 +17,6 @@
   "kbn_references": [
     "@kbn/zod",
     "@kbn/zod-helpers",
+    "@kbn/rison",
   ]
 }