[8.x] [Logs] Use central log sources setting for logs context resolut…

…ion in Discover (#192605) (#193724)

# Backport

This will backport the following commits from `main` to `8.x`:
- [[Logs] Use central log sources setting for logs context resolution in
Discover (#192605)](#192605)

<!--- Backport version: 8.9.8 -->

### Questions ?
Please refer to the [Backport tool
documentation](https://github.com/sqren/backport)

<!--BACKPORT [{"author":{"name":"Kerry
Gallagher","email":"[email protected]"},"sourceCommit":{"committedDate":"2024-09-17T11:20:33Z","message":"[Logs]
Use central log sources setting for logs context resolution in Discover
(#192605)\n\n## Summary\r\n\r\nCloses
https://github.com/elastic/logs-dev/issues/171.\r\n\r\nMost of the noise
in the PR is from making methods async and amending\r\ntest mocks, the
core logic changes are in
`createLogsContextService`.","sha":"a87e7e8e6393f8ca5e3f9772d6c8b458229984ef","branchLabelMapping":{"^v9.0.0$":"main","^v8.16.0$":"8.x","^v(\\d+).(\\d+).\\d+$":"$1.$2"}},"sourcePullRequest":{"labels":["release_note:skip","backport:skip","v9.0.0","ci:project-deploy-observability","Team:obs-ux-logs","Project:OneDiscover"],"number":192605,"url":"https://github.com/elastic/kibana/pull/192605","mergeCommit":{"message":"[Logs]
Use central log sources setting for logs context resolution in Discover
(#192605)\n\n## Summary\r\n\r\nCloses
https://github.com/elastic/logs-dev/issues/171.\r\n\r\nMost of the noise
in the PR is from making methods async and amending\r\ntest mocks, the
core logic changes are in
`createLogsContextService`.","sha":"a87e7e8e6393f8ca5e3f9772d6c8b458229984ef"}},"sourceBranch":"main","suggestedTargetBranches":[],"targetPullRequestStates":[{"branch":"main","label":"v9.0.0","labelRegex":"^v9.0.0$","isSourceBranch":true,"state":"MERGED","url":"https://github.com/elastic/kibana/pull/192605","number":192605,"mergeCommit":{"message":"[Logs]
Use central log sources setting for logs context resolution in Discover
(#192605)\n\n## Summary\r\n\r\nCloses
https://github.com/elastic/logs-dev/issues/171.\r\n\r\nMost of the noise
in the PR is from making methods async and amending\r\ntest mocks, the
core logic changes are in
`createLogsContextService`.","sha":"a87e7e8e6393f8ca5e3f9772d6c8b458229984ef"}}]}]
BACKPORT-->

packages/kbn-discover-utils/src/__mocks__/index.ts

@@ -10,3 +10,4 @@
 export * from './data_view';
 export * from './es_hits';
 export * from './additional_field_groups';
+export * from './logs_context_service';

packages/kbn-discover-utils/src/__mocks__/logs_context_service.ts

@@ -0,0 +1,14 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import { DEFAULT_ALLOWED_LOGS_BASE_PATTERNS_REGEXP, getLogsContextService } from '../data_types';
+
+export const createLogsContextServiceMock = () => {
+  return getLogsContextService([DEFAULT_ALLOWED_LOGS_BASE_PATTERNS_REGEXP]);
+};

packages/kbn-discover-utils/src/data_types/logs/logs_context_service.ts

@@ -8,15 +8,14 @@
  */
 
 import { createRegExpPatternFrom, testPatternAgainstAllowedList } from '@kbn/data-view-utils';
+import type { LogsDataAccessPluginStart } from '@kbn/logs-data-access-plugin/public';
 
 export interface LogsContextService {
   isLogsIndexPattern(indexPattern: unknown): boolean;
 }
 
-// eslint-disable-next-line @typescript-eslint/no-empty-interface
 export interface LogsContextServiceDeps {
-  // We will probably soon add uiSettings as a dependency
-  // to consume user configured indices
+  logsDataAccessPlugin?: LogsDataAccessPluginStart;
 }
 
 export const DEFAULT_ALLOWED_LOGS_BASE_PATTERNS = [
@@ -28,15 +27,36 @@ export const DEFAULT_ALLOWED_LOGS_BASE_PATTERNS = [
   'winlogbeat',
 ];
 
-export const createLogsContextService = (_deps: LogsContextServiceDeps = {}) => {
-  // This is initially an hard-coded set of well-known base patterns,
-  // we can extend this allowed list with any setting coming from uiSettings
-  const ALLOWED_LOGS_DATA_SOURCES = [createRegExpPatternFrom(DEFAULT_ALLOWED_LOGS_BASE_PATTERNS)];
+export const DEFAULT_ALLOWED_LOGS_BASE_PATTERNS_REGEXP = createRegExpPatternFrom(
+  DEFAULT_ALLOWED_LOGS_BASE_PATTERNS
+);
 
+export const createLogsContextService = async ({
+  logsDataAccessPlugin,
+}: LogsContextServiceDeps) => {
+  let logSources: string[] | undefined;
+
+  if (logsDataAccessPlugin) {
+    const logSourcesService = logsDataAccessPlugin.services.logSourcesService;
+    logSources = (await logSourcesService.getLogSources())
+      .map((logSource) => logSource.indexPattern)
+      .join(',') // TODO: Will be replaced by helper in: https://github.com/elastic/kibana/pull/192003
+      .split(',');
+  }
+
+  const ALLOWED_LOGS_DATA_SOURCES = [
+    DEFAULT_ALLOWED_LOGS_BASE_PATTERNS_REGEXP,
+    ...(logSources ? logSources : []),
+  ];
+
+  return getLogsContextService(ALLOWED_LOGS_DATA_SOURCES);
+};
+
+export const getLogsContextService = (allowedDataSources: Array<string | RegExp>) => {
   const isLogsIndexPattern = (indexPattern: unknown) => {
     return (
       typeof indexPattern === 'string' &&
-      testPatternAgainstAllowedList(ALLOWED_LOGS_DATA_SOURCES)(indexPattern)
+      testPatternAgainstAllowedList(allowedDataSources)(indexPattern)
     );
   };
 

packages/kbn-discover-utils/tsconfig.json

@@ -26,6 +26,7 @@
     "@kbn/i18n",
     "@kbn/core-ui-settings-browser",
     "@kbn/ui-theme",
-    "@kbn/expressions-plugin"
+    "@kbn/expressions-plugin",
+    "@kbn/logs-data-access-plugin"
   ]
 }

src/plugins/discover/kibana.jsonc

@@ -41,7 +41,8 @@
       "globalSearch",
       "observabilityAIAssistant",
       "aiops",
-      "fieldsMetadata"
+      "fieldsMetadata",
+      "logsDataAccess"
     ],
     "requiredBundles": ["kibanaUtils", "kibanaReact", "unifiedSearch", "savedObjects"],
     "extraPublicDirs": ["common"]

src/plugins/discover/public/context_awareness/__mocks__/index.tsx

@@ -21,8 +21,8 @@ import {
   RootProfileService,
   SolutionType,
 } from '../profiles';
-import { createProfileProviderServices } from '../profile_providers/profile_provider_services';
 import { ProfilesManager } from '../profiles_manager';
+import { createLogsContextServiceMock } from '@kbn/discover-utils/src/__mocks__';
 
 export const createContextAwarenessMocks = ({
   shouldRegisterProviders = true,
@@ -156,7 +156,7 @@ export const createContextAwarenessMocks = ({
     documentProfileServiceMock
   );
 
-  const profileProviderServices = createProfileProviderServices();
+  const profileProviderServices = createProfileProviderServicesMock();
 
   return {
     rootProfileProviderMock,
@@ -171,3 +171,9 @@ export const createContextAwarenessMocks = ({
     profileProviderServices,
   };
 };
+
+const createProfileProviderServicesMock = () => {
+  return {
+    logsContextService: createLogsContextServiceMock(),
+  };
+};

src/plugins/discover/public/context_awareness/profile_providers/profile_provider_services.ts

@@ -8,14 +8,13 @@
  */
 
 import { createLogsContextService, LogsContextService } from '@kbn/discover-utils';
+import type { LogsDataAccessPluginStart } from '@kbn/logs-data-access-plugin/public';
 
 /**
  * Dependencies required by profile provider implementations
  */
-// eslint-disable-next-line @typescript-eslint/no-empty-interface
 export interface ProfileProviderDeps {
-  // We will probably soon add uiSettings as a dependency
-  // to consume user configured indices
+  logsDataAccessPlugin?: LogsDataAccessPluginStart;
 }
 
 /**
@@ -33,10 +32,12 @@ export interface ProfileProviderServices {
  * @param _deps Profile provider dependencies
  * @returns Profile provider services
  */
-export const createProfileProviderServices = (
-  _deps: ProfileProviderDeps = {}
-): ProfileProviderServices => {
+export const createProfileProviderServices = async (
+  deps: ProfileProviderDeps = {}
+): Promise<ProfileProviderServices> => {
   return {
-    logsContextService: createLogsContextService(),
+    logsContextService: await createLogsContextService({
+      logsDataAccessPlugin: deps.logsDataAccessPlugin,
+    }),
   };
 };

src/plugins/discover/public/context_awareness/profile_providers/register_profile_providers.test.ts

@@ -8,6 +8,7 @@
  */
 
 import { createEsqlDataSource } from '../../../common/data_sources';
+import { DiscoverStartPlugins } from '../../types';
 import { createContextAwarenessMocks } from '../__mocks__';
 import { createExampleRootProfileProvider } from './example/example_root_pofile';
 import { createExampleDataSourceProfileProvider } from './example/example_data_source_profile/profile';
@@ -73,7 +74,8 @@ describe('registerProfileProviders', () => {
       createContextAwarenessMocks({
         shouldRegisterProviders: false,
       });
-    registerProfileProviders({
+    await registerProfileProviders({
+      plugins: {} as DiscoverStartPlugins,
       rootProfileService: rootProfileServiceMock,
       dataSourceProfileService: dataSourceProfileServiceMock,
       documentProfileService: documentProfileServiceMock,
@@ -108,7 +110,8 @@ describe('registerProfileProviders', () => {
       createContextAwarenessMocks({
         shouldRegisterProviders: false,
       });
-    registerProfileProviders({
+    await registerProfileProviders({
+      plugins: {} as DiscoverStartPlugins,
       rootProfileService: rootProfileServiceMock,
       dataSourceProfileService: dataSourceProfileServiceMock,
       documentProfileService: documentProfileServiceMock,

src/plugins/discover/public/context_awareness/profile_providers/register_profile_providers.ts

@@ -23,17 +23,20 @@ import {
   createProfileProviderServices,
   ProfileProviderServices,
 } from './profile_provider_services';
+import type { DiscoverStartPlugins } from '../../types';
 
 /**
  * Register profile providers for root, data source, and document contexts to the profile profile services
  * @param options Register profile provider options
  */
-export const registerProfileProviders = ({
+export const registerProfileProviders = async ({
+  plugins,
   rootProfileService,
   dataSourceProfileService,
   documentProfileService,
   enabledExperimentalProfileIds,
 }: {
+  plugins: DiscoverStartPlugins;
   /**
    * Root profile service
    */
@@ -51,7 +54,9 @@ export const registerProfileProviders = ({
    */
   enabledExperimentalProfileIds: string[];
 }) => {
-  const providerServices = createProfileProviderServices();
+  const providerServices = await createProfileProviderServices({
+    logsDataAccessPlugin: plugins.logsDataAccess,
+  });
   const rootProfileProviders = createRootProfileProviders(providerServices);
   const dataSourceProfileProviders = createDataSourceProfileProviders(providerServices);
   const documentProfileProviders = createDocumentProfileProviders(providerServices);

src/plugins/discover/public/plugin.tsx

@@ -183,7 +183,7 @@ export class DiscoverPlugin
           history: this.historyService.getHistory(),
           scopedHistory: this.scopedHistory,
           urlTracker: this.urlTracker!,
-          profilesManager: await this.createProfilesManager(),
+          profilesManager: await this.createProfilesManager({ plugins: discoverStartPlugins }),
           setHeaderActionMenu: params.setHeaderActionMenu,
         });
 
@@ -302,14 +302,15 @@ export class DiscoverPlugin
     }
   }
 
-  private createProfileServices = once(async () => {
+  private createProfileServices = once(async ({ plugins }: { plugins: DiscoverStartPlugins }) => {
     const { registerProfileProviders } = await import('./context_awareness/profile_providers');
     const rootProfileService = new RootProfileService();
     const dataSourceProfileService = new DataSourceProfileService();
     const documentProfileService = new DocumentProfileService();
     const enabledExperimentalProfileIds = this.experimentalFeatures.enabledProfiles ?? [];
 
-    registerProfileProviders({
+    await registerProfileProviders({
+      plugins,
       rootProfileService,
       dataSourceProfileService,
       documentProfileService,
@@ -319,9 +320,9 @@ export class DiscoverPlugin
     return { rootProfileService, dataSourceProfileService, documentProfileService };
   });
 
-  private async createProfilesManager() {
+  private async createProfilesManager({ plugins }: { plugins: DiscoverStartPlugins }) {
     const { rootProfileService, dataSourceProfileService, documentProfileService } =
-      await this.createProfileServices();
+      await this.createProfileServices({ plugins });
 
     return new ProfilesManager(
       rootProfileService,
@@ -367,7 +368,7 @@ export class DiscoverPlugin
 
     const getDiscoverServicesInternal = async () => {
       const [coreStart, deps] = await core.getStartServices();
-      const profilesManager = await this.createProfilesManager();
+      const profilesManager = await this.createProfilesManager({ plugins: deps });
       return this.getDiscoverServices(coreStart, deps, profilesManager);
     };
 

src/plugins/discover/public/types.ts

@@ -41,6 +41,7 @@ import type {
 import type { AiopsPluginStart } from '@kbn/aiops-plugin/public';
 import type { DataVisualizerPluginStart } from '@kbn/data-visualizer-plugin/public';
 import type { FieldsMetadataPublicStart } from '@kbn/fields-metadata-plugin/public';
+import type { LogsDataAccessPluginStart } from '@kbn/logs-data-access-plugin/public';
 import { DiscoverAppLocator } from '../common';
 import { DiscoverCustomizationContext } from './customizations';
 import { type DiscoverContainerProps } from './components/discover_container';
@@ -170,4 +171,5 @@ export interface DiscoverStartPlugins {
   urlForwarding: UrlForwardingStart;
   usageCollection?: UsageCollectionSetup;
   fieldsMetadata: FieldsMetadataPublicStart;
+  logsDataAccess?: LogsDataAccessPluginStart;
 }

src/plugins/discover/tsconfig.json

@@ -98,6 +98,7 @@
     "@kbn/security-solution-common",
     "@kbn/router-utils",
     "@kbn/management-settings-ids",
+    "@kbn/logs-data-access-plugin"
   ],
   "exclude": [
     "target/**/*"

x-pack/plugins/observability_solution/logs_data_access/common/constants.ts

@@ -5,4 +5,4 @@
  * 2.0.
  */
 
-export const DEFAULT_LOG_SOURCES = ['logs-*-*,logs-*,filebeat-*,kibana_sample_data_logs*'];
+export const DEFAULT_LOG_SOURCES = ['logs-*-*', 'logs-*', 'filebeat-*', 'kibana_sample_data_logs*'];