[Logs] Use central log sources setting in ML jobs (elastic#190071)

## Summary This uses the new central log sources setting when creating or recreating log rate and category jobs. The indices are injected into the job configuration so changes to the advanced setting won't suddenly break the jobs (in the same way that it would potentially break alerting rules). ## UI / UX Small amendment to the text here: ![Screenshot 2024-08-08 at 15 53 28](https://github.com/user-attachments/assets/19f573f3-619e-4c3e-8778-87972dff922e) Callout functionality should still work as expected if the advanced setting is changed after the job has been created: ![Screenshot 2024-08-08 at 15 55 25](https://github.com/user-attachments/assets/a4e54ca3-f31a-4496-9c27-fd2aa8424853) ## Side quests - Logs > Settings page has been updated to use the new `useLogSources` provider in the advanced setting panel. --------- Co-authored-by: Marco Antonio Ghiani <[email protected]>
kapral18 · Aug 9, 2024 · 1f11376 · 1f11376
1 parent b1a2922
commit 1f11376
Show file tree

Hide file tree

Showing 15 changed files with 187 additions and 122 deletions.
diff --git a/.../public/components/logging/log_analysis_job_status/job_configuration_outdated_callout.tsx b/.../public/components/logging/log_analysis_job_status/job_configuration_outdated_callout.tsx
@@ -28,7 +28,7 @@ export const JobConfigurationOutdatedCallout: React.FC<{
   >
     <FormattedMessage
       id="xpack.infra.logs.analysis.jobConfigurationOutdatedCalloutMessage"
-      defaultMessage="The {moduleName} ML job was created using a different source configuration. Recreate the job to apply the current configuration. This removes previously detected anomalies."
+      defaultMessage="The {moduleName} ML job was created using a different configuration. Recreate the job to apply the current configuration. This removes previously detected anomalies."
       values={{
         moduleName,
       }}

diff --git a/...nts/logging/log_analysis_setup/initial_configuration_step/analysis_setup_indices_form.tsx b/...nts/logging/log_analysis_setup/initial_configuration_step/analysis_setup_indices_form.tsx
@@ -5,15 +5,23 @@
  * 2.0.
  */
 
-import { EuiTitle, EuiText, EuiFormRow, EuiFlexGroup, EuiFlexItem } from '@elastic/eui';
+import { EuiTitle, EuiText, EuiFormRow, EuiFlexGroup, EuiFlexItem, EuiLink } from '@elastic/eui';
 import { i18n } from '@kbn/i18n';
 import { FormattedMessage } from '@kbn/i18n-react';
-import React, { useCallback } from 'react';
+import React, { useCallback, useMemo } from 'react';
+import { ApplicationStart } from '@kbn/core-application-browser';
+import { useKibanaContextForPlugin } from '../../../../hooks/use_kibana';
 import { QualityWarning } from '../../../../../common/log_analysis';
 import { LoadingOverlayWrapper } from '../../../loading_overlay_wrapper';
 import { IndexSetupRow } from './index_setup_row';
 import { AvailableIndex, ValidationIndicesError } from './validation';
 
+function getKibanaAdvancedSettingsHref(application: ApplicationStart) {
+  return application.getUrlForApp('management', {
+    path: `/kibana/settings?query=${encodeURIComponent('Log sources')}`,
+  });
+}
+
 export const AnalysisSetupIndicesForm: React.FunctionComponent<{
   disabled?: boolean;
   indices: AvailableIndex[];
@@ -29,6 +37,15 @@ export const AnalysisSetupIndicesForm: React.FunctionComponent<{
   previousQualityWarnings = [],
   validationErrors = [],
 }) => {
+  const {
+    services: { application },
+  } = useKibanaContextForPlugin();
+
+  const advancedSettingsHref = useMemo(
+    () => getKibanaAdvancedSettingsHref(application),
+    [application]
+  );
+
   const changeIsIndexSelected = useCallback(
     (indexName: string, isSelected: boolean) => {
       onChangeSelectedIndices(
@@ -67,7 +84,20 @@ export const AnalysisSetupIndicesForm: React.FunctionComponent<{
         <EuiText size="s" color="subdued">
           <FormattedMessage
             id="xpack.infra.analysisSetup.indicesSelectionDescription"
-            defaultMessage="By default, Machine Learning analyzes log messages in all log indices configured for the source. You can choose to only analyze a subset of the index names. Every selected index name must match at least one index with log entries. You can also choose to only include a certain subset of datasets. Note that the dataset filter applies to all selected indices."
+            defaultMessage="By default, Machine Learning analyzes log messages in all log indices configured for the {advancedSettingsLink}. You can choose to only analyze a subset of the index names. Every selected index name must match at least one index with log entries. You can also choose to only include a certain subset of datasets. Note that the dataset filter applies to all selected indices."
+            values={{
+              advancedSettingsLink: (
+                <EuiLink
+                  data-test-subj="xpack.infra.analysisSetup.logSourcesSettingLink"
+                  href={advancedSettingsHref}
+                >
+                  <FormattedMessage
+                    id="xpack.infra.analysisSetup.logSourcesSettingLinkText"
+                    defaultMessage="log sources setting"
+                  />
+                </EuiLink>
+              ),
+            }}
           />
         </EuiText>
       </EuiFlexItem>

diff --git a/...ainers/logs/log_analysis/modules/log_entry_categories/use_log_entry_categories_module.tsx b/...ainers/logs/log_analysis/modules/log_entry_categories/use_log_entry_categories_module.tsx
@@ -18,14 +18,14 @@ import { useLogEntryCategoriesQuality } from './use_log_entry_categories_quality
 
 export const useLogEntryCategoriesModule = ({
   indexPattern,
-  logViewId,
+  sourceId,
   spaceId,
   idFormat,
   timestampField,
   runtimeMappings,
 }: {
   indexPattern: string;
-  logViewId: string;
+  sourceId: string;
   spaceId: string;
   idFormat: IdFormat;
   timestampField: string;
@@ -34,12 +34,12 @@ export const useLogEntryCategoriesModule = ({
   const sourceConfiguration: ModuleSourceConfiguration = useMemo(
     () => ({
       indices: indexPattern.split(','),
-      sourceId: logViewId,
+      sourceId,
       spaceId,
       timestampField,
       runtimeMappings,
     }),
-    [indexPattern, logViewId, spaceId, timestampField, runtimeMappings]
+    [indexPattern, sourceId, spaceId, timestampField, runtimeMappings]
   );
 
   const logAnalysisModule = useLogAnalysisModule({

diff --git a/.../public/containers/logs/log_analysis/modules/log_entry_rate/use_log_entry_rate_module.tsx b/.../public/containers/logs/log_analysis/modules/log_entry_rate/use_log_entry_rate_module.tsx
@@ -17,14 +17,14 @@ import { logEntryRateModule } from './module_descriptor';
 
 export const useLogEntryRateModule = ({
   indexPattern,
-  logViewId,
+  sourceId,
   spaceId,
   idFormat,
   timestampField,
   runtimeMappings,
 }: {
   indexPattern: string;
-  logViewId: string;
+  sourceId: string;
   spaceId: string;
   idFormat: IdFormat;
   timestampField: string;
@@ -33,12 +33,12 @@ export const useLogEntryRateModule = ({
   const sourceConfiguration: ModuleSourceConfiguration = useMemo(
     () => ({
       indices: indexPattern.split(','),
-      sourceId: logViewId,
+      sourceId,
       spaceId,
       timestampField,
       runtimeMappings,
     }),
-    [indexPattern, logViewId, spaceId, timestampField, runtimeMappings]
+    [indexPattern, sourceId, spaceId, timestampField, runtimeMappings]
   );
 
   const logAnalysisModule = useLogAnalysisModule({

diff --git a/...ns/observability_solution/infra/public/pages/logs/log_entry_categories/page_providers.tsx b/...ns/observability_solution/infra/public/pages/logs/log_entry_categories/page_providers.tsx
@@ -6,26 +6,27 @@
  */
 
 import React, { FC, PropsWithChildren } from 'react';
-import { useLogViewContext } from '@kbn/logs-shared-plugin/public';
+import { useLogSourcesContext } from '@kbn/logs-data-access-plugin/public';
 import { logEntryCategoriesJobType } from '../../../../common/log_analysis';
-import { InlineLogViewSplashPage } from '../../../components/logging/inline_log_view_splash_page';
 import { LogAnalysisSetupFlyoutStateProvider } from '../../../components/logging/log_analysis_setup/setup_flyout';
 import { SourceLoadingPage } from '../../../components/source_loading_page';
 import { LogEntryCategoriesModuleProvider } from '../../../containers/logs/log_analysis/modules/log_entry_categories';
 import { useActiveKibanaSpace } from '../../../hooks/use_kibana_space';
-import { ConnectedLogViewErrorPage } from '../shared/page_log_view_error';
+import { LogSourceErrorPage } from '../shared/page_log_view_error';
 import { useLogMlJobIdFormatsShimContext } from '../shared/use_log_ml_job_id_formats_shim';
 
+const TIMESTAMP_FIELD = '@timestamp';
+const DEFAULT_MODULE_SOURCE_CONFIGURATION_ID = 'default'; // NOTE: Left in for legacy reasons, this used to refer to a log view ID (legacy).
+
 export const LogEntryCategoriesPageProviders: FC<PropsWithChildren<unknown>> = ({ children }) => {
   const {
-    hasFailedLoading,
-    isLoading,
+    logSources,
+    isLoadingLogSources,
     isUninitialized,
-    resolvedLogView,
-    logViewReference,
-    isPersistedLogView,
-    revertToDefaultLogView,
-  } = useLogViewContext();
+    hasFailedLoadingLogSources,
+    logSourcesError,
+    combinedIndices,
+  } = useLogSourcesContext();
   const { space } = useActiveKibanaSpace();
   const { idFormats, isLoadingLogAnalysisIdFormats, hasFailedLoadingLogAnalysisIdFormats } =
     useLogMlJobIdFormatsShimContext();
@@ -35,24 +36,24 @@ export const LogEntryCategoriesPageProviders: FC<PropsWithChildren<unknown>> = (
   // React concurrent mode and Suspense in order to handle that more gracefully.
   if (space == null) {
     return null;
-  } else if (!isPersistedLogView) {
-    return <InlineLogViewSplashPage revertToDefaultLogView={revertToDefaultLogView} />;
-  } else if (hasFailedLoading || hasFailedLoadingLogAnalysisIdFormats) {
-    return <ConnectedLogViewErrorPage />;
-  } else if (isLoading || isUninitialized || isLoadingLogAnalysisIdFormats || !idFormats) {
+  } else if (hasFailedLoadingLogSources || hasFailedLoadingLogAnalysisIdFormats) {
+    return <LogSourceErrorPage errors={logSourcesError !== undefined ? [logSourcesError] : []} />;
+  } else if (
+    isLoadingLogSources ||
+    isUninitialized ||
+    isLoadingLogAnalysisIdFormats ||
+    !idFormats
+  ) {
     return <SourceLoadingPage />;
-  } else if (resolvedLogView != null) {
-    if (logViewReference.type === 'log-view-inline') {
-      throw new Error('Logs ML features only support persisted Log View references');
-    }
+  } else if (logSources.length > 0) {
     return (
       <LogEntryCategoriesModuleProvider
-        indexPattern={resolvedLogView.indices}
-        logViewId={logViewReference.logViewId}
+        indexPattern={combinedIndices}
         spaceId={space.id}
+        sourceId={DEFAULT_MODULE_SOURCE_CONFIGURATION_ID}
         idFormat={idFormats[logEntryCategoriesJobType]}
-        timestampField={resolvedLogView.timestampField}
-        runtimeMappings={resolvedLogView.runtimeMappings}
+        timestampField={TIMESTAMP_FIELD}
+        runtimeMappings={{}}
       >
         <LogAnalysisSetupFlyoutStateProvider>{children}</LogAnalysisSetupFlyoutStateProvider>
       </LogEntryCategoriesModuleProvider>

diff --git a/.../plugins/observability_solution/infra/public/pages/logs/log_entry_rate/page_providers.tsx b/.../plugins/observability_solution/infra/public/pages/logs/log_entry_rate/page_providers.tsx
@@ -6,29 +6,29 @@
  */
 
 import React, { FC, PropsWithChildren } from 'react';
-import { useLogViewContext } from '@kbn/logs-shared-plugin/public';
+import { useLogSourcesContext } from '@kbn/logs-data-access-plugin/public';
 import { logEntryCategoriesJobType, logEntryRateJobType } from '../../../../common/log_analysis';
-import { InlineLogViewSplashPage } from '../../../components/logging/inline_log_view_splash_page';
 import { LogAnalysisSetupFlyoutStateProvider } from '../../../components/logging/log_analysis_setup/setup_flyout';
 import { SourceLoadingPage } from '../../../components/source_loading_page';
 import { LogEntryCategoriesModuleProvider } from '../../../containers/logs/log_analysis/modules/log_entry_categories';
 import { LogEntryRateModuleProvider } from '../../../containers/logs/log_analysis/modules/log_entry_rate';
 import { LogEntryFlyoutProvider } from '../../../containers/logs/log_flyout';
 import { useActiveKibanaSpace } from '../../../hooks/use_kibana_space';
-import { ConnectedLogViewErrorPage } from '../shared/page_log_view_error';
+import { LogSourceErrorPage } from '../shared/page_log_view_error';
 import { useLogMlJobIdFormatsShimContext } from '../shared/use_log_ml_job_id_formats_shim';
 
+const TIMESTAMP_FIELD = '@timestamp';
+const DEFAULT_MODULE_SOURCE_CONFIGURATION_ID = 'default'; // NOTE: Left in for legacy reasons, this used to refer to a log view ID (legacy).
+
 export const LogEntryRatePageProviders: FC<PropsWithChildren<unknown>> = ({ children }) => {
   const {
-    hasFailedLoading,
-    isLoading,
+    logSources,
+    isLoadingLogSources,
     isUninitialized,
-    logViewReference,
-    resolvedLogView,
-    isPersistedLogView,
-    revertToDefaultLogView,
-  } = useLogViewContext();
-
+    hasFailedLoadingLogSources,
+    logSourcesError,
+    combinedIndices,
+  } = useLogSourcesContext();
   const { space } = useActiveKibanaSpace();
 
   const { idFormats, isLoadingLogAnalysisIdFormats, hasFailedLoadingLogAnalysisIdFormats } =
@@ -39,33 +39,33 @@ export const LogEntryRatePageProviders: FC<PropsWithChildren<unknown>> = ({ chil
   // React concurrent mode and Suspense in order to handle that more gracefully.
   if (space == null) {
     return null;
-  } else if (!isPersistedLogView) {
-    return <InlineLogViewSplashPage revertToDefaultLogView={revertToDefaultLogView} />;
-  } else if (isLoading || isUninitialized || isLoadingLogAnalysisIdFormats || !idFormats) {
+  } else if (
+    isLoadingLogSources ||
+    isUninitialized ||
+    isLoadingLogAnalysisIdFormats ||
+    !idFormats
+  ) {
     return <SourceLoadingPage />;
-  } else if (hasFailedLoading || hasFailedLoadingLogAnalysisIdFormats) {
-    return <ConnectedLogViewErrorPage />;
-  } else if (resolvedLogView != null) {
-    if (logViewReference.type === 'log-view-inline') {
-      throw new Error('Logs ML features only support persisted Log Views');
-    }
+  } else if (hasFailedLoadingLogSources || hasFailedLoadingLogAnalysisIdFormats) {
+    return <LogSourceErrorPage errors={logSourcesError !== undefined ? [logSourcesError] : []} />;
+  } else if (logSources.length > 0) {
     return (
       <LogEntryFlyoutProvider>
         <LogEntryRateModuleProvider
-          indexPattern={resolvedLogView.indices}
-          logViewId={logViewReference.logViewId}
+          indexPattern={combinedIndices}
           spaceId={space.id}
+          sourceId={DEFAULT_MODULE_SOURCE_CONFIGURATION_ID}
           idFormat={idFormats[logEntryRateJobType]}
-          timestampField={resolvedLogView.timestampField}
-          runtimeMappings={resolvedLogView.runtimeMappings}
+          timestampField={TIMESTAMP_FIELD}
+          runtimeMappings={{}}
         >
           <LogEntryCategoriesModuleProvider
-            indexPattern={resolvedLogView.indices}
-            logViewId={logViewReference.logViewId}
+            indexPattern={combinedIndices}
             spaceId={space.id}
+            sourceId={DEFAULT_MODULE_SOURCE_CONFIGURATION_ID}
             idFormat={idFormats[logEntryCategoriesJobType]}
-            timestampField={resolvedLogView.timestampField}
-            runtimeMappings={resolvedLogView.runtimeMappings}
+            timestampField={TIMESTAMP_FIELD}
+            runtimeMappings={{}}
           >
             <LogAnalysisSetupFlyoutStateProvider>{children}</LogAnalysisSetupFlyoutStateProvider>
           </LogEntryCategoriesModuleProvider>

diff --git a/x-pack/plugins/observability_solution/infra/public/pages/logs/page_providers.tsx b/x-pack/plugins/observability_solution/infra/public/pages/logs/page_providers.tsx
@@ -12,6 +12,7 @@ import {
   updateContextInUrl as createUpdateContextInUrl,
   listenForUrlChanges as createListenForUrlChanges,
 } from '@kbn/logs-shared-plugin/public';
+import { LogSourcesProvider } from '@kbn/logs-data-access-plugin/public';
 import { LogAnalysisCapabilitiesProvider } from '../../containers/logs/log_analysis';
 import { useKibanaContextForPlugin } from '../../hooks/use_kibana';
 import { useKbnUrlStateStorageFromRouterContext } from '../../containers/kbn_url_state_context';
@@ -21,6 +22,7 @@ export const LogsPageProviders: FC<PropsWithChildren<unknown>> = ({ children })
     services: {
       notifications: { toasts: toastsService },
       logsShared,
+      logsDataAccess,
     },
   } = useKibanaContextForPlugin();
 
@@ -43,7 +45,9 @@ export const LogsPageProviders: FC<PropsWithChildren<unknown>> = ({ children })
       updateContextInUrl={updateContextInUrl}
       listenForUrlChanges={listenForUrlChanges}
     >
-      <LogAnalysisCapabilitiesProvider>{children}</LogAnalysisCapabilitiesProvider>
+      <LogSourcesProvider logSourcesService={logsDataAccess.services.logSourcesService}>
+        <LogAnalysisCapabilitiesProvider>{children}</LogAnalysisCapabilitiesProvider>
+      </LogSourcesProvider>
     </LogViewProvider>
   );
 };