[Dataset quality] Move page to stack management > Data (elastic#184122)

Relates to elastic#183406.

## 📝  Summary
This PR creates a new plugin `data_quality` in order to register dataset
quality as a Stack management page under data section. For now there is
no reference to this new page in the sideNav in stateful or serverless.

In order to navigate to this new page you can use the url
`/app/management/data/data_quality`

Changes included in this PR:
- New plugin created
- Plugin registered in stack management, data section
- Dataset quality plugin is instantiated and the state is in sync with
URL
- Removed references to dataset quality in Logs explorer

## 🎥 Demo


https://github.com/elastic/kibana/assets/1313018/501c9c47-4a1b-4f91-9be6-d022a821e88e

## 🙅🏼 Missing

- Dataset quality locator
- There are still references to logs explorer (table and flyout) that
will be handled in a follow up PR.

---------

Co-authored-by: kibanamachine <[email protected]>

.github/CODEOWNERS

@@ -337,6 +337,7 @@ x-pack/plugins/dashboard_enhanced @elastic/kibana-presentation
 src/plugins/dashboard @elastic/kibana-presentation
 x-pack/packages/kbn-data-forge @elastic/obs-ux-management-team
 src/plugins/data @elastic/kibana-visualizations @elastic/kibana-data-discovery
+x-pack/plugins/data_quality @elastic/obs-ux-logs-team
 test/plugin_functional/plugins/data_search @elastic/kibana-data-discovery
 packages/kbn-data-service @elastic/kibana-visualizations @elastic/kibana-data-discovery
 packages/kbn-data-stream-adapter @elastic/security-threat-hunting-explore

docs/developer/plugin-list.asciidoc

@@ -531,6 +531,10 @@ Plugin server-side only. Plugin has three main functions:
 |Adds drilldown capabilities to dashboard.  Owned by the Kibana App team.
 
 
+|{kib-repo}blob/{branch}/x-pack/plugins/data_quality/README.md[dataQuality]
+|Page where users can see the quality of their log datasets.
+
+
 |{kib-repo}blob/{branch}/x-pack/plugins/observability_solution/dataset_quality/README.md[datasetQuality]
 |In order to make ongoing maintenance of log collection easy we want to introduce the concept of dataset quality, where users can easily get an overview on the datasets they have with information such as integration, size, last activity, among others.
 

package.json

@@ -391,6 +391,7 @@
     "@kbn/dashboard-plugin": "link:src/plugins/dashboard",
     "@kbn/data-forge": "link:x-pack/packages/kbn-data-forge",
     "@kbn/data-plugin": "link:src/plugins/data",
+    "@kbn/data-quality-plugin": "link:x-pack/plugins/data_quality",
     "@kbn/data-search-plugin": "link:test/plugin_functional/plugins/data_search",
     "@kbn/data-service": "link:packages/kbn-data-service",
     "@kbn/data-stream-adapter": "link:packages/kbn-data-stream-adapter",

packages/deeplinks/management/deep_links.ts

@@ -34,6 +34,7 @@ export type ManagementId =
   | 'cases'
   | 'cross_cluster_replication'
   | 'dataViews'
+  | 'data_quality'
   | 'filesManagement'
   | 'license_management'
   | 'index_lifecycle_management'

packages/kbn-optimizer/limits.yml

@@ -28,6 +28,7 @@ pageLoadAssetSize:
   dashboard: 52967
   dashboardEnhanced: 65646
   data: 454087
+  dataQuality: 19384
   datasetQuality: 50624
   dataViewEditor: 28082
   dataViewFieldEditor: 27000

src/plugins/management/public/components/management_sidebar_nav/management_sidebar_nav.tsx

@@ -32,13 +32,16 @@ export const managementSidebarNav = ({
     const sortedManagementSections = sortBy(managementSections, 'order');
 
     return sortedManagementSections.reduce<Array<EuiSideNavItemType<any>>>((acc, section) => {
-      const apps = sortBy(section.getAppsEnabled(), 'order');
+      const apps = sortBy(
+        section.getAppsEnabled().filter((app) => !app.hideFromSidebar),
+        'order'
+      );
 
       if (apps.length) {
         if (!section.hideFromSidebar) {
           acc.push({
             ...createNavItem(section, {
-              items: appsToNavItems(apps.filter((app) => !app.hideFromSidebar)),
+              items: appsToNavItems(apps),
             }),
           });
         }

tsconfig.base.json

@@ -668,6 +668,8 @@
       "@kbn/data-forge/*": ["x-pack/packages/kbn-data-forge/*"],
       "@kbn/data-plugin": ["src/plugins/data"],
       "@kbn/data-plugin/*": ["src/plugins/data/*"],
+      "@kbn/data-quality-plugin": ["x-pack/plugins/data_quality"],
+      "@kbn/data-quality-plugin/*": ["x-pack/plugins/data_quality/*"],
       "@kbn/data-search-plugin": ["test/plugin_functional/plugins/data_search"],
       "@kbn/data-search-plugin/*": ["test/plugin_functional/plugins/data_search/*"],
       "@kbn/data-service": ["packages/kbn-data-service"],

x-pack/.i18nrc.json

@@ -24,6 +24,7 @@
     "xpack.csp": "plugins/cloud_security_posture",
     "xpack.customBranding": "plugins/custom_branding",
     "xpack.dashboard": "plugins/dashboard_enhanced",
+    "xpack.dataQuality": "plugins/data_quality",
     "xpack.datasetQuality": "plugins/observability_solution/dataset_quality",
     "xpack.discover": "plugins/discover_enhanced",
     "xpack.crossClusterReplication": "plugins/cross_cluster_replication",

x-pack/plugins/data_quality/README.md

@@ -0,0 +1,2 @@
+# dataQuality
+Page where users can see the quality of their log datasets.

x-pack/plugins/data_quality/common/index.ts

@@ -0,0 +1,15 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { i18n } from '@kbn/i18n';
+
+export const PLUGIN_ID = 'data_quality';
+export const PLUGIN_NAME = i18n.translate('xpack.dataQuality.name', {
+  defaultMessage: 'Logs data quality',
+});
+
+export { DATA_QUALITY_URL_STATE_KEY, datasetQualityUrlSchemaV1 } from './url_schema';

x-pack/plugins/data_quality/common/url_schema/common.ts

@@ -0,0 +1,8 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export const DATA_QUALITY_URL_STATE_KEY = 'pageState';

x-pack/plugins/data_quality/common/url_schema/index.ts

@@ -0,0 +1,9 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export { DATA_QUALITY_URL_STATE_KEY } from './common';
+export * as datasetQualityUrlSchemaV1 from './url_schema_v1';

x-pack/plugins/data_quality/common/utils/deep_compact_object.ts

@@ -0,0 +1,15 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { isEmpty, isPlainObject, isUndefined } from 'lodash';
+
+export const deepCompactObject = <Value extends Record<string, any>>(obj: Value): Value =>
+  Object.fromEntries(
+    Object.entries(obj)
+      .map(([key, value]) => [key, isPlainObject(value) ? deepCompactObject(value) : value])
+      .filter(([, value]) => !isUndefined(value) && !(isPlainObject(value) && isEmpty(value)))
+  );

x-pack/plugins/data_quality/kibana.jsonc

@@ -0,0 +1,24 @@
+{
+  "type": "plugin",
+  "id": "@kbn/data-quality-plugin",
+  "owner": "@elastic/obs-ux-logs-team",
+  "plugin": {
+    "id": "dataQuality",
+    "server": true,
+    "browser": true,
+    "configPath": ["xpack", "data_quality"],
+    "requiredPlugins": [
+      "datasetQuality",
+      "management",
+      "features",
+  ],
+    "optionalPlugins": [],
+    "requiredBundles": [
+      "kibanaReact",
+      "kibanaUtils",
+    ],
+    "extraPublicDirs": [
+      "common",
+    ]
+  }
+}

x-pack/plugins/data_quality/public/application.tsx

@@ -0,0 +1,63 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { CoreStart } from '@kbn/core/public';
+import { ManagementAppMountParams } from '@kbn/management-plugin/public';
+import React from 'react';
+import ReactDOM from 'react-dom';
+import { KibanaRenderContextProvider } from '@kbn/react-kibana-context-render';
+import { Route, Router, Routes } from '@kbn/shared-ux-router';
+import { KbnUrlStateStorageFromRouterProvider } from './utils/kbn_url_state_context';
+import { useKibanaContextForPluginProvider } from './utils/use_kibana';
+import { AppPluginStartDependencies, DataQualityPluginStart } from './types';
+import { DatasetQualityRoute } from './routes';
+
+export const renderApp = (
+  core: CoreStart,
+  plugins: AppPluginStartDependencies,
+  pluginStart: DataQualityPluginStart,
+  params: ManagementAppMountParams
+) => {
+  ReactDOM.render(
+    <App params={params} core={core} plugins={plugins} pluginStart={pluginStart} />,
+    params.element
+  );
+
+  return () => {
+    ReactDOM.unmountComponentAtNode(params.element);
+  };
+};
+
+interface AppProps {
+  core: CoreStart;
+  plugins: AppPluginStartDependencies;
+  pluginStart: DataQualityPluginStart;
+  params: ManagementAppMountParams;
+}
+
+const App = ({ core, plugins, pluginStart, params }: AppProps) => {
+  const KibanaContextProviderForPlugin = useKibanaContextForPluginProvider(
+    core,
+    plugins,
+    pluginStart,
+    params
+  );
+
+  return (
+    <KibanaRenderContextProvider {...core} {...params}>
+      <KibanaContextProviderForPlugin>
+        <KbnUrlStateStorageFromRouterProvider>
+          <Router history={params.history}>
+            <Routes>
+              <Route path="/" exact={true} render={() => <DatasetQualityRoute />} />
+            </Routes>
+          </Router>
+        </KbnUrlStateStorageFromRouterProvider>
+      </KibanaContextProviderForPlugin>
+    </KibanaRenderContextProvider>
+  );
+};

x-pack/plugins/data_quality/public/index.ts

@@ -0,0 +1,13 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { DataQualityPlugin } from './plugin';
+
+export function plugin() {
+  return new DataQualityPlugin();
+}
+export type { DataQualityPluginSetup, DataQualityPluginStart } from './types';

x-pack/plugins/data_quality/public/plugin.ts

@@ -0,0 +1,56 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { CoreSetup, CoreStart, Plugin } from '@kbn/core/public';
+import { ManagementAppMountParams } from '@kbn/management-plugin/public';
+import {
+  DataQualityPluginSetup,
+  DataQualityPluginStart,
+  AppPluginStartDependencies,
+  AppPluginSetupDependencies,
+} from './types';
+import { PLUGIN_ID, PLUGIN_NAME } from '../common';
+
+export class DataQualityPlugin
+  implements
+    Plugin<
+      DataQualityPluginSetup,
+      DataQualityPluginStart,
+      AppPluginSetupDependencies,
+      AppPluginStartDependencies
+    >
+{
+  public setup(
+    core: CoreSetup<AppPluginStartDependencies, DataQualityPluginStart>,
+    plugins: AppPluginSetupDependencies
+  ): DataQualityPluginSetup {
+    const { management } = plugins;
+
+    management.sections.section.data.registerApp({
+      id: PLUGIN_ID,
+      title: PLUGIN_NAME,
+      order: 2,
+      async mount(params: ManagementAppMountParams) {
+        const [{ renderApp }, [coreStart, pluginsStartDeps, pluginStart]] = await Promise.all([
+          import('./application'),
+          core.getStartServices(),
+        ]);
+
+        return renderApp(coreStart, pluginsStartDeps, pluginStart, params);
+      },
+      hideFromSidebar: true,
+    });
+
+    return {};
+  }
+
+  public start(_core: CoreStart): DataQualityPluginStart {
+    return {};
+  }
+
+  public stop() {}
+}

x-pack/plugins/data_quality/public/routes/dataset_quality/context.tsx

@@ -0,0 +1,78 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { IToasts } from '@kbn/core-notifications-browser';
+import { DatasetQualityPluginStart } from '@kbn/dataset-quality-plugin/public';
+import { DatasetQualityController } from '@kbn/dataset-quality-plugin/public/controller';
+import { IKbnUrlStateStorage } from '@kbn/kibana-utils-plugin/public';
+import React, { createContext, useContext, useEffect, useState } from 'react';
+import {
+  getDatasetQualityStateFromUrl,
+  updateUrlFromDatasetQualityState,
+} from './url_state_storage_service';
+
+const DatasetQualityContext = createContext<{ controller?: DatasetQualityController }>({});
+
+interface ContextProps {
+  children: JSX.Element;
+  urlStateStorageContainer: IKbnUrlStateStorage;
+  toastsService: IToasts;
+  datasetQuality: DatasetQualityPluginStart;
+}
+
+export function DatasetQualityContextProvider({
+  children,
+  urlStateStorageContainer,
+  toastsService,
+  datasetQuality,
+}: ContextProps) {
+  const [controller, setController] = useState<DatasetQualityController>();
+
+  useEffect(() => {
+    async function getDatasetQualityController() {
+      const initialState = getDatasetQualityStateFromUrl({
+        urlStateStorageContainer,
+        toastsService,
+      });
+
+      const datasetQualityController = await datasetQuality.createDatasetQualityController({
+        initialState,
+      });
+      datasetQualityController.service.start();
+      setController(datasetQualityController);
+
+      const datasetQualityStateSubscription = datasetQualityController.state$.subscribe((state) => {
+        updateUrlFromDatasetQualityState({
+          urlStateStorageContainer,
+          datasetQualityState: state,
+        });
+      });
+
+      return () => {
+        datasetQualityController.service.stop();
+        datasetQualityStateSubscription.unsubscribe();
+      };
+    }
+    getDatasetQualityController();
+  }, [datasetQuality, toastsService, urlStateStorageContainer]);
+
+  return (
+    <DatasetQualityContext.Provider value={{ controller }}>
+      {children}
+    </DatasetQualityContext.Provider>
+  );
+}
+
+export const useDatasetQualityContext = () => {
+  const context = useContext(DatasetQualityContext);
+  if (context === undefined) {
+    throw new Error(
+      'useDatasetQualityContext must be used within a <DatasetQualityContextProvider />'
+    );
+  }
+  return context;
+};