[Infra] Add endpoints to manage Custom Dashboards (elastic#176612)

Closes elastic#176069

## Summary

This adds the logic to register a new Saved Object type to store custom
dashboards for Asset Details and adds endpoints to fetch and save custom
dashboards.

Changes highlights:
* Renamed the `enableInfrastructureHostsCustomDashboards` to
`enableInfrastructureAssetCustomDashboards` to make it more generic and
support additional asset types in the future
* Added a new Saved Object type
* Moved initialization of all Infra endpoints to plugin's `start`. This
one one of the points on [the BE tech debt
ticket](elastic#175975). Having
endpoint initialization in `start` makes it more convenient to access
start dependencies which almost all endpoints require.
* Added `savedObjectClient` and `uiSettingsClient` to the custom request
context (also one of the ideas for endpoints improvement). Right now
infra endpoints use custom `libs` object with all dependencies required
for routes, the idea is to rely on the request context instead because
it automatically available for every route handler and by default
includes some useful things like scoped service clients.
* Added a wrapper `handleRouteErrors` to avoid error handling
duplication which we now have in a few routes. In the future we could do
something similar right within `registerRoutes` framework function, but
this would require a bit of refactoring.

## Hot to Test

1. Toggle the UI setting off in Advanced Settings
![CleanShot 2024-02-13 at 16 01
36@2x](https://github.com/elastic/kibana/assets/793851/fc3772a1-a075-42bd-bdc3-2c7e83278844)
2. Go to the Dev Tools and try the endpoints, both should respond with
403
```
GET kbn:api/infra/custom-dashboards/host

POST kbn:api/infra/custom-dashboards
{
  "assetType": "host",
  "dashboardIdList": ["0", "1"]
}
```
3. Toggle the UI setting on
4. Try the endpoints again, now they should work as expected

---------

Co-authored-by: kibanamachine <[email protected]>

docs/management/advanced-options.asciidoc

@@ -465,8 +465,8 @@ preview:[] Enable the APM Trace Explorer feature, that allows you to search and
 [[observability-infrastructure-profiling-integration]]`observability:enableInfrastructureProfilingIntegration`::
 preview:[] Enables the Profiling view in Host details within Infrastructure.
 
-[[observability-infrastructure-hosts-custom-dashboard]]`observability:enableInfrastructureHostsCustomDashboards`::
-preview:[] Enables option to link custom dashboards in the Host Details view.
+[[observability-infrastructure-asset-custom-dashboard]]`observability:enableInfrastructureAssetCustomDashboards`::
+preview:[] Enables option to link custom dashboards in the Asset Details view.
 
 [[observability-profiling-per-vcpu-watt-x86]]`observability:profilingPervCPUWattX86`::
 The average amortized per-core power consumption (based on 100% CPU utilization) for x86 architecture.

packages/kbn-check-mappings-update-cli/current_fields.json

@@ -459,6 +459,11 @@
     "title",
     "type"
   ],
+  "infra-custom-dashboards": [
+    "assetType",
+    "dashboardIdList",
+    "kuery"
+  ],
   "infrastructure-monitoring-log-view": [
     "name"
   ],

packages/kbn-check-mappings-update-cli/current_mappings.json

@@ -1553,6 +1553,19 @@
       }
     }
   },
+  "infra-custom-dashboards": {
+    "properties": {
+      "assetType": {
+        "type": "keyword"
+      },
+      "dashboardIdList": {
+        "type": "keyword"
+      },
+      "kuery": {
+        "type": "text"
+      }
+    }
+  },
   "infrastructure-monitoring-log-view": {
     "dynamic": false,
     "properties": {

packages/kbn-management/settings/setting_ids/index.ts

@@ -123,8 +123,8 @@ export const OBSERVABILITY_ENABLE_COMPARISON_BY_DEFAULT_ID =
   'observability:enableComparisonByDefault';
 export const OBSERVABILITY_ENABLE_INFRASTRUCTURE_HOSTS_VIEW_ID =
   'observability:enableInfrastructureHostsView';
-export const OBSERVABILITY_ENABLE_INFRASTRUCTURE_HOSTS_CUSTOM_DASHBOARDS_ID =
-  'observability:enableInfrastructureHostsCustomDashboards';
+export const OBSERVABILITY_ENABLE_INFRASTRUCTURE_ASSET_CUSTOM_DASHBOARDS_ID =
+  'observability:enableInfrastructureAssetCustomDashboards';
 export const OBSERVABILITY_ENABLE_INSPECT_ES_QUERIES_ID = 'observability:enableInspectEsQueries';
 export const OBSERVABILITY_MAX_SUGGESTIONS_ID = 'observability:maxSuggestions';
 export const OBSERVABILITY_PROFILING_ELASTICSEARCH_PLUGIN_ID =

packages/serverless/settings/observability_project/index.ts

@@ -28,5 +28,5 @@ export const OBSERVABILITY_PROJECT_SETTINGS = [
   settings.OBSERVABILITY_ENABLE_AWS_LAMBDA_METRICS_ID,
   settings.OBSERVABILITY_APM_ENABLE_CRITICAL_PATH_ID,
   settings.OBSERVABILITY_ENABLE_INFRASTRUCTURE_HOSTS_VIEW_ID,
-  settings.OBSERVABILITY_ENABLE_INFRASTRUCTURE_HOSTS_CUSTOM_DASHBOARDS_ID,
+  settings.OBSERVABILITY_ENABLE_INFRASTRUCTURE_ASSET_CUSTOM_DASHBOARDS_ID,
 ];

src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts

@@ -103,6 +103,7 @@ describe('checking migration metadata changes on all registered SO types', () =>
         "guided-onboarding-guide-state": "d338972ed887ac480c09a1a7fbf582d6a3827c91",
         "guided-onboarding-plugin-state": "bc109e5ef46ca594fdc179eda15f3095ca0a37a4",
         "index-pattern": "997108a9ea1e8076e22231e1c95517cdb192b9c5",
+        "infra-custom-dashboards": "b92b6db1c1f8998af6e2951a17b76cf886c6bee5",
         "infrastructure-monitoring-log-view": "5f86709d3c27aed7a8379153b08ee5d3d90d77f5",
         "infrastructure-ui-source": "113182d6895764378dfe7fa9fa027244f3a457c4",
         "ingest-agent-policies": "7633e578f60c074f8267bc50ec4763845e431437",

src/core/server/integration_tests/saved_objects/migrations/group3/type_registrations.test.ts

@@ -73,6 +73,7 @@ const previouslyRegisteredTypes = [
   'index-pattern',
   'infrastructure-monitoring-log-view',
   'infrastructure-ui-source',
+  'infra-custom-dashboards',
   'ingest-agent-policies',
   'ingest-download-sources',
   'ingest-outputs',

src/core/server/integration_tests/saved_objects/migrations/group5/dot_kibana_split.test.ts

@@ -223,6 +223,7 @@ describe('split .kibana index into multiple system indices', () => {
             "guided-onboarding-guide-state",
             "guided-onboarding-plugin-state",
             "index-pattern",
+            "infra-custom-dashboards",
             "infrastructure-monitoring-log-view",
             "infrastructure-ui-source",
             "ingest-agent-policies",

src/plugins/kibana_usage_collection/server/collectors/management/schema.ts

@@ -608,7 +608,7 @@ export const stackManagementSchema: MakeSchemaFrom<UsageStats> = {
     type: 'boolean',
     _meta: { description: 'Non-default value of setting.' },
   },
-  'observability:enableInfrastructureHostsCustomDashboards': {
+  'observability:enableInfrastructureAssetCustomDashboards': {
     type: 'boolean',
     _meta: { description: 'Non-default value of setting.' },
   },

src/plugins/kibana_usage_collection/server/collectors/management/types.ts

@@ -47,7 +47,7 @@ export interface UsageStats {
   'observability:apmAWSLambdaRequestCostPerMillion': number;
   'observability:enableInfrastructureHostsView': boolean;
   'observability:enableInfrastructureProfilingIntegration': boolean;
-  'observability:enableInfrastructureHostsCustomDashboards': boolean;
+  'observability:enableInfrastructureAssetCustomDashboards': boolean;
   'observability:apmAgentExplorerView': boolean;
   'observability:apmEnableTableSearchBar': boolean;
   'visualization:heatmap:maxBuckets': number;

src/plugins/telemetry/schema/oss_plugins.json

@@ -10228,7 +10228,7 @@
             "description": "Non-default value of setting."
           }
         },
-        "observability:enableInfrastructureHostsCustomDashboards": {
+        "observability:enableInfrastructureAssetCustomDashboards": {
           "type": "boolean",
           "_meta": {
             "description": "Non-default value of setting."

x-pack/plugins/infra/common/custom_dashboards.ts

@@ -0,0 +1,16 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { InventoryItemType } from '@kbn/metrics-data-access-plugin/common';
+
+export type InfraCustomDashboardAssetType = InventoryItemType;
+
+export interface InfraCustomDashboard {
+  dashboardIdList: string[];
+  assetType: InfraCustomDashboardAssetType;
+  kuery?: string;
+}

x-pack/plugins/infra/common/http_api/custom_dashboards_api.ts

@@ -0,0 +1,47 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { ItemTypeRT } from '@kbn/metrics-data-access-plugin/common';
+import * as rt from 'io-ts';
+
+const AssetTypeRT = rt.type({
+  assetType: ItemTypeRT,
+});
+
+const CustomDashboardRT = rt.intersection([
+  AssetTypeRT,
+  rt.type({
+    dashboardIdList: rt.array(rt.string),
+  }),
+  rt.partial({
+    kuery: rt.string,
+  }),
+]);
+
+/**
+ GET endpoint
+*/
+export const InfraGetCustomDashboardsRequestParamsRT = AssetTypeRT;
+export const InfraGetCustomDashboardsResponseBodyRT = CustomDashboardRT;
+export type InfraGetCustomDashboardsRequestParams = rt.TypeOf<
+  typeof InfraGetCustomDashboardsRequestParamsRT
+>;
+export type InfraGetCustomDashboardsResponseBody = rt.TypeOf<
+  typeof InfraGetCustomDashboardsResponseBodyRT
+>;
+
+/**
+ * POST endpoint
+ */
+export const InfraSaveCustomDashboardsRequestPayloadRT = CustomDashboardRT;
+export const InfraSaveCustomDashboardsResponseBodyRT = CustomDashboardRT;
+export type InfraSaveCustomDashboardsRequestPayload = rt.TypeOf<
+  typeof InfraSaveCustomDashboardsRequestPayloadRT
+>;
+export type InfraSaveCustomDashboardsResponseBody = rt.TypeOf<
+  typeof InfraSaveCustomDashboardsResponseBodyRT
+>;

x-pack/plugins/infra/server/infra_server.ts

@@ -5,6 +5,7 @@
  * 2.0.
  */
 
+import { CoreStart } from '@kbn/core-lifecycle-server';
 import { InfraBackendLibs } from './lib/infra_types';
 import { initGetHostsAnomaliesRoute, initGetK8sAnomaliesRoute } from './routes/infra_ml';
 import { initInventoryMetaRoute } from './routes/inventory_metadata';
@@ -34,8 +35,14 @@ import { initInfraMetricsRoute } from './routes/infra';
 import { initMetricsExplorerViewRoutes } from './routes/metrics_explorer_views';
 import { initProfilingRoutes } from './routes/profiling';
 import { initServicesRoute } from './routes/services';
+import { initCustomDashboardsRoutes } from './routes/custom_dashboards/custom_dashboards';
+import { type InfraServerPluginStartDeps } from './lib/adapters/framework';
 
-export const initInfraServer = (libs: InfraBackendLibs) => {
+export const initInfraServer = (
+  libs: InfraBackendLibs,
+  coreStart: CoreStart,
+  infraPluginsStart: InfraServerPluginStartDeps
+) => {
   initIpToHostName(libs);
   initGetLogEntryCategoriesRoute(libs);
   initGetLogEntryCategoryDatasetsRoute(libs);
@@ -63,4 +70,5 @@ export const initInfraServer = (libs: InfraBackendLibs) => {
   initInfraMetricsRoute(libs);
   initProfilingRoutes(libs);
   initServicesRoute(libs);
+  initCustomDashboardsRoutes(libs.framework);
 };

x-pack/plugins/infra/server/lib/adapters/framework/kibana_framework_adapter.ts

@@ -7,7 +7,7 @@
 
 import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
 import { TransportRequestParams } from '@elastic/elasticsearch';
-import { ElasticsearchClient, SavedObjectsClientContract } from '@kbn/core/server';
+import { ElasticsearchClient, RouteConfig, SavedObjectsClientContract } from '@kbn/core/server';
 import { CoreSetup, IRouter, KibanaRequest, RequestHandler, RouteMethod } from '@kbn/core/server';
 import { UI_SETTINGS } from '@kbn/data-plugin/server';
 import { TimeseriesVisData } from '@kbn/vis-type-timeseries-plugin/server';
@@ -59,25 +59,30 @@ export class KibanaFramework {
     const routeConfig = {
       path: config.path,
       validate: config.validate,
-      // Currently we have no use of custom options beyond tags, this can be extended
-      // beyond defaultOptions if it's needed.
-      options: defaultOptions,
+      /**
+       * Supported `options` for each type of request method
+       * are a bit different and generic method like this cannot
+       * properly ensure type safety. Hence the need to cast
+       * using `as ...` below to ensure the route config has
+       * the correct options type.
+       */
+      options: { ...config.options, ...defaultOptions },
     };
     switch (config.method) {
       case 'get':
-        this.router.get(routeConfig, handler);
+        this.router.get(routeConfig as RouteConfig<Params, Query, Body, 'get'>, handler);
         break;
       case 'post':
-        this.router.post(routeConfig, handler);
+        this.router.post(routeConfig as RouteConfig<Params, Query, Body, 'post'>, handler);
         break;
       case 'delete':
-        this.router.delete(routeConfig, handler);
+        this.router.delete(routeConfig as RouteConfig<Params, Query, Body, 'delete'>, handler);
         break;
       case 'put':
-        this.router.put(routeConfig, handler);
+        this.router.put(routeConfig as RouteConfig<Params, Query, Body, 'put'>, handler);
         break;
       case 'patch':
-        this.router.patch(routeConfig, handler);
+        this.router.patch(routeConfig as RouteConfig<Params, Query, Body, 'patch'>, handler);
         break;
     }
   }

x-pack/plugins/infra/server/plugin.ts

@@ -38,7 +38,11 @@ import { InfraMetricsDomain } from './lib/domains/metrics_domain';
 import { InfraBackendLibs, InfraDomainLibs } from './lib/infra_types';
 import { infraSourceConfigurationSavedObjectType, InfraSources } from './lib/sources';
 import { InfraSourceStatus } from './lib/source_status';
-import { inventoryViewSavedObjectType, metricsExplorerViewSavedObjectType } from './saved_objects';
+import {
+  infraCustomDashboardsSavedObjectType,
+  inventoryViewSavedObjectType,
+  metricsExplorerViewSavedObjectType,
+} from './saved_objects';
 import { InventoryViewsService } from './services/inventory_views';
 import { MetricsExplorerViewsService } from './services/metrics_explorer_views';
 import { RulesService } from './services/rules';
@@ -199,6 +203,7 @@ export class InfraServerPlugin
     // Register saved object types
     core.savedObjects.registerType(infraSourceConfigurationSavedObjectType);
     core.savedObjects.registerType(inventoryViewSavedObjectType);
+    core.savedObjects.registerType(infraCustomDashboardsSavedObjectType);
     if (this.config.featureFlags.metricsExplorerEnabled) {
       core.savedObjects.registerType(metricsExplorerViewSavedObjectType);
     }
@@ -259,21 +264,27 @@ export class InfraServerPlugin
       ]);
     }
 
-    initInfraServer(this.libs);
     registerRuleTypes(plugins.alerting, this.libs, this.config);
 
     core.http.registerRouteHandlerContext<InfraPluginRequestHandlerContext, 'infra'>(
       'infra',
       async (context, request) => {
-        const soClient = (await context.core).savedObjects.client;
-        const mlSystem = plugins.ml?.mlSystemProvider(request, soClient);
-        const mlAnomalyDetectors = plugins.ml?.anomalyDetectorsProvider(request, soClient);
+        const coreContext = await context.core;
+        const savedObjectsClient = coreContext.savedObjects.client;
+        const uiSettingsClient = coreContext.uiSettings.client;
+        const mlSystem = plugins.ml?.mlSystemProvider(request, savedObjectsClient);
+        const mlAnomalyDetectors = plugins.ml?.anomalyDetectorsProvider(
+          request,
+          savedObjectsClient
+        );
         const spaceId = plugins.spaces?.spacesService.getSpaceId(request) ?? DEFAULT_SPACE_ID;
 
         return {
           mlAnomalyDetectors,
           mlSystem,
           spaceId,
+          savedObjectsClient,
+          uiSettingsClient,
         };
       }
     );
@@ -287,7 +298,7 @@ export class InfraServerPlugin
     } as InfraPluginSetup;
   }
 
-  start(core: CoreStart) {
+  start(core: CoreStart, pluginsStart: InfraServerPluginStartDeps) {
     const inventoryViews = this.inventoryViews.start({
       infraSources: this.libs.sources,
       savedObjects: core.savedObjects,
@@ -298,6 +309,8 @@ export class InfraServerPlugin
       savedObjects: core.savedObjects,
     });
 
+    initInfraServer(this.libs, core, pluginsStart);
+
     return {
       inventoryViews,
       metricsExplorerViews,

x-pack/plugins/infra/server/routes/custom_dashboards/custom_dashboards.ts

@@ -0,0 +1,15 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { KibanaFramework } from '../../lib/adapters/framework/kibana_framework_adapter';
+import { initGetCustomDashboardRoute } from './get_custom_dashboard';
+import { initSaveCustomDashboardRoute } from './save_custom_dashboard';
+
+export function initCustomDashboardsRoutes(framework: KibanaFramework) {
+  initGetCustomDashboardRoute(framework);
+  initSaveCustomDashboardRoute(framework);
+}