[Profiling,Infra,APM] Disable Profiling integration by default (#175201)

Closes #175016

## Summary

This PR disables the Profiling integration in Infra and APM by default
on the plugin configuration level because this integration require users
to first configure the main `profiling` plugin. On-prem users will have
to manually enable both integrations once they enabled the Universal
Profiling for their hosts. Cloud users will have Infra and APM
integrations enabled by default because on Cloud instances Universal
Profiling is already configured. A PR for the default Cloud settings
will follow after this one is merged.

Changes I've made:
* Disabled the Infra integration be default
* Introduced a new APM feature flag for the Profiling integration
* Made sure all the places in APM that rely on Profiling integration
respect the new feature flag
* Fixed a bug in APM when Universal Profiling was shown even though the
integration was disabled in UI settings


https://github.com/elastic/kibana/assets/793851/65dfbb5b-1850-4d18-a92a-6ad59e0436a3

## How To Test

1. Checkout locally
2. Make sure you don't have `xpack.infra.featureFlags.profilingEnabled`
already enabled in `kibana.yml`
3. Open kibana and make sure you don't see "Universal Profiling" tab in
Host and Service details
4. Enabled both flags in `kibana.yml`:
`xpack.infra.featureFlags.profilingEnabled` and
`xpack.apm.featureFlags.profilingIntegrationAvailable: true`
5. Check that now you see "Universal Profiling" tab in the details
screens in both Infra and APM
6. Go to Infra settings view and disable the Profiling integration, make
sure the tab disappears
7. 6. Go to APM settings view and disable the Profiling integration,
make sure the tab disappears

---------

Co-authored-by: Kibana Machine <[email protected]>

test/plugin_functional/test_suites/core_plugins/rendering.ts

@@ -213,6 +213,7 @@ export default function ({ getService }: PluginFunctionalProviderContext) {
         'xpack.apm.featureFlags.migrationToFleetAvailable (any)',
         'xpack.apm.featureFlags.sourcemapApiAvailable (any)',
         'xpack.apm.featureFlags.storageExplorerAvailable (any)',
+        'xpack.apm.featureFlags.profilingIntegrationAvailable (boolean)',
         'xpack.apm.serverless.enabled (any)', // It's a boolean (any because schema.conditional)
         'xpack.assetManager.alphaEnabled (boolean)',
         'xpack.observability_onboarding.serverless.enabled (any)', // It's a boolean (any because schema.conditional)
@@ -282,7 +283,7 @@ export default function ({ getService }: PluginFunctionalProviderContext) {
         'xpack.infra.featureFlags.logThresholdAlertRuleEnabled (any)',
         'xpack.infra.featureFlags.logsUIEnabled (any)',
         'xpack.infra.featureFlags.alertsAndRulesDropdownEnabled (any)',
-        'xpack.infra.featureFlags.profilingEnabled (any)',
+        'xpack.infra.featureFlags.profilingEnabled (boolean)',
 
         'xpack.license_management.ui.enabled (boolean)',
         'xpack.maps.preserveDrawingBuffer (boolean)',

x-pack/plugins/apm/common/apm_feature_flags.ts

@@ -16,6 +16,7 @@ export enum ApmFeatureFlagName {
   MigrationToFleetAvailable = 'migrationToFleetAvailable',
   SourcemapApiAvailable = 'sourcemapApiAvailable',
   StorageExplorerAvailable = 'storageExplorerAvailable',
+  ProfilingIntegrationAvailable = 'profilingIntegrationAvailable',
 }
 
 const apmFeatureFlagMap = {
@@ -47,6 +48,10 @@ const apmFeatureFlagMap = {
     default: true,
     type: t.boolean,
   },
+  [ApmFeatureFlagName.ProfilingIntegrationAvailable]: {
+    default: false,
+    type: t.boolean,
+  },
 };
 
 type ApmFeatureFlagMap = typeof apmFeatureFlagMap;

x-pack/plugins/apm/public/components/app/settings/general_settings/index.tsx

@@ -31,6 +31,8 @@ import {
 } from '@kbn/observability-shared-plugin/public';
 import { FieldRowProvider } from '@kbn/management-settings-components-field-row';
 import { ValueValidation } from '@kbn/core-ui-settings-browser/src/types';
+import { useApmFeatureFlag } from '../../../../hooks/use_apm_feature_flag';
+import { ApmFeatureFlagName } from '../../../../../common/apm_feature_flags';
 import { useApmPluginContext } from '../../../../context/apm_plugin/use_apm_plugin_context';
 
 const LazyFieldRow = React.lazy(async () => ({
@@ -40,24 +42,35 @@ const LazyFieldRow = React.lazy(async () => ({
 
 const FieldRow = withSuspense(LazyFieldRow);
 
-const apmSettingsKeys = [
-  enableComparisonByDefault,
-  defaultApmServiceEnvironment,
-  apmServiceGroupMaxNumberOfServices,
-  enableInspectEsQueries,
-  apmLabsButton,
-  apmAWSLambdaPriceFactor,
-  apmAWSLambdaRequestCostPerMillion,
-  apmEnableServiceMetrics,
-  apmEnableContinuousRollups,
-  enableAgentExplorerView,
-  apmEnableTableSearchBar,
-  apmEnableProfilingIntegration,
-];
+function getApmSettingsKeys(isProfilingIntegrationEnabled: boolean) {
+  const keys = [
+    enableComparisonByDefault,
+    defaultApmServiceEnvironment,
+    apmServiceGroupMaxNumberOfServices,
+    enableInspectEsQueries,
+    apmLabsButton,
+    apmAWSLambdaPriceFactor,
+    apmAWSLambdaRequestCostPerMillion,
+    apmEnableServiceMetrics,
+    apmEnableContinuousRollups,
+    enableAgentExplorerView,
+    apmEnableTableSearchBar,
+  ];
+
+  if (isProfilingIntegrationEnabled) {
+    keys.push(apmEnableProfilingIntegration);
+  }
+
+  return keys;
+}
 
 export function GeneralSettings() {
   const trackApmEvent = useUiTracker({ app: 'apm' });
   const { docLinks, notifications } = useApmPluginContext().core;
+  const isProfilingIntegrationEnabled = useApmFeatureFlag(
+    ApmFeatureFlagName.ProfilingIntegrationAvailable
+  );
+  const apmSettingsKeys = getApmSettingsKeys(isProfilingIntegrationEnabled);
   const {
     fields,
     handleFieldChange,

x-pack/plugins/apm/public/components/app/transaction_details/distribution/index.test.tsx

@@ -25,8 +25,13 @@ import { fromQuery } from '../../../shared/links/url_helpers';
 
 import { TransactionDistribution } from '.';
 
+const coreMock = {
+  settings: { client: { get: () => {} } },
+} as unknown as CoreStart;
+
 function Wrapper({ children }: { children?: ReactNode }) {
   const KibanaReactContext = createKibanaReactContext({
+    ...coreMock,
     usageCollection: { reportUiCounter: () => {} },
   } as Partial<CoreStart>);
 

x-pack/plugins/apm/public/components/routing/templates/apm_service_template/index.tsx

@@ -20,6 +20,7 @@ import { enableAwsLambdaMetrics } from '@kbn/observability-plugin/common';
 import { omit } from 'lodash';
 import React from 'react';
 import { useHistory, useLocation } from 'react-router-dom';
+import { useProfilingIntegrationSetting } from '../../../../hooks/use_profiling_integration_setting';
 import {
   isAWSLambdaAgentName,
   isAzureFunctionsAgentName,
@@ -224,6 +225,8 @@ function useTabs({ selectedTab }: { selectedTab: Tab['key'] }) {
     ApmFeatureFlagName.InfrastructureTabAvailable
   );
 
+  const isProfilingIntegrationEnabled = useProfilingIntegrationSetting();
+
   const isAwsLambdaEnabled = core.uiSettings.get<boolean>(
     enableAwsLambdaMetrics,
     true
@@ -404,6 +407,7 @@ function useTabs({ selectedTab }: { selectedTab: Tab['key'] }) {
         defaultMessage: 'Universal Profiling',
       }),
       hidden:
+        !isProfilingIntegrationEnabled ||
         isRumOrMobileAgentName(agentName) ||
         isAWSLambdaAgentName(serverlessType),
       append: (

x-pack/plugins/apm/public/components/shared/transaction_action_menu/transaction_action_menu.test.tsx

@@ -32,6 +32,7 @@ import { TransactionActionMenu } from './transaction_action_menu';
 import * as Transactions from './__fixtures__/mock_data';
 import { KibanaContextProvider } from '@kbn/kibana-react-plugin/public';
 import * as useAdHocApmDataView from '../../../hooks/use_adhoc_apm_data_view';
+import { useProfilingIntegrationSetting } from '../../../hooks/use_profiling_integration_setting';
 
 const apmContextMock = {
   ...mockApmPluginContextValue,
@@ -60,6 +61,10 @@ const apmContextMock = {
   },
 } as unknown as ApmPluginContextValue;
 
+jest.mock('../../../hooks/use_profiling_integration_setting', () => ({
+  useProfilingIntegrationSetting: jest.fn().mockReturnValue(false),
+}));
+
 const history = createMemoryHistory();
 history.replace(
   '/services/testbeans-go/transactions/view?rangeFrom=now-24h&rangeTo=now&transactionName=GET+%2Ftestbeans-go%2Fapi'
@@ -115,6 +120,10 @@ const expectLogsLocatorsToBeCalled = () => {
 let useAdHocApmDataViewSpy: jest.SpyInstance;
 
 describe('TransactionActionMenu ', () => {
+  beforeEach(() => {
+    jest.clearAllMocks();
+  });
+
   jest.spyOn(hooks, 'useFetcher').mockReturnValue({
     // return as Profiling had been initialized
     data: {
@@ -308,6 +317,10 @@ describe('TransactionActionMenu ', () => {
   });
 
   describe('Profiling items', () => {
+    beforeEach(() => {
+      (useProfilingIntegrationSetting as jest.Mock).mockReturnValue(true);
+    });
+
     it('renders flamegraph item', async () => {
       const component = await renderTransaction(
         Transactions.transactionWithHostData

x-pack/plugins/apm/public/context/apm_plugin/mock_apm_plugin_context.tsx

@@ -85,6 +85,7 @@ const mockConfig: ConfigSchema = {
     migrationToFleetAvailable: true,
     sourcemapApiAvailable: true,
     storageExplorerAvailable: true,
+    profilingIntegrationAvailable: false,
   },
   serverless: { enabled: false },
 };

x-pack/plugins/apm/public/hooks/use_profiling_integration_setting.ts

@@ -0,0 +1,25 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { useUiSetting } from '@kbn/kibana-react-plugin/public';
+import { apmEnableProfilingIntegration } from '@kbn/observability-plugin/common';
+import { ApmFeatureFlagName } from '../../common/apm_feature_flags';
+import { useApmFeatureFlag } from './use_apm_feature_flag';
+
+export function useProfilingIntegrationSetting() {
+  const isProfilingIntegrationFeatureFlagEnabled = useApmFeatureFlag(
+    ApmFeatureFlagName.ProfilingIntegrationAvailable
+  );
+  const isProfilingIntegrationUiSettingEnabled = useUiSetting<boolean>(
+    apmEnableProfilingIntegration
+  );
+
+  return (
+    isProfilingIntegrationFeatureFlagEnabled &&
+    isProfilingIntegrationUiSettingEnabled
+  );
+}

x-pack/plugins/apm/public/hooks/use_profiling_plugin.tsx

@@ -5,16 +5,13 @@
  * 2.0.
  */
 
-import { apmEnableProfilingIntegration } from '@kbn/observability-plugin/common';
 import { useApmPluginContext } from '../context/apm_plugin/use_apm_plugin_context';
 import { isPending, useFetcher } from './use_fetcher';
+import { useProfilingIntegrationSetting } from './use_profiling_integration_setting';
 
 export function useProfilingPlugin() {
-  const { plugins, core } = useApmPluginContext();
-  const isProfilingIntegrationEnabled = core.uiSettings.get<boolean>(
-    apmEnableProfilingIntegration,
-    true
-  );
+  const { plugins } = useApmPluginContext();
+  const isProfilingIntegrationEnabled = useProfilingIntegrationSetting();
 
   const { data, status } = useFetcher((callApmApi) => {
     return callApmApi('GET /internal/apm/profiling/status');

x-pack/plugins/apm/public/index.ts

@@ -24,6 +24,7 @@ export interface ConfigSchema {
     migrationToFleetAvailable: boolean;
     sourcemapApiAvailable: boolean;
     storageExplorerAvailable: boolean;
+    profilingIntegrationAvailable: boolean;
   };
   serverless: {
     enabled: boolean;

x-pack/plugins/apm/server/index.ts

@@ -72,6 +72,12 @@ const configSchema = schema.object({
     migrationToFleetAvailable: disabledOnServerless,
     sourcemapApiAvailable: disabledOnServerless,
     storageExplorerAvailable: disabledOnServerless,
+    /**
+     * Depends on optional "profilingDataAccess" and "profiling"
+     * plugins. Enable both with `xpack.profiling.enabled: true` before
+     * enabling this feature flag.
+     */
+    profilingIntegrationAvailable: schema.boolean({ defaultValue: false }),
   }),
   serverless: schema.object({
     enabled: offeringBasedSchema({

x-pack/plugins/infra/server/plugin.ts

@@ -113,14 +113,11 @@ export const config: PluginConfigDescriptor<InfraConfig> = {
         serverless: schema.boolean({ defaultValue: true }),
       }),
       /**
-       * This flag depends on profilingDataAccess optional plugin,
-       * make sure to enable it with `xpack.profiling.enabled: true`
-       * before enabling this flag.
+       * Depends on optional "profilingDataAccess" and "profiling"
+       * plugins. Enable both with `xpack.profiling.enabled: true` before
+       * enabling this feature flag.
        */
-      profilingEnabled: offeringBasedSchema({
-        traditional: schema.boolean({ defaultValue: true }),
-        serverless: schema.boolean({ defaultValue: false }),
-      }),
+      profilingEnabled: schema.boolean({ defaultValue: false }),
     }),
   }),
   exposeToBrowser: publicConfigKeys,

x-pack/test/functional/apps/infra/config.ts

@@ -13,5 +13,12 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) {
   return {
     ...functionalConfig.getAll(),
     testFiles: [require.resolve('.')],
+    kbnTestServer: {
+      ...functionalConfig.get('kbnTestServer'),
+      serverArgs: [
+        ...functionalConfig.get('kbnTestServer.serverArgs'),
+        `--xpack.infra.featureFlags.profilingEnabled=true`,
+      ],
+    },
   };
 }