Merge branch 'main' into http/remove-replace-buildnr-with-sha

* main:
  use build hash in FTR tests
  [Security Solution] Fix moderate typo (elastic#175883)
  [Fleet] Fix conflicting dynamic template mappings for intermediate objects (elastic#175970)
  [Visualize] Prevent overwriting managed content (elastic#175274)
  [SLO] Add/edit form mark optional fields (elastic#175807)
  skip failing test suite (elastic#175984)
  [data views] Provide method of excluding data tiers when getting field list (elastic#167946)
  [Dataset quality] State management (elastic#174906)
  [Cloud Security] add is_internal config option for outputs (elastic#175546)

docs/settings/fleet-settings.asciidoc

@@ -207,6 +207,8 @@ NOTE: The `xpack.fleet.outputs` settings are intended for advanced configuration
     If `true`, the output specified in `xpack.fleet.outputs` will be the one used to send agent data unless there is another one configured specifically for the agent policy.
   `is_default_monitoring`::: 
     If `true`, the output specified in `xpack.fleet.outputs` will be the one used to send agent monitoring data unless there is another one configured specifically for the agent policy.
+  `is_internal`:::
+    If `true`, the output specified in `xpack.fleet.outputs` will not appear in the UI, and can only be managed via `kibana.yml` or the Fleet API.
   `config`::: 
     Extra config for that output.
   `proxy_id`:::

packages/kbn-check-mappings-update-cli/current_fields.json

@@ -523,6 +523,7 @@
     "hosts",
     "is_default",
     "is_default_monitoring",
+    "is_internal",
     "is_preconfigured",
     "key",
     "name",

packages/kbn-check-mappings-update-cli/current_mappings.json

@@ -1697,6 +1697,10 @@
         "type": "boolean",
         "index": false
       },
+      "is_internal": {
+        "type": "boolean",
+        "index": false
+      },
       "ssl": {
         "type": "binary"
       },

src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts

@@ -107,7 +107,7 @@ describe('checking migration metadata changes on all registered SO types', () =>
         "infrastructure-ui-source": "113182d6895764378dfe7fa9fa027244f3a457c4",
         "ingest-agent-policies": "7633e578f60c074f8267bc50ec4763845e431437",
         "ingest-download-sources": "279a68147e62e4d8858c09ad1cf03bd5551ce58d",
-        "ingest-outputs": "e36a25e789f22b4494be728321f4304a040e286b",
+        "ingest-outputs": "ba8ef97414bc983efdf1c4285afa622df8b4344a",
         "ingest-package-policies": "f4c2767e852b700a8b82678925b86bac08958b43",
         "ingest_manager_settings": "91445219e7115ff0c45d1dabd5d614a80b421797",
         "inventory-view": "b8683c8e352a286b4aca1ab21003115a4800af83",

src/plugins/data_views/common/constants.ts

@@ -69,3 +69,5 @@ export const FIELDS_PATH = '/internal/data_views/fields';
  * @public
  */
 export const EXISTING_INDICES_PATH = '/internal/data_views/_existing_indices';
+
+export const DATA_VIEWS_FIELDS_EXCLUDED_TIERS = 'data_views:fields_excluded_data_tiers';

src/plugins/data_views/common/types.ts

@@ -544,5 +544,6 @@ export interface HasDataService {
 
 export interface ClientConfigType {
   scriptedFieldsEnabled?: boolean;
+  dataTiersExcludedForFields?: string;
   fieldListCachingEnabled?: boolean;
 }

src/plugins/data_views/server/index.ts

@@ -46,6 +46,13 @@ const configSchema = schema.object({
     schema.boolean({ defaultValue: false }),
     schema.never()
   ),
+
+  dataTiersExcludedForFields: schema.conditional(
+    schema.contextRef('serverless'),
+    true,
+    schema.never(),
+    schema.boolean({ defaultValue: true })
+  ),
   fieldListCachingEnabled: schema.conditional(
     schema.contextRef('serverless'),
     true,

src/plugins/data_views/server/plugin.ts

@@ -16,6 +16,7 @@ import { registerIndexPatternsUsageCollector } from './register_index_pattern_us
 import { createScriptedFieldsDeprecationsConfig } from './deprecations';
 import { DATA_VIEW_SAVED_OBJECT_TYPE, LATEST_VERSION } from '../common';
 import type { ClientConfigType } from '../common/types';
+import { dataTiersUiSettingsConfig } from './ui_settings';
 import {
   DataViewsServerPluginSetup,
   DataViewsServerPluginStart,
@@ -50,6 +51,9 @@ export class DataViewsServerPlugin
 
     const config = this.initializerContext.config.get<ClientConfigType>();
 
+    if (config.dataTiersExcludedForFields) {
+      core.uiSettings.register(dataTiersUiSettingsConfig);
+    }
     if (config.fieldListCachingEnabled) {
       core.uiSettings.register(cacheMaxAge);
     }

src/plugins/data_views/server/rest_api_routes/internal/fields.ts

@@ -15,9 +15,10 @@ import type {
   DataViewsServerPluginStartDependencies,
 } from '../../types';
 import type { FieldDescriptorRestResponse } from '../route_types';
-import { FIELDS_PATH as path } from '../../../common/constants';
+import { FIELDS_PATH as path, DATA_VIEWS_FIELDS_EXCLUDED_TIERS } from '../../../common/constants';
 import { parseFields, IBody, IQuery, querySchema, validate } from './fields_for';
 import { DEFAULT_FIELD_CACHE_FRESHNESS } from '../../constants';
+import { getIndexFilterDsl } from './utils';
 
 export function calculateHash(srcBuffer: Buffer) {
   const hash = createHash('sha1');
@@ -31,6 +32,9 @@ const handler: (isRollupsEnabled: () => boolean) => RequestHandler<{}, IQuery, I
     const uiSettings = core.uiSettings.client;
     const { asCurrentUser } = core.elasticsearch.client;
     const indexPatterns = new IndexPatternsFetcher(asCurrentUser, undefined, isRollupsEnabled());
+    const excludedTiers = await core.uiSettings.client.get<string>(
+      DATA_VIEWS_FIELDS_EXCLUDED_TIERS
+    );
     const {
       pattern,
       meta_fields: metaFields,
@@ -59,6 +63,7 @@ const handler: (isRollupsEnabled: () => boolean) => RequestHandler<{}, IQuery, I
           allow_no_indices: allowNoIndex || false,
           includeUnmapped,
         },
+        indexFilter: getIndexFilterDsl({ excludedTiers }),
         ...(parsedFields.length > 0 ? { fields: parsedFields } : {}),
       });
 

src/plugins/data_views/server/rest_api_routes/internal/fields_for.ts

@@ -17,7 +17,11 @@ import type {
   DataViewsServerPluginStartDependencies,
 } from '../../types';
 import type { FieldDescriptorRestResponse } from '../route_types';
-import { FIELDS_FOR_WILDCARD_PATH as path } from '../../../common/constants';
+import {
+  FIELDS_FOR_WILDCARD_PATH as path,
+  DATA_VIEWS_FIELDS_EXCLUDED_TIERS,
+} from '../../../common/constants';
+import { getIndexFilterDsl } from './utils';
 
 /**
  * Accepts one of the following:
@@ -115,8 +119,13 @@ export const validate: FullValidationConfig<any, any, any> = {
 
 const handler: (isRollupsEnabled: () => boolean) => RequestHandler<{}, IQuery, IBody> =
   (isRollupsEnabled) => async (context, request, response) => {
-    const { asCurrentUser } = (await context.core).elasticsearch.client;
+    const core = await context.core;
+    const { asCurrentUser } = core.elasticsearch.client;
+    const excludedTiers = await core.uiSettings.client.get<string>(
+      DATA_VIEWS_FIELDS_EXCLUDED_TIERS
+    );
     const indexPatterns = new IndexPatternsFetcher(asCurrentUser, undefined, isRollupsEnabled());
+
     const {
       pattern,
       meta_fields: metaFields,
@@ -149,7 +158,7 @@ const handler: (isRollupsEnabled: () => boolean) => RequestHandler<{}, IQuery, I
           allow_no_indices: allowNoIndex || false,
           includeUnmapped,
         },
-        indexFilter,
+        indexFilter: getIndexFilterDsl({ indexFilter, excludedTiers }),
         allowHidden,
         ...(parsedFields.length > 0 ? { fields: parsedFields } : {}),
       });

src/plugins/data_views/server/rest_api_routes/internal/utils.test.ts

@@ -0,0 +1,47 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { getIndexFilterDsl } from './utils';
+
+describe('getIndexFilterDsl', () => {
+  const indexFilter = { term: { _id: '1' } };
+  const tiersQuery = {
+    bool: {
+      must_not: [
+        {
+          terms: {
+            _tier: ['data_cold', 'data_frozen'],
+          },
+        },
+      ],
+    },
+  };
+  const excludedTiers = 'data_cold, data_frozen';
+
+  it('no indexFilter, no excluded tiers', () => {
+    expect(getIndexFilterDsl({})).toBeUndefined();
+  });
+
+  it('indexFilter, no excluded tiers', () => {
+    expect(getIndexFilterDsl({ indexFilter })).toEqual(indexFilter);
+  });
+
+  it('excluded tiers, no indexFilter', () => {
+    expect(getIndexFilterDsl({ excludedTiers })).toEqual(tiersQuery);
+  });
+
+  it('indexFilter and excluded tiers', () => {
+    const combinedQuery = {
+      bool: {
+        must: [indexFilter, tiersQuery],
+      },
+    };
+
+    expect(getIndexFilterDsl({ indexFilter, excludedTiers })).toEqual(combinedQuery);
+  });
+});

src/plugins/data_views/server/rest_api_routes/internal/utils.ts

@@ -0,0 +1,46 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import type { QueryDslQueryContainer } from '../../../common/types';
+
+const excludeTiersDsl = (excludedTiers: string) => {
+  const _tier = excludedTiers.split(',').map((tier) => tier.trim());
+  return {
+    bool: {
+      must_not: [
+        {
+          terms: {
+            _tier,
+          },
+        },
+      ],
+    },
+  };
+};
+
+interface GetIndexFilterDslOptions {
+  indexFilter?: QueryDslQueryContainer;
+  excludedTiers?: string;
+}
+
+export const getIndexFilterDsl = ({
+  indexFilter,
+  excludedTiers,
+}: GetIndexFilterDslOptions): QueryDslQueryContainer | undefined => {
+  if (!indexFilter) {
+    return excludedTiers ? excludeTiersDsl(excludedTiers) : undefined;
+  }
+
+  return !excludedTiers
+    ? indexFilter
+    : {
+        bool: {
+          must: [indexFilter, excludeTiersDsl(excludedTiers)],
+        },
+      };
+};

src/plugins/data_views/server/ui_settings.ts

@@ -8,8 +8,25 @@
 
 import { i18n } from '@kbn/i18n';
 import { schema } from '@kbn/config-schema';
+import type { UiSettingsParams } from '@kbn/core/server';
+import { DATA_VIEWS_FIELDS_EXCLUDED_TIERS } from '../common/constants';
 import { DEFAULT_FIELD_CACHE_FRESHNESS } from './constants';
 
+export const dataTiersUiSettingsConfig: Record<string, UiSettingsParams> = {
+  [DATA_VIEWS_FIELDS_EXCLUDED_TIERS]: {
+    name: i18n.translate('dataViews.advancedSettings.dataTiersName', {
+      defaultMessage: 'Data tiers excluded from field requests',
+    }),
+    value: '',
+    type: 'string',
+    description: i18n.translate('dataViews.advancedSettings.dataTiersText', {
+      defaultMessage:
+        'Exclude fields from specified tiers (such as data_frozen) for faster performance. Comma delimit to exclude multiple tiers - data_warm,data_cold',
+    }),
+    schema: schema.string(),
+  },
+};
+
 export const cacheMaxAge = {
   'data_views:cache_max_age': {
     name: i18n.translate('dataViews.advancedSettings.cacheMaxAgeTitle', {

src/plugins/kibana_usage_collection/server/collectors/management/schema.ts

@@ -620,4 +620,8 @@ export const stackManagementSchema: MakeSchemaFrom<UsageStats> = {
     type: 'integer',
     _meta: { description: 'Non-default value of setting.' },
   },
+  'data_views:fields_excluded_data_tiers': {
+    type: 'keyword',
+    _meta: { description: 'Non-default value of setting.' },
+  },
 };

src/plugins/kibana_usage_collection/server/collectors/management/types.ts

@@ -164,4 +164,5 @@ export interface UsageStats {
   'observability:profilingDatacenterPUE': number;
   'observability:profilingCostPervCPUPerHour': number;
   'observability:profilingAWSCostDiscountRate': number;
+  'data_views:fields_excluded_data_tiers': string;
 }

src/plugins/telemetry/schema/oss_plugins.json

@@ -10245,6 +10245,12 @@
           "_meta": {
             "description": "Non-default value of setting."
           }
+        },
+        "data_views:fields_excluded_data_tiers": {
+          "type": "keyword",
+          "_meta": {
+            "description": "Non-default value of setting."
+          }
         }
       }
     },

src/plugins/visualizations/common/content_management/v1/types.ts

@@ -57,6 +57,7 @@ export interface VisualizationSavedObject {
     statusCode: number;
     metadata?: Record<string, unknown>;
   };
+  managed?: boolean;
 }
 
 export type PartialVisualizationSavedObject = Omit<

src/plugins/visualizations/public/types.ts

@@ -55,6 +55,7 @@ export interface VisSavedObject extends ISavedVis {
   searchSource?: ISearchSource;
   version?: string;
   tags?: string[];
+  managed: boolean;
 }
 
 export interface SaveVisOptions {

src/plugins/visualizations/public/utils/saved_visualize_utils.ts

@@ -227,6 +227,7 @@ export async function getSavedVisualization(
     getEsType: () => SAVED_VIS_TYPE,
     getDisplayName: () => SAVED_VIS_TYPE,
     searchSource: opts.searchSource ? services.search.searchSource.createEmpty() : undefined,
+    managed: false,
   } as VisSavedObject;
   const defaultsProps = getDefaults(opts);
 
@@ -255,6 +256,7 @@ export async function getSavedVisualization(
 
   Object.assign(savedObject, attributes);
   savedObject.lastSavedTitle = savedObject.title;
+  savedObject.managed = Boolean(resp.managed);
 
   savedObject.sharingSavedObjectProps = {
     aliasTargetId,

src/plugins/visualizations/public/visualize_app/components/visualize_top_nav.tsx

@@ -14,6 +14,7 @@ import useLocalStorage from 'react-use/lib/useLocalStorage';
 import { useKibana } from '@kbn/kibana-react-plugin/public';
 import type { DataView } from '@kbn/data-views-plugin/public';
 import { switchMap } from 'rxjs';
+import { getManagedContentBadge } from '@kbn/managed-content-badge';
 import type {
   VisualizeServices,
   VisualizeAppState,
@@ -63,7 +64,11 @@ const TopNav = ({
   const { services } = useKibana<VisualizeServices>();
   const { TopNavMenu } = services.navigation.ui;
   const { setHeaderActionMenu, visualizeCapabilities } = services;
-  const { embeddableHandler, vis } = visInstance;
+  const {
+    embeddableHandler,
+    vis,
+    savedVis: { managed },
+  } = visInstance;
   const [inspectorSession, setInspectorSession] = useState<OverlayRef>();
   const [navigateToLens, setNavigateToLens] = useState(false);
   const [displayEditInLensItem, setDisplayEditInLensItem] = useState(false);
@@ -315,6 +320,18 @@ const TopNav = ({
       showDatePicker={showDatePicker()}
       showFilterBar={showFilterBar}
       showQueryInput={showQueryInput}
+      badges={
+        managed
+          ? [
+              getManagedContentBadge(
+                i18n.translate('visualizations.managedBadgeTooltip', {
+                  defaultMessage:
+                    'This visualization is managed by Elastic. Changes made here must be saved to a new visualization.',
+                })
+              ),
+            ]
+          : undefined
+      }
       saveQueryMenuVisibility={
         services.visualizeCapabilities.saveQuery ? 'allowed_by_app_privilege' : 'globally_managed'
       }

src/plugins/visualizations/public/visualize_app/utils/get_top_nav_config.tsx

@@ -605,6 +605,14 @@ export const getTopNavConfig = (
                       }
                     )}
                     onClose={() => {}}
+                    mustCopyOnSaveMessage={
+                      savedVis.managed
+                        ? i18n.translate('visualizations.topNavMenu.mustCopyOnSave', {
+                            defaultMessage:
+                              'This visualization is managed by Elastic. Changes here must be saved to a new visualization.',
+                          })
+                        : undefined
+                    }
                   />
                 );
               }