[UA] Support Deprecated Data Streams Migrations (elastic#202204)

## Summary

- [x] Fix UA currently failing to return upgrade status
- [x] Support surfacing `data_streams` migrations in UA under the ES tab
- [x] Refactor code for better readablity
- [x] Add more test cases across the board for all the es migrations
status feature in UA
- [x] Add a `featureSet.migrateDataStreams` to enable surfacing data
streams migrations
- [x] Surface data streams in UA UI
- [x] Take screenshots for a product review discussions
- [x] Unskip api_integration test cases

### Imporant Notes

ES deprecations are hidden behind the `featureSet` flag and will only be
shown in `8.last` for users.
This gives us time to review the copy and implement the corrective
action for reindexing data streams which is still pending implementaiton
from ES side.

For now we will merge this to unblock upgrades in `8.17` and support
surfacing data_streams deprecations and add tests.

Follow up work for `8.18`
- Add integration Tests
- Update copy of flyout and documentation link
- Reindexing data streams corrective action

closes elastic/kibana-team#1293

## Screenshots

#### Overview Page
<img width="683" alt="image"
src="https://github.com/user-attachments/assets/246d89ac-02cd-4813-ba38-e2e28df00c8d">

####  Elasticsearch deprecation issues Page

<img width="1453" alt="image"
src="https://github.com/user-attachments/assets/b5fd5f15-fa44-4acb-b7ff-4973593dcfbb">


####  Data streams deprecation details flyout
<img width="778" alt="image"
src="https://github.com/user-attachments/assets/af343f69-7e76-4c91-a6e3-cff29e26df59">

---------

Co-authored-by: kibanamachine <[email protected]>

test/plugin_functional/test_suites/core_plugins/rendering.ts

@@ -354,6 +354,7 @@ export default function ({ getService }: PluginFunctionalProviderContext) {
         'xpack.upgrade_assistant.featureSet.migrateSystemIndices (boolean?)',
         'xpack.upgrade_assistant.featureSet.mlSnapshots (boolean?)',
         'xpack.upgrade_assistant.featureSet.reindexCorrectiveActions (boolean?)',
+        'xpack.upgrade_assistant.featureSet.migrateDataStreams (boolean?)',
         'xpack.upgrade_assistant.ui.enabled (boolean?)',
         'xpack.observability.unsafe.alertDetails.metrics.enabled (boolean?)',
         'xpack.observability.unsafe.alertDetails.logs.enabled (boolean?)',

x-pack/plugins/upgrade_assistant/README.md

@@ -15,6 +15,7 @@ Some features of the UA are only needed when upgrading to a new major version. T
 * ML Snapshots (`featureSet.mlSnapshots`): Machine learning Upgrade mode can be toggled from outside Kibana, the purpose of this feature guard is to hide all ML related deprecations from the end user until the next major upgrade.
 When we want to enable ML model snapshot deprecation warnings again we need to change the constant `MachineLearningField.MIN_CHECKED_SUPPORTED_SNAPSHOT_VERSION` to something higher than 7.0.0 in the Elasticsearch code.
 * Migrating system indices (`featureSet.migrateSystemIndices`): Migrating system indices should only be enabled for major version upgrades. This config hides the second step from the UA UI for migrating system indices.
+* Reindex Data Streams (`featureSet.migrateDataStreams`): Migrating deprecated Data streams should only be enabled for major version upgrades. The purpose of this feature guard is to hide all data streams related deprecations from the end user until the next major upgrade.
 * Reindex corrective actions (`featureSet.reindexCorrectiveActions`): Deprecations with reindexing corrective actions are only enabled for major version upgrades. Currently, the reindex actions include some logic that is specific to the [8.0 upgrade](https://github.com/elastic/kibana/blob/main/x-pack/plugins/upgrade_assistant/server/lib/reindexing/index_settings.ts). End users could get into a bad situation if this is enabled before this logic is fixed.
 
 ## Deprecations

x-pack/plugins/upgrade_assistant/__jest__/client_integration/helpers/app_context.mock.ts

@@ -80,6 +80,7 @@ export const getAppContextMock = (kibanaVersion: SemVer) => ({
   featureSet: {
     mlSnapshots: true,
     migrateSystemIndices: true,
+    migrateDataStreams: true,
     reindexCorrectiveActions: true,
   },
   kibanaVersionInfo: {

x-pack/plugins/upgrade_assistant/common/types.ts

@@ -215,7 +215,7 @@ export interface HealthIndicatorAction {
 
 export interface EnrichedDeprecationInfo
   extends Omit<estypes.MigrationDeprecationsDeprecation, 'level'> {
-  type: keyof estypes.MigrationDeprecationsResponse | 'health_indicator';
+  type: keyof estypes.MigrationDeprecationsResponse | 'health_indicator' | 'data_streams';
   isCritical: boolean;
   status?: estypes.HealthReportIndicatorHealthStatus;
   index?: string;
@@ -296,4 +296,5 @@ export interface FeatureSet {
   migrateSystemIndices: boolean;
   mlSnapshots: boolean;
   reindexCorrectiveActions: boolean;
+  migrateDataStreams: boolean;
 }

x-pack/plugins/upgrade_assistant/public/application/components/constants.tsx

@@ -6,8 +6,9 @@
  */
 
 import { i18n } from '@kbn/i18n';
+import { EnrichedDeprecationInfo } from '../../../common/types';
 
-export const DEPRECATION_TYPE_MAP = {
+export const DEPRECATION_TYPE_MAP: Record<EnrichedDeprecationInfo['type'], string> = {
   cluster_settings: i18n.translate(
     'xpack.upgradeAssistant.esDeprecations.clusterDeprecationTypeLabel',
     {
@@ -32,6 +33,9 @@ export const DEPRECATION_TYPE_MAP = {
       defaultMessage: 'Health Indicator',
     }
   ),
+  data_streams: i18n.translate('xpack.upgradeAssistant.esDeprecations.dataStreamsTypeLabel', {
+    defaultMessage: 'Data Stream',
+  }),
 };
 
 export const PAGINATION_CONFIG = {

x-pack/plugins/upgrade_assistant/server/config.ts

@@ -47,6 +47,11 @@ const configSchema = schema.object({
      * End users could get into a bad situation if this is enabled before this logic is fixed.
      */
     reindexCorrectiveActions: schema.boolean({ defaultValue: false }),
+    /**
+     * Migrating deprecated data streams should only be enabled for major version upgrades.
+     * Currently this is manually set to `true` on every `x.last` version.
+     */
+    migrateDataStreams: schema.boolean({ defaultValue: false }),
   }),
   /**
    * This config allows to hide the UI without disabling the plugin.

x-pack/plugins/upgrade_assistant/server/lib/__fixtures__/es_deprecations.ts

@@ -0,0 +1,77 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export const getMockEsDeprecations = () => {
+  return {
+    cluster_settings: [],
+    node_settings: [],
+    ml_settings: [],
+    index_settings: {},
+    data_streams: {},
+  };
+};
+
+export const getMockMlSettingsDeprecations = () => {
+  return {
+    ml_settings: [
+      {
+        level: 'warning',
+        message: 'Datafeed [deprecation-datafeed] uses deprecated query options',
+        url: 'https://www.elastic.co/guide/en/elasticsearch/reference/master/breaking-changes-7.0.html#breaking_70_search_changes',
+        details:
+          '[Deprecated field [use_dis_max] used, replaced by [Set [tie_breaker] to 1 instead]]',
+        // @ts-ignore
+        resolve_during_rolling_upgrade: false,
+      },
+      {
+        level: 'critical',
+        message:
+          'model snapshot [1] for job [deprecation_check_job] needs to be deleted or upgraded',
+        url: '',
+        details: 'details',
+        // @ts-ignore
+        _meta: { snapshot_id: '1', job_id: 'deprecation_check_job' },
+        // @ts-ignore
+        resolve_during_rolling_upgrade: false,
+      },
+    ],
+  };
+};
+
+export const getMockDataStreamDeprecations = () => {
+  return {
+    data_streams: {
+      'my-v7-data-stream': [
+        {
+          level: 'critical',
+          message: 'Old data stream with a compatibility version < 8.0',
+          url: 'https://www.elastic.co/guide/en/elasticsearch/reference/master/breaking-changes-9.0.html',
+          details:
+            'This data stream has backing indices that were created before Elasticsearch 8.0.0',
+          resolve_during_rolling_upgrade: false,
+          _meta: {
+            backing_indices: {
+              count: 52,
+              need_upgrading: {
+                count: 37,
+                searchable_snapshot: {
+                  count: 23,
+                  fully_mounted: {
+                    count: 7,
+                  },
+                  partially_mounted: {
+                    count: 16,
+                  },
+                },
+              },
+            },
+          },
+        },
+      ],
+    },
+  };
+};

x-pack/plugins/upgrade_assistant/server/lib/__fixtures__/fake_deprecations.json

@@ -148,5 +148,31 @@
         "resolve_during_rolling_upgrade": false
       }
     ]
+  },
+  "data_streams": {
+    "my-v7-data-stream" : [{
+      "level" : "critical",
+      "message" : "Old data stream with a compatibility version < 8.0",
+      "url" : "https://www.elastic.co/guide/en/elasticsearch/reference/master/breaking-changes-9.0.html",
+      "details" : "This data stream has backing indices that were created before Elasticsearch 8.0.0",
+      "resolve_during_rolling_upgrade" : false,
+      "_meta": {
+        "backing_indices": {
+          "count": 52,
+          "need_upgrading": {
+            "count": 37,
+            "searchable_snapshot": {
+              "count": 23,
+              "fully_mounted": {
+                "count": 7
+              },
+              "partially_mounted": {
+                "count": 16
+              }
+            }
+          }
+        }
+      }
+    }]
   }
 }