[8.17] [UA] Support Deprecated Data Streams Migrations (elastic#202204)…

… (elastic#202601) # Backport This will backport the following commits from `main` to `8.17`: - [[UA] Support Deprecated Data Streams Migrations (elastic#202204)](elastic#202204)  ### Questions ? Please refer to the [Backport tool documentation](https://github.com/sqren/backport)  --------- Co-authored-by: kibanamachine <[email protected]>
kibanamachine · Dec 3, 2024 · 0acb039 · 0acb039
1 parent d073848
commit 0acb039
Show file tree

Hide file tree

Showing 19 changed files with 743 additions and 502 deletions.
diff --git a/test/plugin_functional/test_suites/core_plugins/rendering.ts b/test/plugin_functional/test_suites/core_plugins/rendering.ts
@@ -353,6 +353,7 @@ export default function ({ getService }: PluginFunctionalProviderContext) {
         'xpack.upgrade_assistant.featureSet.migrateSystemIndices (boolean?)',
         'xpack.upgrade_assistant.featureSet.mlSnapshots (boolean?)',
         'xpack.upgrade_assistant.featureSet.reindexCorrectiveActions (boolean?)',
+        'xpack.upgrade_assistant.featureSet.migrateDataStreams (boolean?)',
         'xpack.upgrade_assistant.ui.enabled (boolean?)',
         'xpack.observability.unsafe.alertDetails.metrics.enabled (boolean?)',
         'xpack.observability.unsafe.alertDetails.logs.enabled (boolean?)',

diff --git a/x-pack/plugins/upgrade_assistant/README.md b/x-pack/plugins/upgrade_assistant/README.md
@@ -15,6 +15,7 @@ Some features of the UA are only needed when upgrading to a new major version. T
 * ML Snapshots (`featureSet.mlSnapshots`): Machine learning Upgrade mode can be toggled from outside Kibana, the purpose of this feature guard is to hide all ML related deprecations from the end user until the next major upgrade.
 When we want to enable ML model snapshot deprecation warnings again we need to change the constant `MachineLearningField.MIN_CHECKED_SUPPORTED_SNAPSHOT_VERSION` to something higher than 7.0.0 in the Elasticsearch code.
 * Migrating system indices (`featureSet.migrateSystemIndices`): Migrating system indices should only be enabled for major version upgrades. This config hides the second step from the UA UI for migrating system indices.
+* Reindex Data Streams (`featureSet.migrateDataStreams`): Migrating deprecated Data streams should only be enabled for major version upgrades. The purpose of this feature guard is to hide all data streams related deprecations from the end user until the next major upgrade.
 * Reindex corrective actions (`featureSet.reindexCorrectiveActions`): Deprecations with reindexing corrective actions are only enabled for major version upgrades. Currently, the reindex actions include some logic that is specific to the [8.0 upgrade](https://github.com/elastic/kibana/blob/main/x-pack/plugins/upgrade_assistant/server/lib/reindexing/index_settings.ts). End users could get into a bad situation if this is enabled before this logic is fixed.
 
 ## Deprecations

diff --git a/x-pack/plugins/upgrade_assistant/__jest__/client_integration/helpers/app_context.mock.ts b/x-pack/plugins/upgrade_assistant/__jest__/client_integration/helpers/app_context.mock.ts
@@ -80,6 +80,7 @@ export const getAppContextMock = (kibanaVersion: SemVer) => ({
   featureSet: {
     mlSnapshots: true,
     migrateSystemIndices: true,
+    migrateDataStreams: true,
     reindexCorrectiveActions: true,
   },
   kibanaVersionInfo: {

diff --git a/x-pack/plugins/upgrade_assistant/common/types.ts b/x-pack/plugins/upgrade_assistant/common/types.ts
@@ -215,7 +215,7 @@ export interface HealthIndicatorAction {
 
 export interface EnrichedDeprecationInfo
   extends Omit<estypes.MigrationDeprecationsDeprecation, 'level'> {
-  type: keyof estypes.MigrationDeprecationsResponse | 'health_indicator';
+  type: keyof estypes.MigrationDeprecationsResponse | 'health_indicator' | 'data_streams';
   isCritical: boolean;
   status?: estypes.HealthReportIndicatorHealthStatus;
   index?: string;
@@ -296,4 +296,5 @@ export interface FeatureSet {
   migrateSystemIndices: boolean;
   mlSnapshots: boolean;
   reindexCorrectiveActions: boolean;
+  migrateDataStreams: boolean;
 }
diff --git a/x-pack/plugins/upgrade_assistant/public/application/components/constants.tsx b/x-pack/plugins/upgrade_assistant/public/application/components/constants.tsx
@@ -6,8 +6,9 @@
  */
 
 import { i18n } from '@kbn/i18n';
+import { EnrichedDeprecationInfo } from '../../../common/types';
 
-export const DEPRECATION_TYPE_MAP = {
+export const DEPRECATION_TYPE_MAP: Record<EnrichedDeprecationInfo['type'], string> = {
   cluster_settings: i18n.translate(
     'xpack.upgradeAssistant.esDeprecations.clusterDeprecationTypeLabel',
     {
@@ -32,6 +33,9 @@ export const DEPRECATION_TYPE_MAP = {
       defaultMessage: 'Health Indicator',
     }
   ),
+  data_streams: i18n.translate('xpack.upgradeAssistant.esDeprecations.dataStreamsTypeLabel', {
+    defaultMessage: 'Data Stream',
+  }),
 };
 
 export const PAGINATION_CONFIG = {

diff --git a/x-pack/plugins/upgrade_assistant/server/config.ts b/x-pack/plugins/upgrade_assistant/server/config.ts
@@ -47,6 +47,11 @@ const configSchema = schema.object({
      * End users could get into a bad situation if this is enabled before this logic is fixed.
      */
     reindexCorrectiveActions: schema.boolean({ defaultValue: false }),
+    /**
+     * Migrating deprecated data streams should only be enabled for major version upgrades.
+     * Currently this is manually set to `true` on every `x.last` version.
+     */
+    migrateDataStreams: schema.boolean({ defaultValue: false }),
   }),
   /**
    * This config allows to hide the UI without disabling the plugin.

diff --git a/x-pack/plugins/upgrade_assistant/server/lib/__fixtures__/es_deprecations.ts b/x-pack/plugins/upgrade_assistant/server/lib/__fixtures__/es_deprecations.ts
@@ -0,0 +1,77 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export const getMockEsDeprecations = () => {
+  return {
+    cluster_settings: [],
+    node_settings: [],
+    ml_settings: [],
+    index_settings: {},
+    data_streams: {},
+  };
+};
+
+export const getMockMlSettingsDeprecations = () => {
+  return {
+    ml_settings: [
+      {
+        level: 'warning',
+        message: 'Datafeed [deprecation-datafeed] uses deprecated query options',
+        url: 'https://www.elastic.co/guide/en/elasticsearch/reference/master/breaking-changes-7.0.html#breaking_70_search_changes',
+        details:
+          '[Deprecated field [use_dis_max] used, replaced by [Set [tie_breaker] to 1 instead]]',
+        // @ts-ignore
+        resolve_during_rolling_upgrade: false,
+      },
+      {
+        level: 'critical',
+        message:
+          'model snapshot [1] for job [deprecation_check_job] needs to be deleted or upgraded',
+        url: '',
+        details: 'details',
+        // @ts-ignore
+        _meta: { snapshot_id: '1', job_id: 'deprecation_check_job' },
+        // @ts-ignore
+        resolve_during_rolling_upgrade: false,
+      },
+    ],
+  };
+};
+
+export const getMockDataStreamDeprecations = () => {
+  return {
+    data_streams: {
+      'my-v7-data-stream': [
+        {
+          level: 'critical',
+          message: 'Old data stream with a compatibility version < 8.0',
+          url: 'https://www.elastic.co/guide/en/elasticsearch/reference/master/breaking-changes-9.0.html',
+          details:
+            'This data stream has backing indices that were created before Elasticsearch 8.0.0',
+          resolve_during_rolling_upgrade: false,
+          _meta: {
+            backing_indices: {
+              count: 52,
+              need_upgrading: {
+                count: 37,
+                searchable_snapshot: {
+                  count: 23,
+                  fully_mounted: {
+                    count: 7,
+                  },
+                  partially_mounted: {
+                    count: 16,
+                  },
+                },
+              },
+            },
+          },
+        },
+      ],
+    },
+  };
+};
diff --git a/x-pack/plugins/upgrade_assistant/server/lib/__fixtures__/fake_deprecations.json b/x-pack/plugins/upgrade_assistant/server/lib/__fixtures__/fake_deprecations.json
@@ -148,5 +148,31 @@
         "resolve_during_rolling_upgrade": false
       }
     ]
+  },
+  "data_streams": {
+    "my-v7-data-stream" : [{
+      "level" : "critical",
+      "message" : "Old data stream with a compatibility version < 8.0",
+      "url" : "https://www.elastic.co/guide/en/elasticsearch/reference/master/breaking-changes-9.0.html",
+      "details" : "This data stream has backing indices that were created before Elasticsearch 8.0.0",
+      "resolve_during_rolling_upgrade" : false,
+      "_meta": {
+        "backing_indices": {
+          "count": 52,
+          "need_upgrading": {
+            "count": 37,
+            "searchable_snapshot": {
+              "count": 23,
+              "fully_mounted": {
+                "count": 7
+              },
+              "partially_mounted": {
+                "count": 16
+              }
+            }
+          }
+        }
+      }
+    }]
   }
 }