[8.7] [ML] Fix retention policy date field should list destination in…

…dex date fields (#155765) (#156047)

# Backport

This will backport the following commits from `main` to `8.7`:
- [[ML] Fix retention policy date field should list destination index
date fields (#155765)](#155765)

<!--- Backport version: 8.9.7 -->

### Questions ?
Please refer to the [Backport tool
documentation](https://github.com/sqren/backport)

<!--BACKPORT [{"author":{"name":"Quynh Nguyen
(Quinn)","email":"[email protected]"},"sourceCommit":{"committedDate":"2023-04-27T16:50:13Z","message":"[ML]
Fix retention policy date field should list destination index date
fields (#155765)\n\nCo-authored-by: Kibana Machine
<[email protected]>","sha":"dcc280b5d81109e10ad8c738c33de2d8c19f6f90","branchLabelMapping":{"^v8.9.0$":"main","^v(\\d+).(\\d+).\\d+$":"$1.$2"}},"sourcePullRequest":{"labels":["release_note:fix",":ml","Feature:Transforms","v8.8.0","v8.7.2","v8.9.0"],"number":155765,"url":"https://github.com/elastic/kibana/pull/155765","mergeCommit":{"message":"[ML]
Fix retention policy date field should list destination index date
fields (#155765)\n\nCo-authored-by: Kibana Machine
<[email protected]>","sha":"dcc280b5d81109e10ad8c738c33de2d8c19f6f90"}},"sourceBranch":"main","suggestedTargetBranches":["8.8","8.7"],"targetPullRequestStates":[{"branch":"8.8","label":"v8.8.0","labelRegex":"^v(\\d+).(\\d+).\\d+$","isSourceBranch":false,"state":"NOT_CREATED"},{"branch":"8.7","label":"v8.7.2","labelRegex":"^v(\\d+).(\\d+).\\d+$","isSourceBranch":false,"state":"NOT_CREATED"},{"branch":"main","label":"v8.9.0","labelRegex":"^v8.9.0$","isSourceBranch":true,"state":"MERGED","url":"https://github.com/elastic/kibana/pull/155765","number":155765,"mergeCommit":{"message":"[ML]
Fix retention policy date field should list destination index date
fields (#155765)\n\nCo-authored-by: Kibana Machine
<[email protected]>","sha":"dcc280b5d81109e10ad8c738c33de2d8c19f6f90"}}]}]
BACKPORT-->

Co-authored-by: Quynh Nguyen (Quinn) <[email protected]>

x-pack/plugins/transform/public/app/sections/create_transform/components/step_details/step_details_form.tsx

@@ -278,7 +278,9 @@ export const StepDetailsForm: FC<StepDetailsFormProps> = React.memo(
     // Reset retention policy settings when the user disables the whole option
     useEffect(() => {
       if (!isRetentionPolicyEnabled) {
-        setRetentionPolicyDateField(isRetentionPolicyAvailable ? dateFieldNames[0] : '');
+        setRetentionPolicyDateField(
+          isRetentionPolicyAvailable ? dataViewAvailableTimeFields[0] : ''
+        );
         setRetentionPolicyMaxAge('');
       }
       // eslint-disable-next-line react-hooks/exhaustive-deps
@@ -713,7 +715,7 @@ export const StepDetailsForm: FC<StepDetailsFormProps> = React.memo(
                 )}
               >
                 <EuiSelect
-                  options={dateFieldNames.map((text: string) => ({ text }))}
+                  options={dataViewAvailableTimeFields.map((text: string) => ({ text }))}
                   value={retentionPolicyDateField}
                   onChange={(e) => setRetentionPolicyDateField(e.target.value)}
                   data-test-subj="transformRetentionPolicyDateFieldSelect"

x-pack/test/functional/apps/transform/edit_clone/cloning.ts

@@ -35,12 +35,19 @@ function getTransformConfig(): TransformPivotConfig {
     source: { index: ['ft_ecommerce'] },
     pivot: {
       group_by: { category: { terms: { field: 'category.keyword' } } },
-      aggregations: { 'products.base_price.avg': { avg: { field: 'products.base_price' } } },
+      aggregations: {
+        'products.base_price.avg': { avg: { field: 'products.base_price' } },
+        'order_date.max': {
+          max: {
+            field: 'order_date',
+          },
+        },
+      },
     },
     description:
       'ecommerce batch transform with avg(products.base_price) grouped by terms(category)',
     frequency: '3s',
-    retention_policy: { time: { field: 'order_date', max_age: '1d' } },
+    retention_policy: { time: { field: 'order_date.max', max_age: '1d' } },
     settings: {
       max_page_search_size: 250,
       num_failure_retries: 0,
@@ -75,11 +82,16 @@ function getTransformConfigWithRuntimeMappings(): TransformPivotConfig {
         'rt_total_charge.avg': { avg: { field: 'rt_total_charge' } },
         'rt_total_charge.min': { min: { field: 'rt_total_charge' } },
         'rt_total_charge.max': { max: { field: 'rt_total_charge' } },
+        max_order_date: {
+          max: {
+            field: 'order_date',
+          },
+        },
       },
     },
     description: 'ecommerce batch transform grouped by terms(rt_gender_lower)',
     frequency: '3s',
-    retention_policy: { time: { field: 'order_date', max_age: '3d' } },
+    retention_policy: { time: { field: 'max_order_date', max_age: '3d' } },
     settings: {
       max_page_search_size: 250,
       num_failure_retries: 5,
@@ -155,11 +167,16 @@ function getTransformConfigWithBoolFilterAgg(): TransformPivotConfig {
             },
           },
         },
+        max_order_date: {
+          max: {
+            field: 'order_date',
+          },
+        },
       },
     },
     description: 'ecommerce batch transform with filter aggregations',
     frequency: '3s',
-    retention_policy: { time: { field: 'order_date', max_age: '3d' } },
+    retention_policy: { time: { field: 'max_order_date', max_age: '3d' } },
     settings: {
       max_page_search_size: 250,
       num_failure_retries: 5,
@@ -253,7 +270,7 @@ export default function ({ getService }: FtrProviderContext) {
             ],
           },
           retentionPolicySwitchEnabled: true,
-          retentionPolicyField: 'order_date',
+          retentionPolicyField: 'order_date.max',
           retentionPolicyMaxAge: '1d',
           numFailureRetries: getNumFailureRetriesStr(
             transformConfigWithPivot.settings?.num_failure_retries
@@ -288,7 +305,7 @@ export default function ({ getService }: FtrProviderContext) {
             values: [`female`, `male`],
           },
           retentionPolicySwitchEnabled: true,
-          retentionPolicyField: 'order_date',
+          retentionPolicyField: 'max_order_date',
           retentionPolicyMaxAge: '3d',
           numFailureRetries: getNumFailureRetriesStr(
             transformConfigWithRuntimeMapping.settings?.num_failure_retries
@@ -341,7 +358,7 @@ export default function ({ getService }: FtrProviderContext) {
             ],
           },
           retentionPolicySwitchEnabled: true,
-          retentionPolicyField: 'order_date',
+          retentionPolicyField: 'max_order_date',
           retentionPolicyMaxAge: '3d',
           numFailureRetries: getNumFailureRetriesStr(
             transformConfigWithBoolFilterAgg.settings?.num_failure_retries