[8.x] [Chore] Isolate CCS check (elastic#195988) (elastic#196869)

# Backport

This will backport the following commits from `main` to `8.x`:
- [[Chore] Isolate CCS check
(elastic#195988)](elastic#195988)

<!--- Backport version: 9.4.3 -->

### Questions ?
Please refer to the [Backport tool
documentation](https://github.com/sqren/backport)

<!--BACKPORT [{"author":{"name":"Thomas
Neirynck","email":"[email protected]"},"sourceCommit":{"committedDate":"2024-10-18T12:40:04Z","message":"[Chore]
Isolate CCS check (elastic#195988)\n\n## Summary\r\n\r\nIsolate CCS check in
single
location.\r\n\r\nCloses\r\nhttps://github.com/elastic/issues/193906#issuecomment-2400370478\r\n\r\n\r\n###
For maintainers\r\n\r\n- [x] This was checked for breaking API changes
and was
[labeled\r\nappropriately](https://www.elastic.co/guide/en/kibana/master/contributing.html#_add_your_labels)\r\n\r\n---------\r\n\r\nCo-authored-by:
kibanamachine
<[email protected]>","sha":"f86d1716b171d846851041d50da074683624fa02","branchLabelMapping":{"^v9.0.0$":"main","^v8.17.0$":"8.x","^v(\\d+).(\\d+).\\d+$":"$1.$2"}},"sourcePullRequest":{"labels":["chore","release_note:skip","v9.0.0","backport:prev-major","ci:project-deploy-observability","Team:obs-ux-infra_services","apm:review","v8.16.0"],"title":"[Chore]
Isolate CCS
check","number":195988,"url":"https://github.com/elastic/kibana/pull/195988","mergeCommit":{"message":"[Chore]
Isolate CCS check (elastic#195988)\n\n## Summary\r\n\r\nIsolate CCS check in
single
location.\r\n\r\nCloses\r\nhttps://github.com/elastic/issues/193906#issuecomment-2400370478\r\n\r\n\r\n###
For maintainers\r\n\r\n- [x] This was checked for breaking API changes
and was
[labeled\r\nappropriately](https://www.elastic.co/guide/en/kibana/master/contributing.html#_add_your_labels)\r\n\r\n---------\r\n\r\nCo-authored-by:
kibanamachine
<[email protected]>","sha":"f86d1716b171d846851041d50da074683624fa02"}},"sourceBranch":"main","suggestedTargetBranches":["8.16"],"targetPullRequestStates":[{"branch":"main","label":"v9.0.0","branchLabelMappingKey":"^v9.0.0$","isSourceBranch":true,"state":"MERGED","url":"https://github.com/elastic/kibana/pull/195988","number":195988,"mergeCommit":{"message":"[Chore]
Isolate CCS check (elastic#195988)\n\n## Summary\r\n\r\nIsolate CCS check in
single
location.\r\n\r\nCloses\r\nhttps://github.com/elastic/issues/193906#issuecomment-2400370478\r\n\r\n\r\n###
For maintainers\r\n\r\n- [x] This was checked for breaking API changes
and was
[labeled\r\nappropriately](https://www.elastic.co/guide/en/kibana/master/contributing.html#_add_your_labels)\r\n\r\n---------\r\n\r\nCo-authored-by:
kibanamachine
<[email protected]>","sha":"f86d1716b171d846851041d50da074683624fa02"}},{"branch":"8.16","label":"v8.16.0","branchLabelMappingKey":"^v(\\d+).(\\d+).\\d+$","isSourceBranch":false,"state":"NOT_CREATED"}]}]
BACKPORT-->

Co-authored-by: Thomas Neirynck <[email protected]>

packages/kbn-es-query/index.ts

@@ -128,6 +128,7 @@ export {
   getDataViewFieldSubtypeNested,
   isDataViewFieldSubtypeMulti,
   isDataViewFieldSubtypeNested,
+  isCCSRemoteIndexName,
 } from './src/utils';
 
 export type { ExecutionContextSearch } from './src/expressions/types';

packages/kbn-es-query/src/utils.test.ts

@@ -0,0 +1,34 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import { isCCSRemoteIndexName } from './utils';
+
+describe('util tests', () => {
+  describe('isCCSRemoteIndexName', () => {
+    it('should not validate empty string', () => {
+      expect(isCCSRemoteIndexName('')).toBe(false);
+    });
+
+    it('should not validate date math expression', () => {
+      expect(isCCSRemoteIndexName('<logstash-{now/d-2d}>')).toBe(false);
+    });
+
+    it('should not validate date math expression with negation', () => {
+      expect(isCCSRemoteIndexName('-<logstash-{now/d-2d}>')).toBe(false);
+    });
+
+    it('should not validate invalid prefix', () => {
+      expect(isCCSRemoteIndexName(':logstash-{now/d-2d}')).toBe(false);
+    });
+
+    it('should validate CCS pattern', () => {
+      expect(isCCSRemoteIndexName('*:logstash-{now/d-2d}')).toBe(true);
+    });
+  });
+});

packages/kbn-es-query/src/utils.ts

@@ -36,3 +36,22 @@ export function isDataViewFieldSubtypeMulti(field: HasSubtype) {
 export function getDataViewFieldSubtypeMulti(field: HasSubtype) {
   return isDataViewFieldSubtypeMulti(field) ? (field.subType as IFieldSubTypeMulti) : undefined;
 }
+
+/**
+ * Check whether the index expression represents a remote index (CCS) or not.
+ * The index name is assumed to be individual index (no commas) but can contain `-`, wildcards,
+ * datemath, remote cluster name and any other syntax permissible in index expression component.
+ *
+ * 2024/10/11 Implementation taken from https://github.com/smalyshev/elasticsearch/blob/main/server/src/main/java/org/elasticsearch/transport/RemoteClusterAware.java
+ *
+ * @param indexExpression
+ */
+export function isCCSRemoteIndexName(indexExpression: string): boolean {
+  if (indexExpression === '' || indexExpression[0] === '<' || indexExpression.startsWith('-<')) {
+    // This is date math, but even if it is not, the remote can't start with '<'.
+    // Thus, whatever it is, this is definitely not a remote index.
+    return false;
+  }
+  // Note remote index name also can not start with ':'
+  return indexExpression.indexOf(':') > 0;
+}

x-pack/plugins/ml/public/application/util/index_utils.ts

@@ -6,6 +6,7 @@
  */
 
 import type { SavedSearch, SavedSearchPublicPluginStart } from '@kbn/saved-search-plugin/public';
+import { isCCSRemoteIndexName } from '@kbn/es-query';
 import type { Query, Filter } from '@kbn/es-query';
 import type { DataView, DataViewField, DataViewsContract } from '@kbn/data-views-plugin/common';
 
@@ -51,7 +52,7 @@ export function getQueryFromSavedSearchObject(savedSearch: SavedSearch) {
  * which means it is cross-cluster
  */
 export function isCcsIndexPattern(indexPattern: string) {
-  return indexPattern.includes(':');
+  return isCCSRemoteIndexName(indexPattern);
 }
 
 export function findMessageField(

x-pack/plugins/monitoring/common/ccs_utils.ts

@@ -5,6 +5,7 @@
  * 2.0.
  */
 
+import { isCCSRemoteIndexName } from '@kbn/es-query';
 import type { MonitoringConfig } from '../server/config';
 
 /**
@@ -67,13 +68,11 @@ export function prefixIndexPatternWithCcs(
  * @return {String} {@code null} if none. Otherwise the cluster prefix.
  */
 export function parseCrossClusterPrefix(indexName: string): string | null {
-  const colonIndex = indexName.indexOf(':');
-
-  if (colonIndex === -1) {
+  const isCcs = isCCSRemoteIndexName(indexName);
+  if (!isCcs) {
     return null;
   }
 
-  // if we found a : in the index name, then cross-cluster search (CCS) was used to find the cluster
-  // and we _should_ use it when we search explicitly for this cluster (to avoid inefficiently checking other monitoring _clusters_)
+  const colonIndex = indexName.indexOf(':');
   return indexName.substr(0, colonIndex);
 }

x-pack/plugins/monitoring/server/lib/alerts/fetch_ccr_read_exceptions.ts

@@ -7,6 +7,7 @@
 
 import { ElasticsearchClient } from '@kbn/core/server';
 import { get } from 'lodash';
+import { isCCSRemoteIndexName } from '@kbn/es-query';
 import { CCS_REMOTE_PATTERN } from '../../../common/constants';
 import { CCRReadExceptionsStats } from '../../../common/types/alerts';
 import { getIndexPatterns, getElasticsearchDataset } from '../cluster/get_index_patterns';
@@ -173,7 +174,7 @@ export async function fetchCCRReadExceptions(
         shardId,
         leaderIndex,
         lastReadException,
-        ccs: monitoringIndexName.includes(':') ? monitoringIndexName.split(':')[0] : null,
+        ccs: isCCSRemoteIndexName(monitoringIndexName) ? monitoringIndexName.split(':')[0] : null,
       });
     }
   }

x-pack/plugins/monitoring/server/lib/alerts/fetch_cluster_health.ts

@@ -5,6 +5,7 @@
  * 2.0.
  */
 import { ElasticsearchClient } from '@kbn/core/server';
+import { isCCSRemoteIndexName } from '@kbn/es-query';
 import { AlertCluster, AlertClusterHealth } from '../../../common/types/alerts';
 import { ElasticsearchSource } from '../../../common/types/es';
 import { createDatasetFilter } from './create_dataset_query_filter';
@@ -87,7 +88,7 @@ export async function fetchClusterHealth(
       health:
         hit._source!.cluster_state?.status || hit._source!.elasticsearch?.cluster?.stats?.status,
       clusterUuid: hit._source!.cluster_uuid || hit._source!.elasticsearch?.cluster?.id,
-      ccs: hit._index.includes(':') ? hit._index.split(':')[0] : undefined,
+      ccs: isCCSRemoteIndexName(hit._index) ? hit._index.split(':')[0] : undefined,
     } as AlertClusterHealth;
   });
 }

x-pack/plugins/observability_solution/apm/public/components/app/diagnostics/summary_tab/index.tsx

@@ -7,6 +7,7 @@
 
 import React from 'react';
 import { EuiFlexGroup, EuiCallOut, EuiDescriptionList, EuiSpacer } from '@elastic/eui';
+import { isCCSRemoteIndexName } from '@kbn/es-query';
 
 import { APIReturnType } from '../../../../services/rest/create_call_apm_api';
 import { ApmIntegrationPackageStatus } from './apm_integration_package_status';
@@ -99,7 +100,7 @@ function PrivilegesCallout({ diagnosticsBundle }: { diagnosticsBundle: Diagnosti
 }
 
 export function getIsCrossCluster(diagnosticsBundle?: DiagnosticsBundle) {
-  return Object.values(diagnosticsBundle?.apmIndices ?? {}).some((indicies) =>
-    indicies.includes(':')
-  );
+  return Object.values(diagnosticsBundle?.apmIndices ?? {}).some((indicies) => {
+    return isCCSRemoteIndexName(indicies);
+  });
 }

x-pack/plugins/observability_solution/apm/server/routes/correlations/queries/fetch_p_values.ts

@@ -5,6 +5,7 @@
  * 2.0.
  */
 
+import { isCCSRemoteIndexName } from '@kbn/es-query';
 import { ERROR_CORRELATION_THRESHOLD } from '../../../../common/correlations/constants';
 import type { FailedTransactionsCorrelation } from '../../../../common/correlations/failed_transactions_correlations/types';
 
@@ -104,7 +105,7 @@ export const fetchPValues = async ({
 
   const index = apmEventClient.indices[eventType as keyof typeof apmEventClient.indices];
 
-  const ccsWarning = rejected.length > 0 && index.includes(':');
+  const ccsWarning = rejected.length > 0 && isCCSRemoteIndexName(index);
 
   return { failedTransactionsCorrelations, ccsWarning, fallbackResult };
 };

x-pack/plugins/observability_solution/apm/server/routes/correlations/queries/fetch_significant_correlations.ts

@@ -8,6 +8,7 @@
 import { range } from 'lodash';
 
 import { termQuery } from '@kbn/observability-plugin/server';
+import { isCCSRemoteIndexName } from '@kbn/es-query';
 import type { LatencyCorrelation } from '../../../../common/correlations/latency_correlations/types';
 import type {
   CommonCorrelationsQueryParams,
@@ -171,7 +172,7 @@ export const fetchSignificantCorrelations = async ({
 
   const index = apmEventClient.indices[eventType as keyof typeof apmEventClient.indices];
 
-  const ccsWarning = rejected.length > 0 && index.includes(':');
+  const ccsWarning = rejected.length > 0 && isCCSRemoteIndexName(index);
 
   return {
     latencyCorrelations,

x-pack/plugins/observability_solution/apm/server/routes/storage_explorer/is_cross_cluster_search.ts

@@ -5,10 +5,12 @@
  * 2.0.
  */
 
+import { isCCSRemoteIndexName } from '@kbn/es-query';
 import { APMEventClient } from '../../lib/helpers/create_es_client/create_apm_event_client';
 import { getApmIndicesCombined } from './indices_stats_helpers';
 
 export function isCrossClusterSearch(apmEventClient: APMEventClient) {
   // Check if a remote cluster is set in APM indices
-  return getApmIndicesCombined(apmEventClient).includes(':');
+  const index = getApmIndicesCombined(apmEventClient);
+  return isCCSRemoteIndexName(index);
 }

x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/utils.ts

@@ -9,6 +9,7 @@ import { chunk, get, invert, isEmpty, partition } from 'lodash';
 import moment from 'moment';
 
 import dateMath from '@kbn/datemath';
+import { isCCSRemoteIndexName } from '@kbn/es-query';
 import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
 import type { TransportResult } from '@elastic/elasticsearch';
 import { ALERT_UUID, ALERT_RULE_UUID, ALERT_RULE_PARAMETERS } from '@kbn/rule-data-utils';
@@ -82,7 +83,9 @@ export const hasReadIndexPrivileges = async (args: {
   const indexNames = Object.keys(privileges.index);
   const filteredIndexNames = isCcsPermissionWarningEnabled
     ? indexNames
-    : indexNames.filter((indexName) => !indexName.includes(':')); // Cross cluster indices uniquely contain `:` in their name
+    : indexNames.filter((indexName) => {
+        return !isCCSRemoteIndexName(indexName);
+      });
 
   const [, indexesWithNoReadPrivileges] = partition(
     filteredIndexNames,

x-pack/plugins/transform/public/app/hooks/use_index_data.ts

@@ -28,6 +28,7 @@ import {
 } from '@kbn/ml-data-grid';
 import type { TimeRange as TimeRangeMs } from '@kbn/ml-date-picker';
 
+import { isCCSRemoteIndexName } from '@kbn/es-query';
 import {
   hasKeywordDuplicate,
   isKeywordDuplicate,
@@ -176,7 +177,7 @@ export const useIndexData = (options: UseIndexDataOptions): UseIndexDataReturnTy
       setErrorMessage(getErrorMessage(dataGridDataError));
       setStatus(INDEX_STATUS.ERROR);
     } else if (!dataGridDataIsLoading && !dataGridDataIsError && dataGridData !== undefined) {
-      const isCrossClusterSearch = indexPattern.includes(':');
+      const isCrossClusterSearch = isCCSRemoteIndexName(indexPattern);
       const isMissingFields = dataGridData.hits.hits.every((d) => typeof d.fields === 'undefined');
 
       const docs = dataGridData.hits.hits.map((d) => getProcessedFields(d.fields ?? {}));