Add otel datastream patterns to APM indices (#190533)

## Summary Part of the OTel effort. This PR adds otel datastream patterns into the default indices that are used by the APM UI. ### Checklist Delete any items that are not applicable to this PR. - [x] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios - [x] This renders correctly on smaller devices using a responsive layout. (You can test this [in your browser](https://www.browserstack.com/guide/responsive-testing-on-local-server)) ### For maintainers - [ ] This was checked for breaking API changes and was [labeled appropriately](https://www.elastic.co/guide/en/kibana/master/contributing.html#kibana-release-notes-process) --------- Co-authored-by: kibanamachine <[email protected]> Co-authored-by: Caue Marcondes <[email protected]> Co-authored-by: Elastic Machine <[email protected]>
elastic · Aug 22, 2024 · 5c0991b · 5c0991b
1 parent a04e5f9
commit 5c0991b
Show file tree

Hide file tree

Showing 8 changed files with 40 additions and 21 deletions.
diff --git a/docs/settings/apm-settings.asciidoc b/docs/settings/apm-settings.asciidoc
@@ -82,19 +82,19 @@ Sets a `fixed_interval` for date histograms in metrics aggregations. Defaults to
 Set to `false` to disable cloud APM migrations. Defaults to `true`.
 
 `xpack.apm.indices.error` {ess-icon}::
-Matcher for all error indices. Defaults to `logs-apm*,apm-*`.
+Matcher for all error indices. Defaults to `logs-apm*,apm-*,traces-*.otel-*`.
 
 `xpack.apm.indices.onboarding` {ess-icon}::
 Matcher for all onboarding indices. Defaults to `apm-*`.
 
 `xpack.apm.indices.span` {ess-icon}::
-Matcher for all span indices. Defaults to `traces-apm*,apm-*`.
+Matcher for all span indices. Defaults to `traces-apm*,apm-*,traces-*.otel-*`.
 
 `xpack.apm.indices.transaction` {ess-icon}::
-Matcher for all transaction indices. Defaults to `traces-apm*,apm-*`.
+Matcher for all transaction indices. Defaults to `traces-apm*,apm-*,traces-*.otel-*`.
 
 `xpack.apm.indices.metric` {ess-icon}::
-Matcher for all metrics indices. Defaults to `metrics-apm*,apm-*`.
+Matcher for all metrics indices. Defaults to `metrics-apm*,apm-*,metrics-*.otel-*`.
 
 `xpack.apm.indices.sourcemap` {ess-icon}::
 Matcher for all source map indices. Defaults to `apm-*`.

diff --git a/x-pack/plugins/observability_solution/apm/scripts/shared/read_kibana_config.ts b/x-pack/plugins/observability_solution/apm/scripts/shared/read_kibana_config.ts
@@ -35,10 +35,10 @@ export const readKibanaConfig = () => {
   };
 
   return {
-    'xpack.apm.indices.transaction': 'traces-apm*,apm-*',
-    'xpack.apm.indices.metric': 'metrics-apm*,apm-*',
-    'xpack.apm.indices.error': 'logs-apm*,apm-*',
-    'xpack.apm.indices.span': 'traces-apm*,apm-*',
+    'xpack.apm.indices.transaction': 'traces-apm*,apm-*,traces-*.otel-*',
+    'xpack.apm.indices.metric': 'metrics-apm*,apm-*,metrics-*.otel-*',
+    'xpack.apm.indices.error': 'logs-apm*,apm-*,logs-*.otel-*',
+    'xpack.apm.indices.span': 'traces-apm*,apm-*,traces-*.otel-*',
     'xpack.apm.indices.onboarding': 'apm-*',
     'elasticsearch.hosts': 'http://localhost:9200',
     ...loadedKibanaConfig,

diff --git a/...ervability_solution/apm/server/routes/storage_explorer/has_storage_explorer_privileges.ts b/...ervability_solution/apm/server/routes/storage_explorer/has_storage_explorer_privileges.ts
@@ -18,12 +18,22 @@ export async function hasStorageExplorerPrivileges({
   apmEventClient: APMEventClient;
 }) {
   const {
-    indices: { transaction, span, metric, error },
+    // Only use apm index patterns and ignore OTel, as the storage explorer only supports APM data
+    indices: {
+      transaction = 'traces-apm*,apm-*',
+      span = 'traces-apm*,apm-*',
+      metric = 'metrics-apm*,apm-*',
+      error = 'logs-apm*,apm-*',
+    },
   } = apmEventClient;
 
   const names = uniq(
     [transaction, span, metric, error].flatMap((indexPatternString) =>
-      indexPatternString.split(',').map((indexPattern) => indexPattern.trim())
+      indexPatternString
+        .split(',')
+        .map((indexPattern) => indexPattern.trim())
+        // At this point we do not do any work for storage explorer + OTel data. So remove any otel related index
+        .filter((indexPattern) => !indexPattern.includes('otel'))
     )
   );
 

diff --git a/x-pack/plugins/observability_solution/apm_data_access/server/index.ts b/x-pack/plugins/observability_solution/apm_data_access/server/index.ts
@@ -10,10 +10,10 @@ import { PluginConfigDescriptor, PluginInitializerContext } from '@kbn/core/serv
 
 const configSchema = schema.object({
   indices: schema.object({
-    transaction: schema.string({ defaultValue: 'traces-apm*,apm-*' }), // TODO: remove apm-* pattern in 9.0
-    span: schema.string({ defaultValue: 'traces-apm*,apm-*' }),
-    error: schema.string({ defaultValue: 'logs-apm*,apm-*' }),
-    metric: schema.string({ defaultValue: 'metrics-apm*,apm-*' }),
+    transaction: schema.string({ defaultValue: 'traces-apm*,apm-*,traces-*.otel-*' }), // TODO: remove apm-* pattern in 9.0
+    span: schema.string({ defaultValue: 'traces-apm*,apm-*,traces-*.otel-*' }),
+    error: schema.string({ defaultValue: 'logs-apm*,apm-*,logs-*.otel-*' }),
+    metric: schema.string({ defaultValue: 'metrics-apm*,apm-*,metrics-*.otel-*' }),
     onboarding: schema.string({ defaultValue: 'apm-*' }), // Unused: to be deleted
     sourcemap: schema.string({ defaultValue: 'apm-*' }), // Unused: to be deleted
   }),

diff --git a/x-pack/test/apm_api_integration/tests/data_view/static.spec.ts b/x-pack/test/apm_api_integration/tests/data_view/static.spec.ts
@@ -21,7 +21,8 @@ export default function ApiTest({ getService }: FtrProviderContext) {
   const supertest = getService('supertest');
   const synthtrace = getService('apmSynthtraceEsClient');
   const logger = getService('log');
-  const dataViewPattern = 'traces-apm*,apm-*,logs-apm*,apm-*,metrics-apm*,apm-*';
+  const dataViewPattern =
+    'traces-apm*,apm-*,traces-*.otel-*,logs-apm*,apm-*,logs-*.otel-*,metrics-apm*,apm-*,metrics-*.otel-*';
 
   function createDataViewWithWriteUser({ spaceId }: { spaceId: string }) {
     return apmApiClient.writeUser({
@@ -116,7 +117,9 @@ export default function ApiTest({ getService }: FtrProviderContext) {
 
         expect(dataView.id).to.be('apm_static_data_view_id_default');
         expect(dataView.name).to.be('APM');
-        expect(dataView.title).to.be('traces-apm*,apm-*,logs-apm*,apm-*,metrics-apm*,apm-*');
+        expect(dataView.title).to.be(
+          'traces-apm*,apm-*,traces-*.otel-*,logs-apm*,apm-*,logs-*.otel-*,metrics-apm*,apm-*,metrics-*.otel-*'
+        );
       });
     });
 

diff --git a/x-pack/test/apm_api_integration/tests/diagnostics/privileges.spec.ts b/x-pack/test/apm_api_integration/tests/diagnostics/privileges.spec.ts
@@ -36,6 +36,9 @@ export default function ApiTest({ getService }: FtrProviderContext) {
           'logs-apm*': { read: true },
           'metrics-apm*': { read: true },
           'traces-apm*': { read: true },
+          'logs-*.otel-*': { read: true },
+          'metrics-*.otel-*': { read: true },
+          'traces-*.otel-*': { read: true },
         });
       });
 
@@ -71,6 +74,9 @@ export default function ApiTest({ getService }: FtrProviderContext) {
           'logs-apm*': { read: true },
           'metrics-apm*': { read: true },
           'traces-apm*': { read: true },
+          'logs-*.otel-*': { read: true },
+          'metrics-*.otel-*': { read: true },
+          'traces-*.otel-*': { read: true },
         });
       });
 

diff --git a/x-pack/test/apm_api_integration/tests/inspect/inspect.spec.ts b/x-pack/test/apm_api_integration/tests/inspect/inspect.spec.ts
@@ -93,7 +93,7 @@ export default function inspectFlagTests({ getService }: FtrProviderContext) {
 
           expect(status).to.be(200);
           expect(body._inspect?.map((res) => res.stats?.indexPattern.value)).to.eql([
-            ['metrics-apm*', 'apm-*'],
+            ['metrics-apm*', 'apm-*', 'metrics-*.otel-*'],
           ]);
         });
       });

diff --git a/x-pack/test/apm_api_integration/tests/settings/apm_indices/apm_indices.spec.ts b/x-pack/test/apm_api_integration/tests/settings/apm_indices/apm_indices.spec.ts
@@ -45,10 +45,10 @@ export default function apmIndicesTests({ getService }: FtrProviderContext) {
       });
       expect(response.status).to.be(200);
       expect(response.body).to.eql({
-        transaction: 'traces-apm*,apm-*',
-        span: 'traces-apm*,apm-*',
-        error: 'logs-apm*,apm-*',
-        metric: 'metrics-apm*,apm-*',
+        transaction: 'traces-apm*,apm-*,traces-*.otel-*',
+        span: 'traces-apm*,apm-*,traces-*.otel-*',
+        error: 'logs-apm*,apm-*,logs-*.otel-*',
+        metric: 'metrics-apm*,apm-*,metrics-*.otel-*',
         onboarding: 'apm-*',
         sourcemap: 'apm-*',
       });