[APM] Migrate traces tests to deployment agnostic (elastic#200561)

## Summary

Closes [elastic#198995](elastic#198995)
Closes elastic#198996
Part of elastic#193245

This PR contains the changes to migrate `traces` test folder to
Deployment-agnostic testing strategy.
It also adds support for deployment-agnostic snapshots validation

### How to test

- Serverless

```
node scripts/functional_tests_server --config x-pack/test/api_integration/deployment_agnostic/configs/serverless/oblt.serverless.config.ts
node scripts/functional_test_runner --config x-pack/test/api_integration/deployment_agnostic/configs/serverless/oblt.serverless.config.ts --grep="APM"
```

It's recommended to be run against
[MKI](https://github.com/crespocarlos/kibana/blob/main/x-pack/test_serverless/README.md#run-tests-on-mki)

- Stateful
```
node scripts/functional_tests_server --config x-pack/test/api_integration/deployment_agnostic/configs/stateful/oblt.stateful.config.ts
node scripts/functional_test_runner --config x-pack/test/api_integration/deployment_agnostic/configs/stateful/oblt.stateful.config.ts --grep="APM"
```

- [ ] ~(OPTIONAL, only if a test has been unskipped) Run flaky test
suite~
- [x] local run for serverless
- [x] local run for stateful
- [x] MKI run for serverless

---------

Co-authored-by: Elastic Machine <[email protected]>
(cherry picked from commit 3917d02)

packages/kbn-test/src/functional_test_runner/lib/mocha/load_tests.ts

@@ -57,7 +57,11 @@ export const loadTests = ({
     updateBaselines,
   };
 
-  decorateSnapshotUi({ lifecycle, updateSnapshots, isCi: !!process.env.CI });
+  decorateSnapshotUi({
+    lifecycle,
+    updateSnapshots,
+    isCi: !!process.env.CI,
+  });
 
   function loadTestFile(path: string) {
     if (typeof path !== 'string' || !isAbsolute(path)) {

packages/kbn-test/src/functional_test_runner/lib/snapshots/decorate_snapshot_ui.ts

@@ -33,11 +33,13 @@ const globalState: {
   registered: boolean;
   currentTest: Test | null;
   snapshotStates: Record<string, ISnapshotState>;
+  deploymentAgnostic: boolean;
 } = {
   updateSnapshot: 'none',
   registered: false,
   currentTest: null,
   snapshotStates: {},
+  deploymentAgnostic: false,
 };
 
 const modifyStackTracePrepareOnce = once(() => {
@@ -125,7 +127,7 @@ export function decorateSnapshotUi({
       const snapshotState = globalState.snapshotStates[file];
 
       if (snapshotState && !test.isPassed()) {
-        snapshotState.markSnapshotsAsCheckedForTest(test.fullTitle());
+        snapshotState.markSnapshotsAsCheckedForTest(getTestTitle(test));
       }
     });
 
@@ -194,7 +196,7 @@ export function expectSnapshot(received: any) {
 
   const context: SnapshotContext = {
     snapshotState,
-    currentTestName: test.fullTitle(),
+    currentTestName: getTestTitle(test),
   };
 
   return {
@@ -204,6 +206,18 @@ export function expectSnapshot(received: any) {
   };
 }
 
+function getTestTitle(test: Test) {
+  return (
+    test
+      .fullTitle()
+      // remove deployment type from test title so that a single snapshot can be used for all deployment types
+      .replace(
+        /^(Serverless|Stateful)\s+([^\-]+)\s*-?\s*Deployment-agnostic/g,
+        'Deployment-agnostic'
+      )
+  );
+}
+
 function expectToMatchSnapshot(snapshotContext: SnapshotContext, received: any) {
   const matcher = toMatchSnapshot.bind(snapshotContext as any);
   const result = matcher(received) as SyncExpectationResult;

x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/index.ts

@@ -40,6 +40,7 @@ export default function apmApiIntegrationTests({
     loadTestFile(require.resolve('./suggestions'));
     loadTestFile(require.resolve('./throughput'));
     loadTestFile(require.resolve('./time_range_metadata'));
+    loadTestFile(require.resolve('./traces'));
     loadTestFile(require.resolve('./transactions'));
   });
 }