From b50eb081b05e3dc2b36f1fad776183ac80e168ff Mon Sep 17 00:00:00 2001
From: Dzmitry Lemechko <dzmitry.lemechko@elastic.co>
Date: Wed, 21 Aug 2024 11:33:25 +0200
Subject: [PATCH] [ftr] re-group configs for deployment agnostic tests
 (#190672)

## Summary

While reviewing #187924 I realised that Solution Teams will have their
own stateful FTR configs and the overall number of configs in
`api_integration/deployment_agnostic` folder might grow up soon and it
is better to re-organise it in advance.

With new structure we have `configs` folder with 2 sub-folders:
`stateful` and `serverless`
<img width="494" alt="image"
src="https://github.com/user-attachments/assets/c844ce2c-51b6-498c-b51f-0cd9203d84ea">

When _Platform_ teams add deployment-agnostic tests, it is expected to
be loaded in `configs/stateful/platform.index.ts` and _at least one_ of
`<project>.serverless.config` under `configs/serverless`

When _Solution_ team adds deployment-agnostic tests (e.g. one of Oblt
teams), it is expected to be loaded _both_ in
`configs/stateful/oblt.index.ts` and `configs/serverless/oblt.index.ts`
---
 .buildkite/ftr_oblt_serverless_configs.yml    |  2 +-
 .buildkite/ftr_oblt_stateful_configs.yml      |  2 ++
 .buildkite/ftr_platform_stateful_configs.yml  |  2 +-
 .buildkite/ftr_search_serverless_configs.yml  |  2 +-
 .../ftr_security_serverless_configs.yml       |  2 +-
 .../deployment_agnostic/README.md             | 34 ++++++++++++-------
 .../{ => configs/serverless}/oblt.index.ts    |  9 ++---
 .../serverless}/oblt.serverless.config.ts     |  2 +-
 .../{ => configs/serverless}/search.index.ts  |  7 ++--
 .../serverless}/search.serverless.config.ts   |  2 +-
 .../configs/serverless/security.index.ts      | 16 +++++++++
 .../serverless}/security.serverless.config.ts |  2 +-
 .../configs/stateful/oblt.index.ts            | 14 ++++++++
 .../configs/stateful/oblt.stateful.config.ts  | 15 ++++++++
 .../stateful/platform.index.ts}               |  9 ++---
 .../stateful/platform.stateful.config.ts}     |  6 ++--
 .../deployment_agnostic/security.index.ts     | 15 --------
 17 files changed, 92 insertions(+), 49 deletions(-)
 rename x-pack/test/api_integration/deployment_agnostic/{ => configs/serverless}/oblt.index.ts (57%)
 rename x-pack/test/api_integration/deployment_agnostic/{ => configs/serverless}/oblt.serverless.config.ts (84%)
 rename x-pack/test/api_integration/deployment_agnostic/{ => configs/serverless}/search.index.ts (62%)
 rename x-pack/test/api_integration/deployment_agnostic/{ => configs/serverless}/search.serverless.config.ts (84%)
 create mode 100644 x-pack/test/api_integration/deployment_agnostic/configs/serverless/security.index.ts
 rename x-pack/test/api_integration/deployment_agnostic/{ => configs/serverless}/security.serverless.config.ts (84%)
 create mode 100644 x-pack/test/api_integration/deployment_agnostic/configs/stateful/oblt.index.ts
 create mode 100644 x-pack/test/api_integration/deployment_agnostic/configs/stateful/oblt.stateful.config.ts
 rename x-pack/test/api_integration/deployment_agnostic/{stateful.index.ts => configs/stateful/platform.index.ts} (54%)
 rename x-pack/test/api_integration/deployment_agnostic/{stateful.config.ts => configs/stateful/platform.stateful.config.ts} (58%)
 delete mode 100644 x-pack/test/api_integration/deployment_agnostic/security.index.ts

diff --git a/.buildkite/ftr_oblt_serverless_configs.yml b/.buildkite/ftr_oblt_serverless_configs.yml
index 8fe505ff0e93e..e49655999ec59 100644
--- a/.buildkite/ftr_oblt_serverless_configs.yml
+++ b/.buildkite/ftr_oblt_serverless_configs.yml
@@ -27,4 +27,4 @@ enabled:
   - x-pack/test_serverless/functional/test_suites/observability/common_configs/config.group6.ts
   - x-pack/test_serverless/functional/test_suites/observability/config.screenshots.ts
   # serverless config files that run deployment-agnostic tests
-  - x-pack/test/api_integration/deployment_agnostic/oblt.serverless.config.ts
+  - x-pack/test/api_integration/deployment_agnostic/configs/serverless/oblt.serverless.config.ts
diff --git a/.buildkite/ftr_oblt_stateful_configs.yml b/.buildkite/ftr_oblt_stateful_configs.yml
index eb8382eacce89..06c99e53e767a 100644
--- a/.buildkite/ftr_oblt_stateful_configs.yml
+++ b/.buildkite/ftr_oblt_stateful_configs.yml
@@ -50,3 +50,5 @@ enabled:
   - x-pack/test/observability_ai_assistant_functional/enterprise/config.ts
   - x-pack/test/profiling_api_integration/cloud/config.ts
   - x-pack/test/functional/apps/apm/config.ts
+  # stateful config files that run deployment-agnostic tests
+  - x-pack/test/api_integration/deployment_agnostic/configs/stateful/oblt.stateful.config.ts
diff --git a/.buildkite/ftr_platform_stateful_configs.yml b/.buildkite/ftr_platform_stateful_configs.yml
index a984afc263170..dd33535c2562a 100644
--- a/.buildkite/ftr_platform_stateful_configs.yml
+++ b/.buildkite/ftr_platform_stateful_configs.yml
@@ -361,4 +361,4 @@ enabled:
   - x-pack/performance/journeys_e2e/infra_hosts_view.ts
   - x-pack/test/custom_branding/config.ts
   # stateful config files that run deployment-agnostic tests
-  - x-pack/test/api_integration/deployment_agnostic/stateful.config.ts
+  - x-pack/test/api_integration/deployment_agnostic/configs/stateful/platform.stateful.config.ts
diff --git a/.buildkite/ftr_search_serverless_configs.yml b/.buildkite/ftr_search_serverless_configs.yml
index 9a5ce6798dbae..7112888f1afce 100644
--- a/.buildkite/ftr_search_serverless_configs.yml
+++ b/.buildkite/ftr_search_serverless_configs.yml
@@ -17,4 +17,4 @@ enabled:
   - x-pack/test_serverless/functional/test_suites/search/common_configs/config.group5.ts
   - x-pack/test_serverless/functional/test_suites/search/common_configs/config.group6.ts
   # serverless config files that run deployment-agnostic tests
-  - x-pack/test/api_integration/deployment_agnostic/search.serverless.config.ts
+  - x-pack/test/api_integration/deployment_agnostic/configs/serverless/search.serverless.config.ts
diff --git a/.buildkite/ftr_security_serverless_configs.yml b/.buildkite/ftr_security_serverless_configs.yml
index 9f54a402a8bbf..d6a92fbb5f446 100644
--- a/.buildkite/ftr_security_serverless_configs.yml
+++ b/.buildkite/ftr_security_serverless_configs.yml
@@ -99,4 +99,4 @@ enabled:
   - x-pack/test/security_solution_endpoint/configs/serverless.endpoint.config.ts
   - x-pack/test/security_solution_endpoint/configs/serverless.integrations.config.ts
   # serverless config files that run deployment-agnostic tests
-  - x-pack/test/api_integration/deployment_agnostic/security.serverless.config.ts
+  - x-pack/test/api_integration/deployment_agnostic/configs/serverless/security.serverless.config.ts
diff --git a/x-pack/test/api_integration/deployment_agnostic/README.md b/x-pack/test/api_integration/deployment_agnostic/README.md
index da0a9d3f3e18b..8cb1c4f7c3344 100644
--- a/x-pack/test/api_integration/deployment_agnostic/README.md
+++ b/x-pack/test/api_integration/deployment_agnostic/README.md
@@ -44,17 +44,25 @@ x-pack/test/<my_own_api_integration_folder>
 │  │  ├─ <api_2>
 │  │  │  ├─ <test_2_1>
 │  │  │  ├─ <test_2_2>
-│  ├─ services
-│  │  ├─ index.ts // only services from 'x-pack/test/api_integration/deployment_agnostic/services'
-│  │  ├─ <deployment_agnostic_service_1>.ts
-│  │  ├─ <deployment_agnostic_service_2>.ts
+│  ├─ configs
+│  │  ├─ stateful
+│  │  │  ├─ <stateful>.index.ts  // e.g., oblt.index.ts
+│  │  │  ├─ <stateful>.config.ts // e.g., oblt.stateful.config.ts
+│  │  ├─ serverless
+│  │     ├─ <serverless_project>.index.ts             // e.g., oblt.index.ts
+│  │     ├─ <serverless_project>.serverless.config.ts // e.g., oblt.serverless.config.ts
 │  ├─ ftr_provider_context.d.ts  // with types of services from './services'
-├─ stateful.index.ts
-├─ stateful.config.ts 
-├─ <serverless_project>.index.ts                // e.g., oblt.index.ts
-├─ <serverless_project>.serverless.config.ts    // e.g., oblt.serverless.config.ts
+│  ├─ services
+│     ├─ index.ts // only services from 'x-pack/test/api_integration/deployment_agnostic/services'
+│     ├─ <deployment_agnostic_service_1>.ts
+│     ├─ <deployment_agnostic_service_2>.ts
 ```
 
+## Loading Your Tests Properly
+When Platform teams add deployment-agnostic tests, it is expected that these tests are loaded in `configs/stateful/platform.index.ts` and at least one of the `<serverless_project>.serverless.config` files under `configs/serverless` folder.
+
+When a Solution team (e.g., one of the Oblt teams) adds deployment-agnostic tests, it is expected that these tests are loaded in both `configs/stateful/oblt.index.ts` and `configs/serverless/oblt.index.ts`.
+
 ## Step-by-Step Guide
 1. Define Deployment-Agnostic Services
 
@@ -121,26 +129,26 @@ Load all test files in `index.ts` under the same folder.
 
 4. Add Tests Entry File and FTR Config File for **Stateful** Deployment
 
-Create `stateful.index.ts` tests entry file and load tests:
+Create `configs/stateful/plaform.index.ts` tests entry file and load tests:
 
 ```ts
 import { DeploymentAgnosticFtrProviderContext } from './ftr_provider_context';
 
 export default function ({ loadTestFile }: DeploymentAgnosticFtrProviderContext) {
   describe('apis', () => {
-    loadTestFile(require.resolve('./apis/<my_api>'));
+    loadTestFile(require.resolve('./../../apis/<my_api>'));
   });
 }
 ```
 
-Create `stateful.config.ts` and link tests entry file:
+Create `configs/stateful/platform.stateful.config.ts` and link tests entry file:
 
 ```ts
 import { createStatefulTestConfig } from './../../api_integration/deployment_agnostic/default_configs/stateful.config.base';
 import { services } from './services';
 
 export default createStatefulTestConfig({
-  testFiles: [require.resolve('./stateful.index.ts')],
+  testFiles: [require.resolve('./platform.index.ts')],
   services,
   junit: {
     reportName: 'Stateful - Deployment-agnostic API Integration Tests',
@@ -157,7 +165,7 @@ import { DeploymentAgnosticFtrProviderContext } from './ftr_provider_context';
 
 export default function ({ loadTestFile }: DeploymentAgnosticFtrProviderContext) {
   describe('Serverless Observability - Deployment-agnostic api integration tests', () => {
-    loadTestFile(require.resolve('./apis/<my_api>'));
+    loadTestFile(require.resolve('./../../apis/<my_api>'));
   });
 }
 ```
diff --git a/x-pack/test/api_integration/deployment_agnostic/oblt.index.ts b/x-pack/test/api_integration/deployment_agnostic/configs/serverless/oblt.index.ts
similarity index 57%
rename from x-pack/test/api_integration/deployment_agnostic/oblt.index.ts
rename to x-pack/test/api_integration/deployment_agnostic/configs/serverless/oblt.index.ts
index d81415e0554dd..3acc5cba6067e 100644
--- a/x-pack/test/api_integration/deployment_agnostic/oblt.index.ts
+++ b/x-pack/test/api_integration/deployment_agnostic/configs/serverless/oblt.index.ts
@@ -4,12 +4,13 @@
  * 2.0; you may not use this file except in compliance with the Elastic License
  * 2.0.
  */
-import { DeploymentAgnosticFtrProviderContext } from './ftr_provider_context';
+import { DeploymentAgnosticFtrProviderContext } from '../../ftr_provider_context';
 
 export default function ({ loadTestFile }: DeploymentAgnosticFtrProviderContext) {
   describe('Serverless Observability - Deployment-agnostic api integration tests', () => {
-    loadTestFile(require.resolve('./apis/console'));
-    loadTestFile(require.resolve('./apis/core'));
-    loadTestFile(require.resolve('./apis/painless_lab'));
+    // load new oblt and platform deployment-agnostic test here
+    loadTestFile(require.resolve('../../apis/console'));
+    loadTestFile(require.resolve('../../apis/core'));
+    loadTestFile(require.resolve('../../apis/painless_lab'));
   });
 }
diff --git a/x-pack/test/api_integration/deployment_agnostic/oblt.serverless.config.ts b/x-pack/test/api_integration/deployment_agnostic/configs/serverless/oblt.serverless.config.ts
similarity index 84%
rename from x-pack/test/api_integration/deployment_agnostic/oblt.serverless.config.ts
rename to x-pack/test/api_integration/deployment_agnostic/configs/serverless/oblt.serverless.config.ts
index 52e1ba2d431ac..245663416243f 100644
--- a/x-pack/test/api_integration/deployment_agnostic/oblt.serverless.config.ts
+++ b/x-pack/test/api_integration/deployment_agnostic/configs/serverless/oblt.serverless.config.ts
@@ -5,7 +5,7 @@
  * 2.0.
  */
 
-import { createServerlessTestConfig } from './default_configs/serverless.config.base';
+import { createServerlessTestConfig } from '../../default_configs/serverless.config.base';
 
 export default createServerlessTestConfig({
   serverlessProject: 'oblt',
diff --git a/x-pack/test/api_integration/deployment_agnostic/search.index.ts b/x-pack/test/api_integration/deployment_agnostic/configs/serverless/search.index.ts
similarity index 62%
rename from x-pack/test/api_integration/deployment_agnostic/search.index.ts
rename to x-pack/test/api_integration/deployment_agnostic/configs/serverless/search.index.ts
index 740520f032f0f..20acbd0d3b610 100644
--- a/x-pack/test/api_integration/deployment_agnostic/search.index.ts
+++ b/x-pack/test/api_integration/deployment_agnostic/configs/serverless/search.index.ts
@@ -4,11 +4,12 @@
  * 2.0; you may not use this file except in compliance with the Elastic License
  * 2.0.
  */
-import { DeploymentAgnosticFtrProviderContext } from './ftr_provider_context';
+import { DeploymentAgnosticFtrProviderContext } from '../../ftr_provider_context';
 
 export default function ({ loadTestFile }: DeploymentAgnosticFtrProviderContext) {
   describe('Serverless Search - Deployment-agnostic api integration tests', () => {
-    loadTestFile(require.resolve('./apis/console'));
-    loadTestFile(require.resolve('./apis/core'));
+    // load new search and platform deployment-agnostic test here
+    loadTestFile(require.resolve('../../apis/console'));
+    loadTestFile(require.resolve('../../apis/core'));
   });
 }
diff --git a/x-pack/test/api_integration/deployment_agnostic/search.serverless.config.ts b/x-pack/test/api_integration/deployment_agnostic/configs/serverless/search.serverless.config.ts
similarity index 84%
rename from x-pack/test/api_integration/deployment_agnostic/search.serverless.config.ts
rename to x-pack/test/api_integration/deployment_agnostic/configs/serverless/search.serverless.config.ts
index 5e90b71d69550..2e6043164af38 100644
--- a/x-pack/test/api_integration/deployment_agnostic/search.serverless.config.ts
+++ b/x-pack/test/api_integration/deployment_agnostic/configs/serverless/search.serverless.config.ts
@@ -5,7 +5,7 @@
  * 2.0.
  */
 
-import { createServerlessTestConfig } from './default_configs/serverless.config.base';
+import { createServerlessTestConfig } from '../../default_configs/serverless.config.base';
 
 export default createServerlessTestConfig({
   serverlessProject: 'es',
diff --git a/x-pack/test/api_integration/deployment_agnostic/configs/serverless/security.index.ts b/x-pack/test/api_integration/deployment_agnostic/configs/serverless/security.index.ts
new file mode 100644
index 0000000000000..c091dfb0999b4
--- /dev/null
+++ b/x-pack/test/api_integration/deployment_agnostic/configs/serverless/security.index.ts
@@ -0,0 +1,16 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+import { DeploymentAgnosticFtrProviderContext } from '../../ftr_provider_context';
+
+export default function ({ loadTestFile }: DeploymentAgnosticFtrProviderContext) {
+  describe('Serverless Security - Deployment-agnostic api integration tests', () => {
+    // load new security and platform deployment-agnostic test here
+    loadTestFile(require.resolve('../../apis/console'));
+    loadTestFile(require.resolve('../../apis/core'));
+    loadTestFile(require.resolve('../../apis/painless_lab'));
+  });
+}
diff --git a/x-pack/test/api_integration/deployment_agnostic/security.serverless.config.ts b/x-pack/test/api_integration/deployment_agnostic/configs/serverless/security.serverless.config.ts
similarity index 84%
rename from x-pack/test/api_integration/deployment_agnostic/security.serverless.config.ts
rename to x-pack/test/api_integration/deployment_agnostic/configs/serverless/security.serverless.config.ts
index d3a30cc95d820..e8e52116f9eff 100644
--- a/x-pack/test/api_integration/deployment_agnostic/security.serverless.config.ts
+++ b/x-pack/test/api_integration/deployment_agnostic/configs/serverless/security.serverless.config.ts
@@ -5,7 +5,7 @@
  * 2.0.
  */
 
-import { createServerlessTestConfig } from './default_configs/serverless.config.base';
+import { createServerlessTestConfig } from '../../default_configs/serverless.config.base';
 
 export default createServerlessTestConfig({
   serverlessProject: 'security',
diff --git a/x-pack/test/api_integration/deployment_agnostic/configs/stateful/oblt.index.ts b/x-pack/test/api_integration/deployment_agnostic/configs/stateful/oblt.index.ts
new file mode 100644
index 0000000000000..d003c3c625b33
--- /dev/null
+++ b/x-pack/test/api_integration/deployment_agnostic/configs/stateful/oblt.index.ts
@@ -0,0 +1,14 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { DeploymentAgnosticFtrProviderContext } from '../../ftr_provider_context';
+
+export default function ({}: DeploymentAgnosticFtrProviderContext) {
+  describe('apis', () => {
+    // load new oblt deployment-agnostic test here
+  });
+}
diff --git a/x-pack/test/api_integration/deployment_agnostic/configs/stateful/oblt.stateful.config.ts b/x-pack/test/api_integration/deployment_agnostic/configs/stateful/oblt.stateful.config.ts
new file mode 100644
index 0000000000000..fdc0926fe9162
--- /dev/null
+++ b/x-pack/test/api_integration/deployment_agnostic/configs/stateful/oblt.stateful.config.ts
@@ -0,0 +1,15 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { createStatefulTestConfig } from '../../default_configs/stateful.config.base';
+
+export default createStatefulTestConfig({
+  testFiles: [require.resolve('./oblt.index.ts')],
+  junit: {
+    reportName: 'Observibility Stateful - Deployment-agnostic API Integration Tests',
+  },
+});
diff --git a/x-pack/test/api_integration/deployment_agnostic/stateful.index.ts b/x-pack/test/api_integration/deployment_agnostic/configs/stateful/platform.index.ts
similarity index 54%
rename from x-pack/test/api_integration/deployment_agnostic/stateful.index.ts
rename to x-pack/test/api_integration/deployment_agnostic/configs/stateful/platform.index.ts
index 3e46c4d5b0472..838c33af03054 100644
--- a/x-pack/test/api_integration/deployment_agnostic/stateful.index.ts
+++ b/x-pack/test/api_integration/deployment_agnostic/configs/stateful/platform.index.ts
@@ -5,12 +5,13 @@
  * 2.0.
  */
 
-import { DeploymentAgnosticFtrProviderContext } from './ftr_provider_context';
+import { DeploymentAgnosticFtrProviderContext } from '../../ftr_provider_context';
 
 export default function ({ loadTestFile }: DeploymentAgnosticFtrProviderContext) {
   describe('apis', () => {
-    loadTestFile(require.resolve('./apis/console'));
-    loadTestFile(require.resolve('./apis/core'));
-    loadTestFile(require.resolve('./apis/painless_lab'));
+    // load new platform deployment-agnostic test here
+    loadTestFile(require.resolve('../../apis/console'));
+    loadTestFile(require.resolve('../../apis/core'));
+    loadTestFile(require.resolve('../../apis/painless_lab'));
   });
 }
diff --git a/x-pack/test/api_integration/deployment_agnostic/stateful.config.ts b/x-pack/test/api_integration/deployment_agnostic/configs/stateful/platform.stateful.config.ts
similarity index 58%
rename from x-pack/test/api_integration/deployment_agnostic/stateful.config.ts
rename to x-pack/test/api_integration/deployment_agnostic/configs/stateful/platform.stateful.config.ts
index 37bc527d7fbc9..a71dfb607572f 100644
--- a/x-pack/test/api_integration/deployment_agnostic/stateful.config.ts
+++ b/x-pack/test/api_integration/deployment_agnostic/configs/stateful/platform.stateful.config.ts
@@ -5,11 +5,11 @@
  * 2.0.
  */
 
-import { createStatefulTestConfig } from './default_configs/stateful.config.base';
+import { createStatefulTestConfig } from '../../default_configs/stateful.config.base';
 
 export default createStatefulTestConfig({
-  testFiles: [require.resolve('./stateful.index.ts')],
+  testFiles: [require.resolve('./platform.index.ts')],
   junit: {
-    reportName: 'Stateful - Deployment-agnostic API Integration Tests',
+    reportName: 'Platform Stateful - Deployment-agnostic API Integration Tests',
   },
 });
diff --git a/x-pack/test/api_integration/deployment_agnostic/security.index.ts b/x-pack/test/api_integration/deployment_agnostic/security.index.ts
deleted file mode 100644
index 9a9db972bcd5d..0000000000000
--- a/x-pack/test/api_integration/deployment_agnostic/security.index.ts
+++ /dev/null
@@ -1,15 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-import { DeploymentAgnosticFtrProviderContext } from './ftr_provider_context';
-
-export default function ({ loadTestFile }: DeploymentAgnosticFtrProviderContext) {
-  describe('Security Search - Deployment-agnostic api integration tests', () => {
-    loadTestFile(require.resolve('./apis/console'));
-    loadTestFile(require.resolve('./apis/core'));
-    loadTestFile(require.resolve('./apis/painless_lab'));
-  });
-}