[FTR] allow to call roleScopedSupertest service with Cookie header (e…

…lastic#192727)

## Summary

During the sync with kibana-security team we agreed on how Kibana APIs
should be tested:

### API Authentication in Kibana: Public vs. Internal APIs

Kibana provides both public and internal APIs, each requiring
authentication with the correct privileges. However, the method of
testing these APIs varies, depending on how they are untilized by end
users.

- **Public APIs**: When testing HTTP requests to public APIs, **API
key-based authentication** should be used. It reflect how end user call
these APIs. Due to existing restrictions, we utilize `Admin` user
credentials to generate API keys for various roles. While the API key
permissions are correctly scoped according to the assigned role, the
user will internally be recognized as `Admin` during authentication.

- **Internal APIs**: Direct HTTP requests to internal APIs are generally
not expected. However, for testing purposes, authentication should be
performed **using the Cookie header**. This approach simulates
client-side behavior during browser interactions, mirroring how internal
APIs are indirectly invoked.

To simplify the process of creating/updating the tests, this PR makes
few changes to `roleScopedSupertest` service

1) testing public APIs (nothing changed)

```ts
      const supertestAdminWithApiKey = await roleScopedSupertest.getSupertestWithRoleScope('admin', {
        withCustomHeaders: { 'accept-encoding': 'gzip' },
      });
      const response = await supertestAdminWithApiKey.get('/app/kibana');
```

2) testing internal APIs
```ts
      const  supertestAdminWithCookieCredentials = await roleScopedSupertest.getSupertestWithRoleScope(
        'admin',
        {
          useCookieHeader: true, // will use Cookie header instead of API key
          withInternalHeaders: true,
        }
      );
      await supertestAdminWithCookieCredentials
        .post(`/internal/kibana/settings/${TEST_SETTING}`)
        .send({ value: 100 })
        .expect(200);
```

I updated some of the existing tests according to the new approach.

Docs for serverless and deployment-agnostic api integration tests were
updated accordingly

(cherry picked from commit d9148f1)

packages/kbn-ftr-common-functional-services/index.ts

@@ -26,7 +26,11 @@ export type Es = ProvidedType<typeof EsProvider>;
 import { SupertestWithoutAuthProvider } from './services/supertest_without_auth';
 export type SupertestWithoutAuthProviderType = ProvidedType<typeof SupertestWithoutAuthProvider>;
 
-export type { InternalRequestHeader, RoleCredentials } from './services/saml_auth';
+export type {
+  InternalRequestHeader,
+  RoleCredentials,
+  CookieCredentials,
+} from './services/saml_auth';
 
 import { SamlAuthProvider } from './services/saml_auth/saml_auth_provider';
 export type SamlAuthProviderType = ProvidedType<typeof SamlAuthProvider>;

packages/kbn-ftr-common-functional-services/services/saml_auth/index.ts

@@ -8,5 +8,5 @@
  */
 
 export { SamlAuthProvider } from './saml_auth_provider';
-export type { RoleCredentials } from './saml_auth_provider';
+export type { RoleCredentials, CookieCredentials } from './saml_auth_provider';
 export type { InternalRequestHeader } from './default_request_headers';

packages/kbn-ftr-common-functional-services/services/saml_auth/saml_auth_provider.ts

@@ -18,7 +18,13 @@ import { InternalRequestHeader } from './default_request_headers';
 export interface RoleCredentials {
   apiKey: { id: string; name: string };
   apiKeyHeader: { Authorization: string };
-  cookieHeader: { Cookie: string };
+}
+
+export interface CookieCredentials {
+  Cookie: string;
+  // supertest.set() expects an object that matches IncomingHttpHeaders type, that needs to accept arbitrary key-value pairs as headers
+  // We extend the interface with an index signature to resolve this.
+  [header: string]: string;
 }
 
 export function SamlAuthProvider({ getService }: FtrProviderContext) {
@@ -60,7 +66,7 @@ export function SamlAuthProvider({ getService }: FtrProviderContext) {
     async getInteractiveUserSessionCookieWithRoleScope(role: string) {
       return sessionManager.getInteractiveUserSessionCookieWithRoleScope(role);
     },
-    async getM2MApiCredentialsWithRoleScope(role: string) {
+    async getM2MApiCookieCredentialsWithRoleScope(role: string): Promise<CookieCredentials> {
       return sessionManager.getApiCredentialsForRole(role);
     },
     async getEmail(role: string) {
@@ -76,7 +82,7 @@ export function SamlAuthProvider({ getService }: FtrProviderContext) {
     },
     async createM2mApiKeyWithRoleScope(role: string): Promise<RoleCredentials> {
       // Get admin credentials in order to create the API key
-      const adminCookieHeader = await this.getM2MApiCredentialsWithRoleScope('admin');
+      const adminCookieHeader = await this.getM2MApiCookieCredentialsWithRoleScope('admin');
 
       // Get the role descrtiptor for the role
       let roleDescriptors = {};
@@ -110,9 +116,12 @@ export function SamlAuthProvider({ getService }: FtrProviderContext) {
       const apiKeyHeader = { Authorization: 'ApiKey ' + apiKey.encoded };
 
       log.debug(`Created api key for role: [${role}]`);
-      return { apiKey, apiKeyHeader, cookieHeader: adminCookieHeader };
+      return { apiKey, apiKeyHeader };
     },
     async invalidateM2mApiKeyWithRoleScope(roleCredentials: RoleCredentials) {
+      // Get admin credentials in order to invalidate the API key
+      const adminCookieHeader = await this.getM2MApiCookieCredentialsWithRoleScope('admin');
+
       const requestBody = {
         apiKeys: [
           {
@@ -126,7 +135,7 @@ export function SamlAuthProvider({ getService }: FtrProviderContext) {
       const { status } = await supertestWithoutAuth
         .post('/internal/security/api_key/invalidate')
         .set(INTERNAL_REQUEST_HEADERS)
-        .set(roleCredentials.cookieHeader)
+        .set(adminCookieHeader)
         .send(requestBody);
 
       expect(status).to.be(200);

x-pack/test/api_integration/apis/cloud_security_posture/helper.ts

@@ -12,12 +12,7 @@ import { ELASTIC_HTTP_VERSION_HEADER } from '@kbn/core-http-common';
 import type { IndexDetails } from '@kbn/cloud-security-posture-common';
 import { CLOUD_SECURITY_PLUGIN_VERSION } from '@kbn/cloud-security-posture-plugin/common/constants';
 import { SecurityService } from '@kbn/ftr-common-functional-ui-services';
-
-export interface RoleCredentials {
-  apiKey: { id: string; name: string };
-  apiKeyHeader: { Authorization: string };
-  cookieHeader: { Cookie: string };
-}
+import { RoleCredentials } from '@kbn/ftr-common-functional-services';
 
 export const deleteIndex = async (es: Client, indexToBeDeleted: string[]) => {
   return Promise.all([

x-pack/test/api_integration/deployment_agnostic/README.md

@@ -97,6 +97,20 @@ export type DeploymentAgnosticFtrProviderContext = GenericFtrProviderContext<typ
 
 3. Add Tests
 
+API Authentication in Kibana: Public vs. Internal APIs
+
+Kibana provides both public and internal APIs, each requiring authentication with the correct privileges. However, the method of testing these APIs varies, depending on how they are untilized by end users.
+
+- Public APIs: When testing HTTP requests to public APIs, API key-based authentication should be used. It reflect how end user call these APIs. Due to existing restrictions, we utilize `Admin` user credentials to generate API keys for various roles. While the API key permissions are correctly scoped according to the assigned role, the user will internally be recognized as `Admin` during authentication.
+
+- Internal APIs: Direct HTTP requests to internal APIs are generally not expected. However, for testing purposes, authentication should be performed using the Cookie header. This approach simulates client-side behavior during browser interactions, mirroring how internal APIs are indirectly invoked.
+
+Recommendations:
+- use `roleScopedSupertest` service to create supertest instance scoped to specific role and pre-defined request headers
+- `roleScopedSupertest.getSupertestWithRoleScope(<role>)` authenticate requests with API key by default
+- pass `withCookieHeader: true` to use Cookie header for requests authentication
+- don't forget to invalidate API key using `destroy()` on supertest scoped instance in `after` hook
+
 Add test files to `x-pack/test/<my_own_api_integration_folder>/deployment_agnostic/apis/<my_api>`:
 
 test example

x-pack/test/api_integration/deployment_agnostic/services/role_scoped_supertest.ts

@@ -7,51 +7,66 @@
 
 import {
   RoleCredentials,
+  CookieCredentials,
   SupertestWithoutAuthProviderType,
   SamlAuthProviderType,
 } from '@kbn/ftr-common-functional-services';
 import { Test } from 'supertest';
 import { DeploymentAgnosticFtrProviderContext } from '../ftr_provider_context';
 
 export interface RequestHeadersOptions {
+  useCookieHeader?: boolean;
   withInternalHeaders?: boolean;
   withCommonHeaders?: boolean;
   withCustomHeaders?: Record<string, string>;
 }
 
 export class SupertestWithRoleScope {
-  private roleAuthc: RoleCredentials | null;
+  private authValue: RoleCredentials | CookieCredentials | null;
   private readonly supertestWithoutAuth: SupertestWithoutAuthProviderType;
   private samlAuth: SamlAuthProviderType;
   private readonly options: RequestHeadersOptions;
 
   constructor(
-    roleAuthc: RoleCredentials,
+    authValue: RoleCredentials | CookieCredentials | null,
     supertestWithoutAuth: SupertestWithoutAuthProviderType,
     samlAuth: SamlAuthProviderType,
     options: RequestHeadersOptions
   ) {
-    this.roleAuthc = roleAuthc;
+    this.authValue = authValue;
     this.supertestWithoutAuth = supertestWithoutAuth;
     this.samlAuth = samlAuth;
     this.options = options;
   }
 
+  private isRoleCredentials(value: any): value is RoleCredentials {
+    return value && typeof value === 'object' && 'apiKey' in value && 'apiKeyHeader' in value;
+  }
+
   async destroy() {
-    if (this.roleAuthc) {
-      await this.samlAuth.invalidateM2mApiKeyWithRoleScope(this.roleAuthc);
-      this.roleAuthc = null;
+    if (this.isRoleCredentials(this.authValue)) {
+      await this.samlAuth.invalidateM2mApiKeyWithRoleScope(this.authValue);
+      this.authValue = null;
     }
   }
 
   private addHeaders(agent: Test): Test {
-    const { withInternalHeaders, withCommonHeaders, withCustomHeaders } = this.options;
+    const { useCookieHeader, withInternalHeaders, withCommonHeaders, withCustomHeaders } =
+      this.options;
 
-    if (!this.roleAuthc) {
-      throw new Error('The instance has already been destroyed.');
+    if (useCookieHeader) {
+      if (!this.authValue || !('Cookie' in this.authValue)) {
+        throw new Error('The instance has already been destroyed or cookieHeader is missing.');
+      }
+      // set cookie header
+      void agent.set(this.authValue);
+    } else {
+      if (!this.authValue || !this.isRoleCredentials(this.authValue)) {
+        throw new Error('The instance has already been destroyed or roleAuthc is missing.');
+      }
+      // set API key header
+      void agent.set(this.authValue.apiKeyHeader);
     }
-    // set role-based API key by default
-    void agent.set(this.roleAuthc.apiKeyHeader);
 
     if (withInternalHeaders) {
       void agent.set(this.samlAuth.getInternalRequestHeader());
@@ -69,7 +84,7 @@ export class SupertestWithRoleScope {
   }
 
   private request(method: 'post' | 'get' | 'put' | 'delete', url: string): Test {
-    if (!this.roleAuthc) {
+    if (!this.authValue) {
       throw new Error('Instance has been destroyed and cannot be used for making requests.');
     }
     const agent = this.supertestWithoutAuth[method](url);
@@ -101,6 +116,9 @@ export class SupertestWithRoleScope {
  *
  * Use this service to easily test API endpoints with role-specific authorization and
  * custom headers, both in serverless and stateful environments.
+ *
+ * Pass '{ useCookieHeader: true }' to use Cookie header for authentication instead of API key.
+ * It is the correct way to perform HTTP requests for internal end-points.
  */
 export function RoleScopedSupertestProvider({ getService }: DeploymentAgnosticFtrProviderContext) {
   const supertestWithoutAuth = getService('supertestWithoutAuth');
@@ -110,10 +128,18 @@ export function RoleScopedSupertestProvider({ getService }: DeploymentAgnosticFt
     async getSupertestWithRoleScope(
       role: string,
       options: RequestHeadersOptions = {
+        useCookieHeader: false,
         withCommonHeaders: false,
         withInternalHeaders: false,
       }
     ) {
+      // if 'useCookieHeader' set to 'true', HTTP requests will be called with cookie Header (like in browser)
+      if (options.useCookieHeader) {
+        const cookieHeader = await samlAuth.getM2MApiCookieCredentialsWithRoleScope(role);
+        return new SupertestWithRoleScope(cookieHeader, supertestWithoutAuth, samlAuth, options);
+      }
+
+      // HTTP requests will be called with API key in header by default
       const roleAuthc = await samlAuth.createM2mApiKeyWithRoleScope(role);
       return new SupertestWithRoleScope(roleAuthc, supertestWithoutAuth, samlAuth, options);
     },

x-pack/test_serverless/README.md

@@ -143,28 +143,60 @@ describe("my test suite", async function() {
 
 #### API integration test example
 
+API Authentication in Kibana: Public vs. Internal APIs
+
+Kibana provides both public and internal APIs, each requiring authentication with the correct privileges. However, the method of testing these APIs varies, depending on how they are untilized by end users.
+
+- Public APIs: When testing HTTP requests to public APIs, API key-based authentication should be used. It reflects how an end user calls these APIs. Due to existing restrictions, we utilize `Admin` user credentials to generate API keys for various roles. While the API key permissions are correctly scoped according to the assigned role, the user will internally be recognized as `Admin` during authentication.
+
+- Internal APIs: Direct HTTP requests to internal APIs are generally not expected. However, for testing purposes, authentication should be performed using the Cookie header. This approach simulates client-side behavior during browser interactions, mirroring how internal APIs are indirectly invoked.
+
 Recommendations:
-- in each test file top level `describe` suite should start with `createM2mApiKeyWithRoleScope` call in `before` hook
-- don't forget to invalidate api key using `invalidateApiKeyWithRoleScope` in `after` hook
-- make api calls using `supertestWithoutAuth` with generated api key header
+- use `roleScopedSupertest` service to create a supertest instance scoped to a specific role and predefined request headers
+- `roleScopedSupertest.getSupertestWithRoleScope(<role>)` authenticates requests with an API key by default
+- pass `withCookieHeader: true` to use Cookie header for request authentication
+- don't forget to invalidate API keys by using `destroy()` on the supertest scoped instance in the `after` hook
 
 ```
-describe("my test suite", async function() {
+describe("my public APIs test suite", async function() {
     before(async () => {
-      roleAuthc = await svlUserManager.createM2mApiKeyWithRoleScope('viewer');
-      commonRequestHeader = svlCommonApi.getCommonRequestHeader();
-      internalRequestHeader = svlCommonApi.getInternalRequestHeader();
+      supertestViewerWithApiKey =
+        await roleScopedSupertest.getSupertestWithRoleScope('viewer', {
+          withInternalHeaders: true,
+        });
     });
 
     after(async () => {
-      await svlUserManager.invalidateApiKeyWithRoleScope(roleAuthc);
+      await supertestViewerWithApiKey.destroy();
     });
 
     it(''test step', async () => {
-      const { body, status } = await supertestWithoutAuth
+      const { body, status } = await supertestViewerWithApiKey
         .delete('/api/spaces/space/default')
-        .set(commonRequestHeader)
-        .set(roleAuthc.apiKeyHeader);
+      ...
+    });
+});
+```
+
+```
+describe("my internal APIs test suite", async function() {
+    before(async () => {
+      supertestViewerWithCookieCredentials =
+        await roleScopedSupertest.getSupertestWithRoleScope('admin', {
+          withCookieHeader: true, // to avoid generating API key and use Cookie header instead
+          withInternalHeaders: true,
+        });
+    });
+
+    after(async () => {
+      // no need to call '.destroy' since we didn't create API key and Cookie persist for the role within FTR run
+    });
+
+    it(''test step', async () => {
+      await supertestAdminWithCookieCredentials
+        .post(`/internal/kibana/settings`)
+        .send({ changes: { [TEST_SETTING]: 500 } })
+        .expect(200);
       ...
     });
 });