[Automatic Import] Add base for ftr api tests (elastic#200169)

## Summary

This PR adds a baseline for FTR API tests for Automatic Import.

- Relates elastic#196063

---------

Co-authored-by: kibanamachine <[email protected]>

.buildkite/ftr_security_stateful_configs.yml

@@ -101,3 +101,4 @@ enabled:
   - x-pack/test/cloud_security_posture_functional/config.ts
   - x-pack/test/cloud_security_posture_functional/config.agentless.ts
   - x-pack/test/cloud_security_posture_functional/data_views/config.ts
+  - x-pack/test/automatic_import_api_integration/security/config_basic.ts

.github/CODEOWNERS

@@ -2459,6 +2459,9 @@ x-pack/plugins/security_solution/common/api/entity_analytics @elastic/security-e
 ## Security Solution sub teams - GenAI
 x-pack/test/security_solution_api_integration/test_suites/genai @elastic/security-generative-ai
 
+## Security Solution sub teams - Automatic Import
+x-pack/test/automatic_import_api_integration @elastic/security-scalability
+
 # Security Defend Workflows - OSQuery Ownership
 /x-pack/test/osquery_cypress @elastic/security-defend-workflows
 /x-pack/plugins/osquery @elastic/security-defend-workflows

x-pack/test/automatic_import_api_integration/common/config.ts

@@ -0,0 +1,89 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { CA_CERT_PATH } from '@kbn/dev-utils';
+import { FtrConfigProviderContext } from '@kbn/test';
+import { services } from './services';
+
+interface CreateTestConfigOptions {
+  license: string;
+  disabledPlugins?: string[];
+  ssl?: boolean;
+  testFiles?: string[];
+  publicBaseUrl?: boolean;
+}
+
+const enabledActionTypes = ['.bedrock', '.gemini', '.gen-ai'];
+
+export function createTestConfig(name: string, options: CreateTestConfigOptions) {
+  const { license = 'trial', disabledPlugins = [], ssl = false, testFiles = [] } = options;
+
+  return async ({ readConfigFile }: FtrConfigProviderContext) => {
+    const xPackApiIntegrationTestsConfig = await readConfigFile(
+      require.resolve('../../api_integration/config.ts')
+    );
+
+    const servers = {
+      ...xPackApiIntegrationTestsConfig.get('servers'),
+      elasticsearch: {
+        ...xPackApiIntegrationTestsConfig.get('servers.elasticsearch'),
+        protocol: ssl ? 'https' : 'http',
+      },
+    };
+
+    return {
+      testFiles,
+      servers,
+      services,
+      junit: {
+        reportName: 'X-Pack Automatic Import API Integration Tests',
+      },
+      esTestCluster: {
+        ...xPackApiIntegrationTestsConfig.get('esTestCluster'),
+        license,
+        ssl,
+        serverArgs: [
+          `xpack.license.self_generated.type=${license}`,
+          `xpack.security.enabled=${
+            !disabledPlugins.includes('security') && ['trial', 'basic'].includes(license)
+          }`,
+        ],
+      },
+      kbnTestServer: {
+        ...xPackApiIntegrationTestsConfig.get('kbnTestServer'),
+        serverArgs: [
+          ...xPackApiIntegrationTestsConfig.get('kbnTestServer.serverArgs'),
+          ...(options.publicBaseUrl ? ['--server.publicBaseUrl=https://localhost:5601'] : []),
+          `--xpack.actions.allowedHosts=${JSON.stringify(['localhost', 'some.non.existent.com'])}`,
+          `--xpack.actions.enabledActionTypes=${JSON.stringify(enabledActionTypes)}`,
+          '--xpack.eventLog.logEntries=true',
+          // Configure a bedrock connector as a default
+          `--xpack.actions.preconfigured=${JSON.stringify({
+            'preconfigured-bedrock': {
+              name: 'preconfigured-bedrock',
+              actionTypeId: '.bedrock',
+              config: {
+                apiUrl: 'https://example.com',
+              },
+              secrets: {
+                username: 'elastic',
+                password: 'elastic',
+              },
+            },
+          })}`,
+          ...(ssl
+            ? [
+                `--elasticsearch.hosts=${servers.elasticsearch.protocol}://${servers.elasticsearch.hostname}:${servers.elasticsearch.port}`,
+                `--elasticsearch.ssl.certificateAuthorities=${CA_CERT_PATH}`,
+              ]
+            : []),
+          '--xpack.integration_assistant.enabled=true',
+        ],
+      },
+    };
+  };
+}

x-pack/test/automatic_import_api_integration/common/ftr_provider_context.d.ts

@@ -0,0 +1,12 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { GenericFtrProviderContext } from '@kbn/test';
+
+import { services } from './services';
+
+export type FtrProviderContext = GenericFtrProviderContext<typeof services, {}>;

x-pack/test/automatic_import_api_integration/common/lib/api/analyze_logs.ts

@@ -0,0 +1,36 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+import type SuperTest from 'supertest';
+import {
+  AnalyzeLogsRequestBody,
+  ANALYZE_LOGS_PATH,
+  AnalyzeLogsResponse,
+} from '@kbn/integration-assistant-plugin/common';
+import { superUser } from '../authentication/users';
+import { User } from '../authentication/types';
+
+export const postAnalyzeLogs = async ({
+  supertest,
+  req,
+  expectedHttpCode = 404,
+  auth = { user: superUser },
+}: {
+  supertest: SuperTest.Agent;
+  req: AnalyzeLogsRequestBody;
+  expectedHttpCode?: number;
+  auth: { user: User };
+}): Promise<AnalyzeLogsResponse> => {
+  const { body: response } = await supertest
+    .post(`${ANALYZE_LOGS_PATH}`)
+    .send(req)
+    .set('kbn-xsrf', 'abc')
+    .set('elastic-api-version', '1')
+    .auth(auth.user.username, auth.user.password)
+    .expect(expectedHttpCode);
+
+  return response;
+};

x-pack/test/automatic_import_api_integration/common/lib/api/categorization.ts

@@ -0,0 +1,36 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+import type SuperTest from 'supertest';
+import {
+  CategorizationRequestBody,
+  CATEGORIZATION_GRAPH_PATH,
+  CategorizationResponse,
+} from '@kbn/integration-assistant-plugin/common';
+import { superUser } from '../authentication/users';
+import { User } from '../authentication/types';
+
+export const postCategorization = async ({
+  supertest,
+  req,
+  expectedHttpCode = 404,
+  auth = { user: superUser },
+}: {
+  supertest: SuperTest.Agent;
+  req: CategorizationRequestBody;
+  expectedHttpCode?: number;
+  auth: { user: User };
+}): Promise<CategorizationResponse> => {
+  const { body: response } = await supertest
+    .post(`${CATEGORIZATION_GRAPH_PATH}`)
+    .send(req)
+    .set('kbn-xsrf', 'abc')
+    .set('elastic-api-version', '1')
+    .auth(auth.user.username, auth.user.password)
+    .expect(expectedHttpCode);
+
+  return response;
+};

x-pack/test/automatic_import_api_integration/common/lib/api/ecs.ts

@@ -0,0 +1,36 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+import type SuperTest from 'supertest';
+import {
+  EcsMappingRequestBody,
+  ECS_GRAPH_PATH,
+  EcsMappingResponse,
+} from '@kbn/integration-assistant-plugin/common';
+import { superUser } from '../authentication/users';
+import { User } from '../authentication/types';
+
+export const postEcsMapping = async ({
+  supertest,
+  req,
+  expectedHttpCode = 404,
+  auth = { user: superUser },
+}: {
+  supertest: SuperTest.Agent;
+  req: EcsMappingRequestBody;
+  expectedHttpCode?: number;
+  auth: { user: User };
+}): Promise<EcsMappingResponse> => {
+  const { body: response } = await supertest
+    .post(`${ECS_GRAPH_PATH}`)
+    .send(req)
+    .set('kbn-xsrf', 'abc')
+    .set('elastic-api-version', '1')
+    .auth(auth.user.username, auth.user.password)
+    .expect(expectedHttpCode);
+
+  return response;
+};

x-pack/test/automatic_import_api_integration/common/lib/api/index.ts

@@ -0,0 +1,7 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+export * from './user_profiles';

x-pack/test/automatic_import_api_integration/common/lib/api/related.ts

@@ -0,0 +1,36 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+import type SuperTest from 'supertest';
+import {
+  RelatedRequestBody,
+  RELATED_GRAPH_PATH,
+  RelatedResponse,
+} from '@kbn/integration-assistant-plugin/common';
+import { superUser } from '../authentication/users';
+import { User } from '../authentication/types';
+
+export const postRelated = async ({
+  supertest,
+  req,
+  expectedHttpCode = 404,
+  auth = { user: superUser },
+}: {
+  supertest: SuperTest.Agent;
+  req: RelatedRequestBody;
+  expectedHttpCode?: number;
+  auth: { user: User };
+}): Promise<RelatedResponse> => {
+  const { body: response } = await supertest
+    .post(`${RELATED_GRAPH_PATH}`)
+    .send(req)
+    .set('kbn-xsrf', 'abc')
+    .set('elastic-api-version', '1')
+    .auth(auth.user.username, auth.user.password)
+    .expect(expectedHttpCode);
+
+  return response;
+};

x-pack/test/automatic_import_api_integration/common/lib/api/user_profiles.ts

@@ -0,0 +1,38 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type SuperTest from 'supertest';
+import { parse as parseCookie, Cookie } from 'tough-cookie';
+import { superUser } from '../authentication/users';
+import { User } from '../authentication/types';
+
+export const loginUsers = async ({
+  supertest,
+  users = [superUser],
+}: {
+  supertest: SuperTest.Agent;
+  users?: User[];
+}) => {
+  const cookies: Cookie[] = [];
+
+  for (const user of users) {
+    const response = await supertest
+      .post('/internal/security/login')
+      .set('kbn-xsrf', 'xxx')
+      .send({
+        providerType: 'basic',
+        providerName: 'basic',
+        currentURL: '/',
+        params: { username: user.username, password: user.password },
+      })
+      .expect(200);
+
+    cookies.push(parseCookie(response.header['set-cookie'][0])!);
+  }
+
+  return cookies;
+};

x-pack/test/automatic_import_api_integration/common/lib/authentication/index.ts

@@ -0,0 +1,85 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { FtrProviderContext as CommonFtrProviderContext } from '../../ftr_provider_context';
+import { Role, User, UserInfo } from './types';
+import { noIntegrationsUser, users } from './users';
+import { roles } from './roles';
+import { loginUsers } from '../api';
+
+export const getUserInfo = (user: User): UserInfo => ({
+  username: user.username,
+  full_name: user.username.replace('_', ' '),
+  email: `${user.username}@elastic.co`,
+});
+
+/**
+ * Creates the users and roles for use in the tests. Defaults to specific users and roles used by the security
+ * scenarios but can be passed specific ones as well.
+ */
+export const createUsersAndRoles = async (
+  getService: CommonFtrProviderContext['getService'],
+  usersToCreate: User[] = users,
+  rolesToCreate: Role[] = roles
+) => {
+  const security = getService('security');
+
+  const createRole = async ({ name, privileges }: Role) => {
+    return await security.role.create(name, privileges);
+  };
+
+  const createUser = async (user: User) => {
+    const userInfo = getUserInfo(user);
+
+    return await security.user.create(user.username, {
+      password: user.password,
+      roles: user.roles,
+      full_name: userInfo.full_name,
+      email: userInfo.email,
+    });
+  };
+
+  await Promise.all(rolesToCreate.map((role) => createRole(role)));
+  await Promise.all(usersToCreate.map((user) => createUser(user)));
+};
+
+export const deleteUsersAndRoles = async (
+  getService: CommonFtrProviderContext['getService'],
+  usersToDelete: User[] = users,
+  rolesToDelete: Role[] = roles
+) => {
+  const security = getService('security');
+
+  try {
+    await Promise.allSettled(usersToDelete.map((user) => security.user.delete(user.username)));
+  } catch (error) {
+    // ignore errors because if a migration is run it will delete the .kibana index which remove the spaces and users
+  }
+
+  try {
+    await Promise.allSettled(rolesToDelete.map((role) => security.role.delete(role.name)));
+  } catch (error) {
+    // ignore errors because if a migration is run it will delete the .kibana index which remove the spaces and users
+  }
+};
+
+export const createUsers = async (getService: CommonFtrProviderContext['getService']) => {
+  await createUsersAndRoles(getService);
+};
+
+export const deleteUsers = async (getService: CommonFtrProviderContext['getService']) => {
+  await deleteUsersAndRoles(getService);
+};
+
+export const activateUserProfiles = async (getService: CommonFtrProviderContext['getService']) => {
+  const supertestWithoutAuth = getService('supertestWithoutAuth');
+
+  await loginUsers({
+    supertest: supertestWithoutAuth,
+    users: [noIntegrationsUser],
+  });
+};