[SLO] [Alerting] deployment agnostic slo burn rate rule tests

.buildkite/ftr_oblt_stateful_configs.yml

@@ -49,3 +49,5 @@ enabled:
   - x-pack/test/observability_ai_assistant_functional/enterprise/config.ts
   - x-pack/test/profiling_api_integration/cloud/config.ts
   - x-pack/test/functional/apps/apm/config.ts
+  # Stateful configs that run deployment-agnostic tests
+  - x-pack/test/api_integration/deployment_agnostic/oblt.stateful.config.ts

.eslintrc.js

@@ -629,6 +629,8 @@ module.exports = {
         'x-pack/test/profiling_api_integration/**/*.ts',
         'x-pack/test/security_solution_api_integration/*/test_suites/**/*',
         'x-pack/test/security_solution_api_integration/**/config*.ts',
+        'x-pack/test/observability_solution_api_integration/*/test_suites/**/*',
+        'x-pack/test/observability_solution_api_integration/**/config*.ts',
       ],
       rules: {
         'import/no-default-export': 'off',

.github/CODEOWNERS

@@ -1133,7 +1133,9 @@ x-pack/test/observability_ai_assistant_functional @elastic/obs-ai-assistant
 /x-pack/test_serverless/**/test_suites/observability/custom_threshold_rule/ @elastic/obs-ux-management-team
 /x-pack/test_serverless/**/test_suites/observability/slos/ @elastic/obs-ux-management-team
 /x-pack/test_serverless/api_integration/test_suites/observability/es_query_rule @elastic/obs-ux-management-team
-/x-pack/test_serverless/api_integration/test_suites/observability/burn_rate_rule @elastic/obs-ux-management-team
+/x-pack/test/api_integration/deployment_agnostic/apis/observability/alerting/burn_rate_rule @elastic/obs-ux-management-team
+/x-pack/test/api_integration/deployment_agnostic/services/alerting_api @elastic/obs-ux-management-team
+/x-pack/test/api_integration/deployment_agnostic/services/slo_api @elastic/obs-ux-management-team
 /x-pack/test_serverless/**/test_suites/observability/infra/ @elastic/obs-ux-infra_services-team
 
 # Elastic Stack Monitoring

x-pack/test_serverless/api_integration/test_suites/observability/burn_rate_rule/burn_rate_rule.ts → x-pack/test/api_integration/deployment_agnostic/apis/observability/alerting/burn_rate_rule.ts

@@ -4,17 +4,10 @@
  * 2.0; you may not use this file except in compliance with the Elastic License
  * 2.0.
  */
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
 
 import { cleanup, Dataset, generate, PartialConfig } from '@kbn/data-forge';
 import expect from '@kbn/expect';
 import { FtrProviderContext } from '../../../ftr_provider_context';
-import { InternalRequestHeader, RoleCredentials } from '../../../../shared/services';
 
 export default function ({ getService }: FtrProviderContext) {
   const esClient = getService('es');
@@ -24,10 +17,9 @@ export default function ({ getService }: FtrProviderContext) {
   const alertingApi = getService('alertingApi');
   const dataViewApi = getService('dataViewApi');
   const sloApi = getService('sloApi');
-  const svlUserManager = getService('svlUserManager');
-  const svlCommonApi = getService('svlCommonApi');
-  let roleAuthc: RoleCredentials;
-  let internalReqHeader: InternalRequestHeader;
+  const config = getService('config');
+  const isServerless = config.get('serverless');
+  const expectedConsumer = isServerless ? 'observability' : 'slo';
 
   describe('Burn rate rule', () => {
     const RULE_TYPE_ID = 'slo.rules.burnRate';
@@ -41,9 +33,14 @@ export default function ({ getService }: FtrProviderContext) {
     let actionId: string;
     let ruleId: string;
 
+    const samlAuth = getService('samlAuth');
+    const supertestWithoutAuth = getService('supertestWithoutAuth');
+    let roleAuthc: RoleCredentials;
+    let internalHeaders: InternalRequestHeader;
+
     before(async () => {
-      roleAuthc = await svlUserManager.createM2mApiKeyWithRoleScope('admin');
-      internalReqHeader = svlCommonApi.getInternalRequestHeader();
+      roleAuthc = await samlAuth.createM2mApiKeyWithRoleScope('admin');
+      internalHeaders = samlAuth.getInternalRequestHeader();
       dataForgeConfig = {
         schedule: [
           {
@@ -60,37 +57,52 @@ export default function ({ getService }: FtrProviderContext) {
         indexing: { dataset: 'fake_hosts' as Dataset, eventsPerCycle: 1, interval: 10000 },
       };
       dataForgeIndices = await generate({ client: esClient, config: dataForgeConfig, logger });
-      await alertingApi.waitForDocumentInIndex({ indexName: DATA_VIEW, docCountTarget: 360 });
+      await alertingApi.waitForDocumentInIndex({
+        indexName: DATA_VIEW,
+        docCountTarget: 360,
+        roleAuthc,
+      });
       await dataViewApi.create({
+        roleAuthc,
         name: DATA_VIEW,
         id: DATA_VIEW_ID,
         title: DATA_VIEW,
       });
-      roleAuthc = await svlUserManager.createM2mApiKeyWithRoleScope('admin');
     });
 
     after(async () => {
-      await supertest.delete(`/api/alerting/rule/${ruleId}`).set(internalReqHeader);
-      await supertest.delete(`/api/actions/connector/${actionId}`).set(internalReqHeader);
+      await supertest
+        .delete(`/api/alerting/rule/${ruleId}`)
+        .set(roleAuthc.apiKeyHeader)
+        .set('kbn-xsrf', 'foo')
+        .set('x-elastic-internal-origin', 'foo');
+      await supertest
+        .delete(`/api/actions/connector/${actionId}`)
+        .set('kbn-xsrf', 'foo')
+        .set(roleAuthc.apiKeyHeader)
+        .set('x-elastic-internal-origin', 'foo');
       await esClient.deleteByQuery({
         index: '.kibana-event-log-*',
         query: { term: { 'rule.id': ruleId } },
         conflicts: 'proceed',
       });
       await dataViewApi.delete({
+        roleAuthc,
         id: DATA_VIEW_ID,
       });
-      await supertest.delete('/api/observability/slos/my-custom-id').set(internalReqHeader);
+      await supertest
+        .delete('/api/observability/slos/my-custom-id')
+        .set('kbn-xsrf', 'foo')
+        .set('x-elastic-internal-origin', 'foo');
 
       await esDeleteAllIndices([ALERT_ACTION_INDEX, ...dataForgeIndices]);
       await cleanup({ client: esClient, config: dataForgeConfig, logger });
-      await svlUserManager.invalidateM2mApiKeyWithRoleScope(roleAuthc);
+      await samlAuth.invalidateM2mApiKeyWithRoleScope(roleAuthc);
     });
 
     describe('Rule creation', () => {
       it('creates rule successfully', async () => {
         actionId = await alertingApi.createIndexConnector({
-          roleAuthc,
           name: 'Index Connector: Slo Burn rate API test',
           indexName: ALERT_ACTION_INDEX,
         });
@@ -123,9 +135,8 @@ export default function ({ getService }: FtrProviderContext) {
         );
 
         const dependencyRule = await alertingApi.createRule({
-          roleAuthc,
           tags: ['observability'],
-          consumer: 'observability',
+          consumer: expectedConsumer,
           name: 'SLO Burn Rate rule - Dependency',
           ruleTypeId: RULE_TYPE_ID,
           schedule: {
@@ -196,9 +207,8 @@ export default function ({ getService }: FtrProviderContext) {
         });
 
         const createdRule = await alertingApi.createRule({
-          roleAuthc,
           tags: ['observability'],
-          consumer: 'observability',
+          consumer: expectedConsumer,
           name: 'SLO Burn Rate rule',
           ruleTypeId: RULE_TYPE_ID,
           schedule: {
@@ -299,9 +309,9 @@ export default function ({ getService }: FtrProviderContext) {
       });
 
       it('should find the created rule with correct information about the consumer', async () => {
-        const match = await alertingApi.findRule(roleAuthc, ruleId);
+        const match = await alertingApi.findRule(ruleId);
         expect(match).not.to.be(undefined);
-        expect(match.consumer).to.be('observability');
+        expect(match.consumer).to.be(expectedConsumer);
       });
     });
   });

x-pack/test/api_integration/deployment_agnostic/apis/observability/alerting/index.ts

@@ -0,0 +1,15 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { FtrProviderContext } from '../../../ftr_provider_context';
+
+export default function ({ loadTestFile }: FtrProviderContext) {
+  describe('Slo - Burn rate rule', function () {
+    this.tags(['ess', 'serverless']);
+    loadTestFile(require.resolve('./burn_rate_rule'));
+  });
+}

x-pack/test/api_integration/deployment_agnostic/oblt.index.ts

@@ -11,5 +11,6 @@ export default function ({ loadTestFile }: DeploymentAgnosticFtrProviderContext)
     loadTestFile(require.resolve('./apis/console'));
     loadTestFile(require.resolve('./apis/core'));
     loadTestFile(require.resolve('./apis/painless_lab'));
+    loadTestFile(require.resolve('./apis/observability/alerting'));
   });
 }

x-pack/test/api_integration/deployment_agnostic/oblt.stateful.config.ts

@@ -0,0 +1,16 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { createStatefulTestConfig } from './default_configs/stateful.config.base';
+
+export default createStatefulTestConfig({
+  serverlessProject: 'oblt',
+  testFiles: [require.resolve('./oblt.index.ts')],
+  junit: {
+    reportName: 'Stateful Observability - Deployment-agnostic API Integration Tests',
+  },
+});

x-pack/test/api_integration/deployment_agnostic/services/alerting_api.ts

@@ -0,0 +1,169 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { RoleCredentials } from '@kbn/ftr-common-functional-services';
+import { DeploymentAgnosticFtrProviderContext } from '../ftr_provider_context';
+
+export function AlertingApiProvider({ getService }: DeploymentAgnosticFtrProviderContext) {
+  const retry = getService('retry');
+  const supertest = getService('supertest');
+  const es = getService('es');
+  const requestTimeout = 30 * 1000;
+  const retryTimeout = 120 * 1000;
+  const logger = getService('log');
+
+  return {
+    async waitForRuleStatus({
+      ruleId,
+      expectedStatus,
+      roleAuthc,
+    }: {
+      ruleId: string;
+      expectedStatus: string;
+      roleAuthc: RoleCredentials;
+    }) {
+      if (!ruleId) {
+        throw new Error(`'ruleId' is undefined`);
+      }
+      return await retry.tryForTime(retryTimeout, async () => {
+        const response = await supertest
+          .get(`/api/alerting/rule/${ruleId}`)
+          .set(roleAuthc.apiKeyHeader)
+          .set(roleAuthc.getInternalRequestHeader())
+          .timeout(requestTimeout);
+        const { execution_status: executionStatus } = response.body || {};
+        const { status } = executionStatus || {};
+        if (status !== expectedStatus) {
+          throw new Error(`waitForStatus(${expectedStatus}): got ${status}`);
+        }
+        return executionStatus?.status;
+      });
+    },
+
+    async waitForDocumentInIndex<T>({
+      indexName,
+      docCountTarget = 1,
+      roleAuthc,
+    }: {
+      indexName: string;
+      docCountTarget?: number;
+      roleAuthc: RoleCredentials;
+    }): Promise<SearchResponse<T, Record<string, AggregationsAggregate>>> {
+      return await retry.tryForTime(retryTimeout, async () => {
+        const response = await es.search<T>({
+          index: indexName,
+          rest_total_hits_as_int: true,
+        });
+        logger.debug(`Found ${response.hits.total} docs, looking for atleast ${docCountTarget}.`);
+        if (!response.hits.total || response.hits.total < docCountTarget) {
+          throw new Error('No hits found');
+        }
+        return response;
+      });
+    },
+
+    async waitForAlertInIndex<T>({
+      indexName,
+      ruleId,
+    }: {
+      indexName: string;
+      ruleId: string;
+    }): Promise<SearchResponse<T, Record<string, AggregationsAggregate>>> {
+      if (!ruleId) {
+        throw new Error(`'ruleId' is undefined`);
+      }
+      return await retry.tryForTime(retryTimeout, async () => {
+        const response = await es.search<T>({
+          index: indexName,
+          body: {
+            query: {
+              term: {
+                'kibana.alert.rule.uuid': ruleId,
+              },
+            },
+          },
+        });
+        if (response.hits.hits.length === 0) {
+          throw new Error('No hits found');
+        }
+        return response;
+      });
+    },
+
+    async createIndexConnector({
+      name,
+      indexName,
+      roleAuthc,
+    }: {
+      name: string;
+      indexName: string;
+      roleAuthC: RoleCredentials;
+    }) {
+      const { body } = await supertest
+        .post(`/api/actions/connector`)
+        .set(roleAuthc.apiKeyHeader)
+        .set(roleAuthC.getInternalRequestHeader())
+        .send({
+          name,
+          config: {
+            index: indexName,
+            refresh: true,
+          },
+          connector_type_id: '.index',
+        });
+      return body.id as string;
+    },
+
+    async createRule({
+      name,
+      ruleTypeId,
+      params,
+      actions = [],
+      tags = [],
+      schedule,
+      consumer,
+      roleAuthc,
+    }: {
+      ruleTypeId: string;
+      name: string;
+      params: MetricThresholdParams | ThresholdParams | SloBurnRateRuleParams;
+      actions?: any[];
+      tags?: any[];
+      schedule?: { interval: string };
+      consumer: string;
+      roleAuthc: RoleCredentials;
+    }) {
+      const { body } = await supertest
+        .post(`/api/alerting/rule`)
+        .set(roleAuthc.apiKeyHeader)
+        .set(roleAuthC.getInternalRequestHeader())
+        .send({
+          params,
+          consumer,
+          schedule: schedule || {
+            interval: '5m',
+          },
+          tags,
+          name,
+          rule_type_id: ruleTypeId,
+          actions,
+        });
+      return body;
+    },
+
+    async findRule(ruleId: string, roleAuthc: RoleCredentials) {
+      if (!ruleId) {
+        throw new Error(`'ruleId' is undefined`);
+      }
+      const response = await supertest
+        .get('/api/alerting/rules/_find')
+        .set(roleAuthc.apiKeyHeader)
+        .set(roleAuthC.getInternalRequestHeader());
+      return response.body.data.find((obj: any) => obj.id === ruleId);
+    },
+  };
+}

x-pack/test/api_integration/deployment_agnostic/services/index.ts

@@ -9,6 +9,8 @@ import { commonFunctionalServices } from '@kbn/ftr-common-functional-services';
 import { deploymentAgnosticServices } from './deployment_agnostic_services';
 import { DataViewApiProvider } from './data_view_api';
 import { SloApiProvider } from './slo_api';
+import { AlertingApiProvider } from './alerting_api';
+import { services as commonServices } from '../../../common/services';
 
 export type {
   InternalRequestHeader,
@@ -17,7 +19,9 @@ export type {
 } from '@kbn/ftr-common-functional-services';
 
 export const services = {
+  ...commonServices,
   ...deploymentAgnosticServices,
+  alertingApi: AlertingApiProvider,
   supertestWithoutAuth: commonFunctionalServices.supertestWithoutAuth,
   samlAuth: commonFunctionalServices.samlAuth,
   dataViewApi: DataViewApiProvider,