From 3b9c56421196254acb358e8a4096f177925ffe82 Mon Sep 17 00:00:00 2001
From: Christos Nasikas <christos.nasikas@elastic.co>
Date: Thu, 21 Dec 2023 14:08:48 +0200
Subject: [PATCH] Retry the execution of the connector

---
 .../connectors/cases/cases_connector.test.ts  | 31 +++++++++++++++++++
 .../connectors/cases/cases_connector.ts       | 14 +++++++++
 2 files changed, 45 insertions(+)

diff --git a/x-pack/plugins/cases/server/connectors/cases/cases_connector.test.ts b/x-pack/plugins/cases/server/connectors/cases/cases_connector.test.ts
index db242ddbbf9c8..7dd88b6ea1ebe 100644
--- a/x-pack/plugins/cases/server/connectors/cases/cases_connector.test.ts
+++ b/x-pack/plugins/cases/server/connectors/cases/cases_connector.test.ts
@@ -15,10 +15,13 @@ import { CasesOracleService } from './cases_oracle_service';
 import { CasesService } from './cases_service';
 import { CasesConnectorError } from './cases_connector_error';
 import { CaseError } from '../../common/error';
+import { fullJitterBackoffFactory } from './full_jitter_backoff';
 
 jest.mock('./cases_connector_executor');
+jest.mock('./full_jitter_backoff');
 
 const CasesConnectorExecutorMock = CasesConnectorExecutor as jest.Mock;
+const fullJitterBackoffFactoryMock = fullJitterBackoffFactory as jest.Mock;
 
 describe('CasesConnector', () => {
   const services = actionsMock.createServices();
@@ -37,11 +40,18 @@ describe('CasesConnector', () => {
 
   const mockExecute = jest.fn();
   const getCasesClient = jest.fn().mockResolvedValue({ foo: 'bar' });
+  // 1ms delay before retrying
+  const nextBackOff = jest.fn().mockReturnValue(1);
+
+  const backOffFactory = {
+    create: () => ({ nextBackOff }),
+  };
 
   let connector: CasesConnector;
 
   beforeEach(() => {
     jest.clearAllMocks();
+    mockExecute.mockResolvedValue({});
 
     CasesConnectorExecutorMock.mockImplementation(() => {
       return {
@@ -49,6 +59,8 @@ describe('CasesConnector', () => {
       };
     });
 
+    fullJitterBackoffFactoryMock.mockReturnValue(backOffFactory);
+
     connector = new CasesConnector({
       casesParams: { getCasesClient },
       connectorParams: {
@@ -156,4 +168,23 @@ describe('CasesConnector', () => {
       })
     ).rejects.toThrowErrorMatchingInlineSnapshot(`"Server error"`);
   });
+
+  it('retries correctly', async () => {
+    mockExecute
+      .mockRejectedValueOnce(new CasesConnectorError('Conflict error', 409))
+      .mockRejectedValueOnce(new CasesConnectorError('ES Unavailable', 503))
+      .mockResolvedValue({});
+
+    await connector.run({
+      alerts: [],
+      groupingBy,
+      owner,
+      rule,
+      timeWindow,
+      reopenClosedCases,
+    });
+
+    expect(nextBackOff).toBeCalledTimes(2);
+    expect(mockExecute).toBeCalledTimes(3);
+  });
 });
diff --git a/x-pack/plugins/cases/server/connectors/cases/cases_connector.ts b/x-pack/plugins/cases/server/connectors/cases/cases_connector.ts
index d5410225aa12c..fa71aca913b2f 100644
--- a/x-pack/plugins/cases/server/connectors/cases/cases_connector.ts
+++ b/x-pack/plugins/cases/server/connectors/cases/cases_connector.ts
@@ -21,6 +21,8 @@ import {
   isCasesConnectorError,
 } from './cases_connector_error';
 import { CasesConnectorExecutor } from './cases_connector_executor';
+import { CaseConnectorRetryService } from './retry_service';
+import { fullJitterBackoffFactory } from './full_jitter_backoff';
 
 interface CasesConnectorParams {
   connectorParams: ServiceParams<CasesConnectorConfig, CasesConnectorSecrets>;
@@ -33,6 +35,7 @@ export class CasesConnector extends SubActionConnector<
 > {
   private readonly casesOracleService: CasesOracleService;
   private readonly casesService: CasesService;
+  private readonly retryService: CaseConnectorRetryService;
   private readonly kibanaRequest: KibanaRequest;
   private readonly casesParams: CasesConnectorParams['casesParams'];
 
@@ -51,6 +54,9 @@ export class CasesConnector extends SubActionConnector<
 
     this.casesService = new CasesService();
 
+    const backOffFactory = fullJitterBackoffFactory({ baseDelay: 5, maxBackoffTime: 2000 });
+    this.retryService = new CaseConnectorRetryService(backOffFactory);
+
     /**
      * TODO: Get request from the actions framework.
      * Should be set in the SubActionConnector's constructor
@@ -80,6 +86,14 @@ export class CasesConnector extends SubActionConnector<
   }
 
   public async run(params: CasesConnectorRunParams) {
+    /**
+     * TODO: Tell the task manager to not retry on non
+     * retryable errors
+     */
+    await this.retryService.retryWithBackoff(() => this._run(params));
+  }
+
+  private async _run(params: CasesConnectorRunParams) {
     try {
       const casesClient = await this.casesParams.getCasesClient(this.kibanaRequest);