[8.x] Wait for backfill SOs to be deleted before running the invalida…

…te API key task (#198422) (#198601)

# Backport

This will backport the following commits from `main` to `8.x`:
- [Wait for backfill SOs to be deleted before running the invalidate API
key task (#198422)](#198422)

<!--- Backport version: 9.4.3 -->

### Questions ?
Please refer to the [Backport tool
documentation](https://github.com/sqren/backport)

<!--BACKPORT [{"author":{"name":"Mike
Côté","email":"[email protected]"},"sourceCommit":{"committedDate":"2024-10-31T18:38:48Z","message":"Wait
for backfill SOs to be deleted before running the invalidate API key
task (#198422)\n\nResolves
https://github.com/elastic/kibana/issues/198168\r\n\r\nFlaky test
runners:\r\n-\r\nhttps://buildkite.com/elastic/kibana-flaky-test-suite-runner/builds/7309\r\n-\r\nhttps://buildkite.com/elastic/kibana-flaky-test-suite-runner/builds/7314","sha":"7b05079d94d4ff596ff9cf49dc448e0390118d1c","branchLabelMapping":{"^v9.0.0$":"main","^v8.17.0$":"8.x","^v(\\d+).(\\d+).\\d+$":"$1.$2"}},"sourcePullRequest":{"labels":["Feature:Alerting","release_note:skip","Team:ResponseOps","v9.0.0","backport:prev-minor","v8.17.0"],"title":"Wait
for backfill SOs to be deleted before running the invalidate API key
task","number":198422,"url":"https://github.com/elastic/kibana/pull/198422","mergeCommit":{"message":"Wait
for backfill SOs to be deleted before running the invalidate API key
task (#198422)\n\nResolves
https://github.com/elastic/kibana/issues/198168\r\n\r\nFlaky test
runners:\r\n-\r\nhttps://buildkite.com/elastic/kibana-flaky-test-suite-runner/builds/7309\r\n-\r\nhttps://buildkite.com/elastic/kibana-flaky-test-suite-runner/builds/7314","sha":"7b05079d94d4ff596ff9cf49dc448e0390118d1c"}},"sourceBranch":"main","suggestedTargetBranches":["8.x"],"targetPullRequestStates":[{"branch":"main","label":"v9.0.0","branchLabelMappingKey":"^v9.0.0$","isSourceBranch":true,"state":"MERGED","url":"https://github.com/elastic/kibana/pull/198422","number":198422,"mergeCommit":{"message":"Wait
for backfill SOs to be deleted before running the invalidate API key
task (#198422)\n\nResolves
https://github.com/elastic/kibana/issues/198168\r\n\r\nFlaky test
runners:\r\n-\r\nhttps://buildkite.com/elastic/kibana-flaky-test-suite-runner/builds/7309\r\n-\r\nhttps://buildkite.com/elastic/kibana-flaky-test-suite-runner/builds/7314","sha":"7b05079d94d4ff596ff9cf49dc448e0390118d1c"}},{"branch":"8.x","label":"v8.17.0","branchLabelMappingKey":"^v8.17.0$","isSourceBranch":false,"state":"NOT_CREATED"}]}]
BACKPORT-->

Co-authored-by: Mike Côté <[email protected]>

x-pack/test/alerting_api_integration/security_and_spaces/group1/tests/alerting/backfill/api_key.ts

@@ -252,6 +252,17 @@ export default function apiKeyBackfillTests({ getService }: FtrProviderContext)
         expect(e?.event?.outcome).to.eql('success');
       }
 
+      // wait for all the ad hoc run SO to be deleted
+      await retry.try(async () => {
+        try {
+          // throws when not found
+          await getAdHocRunSO(backfillId);
+          throw new Error('should have thrown');
+        } catch (e) {
+          expect(e.message).not.to.eql('should have thrown');
+        }
+      });
+
       // invoke the invalidate task
       await runInvalidateTask();