Fixing flaky backfill tests (elastic#194328)

Resolves elastic#192144 and
elastic#181862

## Summary

Unskipped the test that was skipped in
elastic#181862 but it did not fail at
all during the 2400 times total I ran the flaky test runner so I'm going
to unskip and see if it comes back ever

While running the flaky test runner, I saw this test from
elastic#192144 be flaky so I addressed
in this PR. This was caused by the query for the alert documents
sometimes not returning the alert docs in timestamp order, which is
important because I'm trying to compare the actual alert timestamp to
the expected alert timestamp (in order). Added a sort order to the alert
query.

---------

Co-authored-by: Elastic Machine <[email protected]>

x-pack/test/alerting_api_integration/security_and_spaces/group1/tests/alerting/backfill/find.ts

@@ -135,8 +135,7 @@ export default function findBackfillTests({ getService }: FtrProviderContext) {
 
     for (const scenario of UserAtSpaceScenarios) {
       const { user, space } = scenario;
-      // FLAKY: https://github.com/elastic/kibana/issues/181862
-      describe.skip(scenario.id, () => {
+      describe(scenario.id, () => {
         const apiOptions = {
           spaceId: space.id,
           username: user.username,

x-pack/test/alerting_api_integration/security_and_spaces/group1/tests/alerting/backfill/task_runner.ts

@@ -53,6 +53,7 @@ import {
 export default function createBackfillTaskRunnerTests({ getService }: FtrProviderContext) {
   const es = getService('es');
   const retry = getService('retry');
+  const log = getService('log');
   const esTestIndexTool = new ESTestIndexTool(es, retry);
   const supertestWithoutAuth = getService('supertestWithoutAuth');
   const supertest = getService('supertest');
@@ -65,25 +66,25 @@ export default function createBackfillTaskRunnerTests({ getService }: FtrProvide
     moment().utc().subtract(14, 'days').toISOString(),
 
     // backfill execution set 1
-    moment().utc().startOf('day').subtract(13, 'days').add(64, 'seconds').toISOString(),
-    moment().utc().startOf('day').subtract(13, 'days').add(65, 'seconds').toISOString(),
-    moment().utc().startOf('day').subtract(13, 'days').add(66, 'seconds').toISOString(),
+    moment().utc().startOf('day').subtract(13, 'days').add(10, 'minutes').toISOString(),
+    moment().utc().startOf('day').subtract(13, 'days').add(11, 'minutes').toISOString(),
+    moment().utc().startOf('day').subtract(13, 'days').add(12, 'minutes').toISOString(),
 
     // backfill execution set 2
-    moment().utc().startOf('day').subtract(12, 'days').add(89, 'seconds').toISOString(),
+    moment().utc().startOf('day').subtract(12, 'days').add(20, 'minutes').toISOString(),
 
     // backfill execution set 3
-    moment().utc().startOf('day').subtract(11, 'days').add(785, 'seconds').toISOString(),
-    moment().utc().startOf('day').subtract(11, 'days').add(888, 'seconds').toISOString(),
-    moment().utc().startOf('day').subtract(11, 'days').add(954, 'seconds').toISOString(),
-    moment().utc().startOf('day').subtract(11, 'days').add(1045, 'seconds').toISOString(),
-    moment().utc().startOf('day').subtract(11, 'days').add(1145, 'seconds').toISOString(),
+    moment().utc().startOf('day').subtract(11, 'days').add(30, 'minutes').toISOString(),
+    moment().utc().startOf('day').subtract(11, 'days').add(31, 'minutes').toISOString(),
+    moment().utc().startOf('day').subtract(11, 'days').add(32, 'minutes').toISOString(),
+    moment().utc().startOf('day').subtract(11, 'days').add(33, 'minutes').toISOString(),
+    moment().utc().startOf('day').subtract(11, 'days').add(34, 'minutes').toISOString(),
 
     // backfill execution set 4 purposely left empty
 
     // after last backfill
-    moment().utc().startOf('day').subtract(9, 'days').add(666, 'seconds').toISOString(),
-    moment().utc().startOf('day').subtract(9, 'days').add(667, 'seconds').toISOString(),
+    moment().utc().startOf('day').subtract(9, 'days').add(40, 'minutes').toISOString(),
+    moment().utc().startOf('day').subtract(9, 'days').add(41, 'minutes').toISOString(),
   ];
 
   describe('ad hoc backfill task', () => {
@@ -175,6 +176,9 @@ export default function createBackfillTaskRunnerTests({ getService }: FtrProvide
         .send([{ rule_id: ruleId, start, end }])
         .expect(200);
 
+      log.info(`originalDocTimestamps ${JSON.stringify(originalDocTimestamps)}`);
+      log.info(`scheduledBackfill ${JSON.stringify(response2.body)}`);
+
       const scheduleResult = response2.body;
 
       expect(scheduleResult.length).to.eql(1);
@@ -668,7 +672,10 @@ export default function createBackfillTaskRunnerTests({ getService }: FtrProvide
   async function queryForAlertDocs<T>(): Promise<Array<SearchHit<T>>> {
     const searchResult = await es.search({
       index: alertsAsDataIndex,
-      body: { query: { match_all: {} } },
+      body: {
+        sort: [{ [ALERT_ORIGINAL_TIME]: { order: 'asc' } }],
+        query: { match_all: {} },
+      },
     });
     return searchResult.hits.hits as Array<SearchHit<T>>;
   }