[8.x] [ML] Fix overall bucket request for large number of job IDs (#1…

…98292) (#198485)

# Backport

This will backport the following commits from `main` to `8.x`:
- [[ML] Fix overall bucket request for large number of job IDs
(#198292)](#198292)

<!--- Backport version: 9.4.3 -->

### Questions ?
Please refer to the [Backport tool
documentation](https://github.com/sqren/backport)

<!--BACKPORT [{"author":{"name":"Dima
Arnautov","email":"[email protected]"},"sourceCommit":{"committedDate":"2024-10-30T17:30:16Z","message":"[ML]
Fix overall bucket request for large number of job IDs (#198292)\n\n##
Summary\r\n\r\nFixes
https://github.com/elastic/kibana/issues/190591\r\n\r\nAdds chunking for
`over_buckets` request to prevent exceeding the\r\nrequest length
URL.\r\n\r\n### Checklist\r\n\r\n\r\n- [ ] [Unit or
functional\r\ntests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)\r\nwere
updated or added to match the most common
scenarios","sha":"fa0f397a4269a37370d2101a8c5ece03ad0117bd","branchLabelMapping":{"^v9.0.0$":"main","^v8.17.0$":"8.x","^v(\\d+).(\\d+).\\d+$":"$1.$2"}},"sourcePullRequest":{"labels":["release_note:fix",":ml","v9.0.0","Team:ML","v8.16.0","backport:version","v8.17.0"],"title":"[ML]
Fix overall bucket request for large number of job
IDs","number":198292,"url":"https://github.com/elastic/kibana/pull/198292","mergeCommit":{"message":"[ML]
Fix overall bucket request for large number of job IDs (#198292)\n\n##
Summary\r\n\r\nFixes
https://github.com/elastic/kibana/issues/190591\r\n\r\nAdds chunking for
`over_buckets` request to prevent exceeding the\r\nrequest length
URL.\r\n\r\n### Checklist\r\n\r\n\r\n- [ ] [Unit or
functional\r\ntests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)\r\nwere
updated or added to match the most common
scenarios","sha":"fa0f397a4269a37370d2101a8c5ece03ad0117bd"}},"sourceBranch":"main","suggestedTargetBranches":["8.x"],"targetPullRequestStates":[{"branch":"main","label":"v9.0.0","branchLabelMappingKey":"^v9.0.0$","isSourceBranch":true,"state":"MERGED","url":"https://github.com/elastic/kibana/pull/198292","number":198292,"mergeCommit":{"message":"[ML]
Fix overall bucket request for large number of job IDs (#198292)\n\n##
Summary\r\n\r\nFixes
https://github.com/elastic/kibana/issues/190591\r\n\r\nAdds chunking for
`over_buckets` request to prevent exceeding the\r\nrequest length
URL.\r\n\r\n### Checklist\r\n\r\n\r\n- [ ] [Unit or
functional\r\ntests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)\r\nwere
updated or added to match the most common
scenarios","sha":"fa0f397a4269a37370d2101a8c5ece03ad0117bd"}},{"branch":"8.16","label":"v8.16.0","branchLabelMappingKey":"^v(\\d+).(\\d+).\\d+$","isSourceBranch":false,"url":"https://github.com/elastic/kibana/pull/198415","number":198415,"state":"MERGED","mergeCommit":{"sha":"318831e20a039ec3d41d12108a85ced5064116d7","message":"[8.16]
[ML] Fix overall bucket request for large number of job IDs (#198292)
(#198415)\n\n# Backport\n\nThis will backport the following commits from
`main` to `8.16`:\n- [[ML] Fix overall bucket request for large number
of job
IDs\n(#198292)](https://github.com/elastic/kibana/pull/198292)\n\n<!---
Backport version: 9.4.3 -->\n\n### Questions ?\nPlease refer to the
[Backport
tool\ndocumentation](https://github.com/sqren/backport)\n\n<!--BACKPORT
[{\"author\":{\"name\":\"Dima\nArnautov\",\"email\":\"[email protected]\"},\"sourceCommit\":{\"committedDate\":\"2024-10-30T17:30:16Z\",\"message\":\"[ML]\nFix
overall bucket request for large number of job IDs
(#198292)\\n\\n##\nSummary\\r\\n\\r\\nFixes\nhttps://github.com//issues/190591\\r\\n\\r\\nAdds
chunking for\n`over_buckets` request to prevent exceeding
the\\r\\nrequest length\nURL.\\r\\n\\r\\n###
Checklist\\r\\n\\r\\n\\r\\n- [ ] [Unit
or\nfunctional\\r\\ntests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)\\r\\nwere\nupdated
or added to match the most
common\nscenarios\",\"sha\":\"fa0f397a4269a37370d2101a8c5ece03ad0117bd\",\"branchLabelMapping\":{\"^v9.0.0$\":\"main\",\"^v8.17.0$\":\"8.x\",\"^v(\\\\d+).(\\\\d+).\\\\d+$\":\"$1.$2\"}},\"sourcePullRequest\":{\"labels\":[\"release_note:fix\",\":ml\",\"v9.0.0\",\"Team:ML\",\"v8.16.0\",\"backport:version\"],\"title\":\"[ML]\nFix
overall bucket request for large number of
job\nIDs\",\"number\":198292,\"url\":\"https://github.com/elastic/kibana/pull/198292\",\"mergeCommit\":{\"message\":\"[ML]\nFix
overall bucket request for large number of job IDs
(#198292)\\n\\n##\nSummary\\r\\n\\r\\nFixes\nhttps://github.com//issues/190591\\r\\n\\r\\nAdds
chunking for\n`over_buckets` request to prevent exceeding
the\\r\\nrequest length\nURL.\\r\\n\\r\\n###
Checklist\\r\\n\\r\\n\\r\\n- [ ] [Unit
or\nfunctional\\r\\ntests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)\\r\\nwere\nupdated
or added to match the most
common\nscenarios\",\"sha\":\"fa0f397a4269a37370d2101a8c5ece03ad0117bd\"}},\"sourceBranch\":\"main\",\"suggestedTargetBranches\":[\"8.16\"],\"targetPullRequestStates\":[{\"branch\":\"main\",\"label\":\"v9.0.0\",\"branchLabelMappingKey\":\"^v9.0.0$\",\"isSourceBranch\":true,\"state\":\"MERGED\",\"url\":\"https://github.com/elastic/kibana/pull/198292\",\"number\":198292,\"mergeCommit\":{\"message\":\"[ML]\nFix
overall bucket request for large number of job IDs
(#198292)\\n\\n##\nSummary\\r\\n\\r\\nFixes\nhttps://github.com//issues/190591\\r\\n\\r\\nAdds
chunking for\n`over_buckets` request to prevent exceeding
the\\r\\nrequest length\nURL.\\r\\n\\r\\n###
Checklist\\r\\n\\r\\n\\r\\n- [ ] [Unit
or\nfunctional\\r\\ntests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)\\r\\nwere\nupdated
or added to match the most
common\nscenarios\",\"sha\":\"fa0f397a4269a37370d2101a8c5ece03ad0117bd\"}},{\"branch\":\"8.16\",\"label\":\"v8.16.0\",\"branchLabelMappingKey\":\"^v(\\\\d+).(\\\\d+).\\\\d+$\",\"isSourceBranch\":false,\"state\":\"NOT_CREATED\"}]}]\nBACKPORT-->\n\nCo-authored-by:
Dima Arnautov
<[email protected]>"}},{"branch":"8.x","label":"v8.17.0","branchLabelMappingKey":"^v8.17.0$","isSourceBranch":false,"state":"NOT_CREATED"}]}]
BACKPORT-->

Co-authored-by: Dima Arnautov <[email protected]>

x-pack/plugins/ml/public/application/services/ml_api_service/index.ts

@@ -11,7 +11,7 @@ import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
 
 import type { RuntimeMappings } from '@kbn/ml-runtime-field-utils';
 
-import { isNumber } from 'lodash';
+import { chunk, isNumber } from 'lodash';
 import { ML_INTERNAL_BASE_PATH } from '../../../../common/constants/app';
 import type {
   MlServerDefaults,
@@ -397,25 +397,45 @@ export function mlApiProvider(httpService: HttpService) {
       end,
       overallScore,
     }: {
-      jobId: string;
+      jobId: string[];
       topN: string;
       bucketSpan: string;
       start: number;
       end: number;
       overallScore?: number;
-    }) {
+    }): Promise<estypes.MlGetOverallBucketsResponse> {
       const body = JSON.stringify({
         topN,
         bucketSpan,
         start,
         end,
         ...(overallScore ? { overall_score: overallScore } : {}),
       });
-      return httpService.http<any>({
-        path: `${ML_INTERNAL_BASE_PATH}/anomaly_detectors/${jobId}/results/overall_buckets`,
-        method: 'POST',
-        body,
-        version: '1',
+
+      // Max permitted job_id is 64 characters, so we can fit around 30 jobs per request
+      const maxJobsPerRequest = 30;
+
+      return Promise.all(
+        chunk(jobId, maxJobsPerRequest).map((jobIdsChunk) => {
+          return httpService.http<estypes.MlGetOverallBucketsResponse>({
+            path: `${ML_INTERNAL_BASE_PATH}/anomaly_detectors/${jobIdsChunk.join(
+              ','
+            )}/results/overall_buckets`,
+            method: 'POST',
+            body,
+            version: '1',
+          });
+        })
+      ).then((responses) => {
+        // Merge responses
+        return responses.reduce<estypes.MlGetOverallBucketsResponse>(
+          (acc, response) => {
+            acc.count += response.count;
+            acc.overall_buckets.push(...response.overall_buckets);
+            return acc;
+          },
+          { count: 0, overall_buckets: [] }
+        );
       });
     },