[Fleet] Introduce Async deploy policies (elastic#191839)

ersin-erdal · Sep 9, 2024 · ea57fb0 · ea57fb0
1 parent 2c33e97
commit ea57fb0
Show file tree

Hide file tree

Showing 11 changed files with 1,078 additions and 58 deletions.
diff --git a/x-pack/plugins/fleet/common/experimental_features.ts b/x-pack/plugins/fleet/common/experimental_features.ts
@@ -27,6 +27,7 @@ const _allowedExperimentalValues = {
   advancedPolicySettings: true,
   useSpaceAwareness: false,
   enableReusableIntegrationPolicies: true,
+  asyncDeployPolicies: true,
 };
 
 /**

diff --git a/...s/fleet/public/applications/fleet/sections/settings/services/agent_and_policies_count.tsx b/...s/fleet/public/applications/fleet/sections/settings/services/agent_and_policies_count.tsx
@@ -25,6 +25,7 @@ export async function getAgentAndPolicyCountForOutput(output: Output) {
     kuery: agentPolicyKuery,
     page: 1,
     perPage: SO_SEARCH_LIMIT,
+    noAgentCount: true,
   });
 
   if (agentPolicies.error) {

diff --git a/x-pack/plugins/fleet/scripts/create_agent_policies/create_agent_policies.ts b/x-pack/plugins/fleet/scripts/create_agent_policies/create_agent_policies.ts
@@ -7,9 +7,13 @@
 
 import { ToolingLog } from '@kbn/tooling-log';
 import yargs from 'yargs';
+import { chunk } from 'lodash';
 
+import { LEGACY_PACKAGE_POLICY_SAVED_OBJECT_TYPE } from '../../common/constants';
 import { LEGACY_AGENT_POLICY_SAVED_OBJECT_TYPE } from '../../common';
 
+import { packagePolicyFixture } from './fixtures';
+
 const logger = new ToolingLog({
   level: 'info',
   writeTo: process.stdout,
@@ -26,13 +30,17 @@ const printUsage = () =>
 
 const INDEX_BULK_OP = '{ "index":{ "_id": "{{id}}" } }\n';
 
-async function createAgentPoliciesDocsBulk(size: number) {
+function getPolicyId(idx: number | string) {
+  return `test-policy-${idx}`;
+}
+
+async function createAgentPoliciesDocsBulk(range: number[]) {
   const auth = 'Basic ' + Buffer.from(ES_SUPERUSER + ':' + ES_PASSWORD).toString('base64');
-  const body = Array.from({ length: size }, (_, index) => index + 1)
+  const body = range
     .flatMap((idx) => [
       INDEX_BULK_OP.replace(
         /{{id}}/,
-        `${LEGACY_AGENT_POLICY_SAVED_OBJECT_TYPE}:test-policy-${idx}`
+        `${LEGACY_AGENT_POLICY_SAVED_OBJECT_TYPE}:${getPolicyId(idx)}`
       ),
       JSON.stringify({
         [LEGACY_AGENT_POLICY_SAVED_OBJECT_TYPE]: {
@@ -79,17 +87,17 @@ async function createAgentPoliciesDocsBulk(size: number) {
   return data;
 }
 
-async function createEnrollmentToken(size: number) {
+async function createEnrollmentToken(range: number[]) {
   const auth = 'Basic ' + Buffer.from(ES_SUPERUSER + ':' + ES_PASSWORD).toString('base64');
-  const body = Array.from({ length: size }, (_, index) => index + 1)
+  const body = range
     .flatMap((idx) => [
       INDEX_BULK_OP.replace(/{{id}}/, `test-enrollment-token-${idx}`),
       JSON.stringify({
         active: true,
         api_key_id: 'faketest123',
         api_key: 'test==',
         name: `Test Policy ${idx}`,
-        policy_id: `${LEGACY_AGENT_POLICY_SAVED_OBJECT_TYPE}:test-policy-${idx}`,
+        policy_id: `${getPolicyId(idx)}`,
         namespaces: [],
         created_at: new Date().toISOString(),
       }) + '\n',
@@ -113,6 +121,41 @@ async function createEnrollmentToken(size: number) {
   return data;
 }
 
+async function createPackagePolicies(range: number[]) {
+  const auth = 'Basic ' + Buffer.from(ES_SUPERUSER + ':' + ES_PASSWORD).toString('base64');
+  const body = range
+    .flatMap((idx) => [
+      INDEX_BULK_OP.replace(
+        /{{id}}/,
+        `${LEGACY_PACKAGE_POLICY_SAVED_OBJECT_TYPE}:test-policy-${idx}`
+      ),
+      JSON.stringify(
+        packagePolicyFixture({
+          idx,
+          agentPolicyId: getPolicyId(idx),
+        })
+      ) + '\n',
+    ])
+    .join('');
+
+  const res = await fetch(`${ES_URL}/.kibana_ingest/_bulk`, {
+    method: 'post',
+    body,
+    headers: {
+      Authorization: auth,
+      'Content-Type': 'application/x-ndjson',
+    },
+  });
+
+  const data = await res.json();
+
+  if (!data.items) {
+    logger.error('Error creating agent policies docs: ' + JSON.stringify(data));
+    process.exit(1);
+  }
+  return data;
+}
+
 export async function run() {
   const {
     size: sizeArg = 500,
@@ -129,6 +172,15 @@ export async function run() {
 
   const size = Number(sizeArg).valueOf();
   logger.info(`Creating ${size} policies`);
-  await Promise.all([createAgentPoliciesDocsBulk(size), createEnrollmentToken(size)]);
+
+  const range = Array.from({ length: size }, (_ignore, index) => index + 1);
+
+  for (const rangePart of chunk(range, 200)) {
+    await Promise.all([
+      createAgentPoliciesDocsBulk(rangePart),
+      createEnrollmentToken(rangePart),
+      createPackagePolicies(rangePart),
+    ]);
+  }
   logger.info(`Succesfuly created ${size} policies`);
 }