[Fleet] Add new keep_monitoring_alive flag to agent policies (elastic…

…#168865) ## Summary Ref elastic#168629 Adds a new `keep_monitoring_alive` flag to agent policies that allows agent environments to keep the monitoring server if they choose. > [!note] > `monitoring_enabled: []` needs to be explicitly set for this flag to be honored. If `monitoring_enabled` is omitted entirely, Fleet defaults to enabling monitoring for logs and metrics. ## To test Create a preconfigured policy where logs/metrics monitoring is explicitly disabled and include this new flag e.g. ```yml xpack.fleet.agentPolicies: - name: Test preconfigured policy id: test-preconfigured-policy package_policies: [] monitoring_enabled: [] keep_monitoring_alive: true ``` Then, run Kibana and verify the `monitoring` block in the full agent policy has monitoring enabled, but logs/metrics as disabled, e.g. ![image](https://github.com/elastic/kibana/assets/6766512/f5b205ad-96f0-4e88-967a-43546bf2bf84) You can also verify the need for `monitoring_enabled: []` by creating another preconfigured policy e.g. ```yml - name: Test preconfigured policy 2 id: test-preconfigured-policy-2 package_policies: [] keep_monitoring_alive: true ``` This will generate a policy e.g. ![image](https://github.com/elastic/kibana/assets/6766512/013b0381-b605-49a4-8819-3dec713f1e1f) All other combinations of enabling monitoring for logs/metrics should also generate the policies as expected. If you want to verify every combination manually: ```yml xpack.fleet.agentPolicies: - name: Test preconfigured policy id: test-preconfigured-policy package_policies: [] monitoring_enabled: [] keep_monitoring_alive: true - name: Test preconfigured policy 2 id: test-preconfigured-policy-2 package_policies: [] keep_monitoring_alive: true - name: Test preconfigured policy 3 id: test-preconfigured-policy-3 package_policies: [] monitoring_enabled: ['logs'] - name: Test preconfigured policy 4 id: test-preconfigured-policy-4 package_policies: [] monitoring_enabled: ['metrics'] - name: Test preconfigured policy 5 id: test-preconfigured-policy-5 package_policies: [] monitoring_enabled: ['logs', 'metrics'] ``` --------- Co-authored-by: kibanamachine <[email protected]>
Ruhshan · Oct 16, 2023 · eef222f · eef222f
1 parent 9080836
commit eef222f
Show file tree

Hide file tree

Showing 11 changed files with 182 additions and 119 deletions.
diff --git a/docs/settings/fleet-settings.asciidoc b/docs/settings/fleet-settings.asciidoc
@@ -95,6 +95,8 @@ List of agent policies that are configured when the {fleet} app starts.
     String identifying this policy's namespace.
   `monitoring_enabled`::: 
     List of keywords that specify the monitoring data to collect. Valid values include `['logs']`, `['metrics']`, and `['logs', 'metrics']`.
+  `keep_monitoring_alive`:::
+    If `true`, monitoring will be enabled, but logs/metrics collection will be disabled. Use this if you want to keep agent's monitoring server alive even when logs/metrics aren't being collected.
   `is_managed`::: 
     If `true`, this policy is not editable by the user and can only be changed by updating the {kib} config.
   `is_default`::: 

diff --git a/packages/kbn-check-mappings-update-cli/current_mappings.json b/packages/kbn-check-mappings-update-cli/current_mappings.json
@@ -1645,6 +1645,9 @@
       "overrides": {
         "type": "flattened",
         "index": false
+      },
+      "keep_monitoring_alive": {
+        "type": "boolean"
       }
     }
   },
@@ -2594,6 +2597,106 @@
     "dynamic": false,
     "properties": {}
   },
+  "infrastructure-ui-source": {
+    "dynamic": false,
+    "properties": {}
+  },
+  "inventory-view": {
+    "dynamic": false,
+    "properties": {}
+  },
+  "metrics-explorer-view": {
+    "dynamic": false,
+    "properties": {}
+  },
+  "upgrade-assistant-reindex-operation": {
+    "dynamic": false,
+    "properties": {
+      "indexName": {
+        "type": "keyword"
+      },
+      "status": {
+        "type": "integer"
+      }
+    }
+  },
+  "upgrade-assistant-ml-upgrade-operation": {
+    "dynamic": false,
+    "properties": {
+      "snapshotId": {
+        "type": "text",
+        "fields": {
+          "keyword": {
+            "type": "keyword",
+            "ignore_above": 256
+          }
+        }
+      }
+    }
+  },
+  "monitoring-telemetry": {
+    "properties": {
+      "reportedClusterUuids": {
+        "type": "keyword"
+      }
+    }
+  },
+  "apm-telemetry": {
+    "dynamic": false,
+    "properties": {}
+  },
+  "apm-server-schema": {
+    "properties": {
+      "schemaJson": {
+        "type": "text",
+        "index": false
+      }
+    }
+  },
+  "apm-service-group": {
+    "properties": {
+      "groupName": {
+        "type": "keyword"
+      },
+      "kuery": {
+        "type": "text"
+      },
+      "description": {
+        "type": "text"
+      },
+      "color": {
+        "type": "text"
+      }
+    }
+  },
+  "apm-custom-dashboards": {
+    "properties": {
+      "dashboardSavedObjectId": {
+        "type": "keyword"
+      },
+      "kuery": {
+        "type": "text"
+      },
+      "serviceEnvironmentFilterEnabled": {
+        "type": "boolean"
+      },
+      "serviceNameFilterEnabled": {
+        "type": "boolean"
+      }
+    }
+  },
+  "enterprise_search_telemetry": {
+    "dynamic": false,
+    "properties": {}
+  },
+  "app_search_telemetry": {
+    "dynamic": false,
+    "properties": {}
+  },
+  "workplace_search_telemetry": {
+    "dynamic": false,
+    "properties": {}
+  },
   "siem-ui-timeline-note": {
     "properties": {
       "eventId": {
@@ -3055,105 +3158,5 @@
         "index": false
       }
     }
-  },
-  "infrastructure-ui-source": {
-    "dynamic": false,
-    "properties": {}
-  },
-  "inventory-view": {
-    "dynamic": false,
-    "properties": {}
-  },
-  "metrics-explorer-view": {
-    "dynamic": false,
-    "properties": {}
-  },
-  "upgrade-assistant-reindex-operation": {
-    "dynamic": false,
-    "properties": {
-      "indexName": {
-        "type": "keyword"
-      },
-      "status": {
-        "type": "integer"
-      }
-    }
-  },
-  "upgrade-assistant-ml-upgrade-operation": {
-    "dynamic": false,
-    "properties": {
-      "snapshotId": {
-        "type": "text",
-        "fields": {
-          "keyword": {
-            "type": "keyword",
-            "ignore_above": 256
-          }
-        }
-      }
-    }
-  },
-  "monitoring-telemetry": {
-    "properties": {
-      "reportedClusterUuids": {
-        "type": "keyword"
-      }
-    }
-  },
-  "apm-telemetry": {
-    "dynamic": false,
-    "properties": {}
-  },
-  "apm-server-schema": {
-    "properties": {
-      "schemaJson": {
-        "type": "text",
-        "index": false
-      }
-    }
-  },
-  "apm-service-group": {
-    "properties": {
-      "groupName": {
-        "type": "keyword"
-      },
-      "kuery": {
-        "type": "text"
-      },
-      "description": {
-        "type": "text"
-      },
-      "color": {
-        "type": "text"
-      }
-    }
-  },
-  "apm-custom-dashboards": {
-    "properties": {
-      "dashboardSavedObjectId": {
-        "type": "keyword"
-      },
-      "kuery": {
-        "type": "text"
-      },
-      "serviceEnvironmentFilterEnabled": {
-        "type": "boolean"
-      },
-      "serviceNameFilterEnabled": {
-        "type": "boolean"
-      }
-    }
-  },
-  "enterprise_search_telemetry": {
-    "dynamic": false,
-    "properties": {}
-  },
-  "app_search_telemetry": {
-    "dynamic": false,
-    "properties": {}
-  },
-  "workplace_search_telemetry": {
-    "dynamic": false,
-    "properties": {}
   }
 }
diff --git a/...e/server/integration_tests/saved_objects/migrations/group2/check_registered_types.test.ts b/...e/server/integration_tests/saved_objects/migrations/group2/check_registered_types.test.ts
@@ -104,7 +104,7 @@ describe('checking migration metadata changes on all registered SO types', () =>
         "index-pattern": "997108a9ea1e8076e22231e1c95517cdb192b9c5",
         "infrastructure-monitoring-log-view": "5f86709d3c27aed7a8379153b08ee5d3d90d77f5",
         "infrastructure-ui-source": "113182d6895764378dfe7fa9fa027244f3a457c4",
-        "ingest-agent-policies": "f11cc19275f4c3e4ee7c5cd6423b6706b21b989d",
+        "ingest-agent-policies": "7633e578f60c074f8267bc50ec4763845e431437",
         "ingest-download-sources": "279a68147e62e4d8858c09ad1cf03bd5551ce58d",
         "ingest-outputs": "b4e636b13a5d0f89f0400fb67811d4cca4736eb0",
         "ingest-package-policies": "a0c9fb48e04dcd638e593db55f1c6451523f90ea",

diff --git a/x-pack/plugins/fleet/common/openapi/bundled.json b/x-pack/plugins/fleet/common/openapi/bundled.json
@@ -7036,6 +7036,11 @@
               ]
             }
           },
+          "keep_monitoring_alive": {
+            "description": "When set to true, monitoring will be enabled but logs/metrics collection will be disabled",
+            "type": "boolean",
+            "nullable": true
+          },
           "data_output_id": {
             "type": "string",
             "nullable": true

diff --git a/x-pack/plugins/fleet/common/openapi/bundled.yaml b/x-pack/plugins/fleet/common/openapi/bundled.yaml
@@ -4491,6 +4491,12 @@ components:
             enum:
               - metrics
               - logs
+        keep_monitoring_alive:
+          description: >-
+            When set to true, monitoring will be enabled but logs/metrics
+            collection will be disabled
+          type: boolean
+          nullable: true
         data_output_id:
           type: string
           nullable: true

diff --git a/x-pack/plugins/fleet/common/openapi/components/schemas/agent_policy.yaml b/x-pack/plugins/fleet/common/openapi/components/schemas/agent_policy.yaml
@@ -16,6 +16,10 @@ properties:
       enum:
         - metrics
         - logs
+  keep_monitoring_alive:
+    description: When set to true, monitoring will be enabled but logs/metrics collection will be disabled
+    type: boolean
+    nullable: true
   data_output_id:
     type: string
     nullable: true

diff --git a/x-pack/plugins/fleet/common/types/models/agent_policy.ts b/x-pack/plugins/fleet/common/types/models/agent_policy.ts
@@ -52,6 +52,7 @@ export interface AgentPolicy extends Omit<NewAgentPolicy, 'id'> {
   revision: number;
   agents?: number;
   is_protected: boolean;
+  keep_monitoring_alive?: boolean;
 }
 
 export interface FullAgentPolicyInputStream {
@@ -87,6 +88,14 @@ export type FullAgentPolicyOutput = Pick<Output, 'type' | 'hosts' | 'ca_sha256'>
   [key: string]: any;
 };
 
+export interface FullAgentPolicyMonitoring {
+  namespace?: string;
+  use_output?: string;
+  enabled: boolean;
+  metrics: boolean;
+  logs: boolean;
+}
+
 export interface FullAgentPolicy {
   id: string;
   outputs: {
@@ -103,13 +112,7 @@ export interface FullAgentPolicy {
   inputs: FullAgentPolicyInput[];
   revision?: number;
   agent?: {
-    monitoring: {
-      namespace?: string;
-      use_output?: string;
-      enabled: boolean;
-      metrics: boolean;
-      logs: boolean;
-    };
+    monitoring: FullAgentPolicyMonitoring;
     download: { sourceURI: string };
     features: Record<string, { enabled: boolean }>;
     protection?: {

diff --git a/x-pack/plugins/fleet/server/saved_objects/index.ts b/x-pack/plugins/fleet/server/saved_objects/index.ts
@@ -147,6 +147,7 @@ const getSavedObjectTypes = (): { [key: string]: SavedObjectsType } => ({
         },
         is_protected: { type: 'boolean' },
         overrides: { type: 'flattened', index: false },
+        keep_monitoring_alive: { type: 'boolean' },
       },
     },
     migrations: {

diff --git a/x-pack/plugins/fleet/server/services/agent_policies/full_agent_policy.test.ts b/x-pack/plugins/fleet/server/services/agent_policies/full_agent_policy.test.ts
@@ -283,6 +283,20 @@ describe('getFullAgentPolicy', () => {
     });
   });
 
+  it('should return a policy with monitoring enabled but no logs/metrics if keep_monitoring_alive is true', async () => {
+    mockAgentPolicy({
+      keep_monitoring_alive: true,
+    });
+
+    const agentPolicy = await getFullAgentPolicy(savedObjectsClientMock.create(), 'agent-policy');
+
+    expect(agentPolicy?.agent?.monitoring).toEqual({
+      enabled: true,
+      logs: false,
+      metrics: false,
+    });
+  });
+
   it('should get the permissions for monitoring', async () => {
     mockAgentPolicy({
       namespace: 'testnamespace',