From d38c9761087402c23f53e0f004d5838a02e2f292 Mon Sep 17 00:00:00 2001 From: maclin-masterson Date: Tue, 21 Jun 2022 11:04:29 -0400 Subject: [PATCH 01/12] Add module to set fleet urls Adding a new module, kibana_fleet_host, that will allow a user to set server urls and elasticsearch urls for a given kibana fleet. --- plugins/module_utils/kibana.py | 27 +++++ plugins/modules/kibana_fleet_host.py | 141 +++++++++++++++++++++++++++ 2 files changed, 168 insertions(+) create mode 100644 plugins/modules/kibana_fleet_host.py diff --git a/plugins/module_utils/kibana.py b/plugins/module_utils/kibana.py index af3cb95..4de565a 100644 --- a/plugins/module_utils/kibana.py +++ b/plugins/module_utils/kibana.py @@ -552,3 +552,30 @@ def get_agent_list(self): agent_no = agent_no + 1 page_number = page_number + 1 return agent_list_result + + def set_fleet_server_host(self, host): + endpoint = 'fleet/settings' + headers = {'kbn-xsrf': True} + body = { + 'fleet_server_hosts': [ + host + ] + } + + body_json = dumps(body) + + result = self.send_api_request(endpoint, 'PUT', headers=headers, data=body_json) + return result + + def set_fleet_elasticsearch_host(self, host): + endpoint = 'fleet/outputs/fleet-default-output' + body = { + 'hosts': [ + host + ] + } + + body_json = dumps(body) + + result = self.send_api_request(endpoint, 'PUT', data=body_json) + return result diff --git a/plugins/modules/kibana_fleet_host.py b/plugins/modules/kibana_fleet_host.py new file mode 100644 index 0000000..071e40b --- /dev/null +++ b/plugins/modules/kibana_fleet_host.py @@ -0,0 +1,141 @@ +#!/usr/bin/python +# Copyright 2021 Expedient +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# -*- coding: utf-8 -*- + +#from plugins.modules.ece_cluster import DOCUMENTATION + + +ANSIBLE_METADATA = { + 'metadata_version': '1.1', + 'status': ['preview'], + 'supported_by': 'community' +} + +DOCUMENTATION = r''' +--- +module: kibana_fleet_host + +short_description: Add host to kibana fleet + +version_added: 2.3.0 # change this + +author: Mac Masterson (@mgaruccio) + +requirements: + - python3 + +description: + - "This module adds a host endpoint to a kibana fleet" + +options: + host: + description: + - The Kibana Host you're updating + type: str + username: + description: + - Elastic Username + type: str + password: + description: + - Elastic Password + type: str + verify_ssl_cert: + description: + - Whether or not to verify SSL cert on API requests + type: bool + url: + description: + - The url that you want to apply as a fleet server host or an elasticsearch host + type: str + url_type: + description: + - The url type that you want to set for the fleet + - 'server' sets the fleet server host + - 'elasticsearch' sets the fleet elasticsearch host + options: + - server + - elasticsearch + +extends_documentation_fragment: + - expedient.elastic.elastic_auth_options.documentation +''' + + +try: + from ansible_collections.expedient.elastic.plugins.module_utils.kibana import Kibana + import ansible_collections.expedient.elastic.plugins.module_utils.lookups +except: + import sys + import os + util_path = new_path = f'{os.getcwd()}/plugins/module_utils' + sys.path.append(util_path) + from kibana import Kibana + +from ansible.module_utils.basic import AnsibleModule +from json import dumps + +def main(): + module_args=dict( + host=dict(type='str', required=True), + port=dict(type='int', required=True), + username=dict(type='str', required=True), + password=dict(type='str', required=True, no_log=True), + verify_ssl_cert=dict(type='bool', default=True), + url_type=dict(type='str', choices=['server', 'elasticsearch'], required=True), + url=dict(type='str', required=True), + ) + + results = { + 'changed': False, + 'msg': '' + } + + module = AnsibleModule(argument_spec=module_args, supports_check_mode=True) + kibana_fleet = Kibana(module) + + url = module.params.get('url') + url_type = module.params.get('url_type') + + if url_type == 'server': + current_fleet_servers = kibana_fleet.get_fleet_server_hosts() + if url in current_fleet_servers: + results['msg'] += f"\n{url} already exists in the fleet" + else: + fleet_server_add = kibana_fleet.set_fleet_server_host(url) + if 'message' in fleet_server_add: + module.fail_json(msg=fleet_server_add['message']) + else: + results['changed'] = True + results['msg'] += f"Successfully set fleet server host to {url}" + + if url_type == 'elasticsearch': + current_elasticsearch_hosts = kibana_fleet.get_fleet_elasticsearch_hosts() + if url in current_elasticsearch_hosts: + results['msg'] += f"\n{url} already exists in the fleet" + else: + elasticsearch_host_add = kibana_fleet.set_fleet_elasticsearch_host(url) + if 'message' in elasticsearch_host_add: + module.fail_json(msg=elasticsearch_host_add['message']) + else: + results['changed'] = True + results['msg'] += f"\nSuccessfully set fleet elasticsearch server host to {url}" + + module.exit_json(**results) + + +if __name__ == '__main__': + main() From baa6653aa2db3bfa21c74bcf40d717fb01915855 Mon Sep 17 00:00:00 2001 From: maclin-masterson Date: Tue, 21 Jun 2022 16:21:53 -0400 Subject: [PATCH 02/12] Allow headers to be passed to send_api_request() Allow headers to be passed to send_api_request in module_utils/kibana.py. The reasoning for this is that some fleet api calls require a kbn-xsrf header --- plugins/module_utils/kibana.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/module_utils/kibana.py b/plugins/module_utils/kibana.py index 4de565a..22701de 100644 --- a/plugins/module_utils/kibana.py +++ b/plugins/module_utils/kibana.py @@ -37,7 +37,7 @@ def __init__(self, module): self.version = None # this is a hack to make it so that we can run the first request to get the clutser version without erroring out self.version = self.get_cluster_version() - def send_api_request(self, endpoint, method, data=None): + def send_api_request(self, endpoint, method, data=None, headers={}): url = f'https://{self.host}:{self.port}/api/{endpoint}' headers = {} payload = None From 5a83cd32bc1c3e0e1e62287ab7bb4e08ca0b5acb Mon Sep 17 00:00:00 2001 From: maclin-masterson Date: Wed, 22 Jun 2022 10:41:27 -0400 Subject: [PATCH 03/12] Add get_fleet_serveer_hosts() to kibana.py Adding a function to grab a list of current fleet server hosts --- plugins/module_utils/kibana.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/plugins/module_utils/kibana.py b/plugins/module_utils/kibana.py index 22701de..b7110dc 100644 --- a/plugins/module_utils/kibana.py +++ b/plugins/module_utils/kibana.py @@ -553,6 +553,11 @@ def get_agent_list(self): page_number = page_number + 1 return agent_list_result + def get_fleet_server_hosts(self): + endpoint = 'fleet/settings' + result = self.send_api_request(endpoint, 'GET') + return result['item']['fleet_server_hosts'] + def set_fleet_server_host(self, host): endpoint = 'fleet/settings' headers = {'kbn-xsrf': True} From 4b2e801d71bf9a3eefeecff69da46193f773f44c Mon Sep 17 00:00:00 2001 From: maclin-masterson Date: Wed, 22 Jun 2022 10:50:36 -0400 Subject: [PATCH 04/12] add get_fleet_elasticsearch_hosts() to kibana.py Add a function to get current elasticsearch hosts for a given fleet --- plugins/module_utils/kibana.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/plugins/module_utils/kibana.py b/plugins/module_utils/kibana.py index b7110dc..d385451 100644 --- a/plugins/module_utils/kibana.py +++ b/plugins/module_utils/kibana.py @@ -572,6 +572,13 @@ def set_fleet_server_host(self, host): result = self.send_api_request(endpoint, 'PUT', headers=headers, data=body_json) return result + def get_fleet_elasticsearch_hosts(self): + endpoint = 'fleet/outputs' + result = self.send_api_request(endpoint, 'GET') + for item in result['items']: + if item['id'] == "fleet-default-output" and item['type'] == 'elasticsearch': + return item['hosts'] + def set_fleet_elasticsearch_host(self, host): endpoint = 'fleet/outputs/fleet-default-output' body = { From 27b8bf9eefd706eede063b45814c4ef1f0e303ce Mon Sep 17 00:00:00 2001 From: maclin-masterson Date: Tue, 21 Jun 2022 16:29:24 -0400 Subject: [PATCH 05/12] bump galaxy version --- galaxy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/galaxy.yml b/galaxy.yml index f3d6506..ae5e95a 100644 --- a/galaxy.yml +++ b/galaxy.yml @@ -13,7 +13,7 @@ # limitations under the License. --- -version: 2.2.2 +version: 2.3.0 namespace: expedient name: elastic readme: README.md From 32a667444551aa4cb62df15f9bb0ad9311d806f4 Mon Sep 17 00:00:00 2001 From: maclin-masterson Date: Wed, 22 Jun 2022 15:45:07 -0400 Subject: [PATCH 06/12] Remove headers declaration from send_api_request() Since we are taking headers as an argument, we no longer need to define it in the function --- plugins/module_utils/kibana.py | 1 - 1 file changed, 1 deletion(-) diff --git a/plugins/module_utils/kibana.py b/plugins/module_utils/kibana.py index d385451..6dc9aff 100644 --- a/plugins/module_utils/kibana.py +++ b/plugins/module_utils/kibana.py @@ -39,7 +39,6 @@ def __init__(self, module): def send_api_request(self, endpoint, method, data=None, headers={}): url = f'https://{self.host}:{self.port}/api/{endpoint}' - headers = {} payload = None if data: headers['Content-Type'] = 'application/json' From 584c536dfc36325433c27d47a71f004eb0e3e678 Mon Sep 17 00:00:00 2001 From: maclin-masterson Date: Wed, 22 Jun 2022 15:46:02 -0400 Subject: [PATCH 07/12] Fix author user id in kibana_fleet_host docs --- plugins/modules/kibana_fleet_host.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/modules/kibana_fleet_host.py b/plugins/modules/kibana_fleet_host.py index 071e40b..6686766 100644 --- a/plugins/modules/kibana_fleet_host.py +++ b/plugins/modules/kibana_fleet_host.py @@ -32,7 +32,7 @@ version_added: 2.3.0 # change this -author: Mac Masterson (@mgaruccio) +author: Mac Masterson (@maclin-masterson) requirements: - python3 From 074c3aed1114e15c9d938a3a12d4638d0814a7f2 Mon Sep 17 00:00:00 2001 From: maclin-masterson Date: Wed, 22 Jun 2022 15:50:55 -0400 Subject: [PATCH 08/12] change choice value for url_type changing from 'server' to 'fleet_server' for better readability --- plugins/modules/kibana_fleet_host.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/modules/kibana_fleet_host.py b/plugins/modules/kibana_fleet_host.py index 6686766..64855b5 100644 --- a/plugins/modules/kibana_fleet_host.py +++ b/plugins/modules/kibana_fleet_host.py @@ -95,7 +95,7 @@ def main(): username=dict(type='str', required=True), password=dict(type='str', required=True, no_log=True), verify_ssl_cert=dict(type='bool', default=True), - url_type=dict(type='str', choices=['server', 'elasticsearch'], required=True), + url_type=dict(type='str', choices=['fleet_server', 'elasticsearch'], required=True), url=dict(type='str', required=True), ) @@ -110,7 +110,7 @@ def main(): url = module.params.get('url') url_type = module.params.get('url_type') - if url_type == 'server': + if url_type == 'fleet_server': current_fleet_servers = kibana_fleet.get_fleet_server_hosts() if url in current_fleet_servers: results['msg'] += f"\n{url} already exists in the fleet" From 805d49730b83591f22bf5bde6f28cd6fcd6e8e07 Mon Sep 17 00:00:00 2001 From: maclin-masterson Date: Thu, 23 Jun 2022 11:36:46 -0400 Subject: [PATCH 09/12] make urls parameter a list for kibana_fleet_host --- plugins/module_utils/kibana.py | 12 ++++-------- plugins/modules/kibana_fleet_host.py | 3 ++- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/plugins/module_utils/kibana.py b/plugins/module_utils/kibana.py index 6dc9aff..2bdcf13 100644 --- a/plugins/module_utils/kibana.py +++ b/plugins/module_utils/kibana.py @@ -557,13 +557,11 @@ def get_fleet_server_hosts(self): result = self.send_api_request(endpoint, 'GET') return result['item']['fleet_server_hosts'] - def set_fleet_server_host(self, host): + def set_fleet_server_hosts(self, hosts: list): endpoint = 'fleet/settings' headers = {'kbn-xsrf': True} body = { - 'fleet_server_hosts': [ - host - ] + 'fleet_server_hosts': hosts } body_json = dumps(body) @@ -578,12 +576,10 @@ def get_fleet_elasticsearch_hosts(self): if item['id'] == "fleet-default-output" and item['type'] == 'elasticsearch': return item['hosts'] - def set_fleet_elasticsearch_host(self, host): + def set_fleet_elasticsearch_hosts(self, hosts: list): endpoint = 'fleet/outputs/fleet-default-output' body = { - 'hosts': [ - host - ] + 'hosts': hosts } body_json = dumps(body) diff --git a/plugins/modules/kibana_fleet_host.py b/plugins/modules/kibana_fleet_host.py index 64855b5..54d6b00 100644 --- a/plugins/modules/kibana_fleet_host.py +++ b/plugins/modules/kibana_fleet_host.py @@ -96,7 +96,8 @@ def main(): password=dict(type='str', required=True, no_log=True), verify_ssl_cert=dict(type='bool', default=True), url_type=dict(type='str', choices=['fleet_server', 'elasticsearch'], required=True), - url=dict(type='str', required=True), + urls=dict(type='list', elements='str', required=True), + action=dict(type='str', choices=['add', 'overwrite', 'remove'], default='add') ) results = { From fee88aeb480a5fd693716d05e8245096c84e4f90 Mon Sep 17 00:00:00 2001 From: maclin-masterson Date: Thu, 23 Jun 2022 11:37:19 -0400 Subject: [PATCH 10/12] Add KibanaFleet object to kibana_fleet_host module Adding a KibanaFleet object to handle some common operations in the module flow --- plugins/modules/kibana_fleet_host.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/plugins/modules/kibana_fleet_host.py b/plugins/modules/kibana_fleet_host.py index 54d6b00..d298a9f 100644 --- a/plugins/modules/kibana_fleet_host.py +++ b/plugins/modules/kibana_fleet_host.py @@ -88,6 +88,28 @@ from ansible.module_utils.basic import AnsibleModule from json import dumps +class KibanaFleet(Kibana): + def __init__(self, module): + super().__init__(module) + self.url_type = self.module.params.get('url_type') + + def get_current_urls(self): + if self.url_type == 'fleet_server': + current_urls = self.get_fleet_server_hosts() + if self.url_type == 'elasticsearch': + current_urls = self.get_fleet_elasticsearch_hosts() + return current_urls + + def send_urls(self, urls: list): + if self.url_type == 'fleet_server': + result = self.set_fleet_server_hosts(urls) + + if self.url_type == 'elasticsearch': + result = self.set_fleet_elasticsearch_hosts(urls) + + return result + + def main(): module_args=dict( host=dict(type='str', required=True), From b63b9d18be106f05ed43df62cf7ec0b4e7be37a9 Mon Sep 17 00:00:00 2001 From: maclin-masterson Date: Thu, 23 Jun 2022 11:38:31 -0400 Subject: [PATCH 11/12] Handle different action types in kibana_fleet_host Take an action from the user and manipulate the fleet data accordingly. Available actions are add, overwrite and remove. Add will compare the provided urls to the curren urls in kibana, and add them if they don't exist Remove will iterate over the current urls in kibana and omit them if they exist in the provided urls. Overwrite will ignore the current urls, and push the provided urls to the kibana fleet. --- plugins/modules/kibana_fleet_host.py | 58 ++++++++++++++++------------ 1 file changed, 34 insertions(+), 24 deletions(-) diff --git a/plugins/modules/kibana_fleet_host.py b/plugins/modules/kibana_fleet_host.py index d298a9f..80b4b47 100644 --- a/plugins/modules/kibana_fleet_host.py +++ b/plugins/modules/kibana_fleet_host.py @@ -128,34 +128,44 @@ def main(): } module = AnsibleModule(argument_spec=module_args, supports_check_mode=True) - kibana_fleet = Kibana(module) + kibana_fleet = KibanaFleet(module) + action = module.params.get('action') + current_urls = kibana_fleet.get_current_urls() # The urls as they exist in kibana at start + provided_urls = module.params.get('urls') # Urls provided by the user + + # final_urls is a list that gets calculated depending on the provided action. + final_urls = [] + + if action == 'add': + final_urls.extend(current_urls) + for item in provided_urls: + if item in current_urls: + results['msg'] += f"\n{item} already exists in Kibana" + else: + final_urls.append(item) - url = module.params.get('url') - url_type = module.params.get('url_type') + if action == 'overwrite': + final_urls.extend(provided_urls) - if url_type == 'fleet_server': - current_fleet_servers = kibana_fleet.get_fleet_server_hosts() - if url in current_fleet_servers: - results['msg'] += f"\n{url} already exists in the fleet" - else: - fleet_server_add = kibana_fleet.set_fleet_server_host(url) - if 'message' in fleet_server_add: - module.fail_json(msg=fleet_server_add['message']) - else: - results['changed'] = True - results['msg'] += f"Successfully set fleet server host to {url}" + if action == 'remove': + for item in current_urls: + if item in provided_urls: + continue - if url_type == 'elasticsearch': - current_elasticsearch_hosts = kibana_fleet.get_fleet_elasticsearch_hosts() - if url in current_elasticsearch_hosts: - results['msg'] += f"\n{url} already exists in the fleet" + final_urls.append(item) + + # Converting lists to sets for comparison + if set(current_urls) == set(final_urls): + results['msg'] += "\n No action needed" + else: + send_url_result = kibana_fleet.send_urls(final_urls) + if 'message' in send_url_result: + module.fail_json(f"Unable to {action} urls. Error: {send_url_result['message']}") else: - elasticsearch_host_add = kibana_fleet.set_fleet_elasticsearch_host(url) - if 'message' in elasticsearch_host_add: - module.fail_json(msg=elasticsearch_host_add['message']) - else: - results['changed'] = True - results['msg'] += f"\nSuccessfully set fleet elasticsearch server host to {url}" + results['changed'] = True + results['msg'] += f"\nSuccessful {action}" + results['fleet_server_urls'] = kibana_fleet.get_fleet_server_hosts() + results['fleet_elasticsearch_urls'] = kibana_fleet.get_fleet_elasticsearch_hosts() module.exit_json(**results) From 972c9d165a9671f86b1e9fcc2faafc63a0cd967d Mon Sep 17 00:00:00 2001 From: maclin-masterson Date: Thu, 23 Jun 2022 13:51:26 -0400 Subject: [PATCH 12/12] documentation update --- plugins/modules/kibana_fleet_host.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/plugins/modules/kibana_fleet_host.py b/plugins/modules/kibana_fleet_host.py index 80b4b47..73f2033 100644 --- a/plugins/modules/kibana_fleet_host.py +++ b/plugins/modules/kibana_fleet_host.py @@ -57,18 +57,29 @@ description: - Whether or not to verify SSL cert on API requests type: bool - url: + urls: description: - - The url that you want to apply as a fleet server host or an elasticsearch host - type: str + - List of urls that you want to apply as a fleet server host or an elasticsearch host + type: list + element type: str url_type: description: - The url type that you want to set for the fleet - 'server' sets the fleet server host - 'elasticsearch' sets the fleet elasticsearch host options: - - server + - fleet_server - elasticsearch + action: + description: + - The action that you want the module to take against the fleet server + - Add: Add the provided urls to the fleet + - Remove: Remove the provided urls from the fleet + - Overwrite: Replace the urls in the fleet with the provided urls + options: + - Add + - Remove + - Overwrite extends_documentation_fragment: - expedient.elastic.elastic_auth_options.documentation