From 5819a2856e7b99756f7acadc2835d18040ce10f5 Mon Sep 17 00:00:00 2001 From: Nikolaos Kakouros Date: Wed, 7 Apr 2021 12:59:31 +0200 Subject: [PATCH 1/4] wip --- defaults/main.yml | 75 ++++++++++++++++------ molecule/default/converge.yml | 117 ++++++++++++++++++++++++++++------ 2 files changed, 152 insertions(+), 40 deletions(-) diff --git a/defaults/main.yml b/defaults/main.yml index 9a48fe3..470fb82 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -22,20 +22,27 @@ elastic_disable_auto_update: false # true if you want to manually update elasticsearch. This is valid only for # Linux servers that may have some kind of unattended update mechanism. +# TODO rename these to `elastic_files_user` and `elastic_files_group`. elastic_user: root -# The user that will own elasticsearch files. Elasticsearch will be run as the -# `elasticsearch` user as defined in the systemd service file that comes with -# the elasticsearch installation, regardless of this setting. If you want to -# change that user as well, you can use the `elastic_systemd_override` variable +# The user that will own elasticsearch files. By default, when installing +# elasticsearch, the configuration files are owned by user `root` and group +# `elasticsearch`. The default behavior of the role is to replicate this, while +# giving the option to the user to run elasticsearch as an arbitrary user/group +# if they need to. Elasticsearch itself will *run* as the `elasticsearch` user +# as defined in the systemd service file that comes with the elasticsearch +# installation, regardless of this setting. If you want to change the user +# elasticsearch will run as, you can use the `elastic_systemd_override` variable # to change the user that systemd will use to run elasticsearch. elastic_group: elasticsearch -# The group that `elasticsearch_user` will be added to. If the `kibana_user` is +# The group that `elasticsearch_user` will be added to. If the `elastic_user` is # `root`, the `root` user will not be added to this group. By default, when # installing elasticsearch, the configuration files are owned by user `root` and # group `elasticsearch`. The default behavior of the role is to replicate this, # while giving the option to the user to run elasticsearch as an arbitrary -# user/group if they need to. +# user/group if they need to. In the default setup, elasticsearch will be able +# to read the config files that are owned by root (see the `elastic_user` +# variable) because this group will have read permissions. # }}} # Security{{{ elastic_builtin_users_password_backup_file: ~ @@ -49,10 +56,11 @@ elastic_builtin_users_password_backup_file: ~ # format and can be loaded into an ansible variable with the `file` lookup # plugin. If you leave this variable None, the new passwords will be only # printed during provisioning and you will have no way of finding them -# afterwards. In this case, provide a custom password for the `elastic` user in -# `elastic_users` which the role will use to update the `elastic` user before -# exiting. Otherwise, the role will fail in later runs as there will be no user -# to connect to elasticsearch. +# afterwards. In this case, provide a custom password for the `elastic` user +# using the `elastic_users` variable so that the role can update the `elastic` +# user before exiting. Otherwise, the role will fail in later runs as there will +# be no way to know the random password of the elastic user in order to connect +# to elasticsearch. # TODO currently not supported, do not use, no password is set on the keystore. # Need to: @@ -62,14 +70,40 @@ elastic_builtin_users_password_backup_file: ~ elastic_keystore_password: ~ # The password to encrypt the elastic keystore. -# Roles {{{ +## Roles {{{ elastic_custom_roles: [] # Use this variable to define a list of custom roles for users. The syntax of # each role is simply the YAML equivalent of the json syntax for defining a role -# (see the docs). Example: -# }}} -# Realms {{{ -# Native realm {{{ +# (see the elasticsearch docs). Example taken from the docs: +# elastic_custom_roles: +# - name: my_admin_role +# cluster: +# - all +# indices: +# - names: +# - index1 +# - index2 +# privileges: +# - all +# field_security: +# grant: +# - title +# - body +# query: '{"match": {"title": "foo"}}' +# applications: +# - application: myapp +# privileges: +# - admin +# - read +# resources: +# - "*" +# run_as: +# - other_user +# metadata: +# version: 1 +## }}} +## Realms {{{ +### Native realm {{{ elastic_users: [] # Example: # elastic_users: @@ -79,13 +113,13 @@ elastic_users: [] # email: user@email.com # full_name: User Name # roles: # you can also use `groups` instead of `roles` -# - beats_admin +# - my_admin_role # # Use this variable to create custom users on Elasticsearch. Depending on your # workflow, you may also want to use this to add custom passwords for the # built-in users, which will overwrite the random ones created by the role. -# }}} -# }}} +### }}} +## }}} # }}} # Certificates {{{ # You can use the variables in this section to generate certificates and @@ -124,7 +158,10 @@ elastic_ssl_transport: false # enable TLS encryption for the HTTP endpoint and the transport layer # respectively. If you keep these variable to false, you will have to manually # configure elasticsearch (via the `elastic_config` variable) to use the -# certificates. +# certificates. If you don't use any certificates and leave these variables to +# `false`, nothing TLS related will be performed by the role. In this case, make +# sure that you have not enabled xpack security, otherwise, elasticsearch will +# fail to start as xpack security requires TLS to be enabled and configured. elastic_ssl_verification_mode: full # Whether to request connecting clients to provide a valid certificate. This diff --git a/molecule/default/converge.yml b/molecule/default/converge.yml index 4c66d02..88f5a16 100644 --- a/molecule/default/converge.yml +++ b/molecule/default/converge.yml @@ -1,44 +1,111 @@ --- +# TODO test enabling plugins +# TODO test creating snapshots +# TODO test ingest pipelines + - name: Converge hosts: all roles: - nkakouros.elasticsearch vars: - elastic_bind_host: 0.0.0.0 - elastic_cluster_name: watchmen - elastic_node_name: nite-owl - elastic_jvm_extra_config: | - -Des.enforce.bootstrap.checks=true + # Installation + elastic_major_version: 7 + elastic_minor_version: * + elastic_patch_version: * + elastic_disable_auto_update: true + + # Security + elastic_builtin_users_set_random_passwords: true + elastic_builtin_users_password_backup_file: ~/elastic-passwords + + elastic_custom_roles: + - name: my_admin_role + cluster: + - all + indices: + - names: + - index1 + - index2 + privileges: + - all + field_security: + grant: + - title + - body + query: '{"match": {"title": "foo"}}' + applications: + - application: myapp + privileges: + - admin + - read + resources: + - "*" + run_as: + - other_user + metadata: + version: 1 + + elastic_users: + - name: my_user + pass: my_pass + enabled: true + email: user@email.com + full_name: User Name + roles: + - my_admin_role + + # TLS setup + elastic_ssl_http: true + elastic_ssl_transport: true + # TODO add test case to check for password renewal + elastic_certificates_password_renew: false + elastic_ssl_verification_mode: none # TODO enable full elastic_certificates_password: 'nk}$Q%];a3Gy$E!QvT8E' elastic_certificates: ca: "~/pki/ca.crt" crt: "~/pki/issued/elastic.crt" key: "~/pki/private/elastic.key" elastic_certificates_dir: /etc/elasticsearch/certs/ - elastic_builtin_users_set_random_passwords: true - elastic_builtin_users_password_backup_file: ~/elastic-passwords + + # Configuration + elastic_jvm_extra_config: | + -Des.enforce.bootstrap.checks=true + elastic_systemd_override: | + [Service] + LimitMEMLOCK=infinity elastic_config: xpack: security: hide_settings: 'xpack.security.authc.realms.native.*' enabled: false - # authc: - # accept_default_password: false - # realms: - # native: - # native1: - # enabled: true - # order: 0 + authc: + accept_default_password: false + realms: + native: + native1: + enabled: true + order: 0 + elastic_bind_host: 0.0.0.0 + elastic_cluster_name: watchmen + elastic_node_name: nite-owl + elastic_master_node: true + elastic_data_node: true + elastic_ingest_node: true + + # Others + # When using logstash in front of kibana and you use elastic beats to send + # data to logstash (which will then be forwarded to elastic), you will + # need to manually load the index templates that come with the elastic + # beats. This `nkakouros.beats` role allow you to extract the index + # template into a json file. Here, we read the paths on the ansible + # controller where these json files are located in order for the + # `nkakouros.elastic` role to insert them into elasticsearch. template_files: - # When using logstash in front of kibana and you use elastic beats to send - # data to logstash (which will then be forwarded to elastic), you will - # need to manually load the index templates that come with the elastic - # beats. The `nkakouros.beats` role allow you to extract the index - # template into a json file. Here, we read the paths on the ansible - # controller where these json files are located in order for the - # `nkakouros.elastic` role to insert them into elasticsearch. "{{ q('fileglob', '~/elk/beats/*.json') }}" + # TODO do use a json file to test the actual template importing, the lookup + # currently returns the empty list, no json file exists. + elastic_index_templates: >- [ {%- for file in template_files -%} @@ -53,3 +120,11 @@ }, {%- endfor -%} ] + + elastic_stored_scripts: + - name: packetbeat_filter + # Hypothetical script that matches packetbeat logs as they come. + lang: painless + source: "ctx.agent.type == 'packetbeat'" + + elastic_cluster_uuid_backup: '~/elasticsearch-uuid' From b5278c32a584749d8d182fda7686d70c9b35377c Mon Sep 17 00:00:00 2001 From: Nikolaos Kakouros Date: Wed, 7 Apr 2021 13:06:00 +0200 Subject: [PATCH 2/4] wip --- molecule/default/converge.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/molecule/default/converge.yml b/molecule/default/converge.yml index 88f5a16..92672af 100644 --- a/molecule/default/converge.yml +++ b/molecule/default/converge.yml @@ -11,8 +11,8 @@ vars: # Installation elastic_major_version: 7 - elastic_minor_version: * - elastic_patch_version: * + elastic_minor_version: '*' + elastic_patch_version: '*' elastic_disable_auto_update: true # Security From dd356ccf261790b55ac831a831973f1300125416 Mon Sep 17 00:00:00 2001 From: Nikolaos Kakouros Date: Wed, 7 Apr 2021 13:16:17 +0200 Subject: [PATCH 3/4] wip --- molecule/default/converge.yml | 9 +++++---- tasks/core/configure.yml | 26 +++++++++++++++++--------- 2 files changed, 22 insertions(+), 13 deletions(-) diff --git a/molecule/default/converge.yml b/molecule/default/converge.yml index 92672af..b5bbf6a 100644 --- a/molecule/default/converge.yml +++ b/molecule/default/converge.yml @@ -3,6 +3,10 @@ # TODO test enabling plugins # TODO test creating snapshots # TODO test ingest pipelines +# TODO add test case to check for password renewal +# TODO enable full certificate validation +# TODO do use a json file to test the actual template importing, the lookup +# currently returns the empty list, no json file exists. - name: Converge hosts: all @@ -58,9 +62,8 @@ # TLS setup elastic_ssl_http: true elastic_ssl_transport: true - # TODO add test case to check for password renewal elastic_certificates_password_renew: false - elastic_ssl_verification_mode: none # TODO enable full + elastic_ssl_verification_mode: none elastic_certificates_password: 'nk}$Q%];a3Gy$E!QvT8E' elastic_certificates: ca: "~/pki/ca.crt" @@ -103,8 +106,6 @@ # `nkakouros.elastic` role to insert them into elasticsearch. template_files: "{{ q('fileglob', '~/elk/beats/*.json') }}" - # TODO do use a json file to test the actual template importing, the lookup - # currently returns the empty list, no json file exists. elastic_index_templates: >- [ diff --git a/tasks/core/configure.yml b/tasks/core/configure.yml index 66f3bcf..8c58552 100644 --- a/tasks/core/configure.yml +++ b/tasks/core/configure.yml @@ -43,12 +43,20 @@ when: elastic_jvm_extra_config != None notify: elastic-restart-service -- name: Create systemd override file - copy: - content: "{{ elastic_systemd_override }}" - dest: /etc/systemd/system/elasticsearch.service.d/override.conf - owner: root - group: root - mode: 0o644 - when: elastic_systemd_override != None - notify: elastic-restart-service +- name: Override systemd service settings + block: + - name: Create systemd override directory + file: + path: /etc/systemd/system/elasticsearch.service.d + mode: 0o755 + state: directory + + - name: Create systemd override file + copy: + content: "{{ elastic_systemd_override }}" + dest: /etc/systemd/system/elasticsearch.service.d/override.conf + owner: root + group: root + mode: 0o644 + notify: elastic-restart-service + when: elastic_systemd_override is not none From 0a35524c9fa3c0e4589dbd8ce61fda838e93989a Mon Sep 17 00:00:00 2001 From: Nikolaos Kakouros Date: Wed, 7 Apr 2021 17:31:22 +0200 Subject: [PATCH 4/4] wip --- defaults/main.yml | 16 ++++++---------- handlers/main.yml | 2 +- molecule/default/converge.yml | 12 ++++-------- tasks/plugins/repository-gcs.yml | 2 +- tasks/realms/native.yml | 21 +++++++++++---------- tasks/variables.yml | 6 ++++-- vars/main.yml | 23 ----------------------- 7 files changed, 27 insertions(+), 55 deletions(-) diff --git a/defaults/main.yml b/defaults/main.yml index 470fb82..03efc4c 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -74,7 +74,7 @@ elastic_keystore_password: ~ elastic_custom_roles: [] # Use this variable to define a list of custom roles for users. The syntax of # each role is simply the YAML equivalent of the json syntax for defining a role -# (see the elasticsearch docs). Example taken from the docs: +# (see the elasticsearch docs). Example adapted from the docs: # elastic_custom_roles: # - name: my_admin_role # cluster: @@ -85,11 +85,6 @@ elastic_custom_roles: [] # - index2 # privileges: # - all -# field_security: -# grant: -# - title -# - body -# query: '{"match": {"title": "foo"}}' # applications: # - application: myapp # privileges: @@ -147,7 +142,7 @@ elastic_certificates: ~ # elastic_certificates: # ca: path/to/ca.cert # cert: path/to/node/cert -# key: path/to/node/key +# key: path/to/node/key # has to be in PKCS#1 format elastic_certificates_dir: /etc/elasticsearch/certs/ # The folder where `elastic_certificates` will be uploaded into. @@ -195,13 +190,14 @@ elastic_config: {} # configuration can be found in the `vars/main.yml` file of this role. For what # these settings do, consult the elastic search documentation. -elastic_bind_host: localhost -# The address where elastic will listen on for requests to its JSON api. +elastic_bind_host: + - localhost +# The addresses where elastic will listen on for requests to its JSON api. elastic_bind_port: 9200 # The port where elastic will listen on for request to its JSON api. -elastic_transport_host: "{{ elastic_bind_host }}" +elastic_transport_host: "{{ elastic_bind_host[0] }}" # The address elastic search will use to communicate with other nodes in the # cluster. diff --git a/handlers/main.yml b/handlers/main.yml index 22725b5..c056a31 100644 --- a/handlers/main.yml +++ b/handlers/main.yml @@ -45,7 +45,7 @@ url_username: "{{ _elastic_elastic_user['name'] }}" url_password: "{{ _elastic_elastic_user['pass'] }}" validate_certs: false - listen: elastic reload settings + listen: elastic-reload-settings # **Some** settings stored in the keystore are reloadable. This means that ES # does not have to be restarted. Instead, the above call will make the # settings have immediate effect. diff --git a/molecule/default/converge.yml b/molecule/default/converge.yml index b5bbf6a..f7cad02 100644 --- a/molecule/default/converge.yml +++ b/molecule/default/converge.yml @@ -33,11 +33,6 @@ - index2 privileges: - all - field_security: - grant: - - title - - body - query: '{"match": {"title": "foo"}}' applications: - application: myapp privileges: @@ -68,7 +63,7 @@ elastic_certificates: ca: "~/pki/ca.crt" crt: "~/pki/issued/elastic.crt" - key: "~/pki/private/elastic.key" + key: "~/pki/private/elastic.p1" elastic_certificates_dir: /etc/elasticsearch/certs/ # Configuration @@ -81,7 +76,7 @@ xpack: security: hide_settings: 'xpack.security.authc.realms.native.*' - enabled: false + enabled: true authc: accept_default_password: false realms: @@ -89,7 +84,8 @@ native1: enabled: true order: 0 - elastic_bind_host: 0.0.0.0 + elastic_bind_host: + - 0.0.0.0 elastic_cluster_name: watchmen elastic_node_name: nite-owl elastic_master_node: true diff --git a/tasks/plugins/repository-gcs.yml b/tasks/plugins/repository-gcs.yml index e6e704d..534ba80 100644 --- a/tasks/plugins/repository-gcs.yml +++ b/tasks/plugins/repository-gcs.yml @@ -39,7 +39,7 @@ args: chdir: /usr/share/elasticsearch/ when: _elastic_keystore_settings.stdout is not search(_elastic_gpc_credentials) - notify: elastic reload settings + notify: elastic-reload-settings when: _elastic_keystore_settings.stdout is not search('gcs.client.' + elastic_plugin_gcs_credentials.name + 'credentials_file') diff --git a/tasks/realms/native.yml b/tasks/realms/native.yml index 5a3b811..aa80c9e 100644 --- a/tasks/realms/native.yml +++ b/tasks/realms/native.yml @@ -12,15 +12,15 @@ - name: Create users uri: - url: "{{ elastic_node_address }}/_security/user/{{ _elasticsearch__item.name }}" + url: "{{ elastic_node_address }}/_security/user/{{ _elasticsearch__user.name }}" method: POST body: - enabled: "{{ _elasticsearch__item.enabled | default(true) }}" - email: "{{ _elasticsearch__item.email | default(omit) }}" - full_name: "{{ _elasticsearch__item.full_name | default(omit) }}" - password: "{{ _elasticsearch__item.pass }}" - roles: "{{ _elasticsearch__item.roles - | default(_elasticsearch__item.groups + enabled: "{{ _elasticsearch__user.enabled | default(true) }}" + email: "{{ _elasticsearch__user.email | default(omit) }}" + full_name: "{{ _elasticsearch__user.full_name | default(omit) }}" + password: "{{ _elasticsearch__user.pass }}" + roles: "{{ _elasticsearch__user.roles + | default(_elasticsearch__user.groups | default(omit)) }}" body_format: json @@ -31,17 +31,17 @@ url_password: "{{ _elastic_elastic_user['pass'] }}" validate_certs: false return_content: true - when: _elasticsearch__item.name not in _elastic_builtin_users + when: _elasticsearch__user.name not in _elastic_builtin_users loop: "{{ elastic_users }}" loop_control: loop_var: _elasticsearch__user - name: Change user password uri: - url: "{{ elastic_node_address }}/_security/user/{{ _elasticsearch__item.name }}/_password" + url: "{{ elastic_node_address }}/_security/user/{{ _elasticsearch__user.name }}/_password" method: POST body: - password: "{{ _elasticsearch__item.pass }}" + password: "{{ _elasticsearch__user.pass }}" body_format: json headers: Content-Type: application/json @@ -71,3 +71,4 @@ | list | first }} + when: elastic_users | selectattr('name', 'equalto', 'elastic') | length > 0 diff --git a/tasks/variables.yml b/tasks/variables.yml index 15ca271..c95e0f7 100644 --- a/tasks/variables.yml +++ b/tasks/variables.yml @@ -8,8 +8,10 @@ set_fact: elastic_node_address: >- http{{ elastic_ssl_http | ternary('s', '') }}://{{ - elastic_bind_host[0] }}:{{ - elastic_bind_port }} + '127.0.0.1' + if elastic_bind_host[0] == '0.0.0.0' + else elastic_bind_host[0] + }}:{{ elastic_bind_port }} - name: Check if elasticsearch already installed stat: diff --git a/vars/main.yml b/vars/main.yml index e1bff36..5c13810 100644 --- a/vars/main.yml +++ b/vars/main.yml @@ -25,26 +25,3 @@ _elastic_config_defaults: xpack: security: enabled: true - -_elastic_certificates_ca_command: >- - /usr/share/elasticsearch/bin/elasticsearch-certutil - ca - --silent - --pem - --out /etc/elasticsearch/certs/elastic-stack-ca.zip - {{ '--pass ' + elastic_certificates_password - if elastic_certificates_password != None else '' }} - -_elastic_certificates_cert_command: >- - /usr/share/elasticsearch/bin/elasticsearch-certutil - {{ elastic_certificates_generate_csr | ternary('csr', 'cert') }} - --silent - --pem - --in /etc/elasticsearch/certs/elasticsearch-certutil.yml - --out /etc/elasticsearch/certs/elasticsearch-certs.zip - --ca-cert /etc/elasticsearch/certs/ca/ca.crt - --ca-key /etc/elasticsearch/certs/ca/ca.key - {{ '--ca-pass ' + elastic_certificates_password - if elastic_certificates_password != None else '' }} - {{ '--pass ' + elastic_certificates_password - if elastic_certificates_password != None else '' }}