More tests

defaults/main.yml

@@ -22,20 +22,27 @@ elastic_disable_auto_update: false
 # true if you want to manually update elasticsearch. This is valid only for
 # Linux servers that may have some kind of unattended update mechanism.
 
+# TODO rename these to `elastic_files_user` and `elastic_files_group`.
 elastic_user: root
-# The user that will own elasticsearch files. Elasticsearch will be run as the
-# `elasticsearch` user as defined in the systemd service file that comes with
-# the elasticsearch installation, regardless of this setting. If you want to
-# change that user as well, you can use the `elastic_systemd_override` variable
+# The user that will own elasticsearch files. By default, when installing
+# elasticsearch, the configuration files are owned by user `root` and group
+# `elasticsearch`. The default behavior of the role is to replicate this, while
+# giving the option to the user to run elasticsearch as an arbitrary user/group
+# if they need to. Elasticsearch itself will *run* as the `elasticsearch` user
+# as defined in the systemd service file that comes with the elasticsearch
+# installation, regardless of this setting. If you want to change the user
+# elasticsearch will run as, you can use the `elastic_systemd_override` variable
 # to change the user that systemd will use to run elasticsearch.
 
 elastic_group: elasticsearch
-# The group that `elasticsearch_user` will be added to. If the `kibana_user` is
+# The group that `elasticsearch_user` will be added to. If the `elastic_user` is
 # `root`, the `root` user will not be added to this group. By default, when
 # installing elasticsearch, the configuration files are owned by user `root` and
 # group `elasticsearch`. The default behavior of the role is to replicate this,
 # while giving the option to the user to run elasticsearch as an arbitrary
-# user/group if they need to.
+# user/group if they need to. In the default setup, elasticsearch will be able
+# to read the config files that are owned by root (see the `elastic_user`
+# variable) because this group will have read permissions.
 # }}}
 # Security{{{
 elastic_builtin_users_password_backup_file: ~
@@ -49,10 +56,11 @@ elastic_builtin_users_password_backup_file: ~
 # format and can be loaded into an ansible variable with the `file` lookup
 # plugin. If you leave this variable None, the new passwords will be only
 # printed during provisioning and you will have no way of finding them
-# afterwards. In this case, provide a custom password for the `elastic` user in
-# `elastic_users` which the role will use to update the `elastic` user before
-# exiting. Otherwise, the role will fail in later runs as there will be no user
-# to connect to elasticsearch.
+# afterwards. In this case, provide a custom password for the `elastic` user
+# using the `elastic_users` variable so that the role can update the `elastic`
+# user before exiting. Otherwise, the role will fail in later runs as there will
+# be no way to know the random password of the elastic user in order to connect
+# to elasticsearch.
 
 # TODO currently not supported, do not use, no password is set on the keystore.
 # Need to:
@@ -62,14 +70,35 @@ elastic_builtin_users_password_backup_file: ~
 elastic_keystore_password: ~
 # The password to encrypt the elastic keystore.
 
-# Roles {{{
+## Roles {{{
 elastic_custom_roles: []
 # Use this variable to define a list of custom roles for users. The syntax of
 # each role is simply the YAML equivalent of the json syntax for defining a role
-# (see the docs). Example:
-# }}}
-# Realms {{{
-# Native realm {{{
+# (see the elasticsearch docs). Example adapted from the docs:
+# elastic_custom_roles:
+#   - name: my_admin_role
+#     cluster:
+#       - all
+#     indices:
+#       - names:
+#           - index1
+#           - index2
+#         privileges:
+#           - all
+#     applications:
+#       - application: myapp
+#         privileges:
+#           - admin
+#           - read
+#         resources:
+#           - "*"
+#     run_as:
+#       - other_user
+#     metadata:
+#       version: 1
+## }}}
+## Realms {{{
+### Native realm {{{
 elastic_users: []
 # Example:
 # elastic_users:
@@ -79,13 +108,13 @@ elastic_users: []
 #     email: [email protected]
 #     full_name: User Name
 #     roles:         # you can also use `groups` instead of `roles`
-#       - beats_admin
+#       - my_admin_role
 #
 # Use this variable to create custom users on Elasticsearch. Depending on your
 # workflow, you may also want to use this to add custom passwords for the
 # built-in users, which will overwrite the random ones created by the role.
-# }}}
-# }}}
+### }}}
+## }}}
 # }}}
 # Certificates {{{
 # You can use the variables in this section to generate certificates and
@@ -113,7 +142,7 @@ elastic_certificates: ~
 # elastic_certificates:
 #   ca: path/to/ca.cert
 #   cert: path/to/node/cert
-#   key: path/to/node/key
+#   key: path/to/node/key  # has to be in PKCS#1 format
 
 elastic_certificates_dir: /etc/elasticsearch/certs/
 # The folder where `elastic_certificates` will be uploaded into.
@@ -124,7 +153,10 @@ elastic_ssl_transport: false
 # enable TLS encryption for the HTTP endpoint and the transport layer
 # respectively. If you keep these variable to false, you will have to manually
 # configure elasticsearch (via the `elastic_config` variable) to use the
-# certificates.
+# certificates. If you don't use any certificates and leave these variables to
+# `false`, nothing TLS related will be performed by the role. In this case, make
+# sure that you have not enabled xpack security, otherwise, elasticsearch will
+# fail to start as xpack security requires TLS to be enabled and configured.
 
 elastic_ssl_verification_mode: full
 # Whether to request connecting clients to provide a valid certificate. This
@@ -158,13 +190,14 @@ elastic_config: {}
 # configuration can be found in the `vars/main.yml` file of this role. For what
 # these settings do, consult the elastic search documentation.
 
-elastic_bind_host: localhost
-# The address where elastic will listen on for requests to its JSON api.
+elastic_bind_host:
+  - localhost
+# The addresses where elastic will listen on for requests to its JSON api.
 
 elastic_bind_port: 9200
 # The port where elastic will listen on for request to its JSON api.
 
-elastic_transport_host: "{{ elastic_bind_host }}"
+elastic_transport_host: "{{ elastic_bind_host[0] }}"
 # The address elastic search will use to communicate with other nodes in the
 # cluster.
 

handlers/main.yml

@@ -45,7 +45,7 @@
     url_username: "{{ _elastic_elastic_user['name'] }}"
     url_password: "{{ _elastic_elastic_user['pass'] }}"
     validate_certs: false
-  listen: elastic reload settings
+  listen: elastic-reload-settings
   # **Some** settings stored in the keystore are reloadable. This means that ES
   # does not have to be restarted. Instead, the above call will make the
   # settings have immediate effect.

molecule/default/converge.yml

@@ -1,44 +1,108 @@
 ---
 
+# TODO test enabling plugins
+# TODO test creating snapshots
+# TODO test ingest pipelines
+# TODO add test case to check for password renewal
+# TODO enable full certificate validation
+# TODO do use a json file to test the actual template importing, the lookup
+# currently returns the empty list, no json file exists.
+
 - name: Converge
   hosts: all
   roles:
     - nkakouros.elasticsearch
   vars:
-    elastic_bind_host: 0.0.0.0
-    elastic_cluster_name: watchmen
-    elastic_node_name: nite-owl
-    elastic_jvm_extra_config: |
-      -Des.enforce.bootstrap.checks=true
+    # Installation
+    elastic_major_version: 7
+    elastic_minor_version: '*'
+    elastic_patch_version: '*'
+    elastic_disable_auto_update: true
+
+    # Security
+    elastic_builtin_users_set_random_passwords: true
+    elastic_builtin_users_password_backup_file: ~/elastic-passwords
+
+    elastic_custom_roles:
+      - name: my_admin_role
+        cluster:
+          - all
+        indices:
+          - names:
+              - index1
+              - index2
+            privileges:
+              - all
+        applications:
+          - application: myapp
+            privileges:
+              - admin
+              - read
+            resources:
+              - "*"
+        run_as:
+          - other_user
+        metadata:
+          version: 1
+
+    elastic_users:
+      - name: my_user
+        pass: my_pass
+        enabled: true
+        email: [email protected]
+        full_name: User Name
+        roles:
+          - my_admin_role
+
+    # TLS setup
+    elastic_ssl_http: true
+    elastic_ssl_transport: true
+    elastic_certificates_password_renew: false
+    elastic_ssl_verification_mode: none
     elastic_certificates_password: 'nk}$Q%];a3Gy$E!QvT8E'
     elastic_certificates:
       ca: "~/pki/ca.crt"
       crt: "~/pki/issued/elastic.crt"
-      key: "~/pki/private/elastic.key"
+      key: "~/pki/private/elastic.p1"
     elastic_certificates_dir: /etc/elasticsearch/certs/
-    elastic_builtin_users_set_random_passwords: true
-    elastic_builtin_users_password_backup_file: ~/elastic-passwords
+
+    # Configuration
+    elastic_jvm_extra_config: |
+      -Des.enforce.bootstrap.checks=true
+    elastic_systemd_override: |
+      [Service]
+      LimitMEMLOCK=infinity
     elastic_config:
       xpack:
         security:
           hide_settings: 'xpack.security.authc.realms.native.*'
-          enabled: false
-          # authc:
-          #   accept_default_password: false
-          #   realms:
-          #     native:
-          #       native1:
-          #         enabled: true
-          #         order: 0
+          enabled: true
+          authc:
+            accept_default_password: false
+            realms:
+              native:
+                native1:
+                  enabled: true
+                  order: 0
+    elastic_bind_host:
+      - 0.0.0.0
+    elastic_cluster_name: watchmen
+    elastic_node_name: nite-owl
+    elastic_master_node: true
+    elastic_data_node: true
+    elastic_ingest_node: true
+
+    # Others
+    # When using logstash in front of kibana and you use elastic beats to send
+    # data to logstash (which will then be forwarded to elastic), you will
+    # need to manually load the index templates that come with the elastic
+    # beats. This `nkakouros.beats` role allow you to extract the index
+    # template into a json file. Here, we read the paths on the ansible
+    # controller where these json files are located in order for the
+    # `nkakouros.elastic` role to insert them into elasticsearch.
     template_files:
-      # When using logstash in front of kibana and you use elastic beats to send
-      # data to logstash (which will then be forwarded to elastic), you will
-      # need to manually load the index templates that come with the elastic
-      # beats. The `nkakouros.beats` role allow you to extract the index
-      # template into a json file. Here, we read the paths on the ansible
-      # controller where these json files are located in order for the
-      # `nkakouros.elastic` role to insert them into elasticsearch.
       "{{ q('fileglob', '~/elk/beats/*.json') }}"
+
     elastic_index_templates: >-
       [
         {%- for file in template_files -%}
@@ -53,3 +117,11 @@
           },
         {%- endfor -%}
       ]
+
+    elastic_stored_scripts:
+      - name: packetbeat_filter
+        # Hypothetical script that matches packetbeat logs as they come.
+        lang: painless
+        source: "ctx.agent.type == 'packetbeat'"
+
+    elastic_cluster_uuid_backup: '~/elasticsearch-uuid'

tasks/core/configure.yml

@@ -43,12 +43,20 @@
   when: elastic_jvm_extra_config != None
   notify: elastic-restart-service
 
-- name: Create systemd override file
-  copy:
-    content: "{{ elastic_systemd_override }}"
-    dest: /etc/systemd/system/elasticsearch.service.d/override.conf
-    owner: root
-    group: root
-    mode: 0o644
-  when: elastic_systemd_override != None
-  notify: elastic-restart-service
+- name: Override systemd service settings
+  block:
+    - name: Create systemd override directory
+      file:
+        path: /etc/systemd/system/elasticsearch.service.d
+        mode: 0o755
+        state: directory
+
+    - name: Create systemd override file
+      copy:
+        content: "{{ elastic_systemd_override }}"
+        dest: /etc/systemd/system/elasticsearch.service.d/override.conf
+        owner: root
+        group: root
+        mode: 0o644
+      notify: elastic-restart-service
+  when: elastic_systemd_override is not none

tasks/plugins/repository-gcs.yml

@@ -39,7 +39,7 @@
       args:
         chdir: /usr/share/elasticsearch/
       when: _elastic_keystore_settings.stdout is not search(_elastic_gpc_credentials)
-      notify: elastic reload settings
+      notify: elastic-reload-settings
   when:
     _elastic_keystore_settings.stdout is not search('gcs.client.' + elastic_plugin_gcs_credentials.name + 'credentials_file')
 

tasks/realms/native.yml

@@ -12,15 +12,15 @@
 
 - name: Create users
   uri:
-    url: "{{ elastic_node_address }}/_security/user/{{ _elasticsearch__item.name }}"
+    url: "{{ elastic_node_address }}/_security/user/{{ _elasticsearch__user.name }}"
     method: POST
     body:
-      enabled: "{{ _elasticsearch__item.enabled | default(true) }}"
-      email: "{{ _elasticsearch__item.email | default(omit) }}"
-      full_name: "{{ _elasticsearch__item.full_name | default(omit) }}"
-      password: "{{ _elasticsearch__item.pass }}"
-      roles: "{{ _elasticsearch__item.roles
-                 | default(_elasticsearch__item.groups
+      enabled: "{{ _elasticsearch__user.enabled | default(true) }}"
+      email: "{{ _elasticsearch__user.email | default(omit) }}"
+      full_name: "{{ _elasticsearch__user.full_name | default(omit) }}"
+      password: "{{ _elasticsearch__user.pass }}"
+      roles: "{{ _elasticsearch__user.roles
+                 | default(_elasticsearch__user.groups
                  | default(omit))
               }}"
     body_format: json
@@ -31,17 +31,17 @@
     url_password: "{{ _elastic_elastic_user['pass'] }}"
     validate_certs: false
     return_content: true
-  when: _elasticsearch__item.name not in _elastic_builtin_users
+  when: _elasticsearch__user.name not in _elastic_builtin_users
   loop: "{{ elastic_users }}"
   loop_control:
     loop_var: _elasticsearch__user
 
 - name: Change user password
   uri:
-    url: "{{ elastic_node_address }}/_security/user/{{ _elasticsearch__item.name }}/_password"
+    url: "{{ elastic_node_address }}/_security/user/{{ _elasticsearch__user.name }}/_password"
     method: POST
     body:
-      password: "{{ _elasticsearch__item.pass }}"
+      password: "{{ _elasticsearch__user.pass }}"
     body_format: json
     headers:
       Content-Type: application/json
@@ -71,3 +71,4 @@
         | list
         | first
       }}
+  when: elastic_users | selectattr('name', 'equalto', 'elastic') | length > 0