Adding opensearch as an option to 1-clicks

This adds a 1 click recipe that installs opensearch and
opensearch dashboards on a single droplet with TLS enabled.

The apps use a self signed certificate and the passwords for the admin
and kibana user are at `/home/opensearch/admin_password.txt` and
`/home/opensearch/kibana_password.txt` files.

opensearch-20-04/ansible/opensearch-playbook.yml

@@ -0,0 +1,6 @@
+---
+- hosts: default
+  become: true
+  serial: 1
+  roles:
+    - role: opensearch

opensearch-20-04/ansible/roles/opensearch/defaults/main.yml

@@ -0,0 +1,21 @@
+---
+opensearch_home: /opt/opensearch
+opensearch_config_home: /opt/opensearch/config
+opensearch_config_file: /opt/opensearch/config/opensearch.yml
+opensearch_url: https://artifacts.opensearch.org/releases/bundle/opensearch/1.3.1/opensearch-1.3.1-linux-x64.tar.gz
+
+opensearch_dashboards_home: /opt/opensearch-dashboards
+opensearch_dashboards_config_home: /opt/opensearch-dashboards/config
+opensearch_dashboards_config_file: /opt/opensearch-dashboards/config/opensearch_dashboards.yml
+opensearch_dashboards_url: https://artifacts.opensearch.org/releases/bundle/opensearch-dashboards/1.3.1/opensearch-dashboards-1.3.1-linux-x64.tar.gz
+
+opensearch_user: opensearch
+opensearch_group: opensearch
+
+opensearch_plugin_bin_path: /opt/opensearch/bin/opensearch-plugin
+opensearch_security_plugin_conf_path: /opt/opensearch/plugins/opensearch-security/securityconfig
+opensearch_security_plugin_tools_path: /opt/opensearch/plugins/opensearch-security/tools
+
+systemctl_path: /etc/systemd/system
+
+nodecerts_home: /opt/opensearch-nodecerts

opensearch-20-04/ansible/roles/opensearch/files/internal_users.yml.template

@@ -0,0 +1,21 @@
+---
+# This is the internal user database
+# The hash value is a bcrypt hash and can be generated with plugin/tools/hash.sh
+
+_meta:
+  type: "internalusers"
+  config_version: 2
+
+# Define your internal users here
+
+admin:
+  hash: "{{ datasource "admin_password" }}"
+  reserved: true
+  backend_roles:
+    - "admin"
+  description: "admin user"
+
+kibanaserver:
+  hash: "{{ datasource "kibanaserver_password" }}"
+  reserved: true
+  description: "kibanaserver user"

opensearch-20-04/ansible/roles/opensearch/files/jvm.options.template

@@ -0,0 +1,80 @@
+## JVM configuration
+
+################################################################
+## IMPORTANT: JVM heap size
+################################################################
+##
+## You should always set the min and max JVM heap
+## size to the same value. For example, to set
+## the heap to 4 GB, set:
+##
+## -Xms4g
+## -Xmx4g
+##
+## See https://opensearch.org/docs/opensearch/install/important-settings/
+## for more information
+##
+################################################################
+
+# Xms represents the initial size of total heap space
+# Xmx represents the maximum size of total heap space
+
+-Xms{{datasource "max_memory" }}m
+-Xmx{{datasource "max_memory" }}m
+
+################################################################
+## Expert settings
+################################################################
+##
+## All settings below this section are considered
+## expert settings. Don't tamper with them unless
+## you understand what you are doing
+##
+################################################################
+
+## GC configuration
+8-13:-XX:+UseConcMarkSweepGC
+8-13:-XX:CMSInitiatingOccupancyFraction=75
+8-13:-XX:+UseCMSInitiatingOccupancyOnly
+
+## G1GC Configuration
+# NOTE: G1 GC is only supported on JDK version 10 or later
+# to use G1GC, uncomment the next two lines and update the version on the
+# following three lines to your version of the JDK
+# 10-13:-XX:-UseConcMarkSweepGC
+# 10-13:-XX:-UseCMSInitiatingOccupancyOnly
+14-:-XX:+UseG1GC
+14-:-XX:G1ReservePercent=25
+14-:-XX:InitiatingHeapOccupancyPercent=30
+
+## JVM temporary directory
+-Djava.io.tmpdir=${OPENSEARCH_TMPDIR}
+
+## heap dumps
+
+# generate a heap dump when an allocation from the Java heap fails
+# heap dumps are created in the working directory of the JVM
+-XX:+HeapDumpOnOutOfMemoryError
+
+# specify an alternative path for heap dumps; ensure the directory exists and
+# has sufficient space
+-XX:HeapDumpPath=data
+
+# specify an alternative path for JVM fatal error logs
+-XX:ErrorFile=logs/hs_err_pid%p.log
+
+## JDK 8 GC logging
+8:-XX:+PrintGCDetails
+8:-XX:+PrintGCDateStamps
+8:-XX:+PrintTenuringDistribution
+8:-XX:+PrintGCApplicationStoppedTime
+8:-Xloggc:logs/gc.log
+8:-XX:+UseGCLogFileRotation
+8:-XX:NumberOfGCLogFiles=32
+8:-XX:GCLogFileSize=64m
+
+# JDK 9+ GC logging
+9-:-Xlog:gc*,gc+age=trace,safepoint:file=logs/gc.log:utctime,pid,tags:filecount=32,filesize=64m
+
+# Explicitly allow security manager (https://bugs.openjdk.java.net/browse/JDK-8270380)
+18-:-Djava.security.manager=allow

opensearch-20-04/ansible/roles/opensearch/files/opensearch.yml.template

@@ -0,0 +1,31 @@
+cluster.name: {{ datasource "droplet_name" }}
+
+node.name: {{ datasource "droplet_name" }}
+
+network.host: {{ datasource "ip_address" }}
+
+http.port: 9200
+
+discovery.type: single-node
+
+bootstrap.memory_lock: true
+
+plugins.security.allow_default_init_securityindex: true
+plugins.security.audit.type: internal_opensearch
+plugins.security.enable_snapshot_restore_privilege: true
+plugins.security.check_snapshot_restore_write_privileges: true
+plugins.security.restapi.roles_enabled: ["all_access", "security_rest_api_access"]
+
+plugins.security.ssl.transport.pemcert_filepath: opensearch.pem
+plugins.security.ssl.transport.pemkey_filepath: opensearch.key
+plugins.security.ssl.transport.pemtrustedcas_filepath: root-ca.pem
+plugins.security.ssl.transport.enforce_hostname_verification: false
+plugins.security.ssl.transport.resolve_hostname: false
+plugins.security.ssl.http.enabled: true
+plugins.security.ssl.http.pemcert_filepath: opensearch_http.pem
+plugins.security.ssl.http.pemkey_filepath: opensearch_http.key
+plugins.security.ssl.http.pemtrustedcas_filepath: root-ca.pem
+plugins.security.nodes_dn:
+- CN=opensearch,OU=Ops,O=opensearch\, Inc.,DC=opensearch
+plugins.security.authcz.admin_dn:
+- CN=admin.opensearch,OU=Ops,O=opensearch\, Inc.,DC=opensearch

opensearch-20-04/ansible/roles/opensearch/files/opensearch_dashboards.yml.template

@@ -0,0 +1,21 @@
+server.port: 5601
+server.host: "{{ datasource "ip_address" }}"
+opensearch.hosts: ["https://{{ datasource "ip_address" }}:9200"]
+opensearch.ssl.verificationMode: none
+opensearch.username: "kibanaserver"
+opensearch.password: "{{ datasource "kibanaserver_password" }}"
+opensearch.requestHeadersWhitelist: [ authorization,securitytenant ]
+
+opensearch_security.multitenancy.enabled: true
+opensearch_security.multitenancy.tenants.preferred: ["Private", "Global"]
+opensearch_security.readonly_mode.roles: ["kibana_read_only"]
+opensearch_security.cookie.secure: true
+
+server.ssl.enabled: true
+server.ssl.certificate: {{ datasource "opensearch_config_home" }}/opensearch_http.pem
+server.ssl.key: {{ datasource "opensearch_config_home" }}/opensearch_http.key
+
+opensearch.ssl.certificateAuthorities: [ "{{ datasource "opensearch_config_home" }}/root-ca.pem" ]
+
+opensearch.ssl.certificate: {{ datasource "opensearch_config_home" }}/opensearch.pem
+opensearch.ssl.key: {{ datasource "opensearch_config_home" }}/opensearch.key

opensearch-20-04/ansible/roles/opensearch/files/tlsconfig.yml.template

@@ -0,0 +1,45 @@
+ca:
+  root:
+    dn: CN=root.ca.opensearch,OU=CA,O=opensearch\, Inc.,DC=opensearch
+    keysize: 2048
+    validityDays: 3650
+    pkPassword: none
+    file: root-ca.pem
+
+### Default values and global settings
+defaults:
+  validityDays: 3650
+  pkPassword: none
+  # Set this to true in order to generate config and certificates for
+  # the HTTP interface of nodes
+  httpsEnabled: true
+  reuseTransportCertificatesForHttp: false
+  verifyHostnames: false
+  resolveHostnames: false
+
+
+###
+### Nodes
+###
+#
+# Specify the nodes of your ES cluster here
+#
+nodes:
+- name: opensearch
+  dn: CN=opensearch,OU=Ops,O=opensearch\, Inc.,DC=opensearch
+  dns: opensearch
+  ip: {{ datasource "ip_address" }}
+
+###
+### Clients
+###
+#
+# Specify the clients that shall access your ES cluster with certificate authentication here
+#
+# At least one client must be an admin user (i.e., a super-user). Admin users can
+# be specified with the attribute admin: true
+#
+clients:
+  - name: admin
+    dn: CN=admin.opensearch,OU=Ops,O=opensearch\, Inc.,DC=opensearch
+    admin: true

opensearch-20-04/ansible/roles/opensearch/tasks/dashboards.yml

@@ -0,0 +1,40 @@
+---
+
+- name: Download dashboards
+  get_url:
+    url: "{{ opensearch_dashboards_url }}"
+    dest: "/tmp/dashboards.tar.gz"
+
+- name: Unpack dashboards tarball
+  unarchive:
+    copy: no
+    src: "/tmp/dashboards.tar.gz"
+    dest: "{{ opensearch_dashboards_home }}"
+    owner: "{{ opensearch_user }}"
+    group: "{{ opensearch_group }}"
+    extra_opts:
+      - --strip-components=1
+
+- name: Dashboards Install | create systemd service
+  template:
+    src: opensearch_dashboards.service
+    dest: "{{ systemctl_path }}/opensearch_dashboards.service"
+
+- name: Add dashboards jvm options template
+  copy:
+    src: jvm.options.template
+    dest: "{{ opensearch_dashboards_config_home }}/jvm.options.template"
+    owner: "{{ opensearch_user }}"
+    group: "{{ opensearch_group }}"
+
+- name: Copy template config file
+  copy:
+    src: "opensearch_dashboards.yml.template"
+    dest: "{{ opensearch_dashboards_config_file }}.template"
+    owner: "{{ opensearch_user }}"
+    group: "{{ opensearch_group }}"
+
+- name: Remove dashboards config file
+  file:
+    path: "{{ opensearch_dashboards_config_file }}"
+    state: absent

opensearch-20-04/ansible/roles/opensearch/tasks/main.yml

@@ -0,0 +1,67 @@
+---
+
+- name: setup group
+  group:
+    name: "{{ opensearch_group }}"
+    system: true
+    state: present
+
+- name: setup user
+  user:
+    name: "{{ opensearch_user }}"
+    group: "{{ opensearch_group }}"
+    system: true
+
+- name: Set vm.max_map_count in sysctl.conf
+  sysctl:
+    name: vm.max_map_count
+    value: 262144
+    state: present
+
+- name: Set open files limit in sysctl.conf
+  sysctl:
+    name: fs.file-max
+    value: 65536
+    state: present
+
+- name: create base directories
+  file:
+    path: "{{ item }}"
+    state: directory
+    owner: "{{ opensearch_user }}"
+    group: "{{ opensearch_group }}"
+    mode: 0744
+  loop:
+    - "{{ opensearch_home }}"
+    - "{{ opensearch_dashboards_home }}"
+    - "{{ nodecerts_home }}"
+
+- name: Download gomplate
+  get_url:
+    url: "https://github.com/hairyhenderson/gomplate/releases/download/v3.10.0/gomplate_linux-amd64"
+    dest: "/usr/bin/gomplate"
+    mode: a+x
+
+- name: include opensearch
+  include: opensearch.yml
+
+- name: include dashboards
+  include: dashboards.yml
+
+- name: include security
+  include: security.yml
+
+- name: Add opensearch init script
+  template:
+    src: 001_onboot
+    dest: "/var/lib/cloud/scripts/per-instance/001_onboot"
+    owner: "{{ opensearch_user }}"
+    group: "{{ opensearch_group }}"
+    mode: a+x
+
+- name: Run config script on boot
+  cron:
+    name: "configure-opensearch"
+    special_time: "reboot"
+    job: "/bin/bash /var/lib/cloud/scripts/per-instance/001_onboot"
+    user: "{{ opensearch_user }}"

opensearch-20-04/ansible/roles/opensearch/tasks/opensearch.yml

@@ -0,0 +1,41 @@
+---
+
+- name: Download opensearch
+  get_url:
+    url: "{{ opensearch_url }}"
+    dest: "/tmp/opensearch.tar.gz"
+
+- name: Unpack opensearch tarball
+  unarchive:
+    copy: no
+    src: "/tmp/opensearch.tar.gz"
+    dest: "{{ opensearch_home }}"
+    owner: "{{ opensearch_user }}"
+    group: "{{ opensearch_group }}"
+    mode: 0700
+    extra_opts:
+      - --strip-components=1
+
+- name: Add opensearch config template
+  copy:
+    src: opensearch.yml.template
+    dest: "{{ opensearch_config_file }}.template"
+    owner: "{{ opensearch_user }}"
+    group: "{{ opensearch_group }}"
+
+- name: Add opensearch jvm options template
+  copy:
+    src: jvm.options.template
+    dest: "{{ opensearch_config_home }}/jvm.options.template"
+    owner: "{{ opensearch_user }}"
+    group: "{{ opensearch_group }}"
+
+- name: Remove opensearch config file
+  file:
+    path: "{{ opensearch_config_file }}"
+    state: absent
+
+- name: OpenSearch Install | create systemd service
+  template:
+    src: opensearch.service
+    dest: "{{ systemctl_path }}/opensearch.service"