update to elk8.3

.env

@@ -1 +1,21 @@
-ELK_VERSION=7.16.2
+ELASTIC_VERSION=8.3.3
+
+## Passwords for stack users
+#
+
+# User 'elastic' (built-in)
+#
+# Superuser role, full access to cluster management and data indices.
+# https://www.elastic.co/guide/en/elasticsearch/reference/current/built-in-users.html
+ELASTIC_PASSWORD='changeme'
+# User 'logstash_internal' (custom)
+#
+# The user Logstash uses to connect and send data to Elasticsearch.
+# https://www.elastic.co/guide/en/logstash/current/ls-security.html
+LOGSTASH_INTERNAL_PASSWORD='changeme'
+
+# User 'kibana_system' (built-in)
+#
+# The user Kibana uses to connect and communicate with Elasticsearch.
+# https://www.elastic.co/guide/en/elasticsearch/reference/current/built-in-users.html
+KIBANA_SYSTEM_PASSWORD='changeme'

docker-compose.yml

@@ -1,11 +1,27 @@
 version: '3.2'
 
 services:
+  setup:
+    build:
+      context: setup/
+      args:
+        ELASTIC_VERSION: ${ELASTIC_VERSION}
+    init: true
+    volumes:
+      - setup:/state:Z
+    environment:
+      ELASTIC_PASSWORD: ${ELASTIC_PASSWORD:-}
+      LOGSTASH_INTERNAL_PASSWORD: ${LOGSTASH_INTERNAL_PASSWORD:-}
+      KIBANA_SYSTEM_PASSWORD: ${KIBANA_SYSTEM_PASSWORD:-}
+    networks:
+      - elk
+    depends_on:
+      - elasticsearch
   elasticsearch:
     build:
       context: elasticsearch/
       args:
-        ELK_VERSION: $ELK_VERSION
+        ELASTIC_VERSION: $ELASTIC_VERSION
     volumes:
       - type: bind
         source: ./elasticsearch/config/elasticsearch.yml
@@ -19,7 +35,7 @@ services:
       - "9300:9300"
     environment:
       ES_JAVA_OPTS: "-Xmx1g -Xms256m"
-      ELASTIC_PASSWORD: changeme
+      ELASTIC_PASSWORD: ${ELASTIC_PASSWORD:-}
       # Use single node discovery in order to disable production mode and avoid bootstrap checks.
       # see: https://www.elastic.co/guide/en/elasticsearch/reference/current/bootstrap-checks.html
       discovery.type: single-node
@@ -30,22 +46,18 @@ services:
     build:
       context: logstash/
       args:
-        ELK_VERSION: $ELK_VERSION
+        ELASTIC_VERSION: $ELASTIC_VERSION
     volumes:
-      - type: bind
-        source: ./logstash/config/logstash.yml
-        target: /usr/share/logstash/config/logstash.yml
-        read_only: true
-      - type: bind
-        source: ./logstash/pipeline
-        target: /usr/share/logstash/pipeline
-        read_only: true
+      - ./logstash/config/logstash.yml:/usr/share/logstash/config/logstash.yml:ro,Z
+      - ./logstash/pipeline:/usr/share/logstash/pipeline:ro,Z
     ports:
       - "5044:5044"
-      - "6100:5000"
+      - "50000:50000/tcp"
+      - "50000:50000/udp"
       - "9600:9600"
     environment:
       LS_JAVA_OPTS: "-Xmx256m -Xms256m"
+      LOGSTASH_INTERNAL_PASSWORD: ${LOGSTASH_INTERNAL_PASSWORD:-}
     networks:
       - elk
     depends_on:
@@ -55,14 +67,13 @@ services:
     build:
       context: kibana/
       args:
-        ELK_VERSION: $ELK_VERSION
+        ELASTIC_VERSION: $ELASTIC_VERSION
     volumes:
-      - type: bind
-        source: ./kibana/config/kibana.yml
-        target: /usr/share/kibana/config/kibana.yml
-        read_only: true
+      - ./kibana/config/kibana.yml:/usr/share/kibana/config/kibana.yml:ro,Z
     ports:
       - "5601:5601"
+    environment:
+      KIBANA_SYSTEM_PASSWORD: ${KIBANA_SYSTEM_PASSWORD:-}
     networks:
       - elk
     depends_on:
@@ -74,3 +85,4 @@ networks:
 
 volumes:
   elasticsearch:
+  setup:

elasticsearch/Dockerfile

@@ -1,7 +1,7 @@
-ARG ELK_VERSION
+ARG ELASTIC_VERSION
 
 # https://www.docker.elastic.co/
-FROM docker.elastic.co/elasticsearch/elasticsearch:${ELK_VERSION}
+FROM docker.elastic.co/elasticsearch/elasticsearch:${ELASTIC_VERSION}
 
 # Add your elasticsearch plugins setup here
 # Example: RUN elasticsearch-plugin install analysis-icu

elasticsearch/config/elasticsearch.yml

@@ -10,4 +10,3 @@ network.host: 0.0.0.0
 #
 xpack.license.self_generated.type: trial
 xpack.security.enabled: true
-xpack.monitoring.collection.enabled: true

extensions/apm-server/.dockerignore

@@ -0,0 +1,6 @@
+# Ignore Docker build files
+Dockerfile
+.dockerignore
+
+# Ignore OS artifacts
+**/.DS_Store

extensions/apm-server/Dockerfile

@@ -1,3 +1,3 @@
-ARG ELK_VERSION
+ARG ELASTIC_VERSION
 
-FROM docker.elastic.co/apm/apm-server:${ELK_VERSION}
+FROM docker.elastic.co/apm/apm-server:${ELASTIC_VERSION}

extensions/apm-server/README.md

@@ -52,5 +52,5 @@ Configuration settings for each supported language are available in the APM docu
 
 [Running APM Server on Docker][apm-docker]
 
-[apm-agents]: https://www.elastic.co/guide/en/apm/get-started/current/components.html#_apm_agents
-[apm-docker]: https://www.elastic.co/guide/en/apm/server/current/running-on-docker.html
+[apm-agents]: https://www.elastic.co/guide/en/apm/guide/current/components.html
+[apm-docker]: https://www.elastic.co/guide/en/apm/guide/current/running-on-docker.html

extensions/apm-server/apm-server-compose.yml

@@ -1,23 +1,23 @@
-version: '3.2'
+version: '3.7'
 
 services:
   apm-server:
     build:
       context: extensions/apm-server/
       args:
-        ELK_VERSION: $ELK_VERSION
+        ELASTIC_VERSION: ${ELASTIC_VERSION}
     command:
         # Disable strict permission checking on 'apm-server.yml' configuration file
         # https://www.elastic.co/guide/en/beats/libbeat/current/config-file-permissions.html
       - --strict.perms=false
     volumes:
-      - type: bind
-        source: ./extensions/apm-server/config/apm-server.yml
-        target: /usr/share/apm-server/apm-server.yml
-        read_only: true
+      - ./extensions/apm-server/config/apm-server.yml:/usr/share/apm-server/apm-server.yml:ro,Z
     ports:
       - '8200:8200'
+    environment:
+      ELASTIC_PASSWORD: ${ELASTIC_PASSWORD:-}
     networks:
       - elk
     depends_on:
       - elasticsearch
+

extensions/apm-server/config/apm-server.yml

@@ -5,4 +5,4 @@ output:
   elasticsearch:
     hosts: ['http://elasticsearch:9200']
     username: elastic
-    password: changeme
+    password: ${ELASTIC_PASSWORD}

extensions/curator/.dockerignore

@@ -0,0 +1,6 @@
+# Ignore Docker build files
+Dockerfile
+.dockerignore
+
+# Ignore OS artifacts
+**/.DS_Store

extensions/curator/config/curator.yml

@@ -5,7 +5,7 @@ client:
   hosts:
     - elasticsearch
   port: 9200
-  http_auth: elastic:changeme
+  http_auth: 'elastic:changeme'
 
 logging:
   loglevel: INFO

extensions/curator/curator-compose.yml

@@ -1,19 +1,13 @@
-version: '3.2'
+version: '3.7'
 
 services:
   curator:
     build:
       context: extensions/curator/
     init: true
     volumes:
-      - type: bind
-        source: ./extensions/curator/config/curator.yml
-        target: /usr/share/curator/config/curator.yml
-        read_only: true
-      - type: bind
-        source: ./extensions/curator/config/delete_log_files_curator.yml
-        target: /usr/share/curator/config/delete_log_files_curator.yml
-        read_only: true
+      - ./extensions/curator/config/curator.yml:/usr/share/curator/config/curator.yml:ro,Z
+      - ./extensions/curator/config/delete_log_files_curator.yml:/usr/share/curator/config/delete_log_files_curator.yml:ro,Z
     networks:
       - elk
     depends_on:

extensions/enterprise-search/.dockerignore

@@ -0,0 +1,6 @@
+# Ignore Docker build files
+Dockerfile
+.dockerignore
+
+# Ignore OS artifacts
+**/.DS_Store

extensions/enterprise-search/Dockerfile

@@ -1,4 +1,4 @@
-ARG ELK_VERSION
+ARG ELASTIC_VERSION
 
 # https://www.docker.elastic.co/
-FROM docker.elastic.co/enterprise-search/enterprise-search:${ELK_VERSION}
+FROM docker.elastic.co/enterprise-search/enterprise-search:${ELASTIC_VERSION}

extensions/enterprise-search/README.md

@@ -31,8 +31,9 @@ secret_management.encryption_keys:
 secret_management.encryption_keys: [my_first_encryption_key, my_second_encryption_key, ...]
 ```
 
-> :information_source: To generate a strong encryption key, for example using the AES-256 cipher, you can use the
-> OpenSSL utility or any other online/offline tool of your choice:
+> **Note**  
+> To generate a strong encryption key, for example using the AES-256 cipher, you can use the OpenSSL utility or any
+> other online/offline tool of your choice:
 >
 > ```console
 > $ openssl enc -aes-256 -P
@@ -57,6 +58,17 @@ add the following setting:
 xpack.security.authc.api_key.enabled: true
 ```
 
+### Configure the Enterprise Search host in Kibana
+
+Kibana acts as the [management interface][enterprisesearch-ui] to Enterprise Search.
+
+To enable the management experience for Enterprise Search, modify the Kibana configuration file in
+[`kibana/config/kibana.yml`][config-kbn] and add the following setting:
+
+```yaml
+enterpriseSearch.host: http://enterprise-search:3002
+```
+
 ### Start the server
 
 To include Enterprise Search in the stack, run Docker Compose from the root of the repository with an additional command
@@ -89,8 +101,9 @@ enterprise-search:
     ENT_SEARCH_DEFAULT_PASSWORD: {{some strong password}}
 ```
 
-> :warning: The default Enterprise Search password can only be set during the initial boot. Once the password is
-> persisted in Elasticsearch, it can only be changed via the Elasticsearch API.
+> **Warning**  
+> The default Enterprise Search password can only be set during the initial boot. Once the password is persisted in
+> Elasticsearch, it can only be changed via the Elasticsearch API.
 
 For more information, please refer to [User Management and Security][enterprisesearch-security].
 
@@ -129,6 +142,8 @@ Docker container: [Running Enterprise Search Using Docker][enterprisesearch-dock
 [enterprisesearch-config]: https://www.elastic.co/guide/en/enterprise-search/current/configuration.html
 [enterprisesearch-docker]: https://www.elastic.co/guide/en/enterprise-search/current/docker.html
 [enterprisesearch-docs]: https://www.elastic.co/guide/en/enterprise-search/current/index.html
+[enterprisesearch-ui]: https://www.elastic.co/guide/en/enterprise-search/current/user-interfaces.html
 
 [es-security]: https://www.elastic.co/guide/en/elasticsearch/reference/current/security-settings.html#api-key-service-settings
 [config-es]: ../../elasticsearch/config/elasticsearch.yml
+[config-kbn]: ../../kibana/config/kibana.yml

extensions/enterprise-search/config/enterprise-search.yml

@@ -15,13 +15,14 @@ secret_management.encryption_keys:
 # IP address Enterprise Search listens on
 ent_search.listen_host: 0.0.0.0
 
-# URL at which users reach Enterprise Search
+# URL at which users reach Enterprise Search / Kibana
 ent_search.external_url: http://localhost:3002
+kibana.host: http://localhost:5601
 
 # Elasticsearch URL and credentials
 elasticsearch.host: http://elasticsearch:9200
 elasticsearch.username: elastic
-elasticsearch.password: changeme
+elasticsearch.password: ${ELASTIC_PASSWORD}
 
 # Allow Enterprise Search to modify Elasticsearch settings. Used to enable auto-creation of Elasticsearch indexes.
 allow_es_settings_modification: true

extensions/enterprise-search/enterprise-search-compose.yml

@@ -1,19 +1,17 @@
-version: '3.2'
+version: '3.7'
 
 services:
   enterprise-search:
     build:
       context: extensions/enterprise-search/
       args:
-        ELK_VERSION: $ELK_VERSION
+        ELASTIC_VERSION: ${ELASTIC_VERSION}
     volumes:
-      - type: bind
-        source: ./extensions/enterprise-search/config/enterprise-search.yml
-        target: /usr/share/enterprise-search/config/enterprise-search.yml
-        read_only: true
+      - ./extensions/enterprise-search/config/enterprise-search.yml:/usr/share/enterprise-search/config/enterprise-search.yml:ro,Z
     environment:
-      JAVA_OPTS: -Xmx2g -Xms2g
-      ENT_SEARCH_DEFAULT_PASSWORD: changeme
+      JAVA_OPTS: -Xms2g -Xmx2g
+      ENT_SEARCH_DEFAULT_PASSWORD: 'changeme'
+      ELASTIC_PASSWORD: ${ELASTIC_PASSWORD:-}
     ports:
       - '3002:3002'
     networks:

extensions/filebeat/.dockerignore

@@ -0,0 +1,6 @@
+# Ignore Docker build files
+Dockerfile
+.dockerignore
+
+# Ignore OS artifacts
+**/.DS_Store

extensions/filebeat/Dockerfile

@@ -0,0 +1,3 @@
+ARG ELASTIC_VERSION
+
+FROM docker.elastic.co/beats/filebeat:${ELASTIC_VERSION}

extensions/filebeat/README.md

@@ -0,0 +1,36 @@
+# Filebeat
+
+Filebeat is a lightweight shipper for forwarding and centralizing log data. Installed as an agent on your servers,
+Filebeat monitors the log files or locations that you specify, collects log events, and forwards them either to
+Elasticsearch or Logstash for indexing.
+
+## Usage
+
+To include Filebeat in the stack, run Docker Compose from the root of the repository with an additional command line
+argument referencing the `filebeat-compose.yml` file:
+
+```console
+$ docker-compose -f docker-compose.yml -f extensions/filebeat/filebeat-compose.yml up
+```
+
+## Configuring Filebeat
+
+The Filebeat configuration is stored in [`config/filebeat.yml`](./config/filebeat.yml). You can modify this file with
+the help of the [Configuration reference][filebeat-config].
+
+Any change to the Filebeat configuration requires a restart of the Filebeat container:
+
+```console
+$ docker-compose -f docker-compose.yml -f extensions/filebeat/filebeat-compose.yml restart filebeat
+```
+
+Please refer to the following documentation page for more details about how to configure Filebeat inside a Docker
+container: [Run Filebeat on Docker][filebeat-docker].
+
+## See also
+
+[Filebeat documentation][filebeat-doc]
+
+[filebeat-config]: https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-reference-yml.html
+[filebeat-docker]: https://www.elastic.co/guide/en/beats/filebeat/current/running-on-docker.html
+[filebeat-doc]: https://www.elastic.co/guide/en/beats/filebeat/current/index.html