From ef71e888d2922024d699c398d506b81249c2d273 Mon Sep 17 00:00:00 2001 From: Alessandro Fiori Date: Mon, 1 Aug 2022 13:47:15 +0200 Subject: [PATCH] update to elk8.3 --- .env | 22 +- docker-compose.yml | 46 +++-- elasticsearch/Dockerfile | 4 +- elasticsearch/config/elasticsearch.yml | 1 - extensions/.DS_Store | Bin 6148 -> 0 bytes extensions/apm-server/.dockerignore | 6 + extensions/apm-server/Dockerfile | 4 +- extensions/apm-server/README.md | 4 +- extensions/apm-server/apm-server-compose.yml | 12 +- extensions/apm-server/config/apm-server.yml | 2 +- extensions/curator/.dockerignore | 6 + extensions/curator/config/curator.yml | 2 +- extensions/curator/curator-compose.yml | 12 +- extensions/enterprise-search/.dockerignore | 6 + extensions/enterprise-search/Dockerfile | 4 +- extensions/enterprise-search/README.md | 23 ++- .../config/enterprise-search.yml | 5 +- .../enterprise-search-compose.yml | 14 +- extensions/filebeat/.dockerignore | 6 + extensions/filebeat/Dockerfile | 3 + extensions/filebeat/README.md | 36 ++++ extensions/filebeat/config/filebeat.yml | 30 +++ extensions/filebeat/filebeat-compose.yml | 34 +++ extensions/heartbeat/.dockerignore | 6 + extensions/heartbeat/Dockerfile | 3 + extensions/heartbeat/README.md | 35 ++++ extensions/heartbeat/config/heartbeat.yml | 31 +++ extensions/heartbeat/heartbeat-compose.yml | 23 +++ extensions/logspout/.dockerignore | 6 + extensions/logspout/README.md | 2 +- extensions/logspout/logspout-compose.yml | 9 +- extensions/metricbeat/.dockerignore | 6 + extensions/metricbeat/Dockerfile | 4 +- extensions/metricbeat/config/metricbeat.yml | 2 +- extensions/metricbeat/metricbeat-compose.yml | 23 +-- kibana/Dockerfile | 4 +- kibana/config/kibana.yml | 4 +- logstash/Dockerfile | 4 +- logstash/config/logstash.yml | 7 - logstash/pipeline/logstash.conf | 9 +- setup/Dockerfile | 17 ++ setup/entrypoint.sh | 107 ++++++++++ setup/helpers.sh | 193 ++++++++++++++++++ setup/roles/logstash_writer.json | 33 +++ 44 files changed, 715 insertions(+), 95 deletions(-) delete mode 100644 extensions/.DS_Store create mode 100644 extensions/apm-server/.dockerignore create mode 100644 extensions/curator/.dockerignore create mode 100644 extensions/enterprise-search/.dockerignore create mode 100644 extensions/filebeat/.dockerignore create mode 100644 extensions/filebeat/Dockerfile create mode 100644 extensions/filebeat/README.md create mode 100644 extensions/filebeat/config/filebeat.yml create mode 100644 extensions/filebeat/filebeat-compose.yml create mode 100644 extensions/heartbeat/.dockerignore create mode 100644 extensions/heartbeat/Dockerfile create mode 100644 extensions/heartbeat/README.md create mode 100644 extensions/heartbeat/config/heartbeat.yml create mode 100644 extensions/heartbeat/heartbeat-compose.yml create mode 100644 extensions/logspout/.dockerignore create mode 100644 extensions/metricbeat/.dockerignore create mode 100644 setup/Dockerfile create mode 100644 setup/entrypoint.sh create mode 100644 setup/helpers.sh create mode 100644 setup/roles/logstash_writer.json diff --git a/.env b/.env index 75a20a1..49ad31e 100644 --- a/.env +++ b/.env @@ -1 +1,21 @@ -ELK_VERSION=7.16.2 \ No newline at end of file +ELASTIC_VERSION=8.3.3 + +## Passwords for stack users +# + +# User 'elastic' (built-in) +# +# Superuser role, full access to cluster management and data indices. +# https://www.elastic.co/guide/en/elasticsearch/reference/current/built-in-users.html +ELASTIC_PASSWORD='changeme' +# User 'logstash_internal' (custom) +# +# The user Logstash uses to connect and send data to Elasticsearch. +# https://www.elastic.co/guide/en/logstash/current/ls-security.html +LOGSTASH_INTERNAL_PASSWORD='changeme' + +# User 'kibana_system' (built-in) +# +# The user Kibana uses to connect and communicate with Elasticsearch. +# https://www.elastic.co/guide/en/elasticsearch/reference/current/built-in-users.html +KIBANA_SYSTEM_PASSWORD='changeme' \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index db98864..31650e2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,11 +1,27 @@ version: '3.2' services: + setup: + build: + context: setup/ + args: + ELASTIC_VERSION: ${ELASTIC_VERSION} + init: true + volumes: + - setup:/state:Z + environment: + ELASTIC_PASSWORD: ${ELASTIC_PASSWORD:-} + LOGSTASH_INTERNAL_PASSWORD: ${LOGSTASH_INTERNAL_PASSWORD:-} + KIBANA_SYSTEM_PASSWORD: ${KIBANA_SYSTEM_PASSWORD:-} + networks: + - elk + depends_on: + - elasticsearch elasticsearch: build: context: elasticsearch/ args: - ELK_VERSION: $ELK_VERSION + ELASTIC_VERSION: $ELASTIC_VERSION volumes: - type: bind source: ./elasticsearch/config/elasticsearch.yml @@ -19,7 +35,7 @@ services: - "9300:9300" environment: ES_JAVA_OPTS: "-Xmx1g -Xms256m" - ELASTIC_PASSWORD: changeme + ELASTIC_PASSWORD: ${ELASTIC_PASSWORD:-} # Use single node discovery in order to disable production mode and avoid bootstrap checks. # see: https://www.elastic.co/guide/en/elasticsearch/reference/current/bootstrap-checks.html discovery.type: single-node @@ -30,22 +46,18 @@ services: build: context: logstash/ args: - ELK_VERSION: $ELK_VERSION + ELASTIC_VERSION: $ELASTIC_VERSION volumes: - - type: bind - source: ./logstash/config/logstash.yml - target: /usr/share/logstash/config/logstash.yml - read_only: true - - type: bind - source: ./logstash/pipeline - target: /usr/share/logstash/pipeline - read_only: true + - ./logstash/config/logstash.yml:/usr/share/logstash/config/logstash.yml:ro,Z + - ./logstash/pipeline:/usr/share/logstash/pipeline:ro,Z ports: - "5044:5044" - - "6100:5000" + - "50000:50000/tcp" + - "50000:50000/udp" - "9600:9600" environment: LS_JAVA_OPTS: "-Xmx256m -Xms256m" + LOGSTASH_INTERNAL_PASSWORD: ${LOGSTASH_INTERNAL_PASSWORD:-} networks: - elk depends_on: @@ -55,14 +67,13 @@ services: build: context: kibana/ args: - ELK_VERSION: $ELK_VERSION + ELASTIC_VERSION: $ELASTIC_VERSION volumes: - - type: bind - source: ./kibana/config/kibana.yml - target: /usr/share/kibana/config/kibana.yml - read_only: true + - ./kibana/config/kibana.yml:/usr/share/kibana/config/kibana.yml:ro,Z ports: - "5601:5601" + environment: + KIBANA_SYSTEM_PASSWORD: ${KIBANA_SYSTEM_PASSWORD:-} networks: - elk depends_on: @@ -74,3 +85,4 @@ networks: volumes: elasticsearch: + setup: diff --git a/elasticsearch/Dockerfile b/elasticsearch/Dockerfile index 3928544..22528c6 100644 --- a/elasticsearch/Dockerfile +++ b/elasticsearch/Dockerfile @@ -1,7 +1,7 @@ -ARG ELK_VERSION +ARG ELASTIC_VERSION # https://www.docker.elastic.co/ -FROM docker.elastic.co/elasticsearch/elasticsearch:${ELK_VERSION} +FROM docker.elastic.co/elasticsearch/elasticsearch:${ELASTIC_VERSION} # Add your elasticsearch plugins setup here # Example: RUN elasticsearch-plugin install analysis-icu diff --git a/elasticsearch/config/elasticsearch.yml b/elasticsearch/config/elasticsearch.yml index 86822dd..2a0c2af 100644 --- a/elasticsearch/config/elasticsearch.yml +++ b/elasticsearch/config/elasticsearch.yml @@ -10,4 +10,3 @@ network.host: 0.0.0.0 # xpack.license.self_generated.type: trial xpack.security.enabled: true -xpack.monitoring.collection.enabled: true diff --git a/extensions/.DS_Store b/extensions/.DS_Store deleted file mode 100644 index 4dfae7bd370ec8ef7e55181ce15b3f0a97466b5d..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6148 zcmeHKK}y3w6#b)x*0`{W3vm^3tH9%{Z7Z7UKmG-(d)d|@uJxGgJ~RKyfu642aD&It9tHm zV{S6Z&+p-t)TEG>*k1;;pqS=Gg?q1MMqXQgm1+C(o+w_cs%I7T+_0WQ>~k|{;XN-w zdC>kD7YBF_X;Rd9dbU|sUUyU7etNpWeEGHfF0*2NEtP$r4|Ro0dXL6fcFO%h0(K^8q`|{Dt!e2HqfmDZN5uzj%(4gm>R?gLYY*cNmcfW zp-ejZUCZ++k=?BzY1C z#DRb1fNJ :information_source: To generate a strong encryption key, for example using the AES-256 cipher, you can use the -> OpenSSL utility or any other online/offline tool of your choice: +> **Note** +> To generate a strong encryption key, for example using the AES-256 cipher, you can use the OpenSSL utility or any +> other online/offline tool of your choice: > > ```console > $ openssl enc -aes-256 -P @@ -57,6 +58,17 @@ add the following setting: xpack.security.authc.api_key.enabled: true ``` +### Configure the Enterprise Search host in Kibana + +Kibana acts as the [management interface][enterprisesearch-ui] to Enterprise Search. + +To enable the management experience for Enterprise Search, modify the Kibana configuration file in +[`kibana/config/kibana.yml`][config-kbn] and add the following setting: + +```yaml +enterpriseSearch.host: http://enterprise-search:3002 +``` + ### Start the server To include Enterprise Search in the stack, run Docker Compose from the root of the repository with an additional command @@ -89,8 +101,9 @@ enterprise-search: ENT_SEARCH_DEFAULT_PASSWORD: {{some strong password}} ``` -> :warning: The default Enterprise Search password can only be set during the initial boot. Once the password is -> persisted in Elasticsearch, it can only be changed via the Elasticsearch API. +> **Warning** +> The default Enterprise Search password can only be set during the initial boot. Once the password is persisted in +> Elasticsearch, it can only be changed via the Elasticsearch API. For more information, please refer to [User Management and Security][enterprisesearch-security]. @@ -129,6 +142,8 @@ Docker container: [Running Enterprise Search Using Docker][enterprisesearch-dock [enterprisesearch-config]: https://www.elastic.co/guide/en/enterprise-search/current/configuration.html [enterprisesearch-docker]: https://www.elastic.co/guide/en/enterprise-search/current/docker.html [enterprisesearch-docs]: https://www.elastic.co/guide/en/enterprise-search/current/index.html +[enterprisesearch-ui]: https://www.elastic.co/guide/en/enterprise-search/current/user-interfaces.html [es-security]: https://www.elastic.co/guide/en/elasticsearch/reference/current/security-settings.html#api-key-service-settings [config-es]: ../../elasticsearch/config/elasticsearch.yml +[config-kbn]: ../../kibana/config/kibana.yml diff --git a/extensions/enterprise-search/config/enterprise-search.yml b/extensions/enterprise-search/config/enterprise-search.yml index 891b510..40bb592 100644 --- a/extensions/enterprise-search/config/enterprise-search.yml +++ b/extensions/enterprise-search/config/enterprise-search.yml @@ -15,13 +15,14 @@ secret_management.encryption_keys: # IP address Enterprise Search listens on ent_search.listen_host: 0.0.0.0 -# URL at which users reach Enterprise Search +# URL at which users reach Enterprise Search / Kibana ent_search.external_url: http://localhost:3002 +kibana.host: http://localhost:5601 # Elasticsearch URL and credentials elasticsearch.host: http://elasticsearch:9200 elasticsearch.username: elastic -elasticsearch.password: changeme +elasticsearch.password: ${ELASTIC_PASSWORD} # Allow Enterprise Search to modify Elasticsearch settings. Used to enable auto-creation of Elasticsearch indexes. allow_es_settings_modification: true diff --git a/extensions/enterprise-search/enterprise-search-compose.yml b/extensions/enterprise-search/enterprise-search-compose.yml index cd58caa..ad25739 100644 --- a/extensions/enterprise-search/enterprise-search-compose.yml +++ b/extensions/enterprise-search/enterprise-search-compose.yml @@ -1,19 +1,17 @@ -version: '3.2' +version: '3.7' services: enterprise-search: build: context: extensions/enterprise-search/ args: - ELK_VERSION: $ELK_VERSION + ELASTIC_VERSION: ${ELASTIC_VERSION} volumes: - - type: bind - source: ./extensions/enterprise-search/config/enterprise-search.yml - target: /usr/share/enterprise-search/config/enterprise-search.yml - read_only: true + - ./extensions/enterprise-search/config/enterprise-search.yml:/usr/share/enterprise-search/config/enterprise-search.yml:ro,Z environment: - JAVA_OPTS: -Xmx2g -Xms2g - ENT_SEARCH_DEFAULT_PASSWORD: changeme + JAVA_OPTS: -Xms2g -Xmx2g + ENT_SEARCH_DEFAULT_PASSWORD: 'changeme' + ELASTIC_PASSWORD: ${ELASTIC_PASSWORD:-} ports: - '3002:3002' networks: diff --git a/extensions/filebeat/.dockerignore b/extensions/filebeat/.dockerignore new file mode 100644 index 0000000..37eef9d --- /dev/null +++ b/extensions/filebeat/.dockerignore @@ -0,0 +1,6 @@ +# Ignore Docker build files +Dockerfile +.dockerignore + +# Ignore OS artifacts +**/.DS_Store diff --git a/extensions/filebeat/Dockerfile b/extensions/filebeat/Dockerfile new file mode 100644 index 0000000..b8dd5f3 --- /dev/null +++ b/extensions/filebeat/Dockerfile @@ -0,0 +1,3 @@ +ARG ELASTIC_VERSION + +FROM docker.elastic.co/beats/filebeat:${ELASTIC_VERSION} diff --git a/extensions/filebeat/README.md b/extensions/filebeat/README.md new file mode 100644 index 0000000..43c5d67 --- /dev/null +++ b/extensions/filebeat/README.md @@ -0,0 +1,36 @@ +# Filebeat + +Filebeat is a lightweight shipper for forwarding and centralizing log data. Installed as an agent on your servers, +Filebeat monitors the log files or locations that you specify, collects log events, and forwards them either to +Elasticsearch or Logstash for indexing. + +## Usage + +To include Filebeat in the stack, run Docker Compose from the root of the repository with an additional command line +argument referencing the `filebeat-compose.yml` file: + +```console +$ docker-compose -f docker-compose.yml -f extensions/filebeat/filebeat-compose.yml up +``` + +## Configuring Filebeat + +The Filebeat configuration is stored in [`config/filebeat.yml`](./config/filebeat.yml). You can modify this file with +the help of the [Configuration reference][filebeat-config]. + +Any change to the Filebeat configuration requires a restart of the Filebeat container: + +```console +$ docker-compose -f docker-compose.yml -f extensions/filebeat/filebeat-compose.yml restart filebeat +``` + +Please refer to the following documentation page for more details about how to configure Filebeat inside a Docker +container: [Run Filebeat on Docker][filebeat-docker]. + +## See also + +[Filebeat documentation][filebeat-doc] + +[filebeat-config]: https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-reference-yml.html +[filebeat-docker]: https://www.elastic.co/guide/en/beats/filebeat/current/running-on-docker.html +[filebeat-doc]: https://www.elastic.co/guide/en/beats/filebeat/current/index.html diff --git a/extensions/filebeat/config/filebeat.yml b/extensions/filebeat/config/filebeat.yml new file mode 100644 index 0000000..dfbd0c3 --- /dev/null +++ b/extensions/filebeat/config/filebeat.yml @@ -0,0 +1,30 @@ +## Filebeat configuration +## https://github.com/elastic/beats/blob/master/deploy/docker/filebeat.docker.yml +# + +filebeat.config: + modules: + path: ${path.config}/modules.d/*.yml + reload.enabled: false + +filebeat.autodiscover: + providers: + # The Docker autodiscover provider automatically retrieves logs from Docker + # containers as they start and stop. + - type: docker + hints.enabled: true + +processors: + - add_cloud_metadata: ~ + +output.elasticsearch: + hosts: ['http://elasticsearch:9200'] + username: elastic + password: ${ELASTIC_PASSWORD} + +## HTTP endpoint for health checking +## https://www.elastic.co/guide/en/beats/filebeat/current/http-endpoint.html +# + +http.enabled: true +http.host: 0.0.0.0 diff --git a/extensions/filebeat/filebeat-compose.yml b/extensions/filebeat/filebeat-compose.yml new file mode 100644 index 0000000..04ff104 --- /dev/null +++ b/extensions/filebeat/filebeat-compose.yml @@ -0,0 +1,34 @@ +version: '3.7' + +services: + filebeat: + build: + context: extensions/filebeat/ + args: + ELASTIC_VERSION: ${ELASTIC_VERSION} + # Run as 'root' instead of 'filebeat' (uid 1000) to allow reading + # 'docker.sock' and the host's filesystem. + user: root + command: + # Log to stderr. + - -e + # Disable config file permissions checks. Allows mounting + # 'config/filebeat.yml' even if it's not owned by root. + # see: https://www.elastic.co/guide/en/beats/libbeat/current/config-file-permissions.html + - --strict.perms=false + volumes: + - ./extensions/filebeat/config/filebeat.yml:/usr/share/filebeat/filebeat.yml:ro,Z + - type: bind + source: /var/lib/docker/containers + target: /var/lib/docker/containers + read_only: true + - type: bind + source: /var/run/docker.sock + target: /var/run/docker.sock + read_only: true + environment: + ELASTIC_PASSWORD: ${ELASTIC_PASSWORD:-} + networks: + - elk + depends_on: + - elasticsearch diff --git a/extensions/heartbeat/.dockerignore b/extensions/heartbeat/.dockerignore new file mode 100644 index 0000000..37eef9d --- /dev/null +++ b/extensions/heartbeat/.dockerignore @@ -0,0 +1,6 @@ +# Ignore Docker build files +Dockerfile +.dockerignore + +# Ignore OS artifacts +**/.DS_Store diff --git a/extensions/heartbeat/Dockerfile b/extensions/heartbeat/Dockerfile new file mode 100644 index 0000000..0d7de19 --- /dev/null +++ b/extensions/heartbeat/Dockerfile @@ -0,0 +1,3 @@ +ARG ELASTIC_VERSION + +FROM docker.elastic.co/beats/heartbeat:${ELASTIC_VERSION} diff --git a/extensions/heartbeat/README.md b/extensions/heartbeat/README.md new file mode 100644 index 0000000..da02f92 --- /dev/null +++ b/extensions/heartbeat/README.md @@ -0,0 +1,35 @@ +# Heartbeat + +Heartbeat is a lightweight daemon that periodically checks the status of your services and determines whether they are +available. + +## Usage + +To include Heartbeat in the stack, run Docker Compose from the root of the repository with an additional command line +argument referencing the `heartbeat-compose.yml` file: + +```console +$ docker-compose -f docker-compose.yml -f extensions/heartbeat/heartbeat-compose.yml up +``` + +## Configuring Heartbeat + +The Heartbeat configuration is stored in [`config/heartbeat.yml`](./config/heartbeat.yml). You can modify this file +with the help of the [Configuration reference][heartbeat-config]. + +Any change to the Heartbeat configuration requires a restart of the Heartbeat container: + +```console +$ docker-compose -f docker-compose.yml -f extensions/heartbeat/heartbeat-compose.yml restart heartbeat +``` + +Please refer to the following documentation page for more details about how to configure Heartbeat inside a +Docker container: [Run Heartbeat on Docker][heartbeat-docker]. + +## See also + +[Heartbeat documentation][heartbeat-doc] + +[heartbeat-config]: https://www.elastic.co/guide/en/beats/heartbeat/current/heartbeat-reference-yml.html +[heartbeat-docker]: https://www.elastic.co/guide/en/beats/heartbeat/current/running-on-docker.html +[heartbeat-doc]: https://www.elastic.co/guide/en/beats/heartbeat/current/index.html diff --git a/extensions/heartbeat/config/heartbeat.yml b/extensions/heartbeat/config/heartbeat.yml new file mode 100644 index 0000000..b5276c0 --- /dev/null +++ b/extensions/heartbeat/config/heartbeat.yml @@ -0,0 +1,31 @@ +## Heartbeat configuration +## https://github.com/elastic/beats/blob/main/deploy/docker/heartbeat.docker.yml +# + +heartbeat.monitors: +- type: http + schedule: '@every 5s' + urls: + - http://elasticsearch:9200 + username: elastic + password: ${ELASTIC_PASSWORD} + +- type: icmp + schedule: '@every 5s' + hosts: + - elasticsearch + +processors: +- add_cloud_metadata: ~ + +output.elasticsearch: + hosts: ['http://elasticsearch:9200'] + username: elastic + password: ${ELASTIC_PASSWORD} + +## HTTP endpoint for health checking +## https://www.elastic.co/guide/en/beats/heartbeat/current/http-endpoint.html +# + +http.enabled: true +http.host: 0.0.0.0 diff --git a/extensions/heartbeat/heartbeat-compose.yml b/extensions/heartbeat/heartbeat-compose.yml new file mode 100644 index 0000000..a6fe2ad --- /dev/null +++ b/extensions/heartbeat/heartbeat-compose.yml @@ -0,0 +1,23 @@ +version: '3.7' + +services: + heartbeat: + build: + context: extensions/heartbeat/ + args: + ELASTIC_VERSION: ${ELASTIC_VERSION} + command: + # Log to stderr. + - -e + # Disable config file permissions checks. Allows mounting + # 'config/heartbeat.yml' even if it's not owned by root. + # see: https://www.elastic.co/guide/en/beats/libbeat/current/config-file-permissions.html + - --strict.perms=false + volumes: + - ./extensions/heartbeat/config/heartbeat.yml:/usr/share/heartbeat/heartbeat.yml:ro,Z + environment: + ELASTIC_PASSWORD: ${ELASTIC_PASSWORD:-} + networks: + - elk + depends_on: + - elasticsearch diff --git a/extensions/logspout/.dockerignore b/extensions/logspout/.dockerignore new file mode 100644 index 0000000..37eef9d --- /dev/null +++ b/extensions/logspout/.dockerignore @@ -0,0 +1,6 @@ +# Ignore Docker build files +Dockerfile +.dockerignore + +# Ignore OS artifacts +**/.DS_Store diff --git a/extensions/logspout/README.md b/extensions/logspout/README.md index f6a4d2b..2e34648 100644 --- a/extensions/logspout/README.md +++ b/extensions/logspout/README.md @@ -17,7 +17,7 @@ In your Logstash pipeline configuration, enable the `udp` input and set the inpu ```logstash input { udp { - port => 5000 + port => 50000 codec => json } } diff --git a/extensions/logspout/logspout-compose.yml b/extensions/logspout/logspout-compose.yml index 1c349f2..8af149d 100644 --- a/extensions/logspout/logspout-compose.yml +++ b/extensions/logspout/logspout-compose.yml @@ -1,13 +1,16 @@ -version: '3.2' +version: '3.7' services: logspout: build: context: extensions/logspout volumes: - - /var/run/docker.sock:/var/run/docker.sock:ro + - type: bind + source: /var/run/docker.sock + target: /var/run/docker.sock + read_only: true environment: - ROUTE_URIS: logstash://logstash:5000 + ROUTE_URIS: logstash://logstash:50000 LOGSTASH_TAGS: docker-elk networks: - elk diff --git a/extensions/metricbeat/.dockerignore b/extensions/metricbeat/.dockerignore new file mode 100644 index 0000000..37eef9d --- /dev/null +++ b/extensions/metricbeat/.dockerignore @@ -0,0 +1,6 @@ +# Ignore Docker build files +Dockerfile +.dockerignore + +# Ignore OS artifacts +**/.DS_Store diff --git a/extensions/metricbeat/Dockerfile b/extensions/metricbeat/Dockerfile index e55f625..6d05bf5 100644 --- a/extensions/metricbeat/Dockerfile +++ b/extensions/metricbeat/Dockerfile @@ -1,3 +1,3 @@ -ARG ELK_VERSION +ARG ELASTIC_VERSION -FROM docker.elastic.co/beats/metricbeat:${ELK_VERSION} +FROM docker.elastic.co/beats/metricbeat:${ELASTIC_VERSION} diff --git a/extensions/metricbeat/config/metricbeat.yml b/extensions/metricbeat/config/metricbeat.yml index eac94d6..f7c07ca 100644 --- a/extensions/metricbeat/config/metricbeat.yml +++ b/extensions/metricbeat/config/metricbeat.yml @@ -34,7 +34,7 @@ processors: output.elasticsearch: hosts: ['http://elasticsearch:9200'] username: elastic - password: changeme + password: ${ELASTIC_PASSWORD} ## HTTP endpoint for health checking ## https://www.elastic.co/guide/en/beats/metricbeat/current/http-endpoint.html diff --git a/extensions/metricbeat/metricbeat-compose.yml b/extensions/metricbeat/metricbeat-compose.yml index c8f9032..da62833 100644 --- a/extensions/metricbeat/metricbeat-compose.yml +++ b/extensions/metricbeat/metricbeat-compose.yml @@ -1,29 +1,26 @@ -version: '3.2' +version: '3.7' services: metricbeat: build: context: extensions/metricbeat/ args: - ELK_VERSION: $ELK_VERSION + ELASTIC_VERSION: ${ELASTIC_VERSION} # Run as 'root' instead of 'metricbeat' (uid 1000) to allow reading # 'docker.sock' and the host's filesystem. user: root command: - # Log to stderr. + # Log to stderr. - -e - # Disable config file permissions checks. Allows mounting - # 'config/metricbeat.yml' even if it's not owned by root. - # see: https://www.elastic.co/guide/en/beats/libbeat/current/config-file-permissions.html + # Disable config file permissions checks. Allows mounting + # 'config/metricbeat.yml' even if it's not owned by root. + # see: https://www.elastic.co/guide/en/beats/libbeat/current/config-file-permissions.html - --strict.perms=false - # Mount point of the host’s filesystem. Required to monitor the host - # from within a container. + # Mount point of the host’s filesystem. Required to monitor the host + # from within a container. - --system.hostfs=/hostfs volumes: - - type: bind - source: ./extensions/metricbeat/config/metricbeat.yml - target: /usr/share/metricbeat/metricbeat.yml - read_only: true + - ./extensions/metricbeat/config/metricbeat.yml:/usr/share/metricbeat/metricbeat.yml:ro,Z - type: bind source: / target: /hostfs @@ -40,6 +37,8 @@ services: source: /var/run/docker.sock target: /var/run/docker.sock read_only: true + environment: + ELASTIC_PASSWORD: ${ELASTIC_PASSWORD:-} networks: - elk depends_on: diff --git a/kibana/Dockerfile b/kibana/Dockerfile index 2fb3659..9a075be 100644 --- a/kibana/Dockerfile +++ b/kibana/Dockerfile @@ -1,7 +1,7 @@ -ARG ELK_VERSION +ARG ELASTIC_VERSION # https://www.docker.elastic.co/ -FROM docker.elastic.co/kibana/kibana:${ELK_VERSION} +FROM docker.elastic.co/kibana/kibana:${ELASTIC_VERSION} # Add your kibana plugins setup here # Example: RUN kibana-plugin install diff --git a/kibana/config/kibana.yml b/kibana/config/kibana.yml index 0e1dc60..07ab33e 100644 --- a/kibana/config/kibana.yml +++ b/kibana/config/kibana.yml @@ -9,5 +9,5 @@ monitoring.ui.container.elasticsearch.enabled: true ## X-Pack security credentials # -elasticsearch.username: elastic -elasticsearch.password: changeme +elasticsearch.username: kibana_system +elasticsearch.password: ${KIBANA_SYSTEM_PASSWORD} diff --git a/logstash/Dockerfile b/logstash/Dockerfile index 6a444e7..bde5808 100644 --- a/logstash/Dockerfile +++ b/logstash/Dockerfile @@ -1,7 +1,7 @@ -ARG ELK_VERSION +ARG ELASTIC_VERSION # https://www.docker.elastic.co/ -FROM docker.elastic.co/logstash/logstash:${ELK_VERSION} +FROM docker.elastic.co/logstash/logstash:${ELASTIC_VERSION} # Add your logstash plugins setup here # Example: RUN logstash-plugin install logstash-filter-json diff --git a/logstash/config/logstash.yml b/logstash/config/logstash.yml index a48c35f..47722ea 100644 --- a/logstash/config/logstash.yml +++ b/logstash/config/logstash.yml @@ -3,10 +3,3 @@ ## https://github.com/elastic/logstash/blob/master/docker/data/logstash/config/logstash-full.yml # http.host: "0.0.0.0" -xpack.monitoring.elasticsearch.hosts: [ "http://elasticsearch:9200" ] - -## X-Pack security credentials -# -xpack.monitoring.enabled: true -xpack.monitoring.elasticsearch.username: elastic -xpack.monitoring.elasticsearch.password: changeme diff --git a/logstash/pipeline/logstash.conf b/logstash/pipeline/logstash.conf index 7d5918b..8c2bd20 100644 --- a/logstash/pipeline/logstash.conf +++ b/logstash/pipeline/logstash.conf @@ -4,7 +4,7 @@ input { } tcp { - port => 5000 + port => 50000 } } @@ -13,8 +13,7 @@ input { output { elasticsearch { hosts => "elasticsearch:9200" - user => "elastic" - password => "changeme" - ecs_compatibility => disabled + user => "logstash_internal" + password => "${LOGSTASH_INTERNAL_PASSWORD}" } -} +} \ No newline at end of file diff --git a/setup/Dockerfile b/setup/Dockerfile new file mode 100644 index 0000000..a6e92bc --- /dev/null +++ b/setup/Dockerfile @@ -0,0 +1,17 @@ +ARG ELASTIC_VERSION + +# https://www.docker.elastic.co/ +FROM docker.elastic.co/elasticsearch/elasticsearch:${ELASTIC_VERSION} + +USER root + +COPY . / + +RUN set -eux; \ + mkdir /state; \ + chown elasticsearch /state; \ + chmod +x /entrypoint.sh + +USER elasticsearch:root + +ENTRYPOINT ["/entrypoint.sh"] \ No newline at end of file diff --git a/setup/entrypoint.sh b/setup/entrypoint.sh new file mode 100644 index 0000000..7ba40fb --- /dev/null +++ b/setup/entrypoint.sh @@ -0,0 +1,107 @@ +#!/usr/bin/env bash + +set -eu +set -o pipefail + +source "$(dirname "${BASH_SOURCE[0]}")/helpers.sh" + + +# -------------------------------------------------------- +# Users declarations + +declare -A users_passwords +users_passwords=( + [logstash_internal]="${LOGSTASH_INTERNAL_PASSWORD:-}" + [kibana_system]="${KIBANA_SYSTEM_PASSWORD:-}" +) + +declare -A users_roles +users_roles=( + [logstash_internal]='logstash_writer' +) + +# -------------------------------------------------------- +# Roles declarations + +declare -A roles_files +roles_files=( + [logstash_writer]='logstash_writer.json' +) + +# -------------------------------------------------------- + + +echo "-------- $(date) --------" + +state_file="$(dirname "${BASH_SOURCE[0]}")/state/.done" +if [[ -e "$state_file" ]]; then + log "State file exists at '${state_file}', skipping setup" + exit 0 +fi + +log 'Waiting for availability of Elasticsearch. This can take several minutes.' + +declare -i exit_code=0 +wait_for_elasticsearch || exit_code=$? + +if ((exit_code)); then + case $exit_code in + 6) + suberr 'Could not resolve host. Is Elasticsearch running?' + ;; + 7) + suberr 'Failed to connect to host. Is Elasticsearch healthy?' + ;; + 28) + suberr 'Timeout connecting to host. Is Elasticsearch healthy?' + ;; + *) + suberr "Connection to Elasticsearch failed. Exit code: ${exit_code}" + ;; + esac + + exit $exit_code +fi + +sublog 'Elasticsearch is running' + +for role in "${!roles_files[@]}"; do + log "Role '$role'" + + declare body_file + body_file="$(dirname "${BASH_SOURCE[0]}")/roles/${roles_files[$role]:-}" + if [[ ! -f "${body_file:-}" ]]; then + sublog "No role body found at '${body_file}', skipping" + continue + fi + + sublog 'Creating/updating' + ensure_role "$role" "$(<"${body_file}")" +done + +for user in "${!users_passwords[@]}"; do + log "User '$user'" + if [[ -z "${users_passwords[$user]:-}" ]]; then + sublog 'No password defined, skipping' + continue + fi + + declare -i user_exists=0 + user_exists="$(check_user_exists "$user")" + + if ((user_exists)); then + sublog 'User exists, setting password' + set_user_password "$user" "${users_passwords[$user]}" + else + if [[ -z "${users_roles[$user]:-}" ]]; then + err ' No role defined, skipping creation' + continue + fi + + sublog 'User does not exist, creating' + create_user "$user" "${users_passwords[$user]}" "${users_roles[$user]}" + fi +done + +mkdir -p "$(dirname "${state_file}")" +touch "$state_file" \ No newline at end of file diff --git a/setup/helpers.sh b/setup/helpers.sh new file mode 100644 index 0000000..0f304d0 --- /dev/null +++ b/setup/helpers.sh @@ -0,0 +1,193 @@ +#!/usr/bin/env bash + +# Log a message. +function log { + echo "[+] $1" +} + +# Log a message at a sub-level. +function sublog { + echo " ⠿ $1" +} + +# Log an error. +function err { + echo "[x] $1" >&2 +} + +# Log an error at a sub-level. +function suberr { + echo " ⠍ $1" >&2 +} + +# Poll the 'elasticsearch' service until it responds with HTTP code 200. +function wait_for_elasticsearch { + local elasticsearch_host="${ELASTICSEARCH_HOST:-elasticsearch}" + + local -a args=( '-s' '-D-' '-m15' '-w' '%{http_code}' "http://${elasticsearch_host}:9200/" ) + + if [[ -n "${ELASTIC_PASSWORD:-}" ]]; then + args+=( '-u' "elastic:${ELASTIC_PASSWORD}" ) + fi + + local -i result=1 + local output + + # retry for max 300s (60*5s) + for _ in $(seq 1 60); do + local -i exit_code=0 + output="$(curl "${args[@]}")" || exit_code=$? + + if ((exit_code)); then + result=$exit_code + fi + + if [[ "${output: -3}" -eq 200 ]]; then + result=0 + break + fi + + sleep 5 + done + + if ((result)) && [[ "${output: -3}" -ne 000 ]]; then + echo -e "\n${output::-3}" + fi + + return $result +} + +# Verify that the given Elasticsearch user exists. +function check_user_exists { + local username=$1 + + local elasticsearch_host="${ELASTICSEARCH_HOST:-elasticsearch}" + + local -a args=( '-s' '-D-' '-m15' '-w' '%{http_code}' + "http://${elasticsearch_host}:9200/_security/user/${username}" + ) + + if [[ -n "${ELASTIC_PASSWORD:-}" ]]; then + args+=( '-u' "elastic:${ELASTIC_PASSWORD}" ) + fi + + local -i result=1 + local -i exists=0 + local output + + output="$(curl "${args[@]}")" + if [[ "${output: -3}" -eq 200 || "${output: -3}" -eq 404 ]]; then + result=0 + fi + if [[ "${output: -3}" -eq 200 ]]; then + exists=1 + fi + + if ((result)); then + echo -e "\n${output::-3}" + else + echo "$exists" + fi + + return $result +} + +# Set password of a given Elasticsearch user. +function set_user_password { + local username=$1 + local password=$2 + + local elasticsearch_host="${ELASTICSEARCH_HOST:-elasticsearch}" + + local -a args=( '-s' '-D-' '-m15' '-w' '%{http_code}' + "http://${elasticsearch_host}:9200/_security/user/${username}/_password" + '-X' 'POST' + '-H' 'Content-Type: application/json' + '-d' "{\"password\" : \"${password}\"}" + ) + + if [[ -n "${ELASTIC_PASSWORD:-}" ]]; then + args+=( '-u' "elastic:${ELASTIC_PASSWORD}" ) + fi + + local -i result=1 + local output + + output="$(curl "${args[@]}")" + if [[ "${output: -3}" -eq 200 ]]; then + result=0 + fi + + if ((result)); then + echo -e "\n${output::-3}\n" + fi + + return $result +} + +# Create the given Elasticsearch user. +function create_user { + local username=$1 + local password=$2 + local role=$3 + + local elasticsearch_host="${ELASTICSEARCH_HOST:-elasticsearch}" + + local -a args=( '-s' '-D-' '-m15' '-w' '%{http_code}' + "http://${elasticsearch_host}:9200/_security/user/${username}" + '-X' 'POST' + '-H' 'Content-Type: application/json' + '-d' "{\"password\":\"${password}\",\"roles\":[\"${role}\"]}" + ) + + if [[ -n "${ELASTIC_PASSWORD:-}" ]]; then + args+=( '-u' "elastic:${ELASTIC_PASSWORD}" ) + fi + + local -i result=1 + local output + + output="$(curl "${args[@]}")" + if [[ "${output: -3}" -eq 200 ]]; then + result=0 + fi + + if ((result)); then + echo -e "\n${output::-3}\n" + fi + + return $result +} + +# Ensure that the given Elasticsearch role is up-to-date, create it if required. +function ensure_role { + local name=$1 + local body=$2 + + local elasticsearch_host="${ELASTICSEARCH_HOST:-elasticsearch}" + + local -a args=( '-s' '-D-' '-m15' '-w' '%{http_code}' + "http://${elasticsearch_host}:9200/_security/role/${name}" + '-X' 'POST' + '-H' 'Content-Type: application/json' + '-d' "$body" + ) + + if [[ -n "${ELASTIC_PASSWORD:-}" ]]; then + args+=( '-u' "elastic:${ELASTIC_PASSWORD}" ) + fi + + local -i result=1 + local output + + output="$(curl "${args[@]}")" + if [[ "${output: -3}" -eq 200 ]]; then + result=0 + fi + + if ((result)); then + echo -e "\n${output::-3}\n" + fi + + return $result +} \ No newline at end of file diff --git a/setup/roles/logstash_writer.json b/setup/roles/logstash_writer.json new file mode 100644 index 0000000..dcb47bb --- /dev/null +++ b/setup/roles/logstash_writer.json @@ -0,0 +1,33 @@ +{ + "cluster": [ + "manage_index_templates", + "monitor", + "manage_ilm" + ], + "indices": [ + { + "names": [ + "logs-generic-default", + "logstash-*", + "ecs-logstash-*" + ], + "privileges": [ + "write", + "create", + "create_index", + "manage", + "manage_ilm" + ] + }, + { + "names": [ + "logstash", + "ecs-logstash" + ], + "privileges": [ + "write", + "manage" + ] + } + ] + } \ No newline at end of file