From c911d3ce3bf5f3be5fd83e663e9e8c071562d47c Mon Sep 17 00:00:00 2001 From: Bram van Dartel Date: Mon, 15 Apr 2024 09:50:38 +0200 Subject: [PATCH] Development (#357) * Switched from S6-overlay v2 to S6-overlay v3 * Base image Alpine 3.19 * PostrgeSQL 16 support * HASSIO support (PostgreSQL 16 and s6-overlay v3 dependency). (thanks for all the great help @sanderdw and @Alfagek!) * Support for Docker secrets (thank for your input @frankforpresident) --- Dockerfile | 3 +- Makefile | 4 + README.md | 19 ++++ .../.env-from-docker-secrets | 56 ----------- docker/entrypoint/docker-entrypoint.sh | 32 ------- examples/docker-compose.example.yaml | 9 +- examples/docker-compose.secrets-example.yaml | 93 ------------------- examples/secret.txt | 1 - .../s6-overlay/s6-rc.d/docker-entrypoint/up | 1 - .../branding | 0 .../run | 41 +++++++- .../type | 0 .../s6-rc.d/init-docker-entrypoint/up | 1 + .../s6-rc.d/svc-dsmr-backend/dependencies | 2 +- .../s6-rc.d/svc-dsmr-datalogger/dependencies | 2 +- .../svc-dsmr-remote-datalogger/dependencies | 2 +- .../svc-dsmr-webinterface/dependencies | 2 +- .../s6-overlay/s6-rc.d/svc-nginx/dependencies | 2 +- .../s6-rc.d/user/contents.d/docker-entrypoint | 0 .../user/contents.d/init-docker-entrypoint | 0 20 files changed, 77 insertions(+), 193 deletions(-) delete mode 100644 docker/entrypoint/docker-entrypoint.d/.env-from-docker-secrets delete mode 100755 docker/entrypoint/docker-entrypoint.sh delete mode 100755 examples/docker-compose.secrets-example.yaml delete mode 100644 examples/secret.txt delete mode 100755 rootfs/etc/s6-overlay/s6-rc.d/docker-entrypoint/up rename rootfs/etc/s6-overlay/s6-rc.d/{docker-entrypoint => init-docker-entrypoint}/branding (100%) rename rootfs/etc/s6-overlay/s6-rc.d/{docker-entrypoint => init-docker-entrypoint}/run (88%) rename rootfs/etc/s6-overlay/s6-rc.d/{docker-entrypoint => init-docker-entrypoint}/type (100%) create mode 100755 rootfs/etc/s6-overlay/s6-rc.d/init-docker-entrypoint/up delete mode 100755 rootfs/etc/s6-overlay/s6-rc.d/user/contents.d/docker-entrypoint rename docker/entrypoint/docker-entrypoint.d/.gitkeep => rootfs/etc/s6-overlay/s6-rc.d/user/contents.d/init-docker-entrypoint (100%) mode change 100644 => 100755 diff --git a/Dockerfile b/Dockerfile index 51e1ca2..6cbba73 100644 --- a/Dockerfile +++ b/Dockerfile @@ -133,7 +133,6 @@ RUN echo "**** configure nginx package ****" \ FROM base as final COPY rootfs / -COPY ./docker/entrypoint / # TODO: Improve healtcheck to respond on 200 only # TODO: Improve healtcheck so it's only valid for containers with the webinterface enabled @@ -141,4 +140,4 @@ HEALTHCHECK --interval=15s --timeout=3s --retries=10 CMD curl -Lsf http://127.0. WORKDIR /app -ENTRYPOINT ["/docker-entrypoint.sh", "/init"] +ENTRYPOINT ["/init"] diff --git a/Makefile b/Makefile index ca556dc..fe81c4c 100644 --- a/Makefile +++ b/Makefile @@ -9,3 +9,7 @@ test: build shell: exec docker exec -ti dsmr bash + + +# docker build --pull --rm --format docker --build-arg DSMR_VERSION="5.11.0" --platform="linux/amd64" --build-arg QEMU_ARCH="x86_64" --build-arg DOCKER_TARGET_RELEASE="2099.09.09" -t dsmr_test_image .; docker save localhost/dsmr_test_image:latest > dsmr_dev; scp -O dsmr_dev xirixiz@nas.skynet:/volume1/onedrive/smarthome +# docker image load < dsmr_dev ; ./docker.sh dsmr_dev; docker logs -f dsmr \ No newline at end of file diff --git a/README.md b/README.md index 24841d6..c45f221 100755 --- a/README.md +++ b/README.md @@ -272,6 +272,25 @@ It's not possible to combine those settings!!!: The ```--no-healthcheck``` argument should only be used when the containers function NOT presenting the DSMR Reader webinterface, for example the datalogger sender mode. By default this argument should not be used! +* ##### Environment variables from files (Docker secrets) +You can set any environment variable from a file by using a special prepend `FILE__`. + +As an example: +```yaml +services: + some_service: + image: some_image + environment: + FILE__SECRET: /run/secrets/a_secret_file + secrets: - a_secret_file + +secrets: + a_secret_file: + file : somedir/my_secret.txt +``` + +Basiccally, the bottom secrets section mounts `my_secrets.txt` as `/run/secrets/a_secret_file`. The secrets section under the service authorize the service to use the `a_secret_file secret`. The environment variable FILE__SECRET tells the service what file to read to set/get the value of the environment variable `SECRET`. + *** #### Features * ##### DSMR Reader - Database cleanup/vacuum diff --git a/docker/entrypoint/docker-entrypoint.d/.env-from-docker-secrets b/docker/entrypoint/docker-entrypoint.d/.env-from-docker-secrets deleted file mode 100644 index 3e61ff8..0000000 --- a/docker/entrypoint/docker-entrypoint.d/.env-from-docker-secrets +++ /dev/null @@ -1,56 +0,0 @@ -#!/usr/bin/env sh - -# EXPANDING VARIABLES FROM DOCKER SECRETS -: ${ENV_SECRETS_DIR:=/run/secrets} - -# Function to print debug messages for environment secrets -env_secret_debug() { - if [ ! -z "$ENV_SECRETS_DEBUG" ]; then - echo -e "\033[1m$@\033[0m" - fi -} - -# This function populates environment variables from variables containing file paths -populate_file_variables() { - for env_var in $(env | grep '_FILE=' | cut -d '=' -f 1); do - var_name="${env_var%_FILE}" # Remove the '_FILE' suffix - file_path=$(eval echo "\$$env_var") # Get the file path from the environment variable - if [ -s "$file_path" ]; then - val=$(cat "$file_path") # Read the file contents into the environment variable - export "$var_name"="$val" - env_secret_debug "Populated variable: $var_name" - else - env_secret_debug "File is empty or does not exist: $file_path" - fi - done -} - -# This function expands variables from docker secrets -expand_docker_secrets() { - for env_var in $(printenv | cut -f1 -d"="); do - var_value=$(eval echo "\$$env_var") # Get the value of the environment variable - if secret_key=$(expr match "$var_value" "DOCKER-SECRET->\([^}]\+\)$"); then - secret="${ENV_SECRETS_DIR}/${secret_key}" - if [ -f "$secret" ]; then - val=$(cat "$secret") - export "$env_var"="$val" # Expand the variable with the secret value - env_secret_debug "Expanded variable: $env_var" - else - env_secret_debug "Secret file does not exist! $secret" - fi - fi - done -} - -# Populate environment variables from variables containing file paths -# Conditionally expand variables from docker secrets -if [ ! -z "$ENV_SECRETS_DIR" ]; then - populate_file_variables - expand_docker_secrets -else - env_secret_debug "No secrets found in /run/secrets" -fi - - -# Execute the command provided as arguments to the script -exec "$@" diff --git a/docker/entrypoint/docker-entrypoint.sh b/docker/entrypoint/docker-entrypoint.sh deleted file mode 100755 index c0abbd7..0000000 --- a/docker/entrypoint/docker-entrypoint.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/bin/sh -# vim:sw=4:ts=4:et - -set -e - -. /docker-entrypoint.d/.env-from-docker-secrets - -if /usr/bin/find "/docker-entrypoint.d/" -mindepth 1 -maxdepth 1 -type f -print -quit 2>/dev/null | read v; then - echo "$0: /docker-entrypoint.d/ is not empty, will attempt to perform configuration" - - echo "$0: Looking for shell scripts in /docker-entrypoint.d/" - find "/docker-entrypoint.d/" -follow -type f -print | sort -V | while read -r f; do - case "$f" in - *.sh) - if [ -x "$f" ]; then - echo "$0: Launching $f"; - "$f" - else - # warn on shell scripts without exec bit - echo "$0: Ignoring $f, not executable"; - fi - ;; - *) echo "$0: Ignoring $f";; - esac -done - -echo "$0: Configuration complete; ready for start up" -else - echo "$0: No files found in /docker-entrypoint.d/, skipping configuration" -fi - -exec "$@" diff --git a/examples/docker-compose.example.yaml b/examples/docker-compose.example.yaml index 7ee3bdb..4d58e08 100755 --- a/examples/docker-compose.example.yaml +++ b/examples/docker-compose.example.yaml @@ -2,7 +2,7 @@ version: '3' services: dsmrdb: - # When using Postgres, release 13.x, 14.x and 15.x are supported only + # When using Postgres, release 13.x, 14.x, 15.x, and 16.x are supported only # due to the limited availability of client packages, especially for arm32v7 image: postgres:16-alpine container_name: dsmrdb @@ -42,6 +42,9 @@ services: - TZ=Europe/Amsterdam - DJANGO_TIME_ZONE=Europe/Amsterdam - VIRTUAL_HOST=localhost + - FILE__SECRET=/run/secrets/a_secret_file + secrets: + - a_secret_file ports: - 7777:80 - 7779:443 @@ -65,6 +68,10 @@ services: timeout: 5s retries: 10 +secrets: + a_secret_file: + file : somedir/my_secret.txt + volumes: dsmrdb: null dsmrdb_backups: null diff --git a/examples/docker-compose.secrets-example.yaml b/examples/docker-compose.secrets-example.yaml deleted file mode 100755 index ac5dc44..0000000 --- a/examples/docker-compose.secrets-example.yaml +++ /dev/null @@ -1,93 +0,0 @@ -version: '3' - -services: - dsmrdb: - # When using Postgres, release 13.x, 14.x, and 15.x are supported only - # due to the limited availability of client packages, especially for arm32v7 - image: postgres:16-alpine - container_name: dsmrdb - restart: always - secrets: - - postgres_user - - postgres_password - volumes: - - ./dsmrdb:/var/lib/postgresql/data - environment: - - TZ=Europe/Amsterdam - - PG_TZ=Europe/Amsterdam - - POSTGRES_DB=dsmrreader - - POSTGRES_USER_FILE=/run/secrets/postgres_user - - POSTGRES_PASSWORD_FILE=/run/secrets/postgres_password - healthcheck: - # postgres is the default user, please update with - # the DJANGO_DATABASE_USER used for dsmr-reader-docker - # default for DSMR Reader is dsmrreader - test: [ "CMD-SHELL", "pg_isready -U dsmrreader" ] - interval: 10s - timeout: 5s - retries: 10 - - dsmr: - image: ghcr.io/xirixiz/dsmr-reader-docker:latest - depends_on: - dsmrdb: - condition: service_healthy - container_name: dsmr - links: - - dsmrdb - cap_add: - - NET_ADMIN - restart: always - secrets: - - postgres_user - - postgres_password - - dsmrreader_admin_user - - dsmrreader_admin_password - volumes: - - /etc/localtime:/etc/localtime:ro - - ./dsmr_backups:/app/backups - environment: - - TZ=Europe/Amsterdam - - DJANGO_TIME_ZONE=Europe/Amsterdam - - VIRTUAL_HOST=localhost - # Postfix the environment variable with _FILE to read the secret from a file - - DJANGO_DATABASE_NAME=dsmrreader - - DJANGO_DATABASE_USER_FILE=/run/secrets/postgres_user - - DJANGO_DATABASE_PASSWORD_FILE=/run/secrets/postgres_password - # Alternatively, you can use DOCKER-SECRET->secret_key environment variables - - DSMRREADER_ADMIN_USER=DOCKER-SECRET->dsmrreader_admin_user - - DSMRREADER_ADMIN_PASSWORD=DOCKER-SECRET->dsmrreader_admin_password - ports: - - 7777:80 - - 7779:443 - devices: - - "/dev/ttyUSB1:/dev/ttyUSB0" - healthcheck: - test: - [ - "CMD", - "curl", - "-Lsf", - "http://127.0.0.1/about", - "-o", - "/dev/null", - "-w", - "HTTP_%{http_code}" - ] - interval: 10s - timeout: 5s - retries: 10 - -secrets: - postgres_user: - file: ./secret.txt - postgres_password: - file: ./secret.txt - dsmrreader_admin_user: - file: ./secret.txt - dsmrreader_admin_password: - file: ./secret.txt - -volumes: - dsmrdb: null - dsmrdb_backups: null diff --git a/examples/secret.txt b/examples/secret.txt deleted file mode 100644 index 3ef8118..0000000 --- a/examples/secret.txt +++ /dev/null @@ -1 +0,0 @@ -dsmrreader \ No newline at end of file diff --git a/rootfs/etc/s6-overlay/s6-rc.d/docker-entrypoint/up b/rootfs/etc/s6-overlay/s6-rc.d/docker-entrypoint/up deleted file mode 100755 index a1a9df3..0000000 --- a/rootfs/etc/s6-overlay/s6-rc.d/docker-entrypoint/up +++ /dev/null @@ -1 +0,0 @@ -/etc/s6-overlay/s6-rc.d/docker-entrypoint/run \ No newline at end of file diff --git a/rootfs/etc/s6-overlay/s6-rc.d/docker-entrypoint/branding b/rootfs/etc/s6-overlay/s6-rc.d/init-docker-entrypoint/branding similarity index 100% rename from rootfs/etc/s6-overlay/s6-rc.d/docker-entrypoint/branding rename to rootfs/etc/s6-overlay/s6-rc.d/init-docker-entrypoint/branding diff --git a/rootfs/etc/s6-overlay/s6-rc.d/docker-entrypoint/run b/rootfs/etc/s6-overlay/s6-rc.d/init-docker-entrypoint/run similarity index 88% rename from rootfs/etc/s6-overlay/s6-rc.d/docker-entrypoint/run rename to rootfs/etc/s6-overlay/s6-rc.d/init-docker-entrypoint/run index cf0d26b..adb7fcf 100755 --- a/rootfs/etc/s6-overlay/s6-rc.d/docker-entrypoint/run +++ b/rootfs/etc/s6-overlay/s6-rc.d/init-docker-entrypoint/run @@ -24,7 +24,7 @@ function _pre_reqs() { groupmod -o -g "${DGID}" app >/dev/null 2>&1 usermod -o -u "${DUID}" app >/dev/null 2>&1 - cat /etc/s6-overlay/s6-rc.d/docker-entrypoint/branding + cat /etc/s6-overlay/s6-rc.d/init-docker-entrypoint/branding echo " User UID: $(id -u app) User GID: $(id -g app) @@ -324,7 +324,7 @@ function _generate_clientcert_auth_configuration() { _info "CLIENT CERT AUTHENTICATION configured and enabled" return else - _error "NGINX configuration error" + _error "NGINX configuration error!" exit 1 fi fi @@ -332,6 +332,42 @@ function _generate_clientcert_auth_configuration() { _info "ENABLE_CLIENTCERT_AUTH is disabled, nothing to see here. Continuing..." } +# Todo: improve docker secrets scripts do check on hex for newline +# filename="${FILENAME##*/}" +# last_bytes=$(tail -c 2 "${SECRETFILE}" | od -An -tx1 | tr -d ' ') + +# if [[ "$last_bytes" == "0a" ]]; then # Only LF +# _info "Warning: Docker secret file '$filename' contains a trailing Line Feed (LF) newline ('\\n') which may cause issues." +# echo "Consider removing it with 'sed -i '' ':a;N;$!ba;s/\\n$//' '$SECRETFILE'' if you encounter problems." +# elif [[ "$last_bytes" == "0d0a" ]]; then # CRLF +# _info "Warning: Docker secret file '$filename' contains a trailing Windows-style newline (CRLF) which may cause issues." +# echo "Consider removing it with 'sed -i '' 's/\\r\\n$//' '$SECRETFILE'' if you encounter problems." +# elif [[ "$last_bytes" =~ "0d" ]]; then # Only CR +# _info "Warning: Docker secret file '$filename' contains a trailing Carriage Return (CR) newline ('\\r') which may cause issues." +# echo "Consider removing it with 'sed -i '' 's/\\r$//' '$SECRETFILE'' if you encounter problems." +# else +# _info "Docker secret file '$filename' is properly formatted without any trailing newline." +# fi + +function _docker_secrets { + if find /run/s6/container_environment/FILE__* -maxdepth 1 > /dev/null 2>&1; then + _info "Enabling Docker secrets..." + for FILENAME in /run/s6/container_environment/FILE__*; do + SECRETFILE=$(cat "${FILENAME}") + if [[ -f ${SECRETFILE} ]]; then + FILESTRIP=${FILENAME//FILE__/} + if [[ $(tail -n1 "${SECRETFILE}" | wc -l) != 0 ]]; then + _info "Docker secret: ${FILENAME##*/} contains a trailing newline and may not work as expected!" + fi + cat "${SECRETFILE}" >"${FILESTRIP}" + _info "Docker secret ${FILESTRIP##*/} set from ${FILENAME##*/}..." + else + _info "Cannot find Docker secret in ${FILENAME##*/}..." + fi + done + fi +} + function _iframe { if [[ "${ENABLE_IFRAME}" = true ]]; then _info "Enabling IFrame..." @@ -362,4 +398,5 @@ if [[ "${DSMRREADER_OPERATION_MODE}" != api_client ]]; then _generate_auth_configuration _dsmr_datalogger_mode _optional_settings + _docker_secrets fi diff --git a/rootfs/etc/s6-overlay/s6-rc.d/docker-entrypoint/type b/rootfs/etc/s6-overlay/s6-rc.d/init-docker-entrypoint/type similarity index 100% rename from rootfs/etc/s6-overlay/s6-rc.d/docker-entrypoint/type rename to rootfs/etc/s6-overlay/s6-rc.d/init-docker-entrypoint/type diff --git a/rootfs/etc/s6-overlay/s6-rc.d/init-docker-entrypoint/up b/rootfs/etc/s6-overlay/s6-rc.d/init-docker-entrypoint/up new file mode 100755 index 0000000..280876d --- /dev/null +++ b/rootfs/etc/s6-overlay/s6-rc.d/init-docker-entrypoint/up @@ -0,0 +1 @@ +/etc/s6-overlay/s6-rc.d/init-docker-entrypoint/run \ No newline at end of file diff --git a/rootfs/etc/s6-overlay/s6-rc.d/svc-dsmr-backend/dependencies b/rootfs/etc/s6-overlay/s6-rc.d/svc-dsmr-backend/dependencies index 024f80d..6c2437a 100644 --- a/rootfs/etc/s6-overlay/s6-rc.d/svc-dsmr-backend/dependencies +++ b/rootfs/etc/s6-overlay/s6-rc.d/svc-dsmr-backend/dependencies @@ -1 +1 @@ -docker-entrypoint \ No newline at end of file +init-docker-entrypoint \ No newline at end of file diff --git a/rootfs/etc/s6-overlay/s6-rc.d/svc-dsmr-datalogger/dependencies b/rootfs/etc/s6-overlay/s6-rc.d/svc-dsmr-datalogger/dependencies index 9684dd2..aeb4dd8 100644 --- a/rootfs/etc/s6-overlay/s6-rc.d/svc-dsmr-datalogger/dependencies +++ b/rootfs/etc/s6-overlay/s6-rc.d/svc-dsmr-datalogger/dependencies @@ -1,2 +1,2 @@ -docker-entrypoint +init-docker-entrypoint svc-nginx \ No newline at end of file diff --git a/rootfs/etc/s6-overlay/s6-rc.d/svc-dsmr-remote-datalogger/dependencies b/rootfs/etc/s6-overlay/s6-rc.d/svc-dsmr-remote-datalogger/dependencies index 9684dd2..aeb4dd8 100644 --- a/rootfs/etc/s6-overlay/s6-rc.d/svc-dsmr-remote-datalogger/dependencies +++ b/rootfs/etc/s6-overlay/s6-rc.d/svc-dsmr-remote-datalogger/dependencies @@ -1,2 +1,2 @@ -docker-entrypoint +init-docker-entrypoint svc-nginx \ No newline at end of file diff --git a/rootfs/etc/s6-overlay/s6-rc.d/svc-dsmr-webinterface/dependencies b/rootfs/etc/s6-overlay/s6-rc.d/svc-dsmr-webinterface/dependencies index 024f80d..6c2437a 100644 --- a/rootfs/etc/s6-overlay/s6-rc.d/svc-dsmr-webinterface/dependencies +++ b/rootfs/etc/s6-overlay/s6-rc.d/svc-dsmr-webinterface/dependencies @@ -1 +1 @@ -docker-entrypoint \ No newline at end of file +init-docker-entrypoint \ No newline at end of file diff --git a/rootfs/etc/s6-overlay/s6-rc.d/svc-nginx/dependencies b/rootfs/etc/s6-overlay/s6-rc.d/svc-nginx/dependencies index 024f80d..6c2437a 100644 --- a/rootfs/etc/s6-overlay/s6-rc.d/svc-nginx/dependencies +++ b/rootfs/etc/s6-overlay/s6-rc.d/svc-nginx/dependencies @@ -1 +1 @@ -docker-entrypoint \ No newline at end of file +init-docker-entrypoint \ No newline at end of file diff --git a/rootfs/etc/s6-overlay/s6-rc.d/user/contents.d/docker-entrypoint b/rootfs/etc/s6-overlay/s6-rc.d/user/contents.d/docker-entrypoint deleted file mode 100755 index e69de29..0000000 diff --git a/docker/entrypoint/docker-entrypoint.d/.gitkeep b/rootfs/etc/s6-overlay/s6-rc.d/user/contents.d/init-docker-entrypoint old mode 100644 new mode 100755 similarity index 100% rename from docker/entrypoint/docker-entrypoint.d/.gitkeep rename to rootfs/etc/s6-overlay/s6-rc.d/user/contents.d/init-docker-entrypoint