From 51317a26cb0207174aa8ef3de449bbb38d6a2b73 Mon Sep 17 00:00:00 2001 From: LoveIsGrief Date: Sun, 11 Aug 2024 11:10:44 +0200 Subject: [PATCH 01/43] chore(docker): add .env to redis and db services --- docker/compose.yml | 4 ++++ notes.md | 6 ++++++ 2 files changed, 10 insertions(+) create mode 100644 notes.md diff --git a/docker/compose.yml b/docker/compose.yml index bc6bf334c..f46b1d9cc 100644 --- a/docker/compose.yml +++ b/docker/compose.yml @@ -102,6 +102,8 @@ services: command: /bin/sh -c "redis-server --requirepass $${REDIS_PASSWORD}" volumes: - ./storage/redis:/data + env_file: + - .env healthcheck: test: ["CMD", "redis-cli", "ping"] @@ -116,6 +118,8 @@ services: environment: - POSTGRES_DB=kbin - POSTGRES_USER=kbin + env_file: + - .env rabbitmq: image: rabbitmq:3.13.6-management-alpine diff --git a/notes.md b/notes.md new file mode 100644 index 000000000..5256cae48 --- /dev/null +++ b/notes.md @@ -0,0 +1,6 @@ +- [ ] add link to [dev server docs](docs/04-contributing/development_server.md) in `CONTRIBUTING.md` +- [ ] mention how to activate debug logs for symfony (`bin/console -vvv some-command`) +# Docker dev + +- [ ] Make a separate `.env.dev` for development +- [ ] Indicate that `.env.dev` should be copied into [/docker/](/docker/) From f92c58a603ed909ab292bea86ab83fdadd4958a2 Mon Sep 17 00:00:00 2001 From: LoveIsGrief Date: Sun, 11 Aug 2024 12:54:37 +0200 Subject: [PATCH 02/43] chore: make docker runnable as dev app --- .env.dev_docker | 182 +++++++++++++++++++++++++++++++++++++++++++++ docker/compose.yml | 14 ++-- notes.md | 6 ++ 3 files changed, 193 insertions(+), 9 deletions(-) create mode 100644 .env.dev_docker diff --git a/.env.dev_docker b/.env.dev_docker new file mode 100644 index 000000000..a72276e3c --- /dev/null +++ b/.env.dev_docker @@ -0,0 +1,182 @@ +# In all environments, the following files are loaded if they exist, +# the latter taking precedence over the former: +# +# * .env contains default values for the environment variables needed by the app +# * .env.local uncommitted file with local overrides +# * .env.$APP_ENV committed environment-specific defaults +# * .env.$APP_ENV.local uncommitted environment-specific overrides +# +# Real environment variables win over .env files. +# +# DO NOT DEFINE PRODUCTION SECRETS IN THIS FILE NOR IN ANY OTHER COMMITTED FILES. +# +# Run "composer dump-env prod" to compile .env files for production use (requires symfony/flex >=1.2). +# https://symfony.com/doc/current/best_practices.html#use-environment-variables-for-infrastructure-configuration + +# Mbin variables +SERVER_NAME=127.0.0.1:8000 +KBIN_DOMAIN=127.0.0.1:8000 +KBIN_TITLE=Mbin +KBIN_DEFAULT_LANG=en +KBIN_FEDERATION_ENABLED=true +KBIN_CONTACT_EMAIL=contact@mbin.domain.tdl +KBIN_SENDER_EMAIL=noreply@mbin.domain.tdl +KBIN_JS_ENABLED=true +KBIN_REGISTRATIONS_ENABLED=true +KBIN_API_ITEMS_PER_PAGE=25 +KBIN_STORAGE_URL=http://127.0.0.1:8000/media +KBIN_META_TITLE="Mbin" +KBIN_META_DESCRIPTION="content aggregator, content voting, discussion and micro-blogging platform on the fediverse" +KBIN_META_KEYWORDS="mbin, content aggregator, open source, fediverse" +KBIN_HEADER_LOGO=false +KBIN_FEDERATION_PAGE_ENABLED=true +MBIN_DEFAULT_THEME=default + +# If you are running Mbin behind a reverse proxy, uncomment the line below and adjust the proxy address/range below +# to your server's IP address if it does not already fall within the private IP spaces specified. +TRUSTED_PROXIES=::1,127.0.0.1,10.0.0.0/8,172.16.0.0/12,192.168.0.0/16 +#TRUSTED_PROXIES= + +# Max image filesize (in bytes) +# This should be set to <= `upload_max_filesize` and `post_max_size` in the server's php.ini file +MAX_IMAGE_BYTES=6000000 + +# Captcha (also enable in admin panel/settings) +KBIN_CAPTCHA_ENABLED=false + +###> meteo-concept/hcaptcha-bundle ### +HCAPTCHA_SITE_KEY= +HCAPTCHA_SECRET= +###< meteo-concept/hcaptcha-bundle ### + +# Redis +REDIS_PASSWORD=!ChangeThisRedisPass! +REDIS_DNS=redis://${REDIS_PASSWORD}@redis:6379 + +# S3 storage (optional) +S3_KEY= +S3_SECRET= +S3_BUCKET= +S3_REGION= +S3_ENDPOINT= +S3_VERSION= + +# Only let admins generate oauth clients +KBIN_ADMIN_ONLY_OAUTH_CLIENTS=false + +# oAuth (optional) +OAUTH_AZURE_ID= +OAUTH_AZURE_SECRET= +# If you want people from an enterprise to connect your instance, set the tenant id here. +# If you want people from anywhere to connect with either their personnal or professionnal microsoft account, use "common" +OAUTH_AZURE_TENANT= +OAUTH_FACEBOOK_ID= +OAUTH_FACEBOOK_SECRET= +OAUTH_GOOGLE_ID= +OAUTH_GOOGLE_SECRET= +OAUTH_DISCORD_ID= +OAUTH_DISCORD_SECRET= +OAUTH_GITHUB_ID= +OAUTH_GITHUB_SECRET= +OAUTH_KEYCLOAK_ID= +OAUTH_KEYCLOAK_SECRET= +OAUTH_KEYCLOAK_URI= +OAUTH_KEYCLOAK_REALM= +OAUTH_KEYCLOAK_VERSION= +OAUTH_SIMPLELOGIN_ID= +OAUTH_SIMPLELOGIN_SECRET= +OAUTH_ZITADEL_ID= +OAUTH_ZITADEL_SECRET= +OAUTH_ZITADEL_BASE_URL= +OAUTH_AUTHENTIK_ID= +OAUTH_AUTHENTIK_SECRET= +OAUTH_AUTHENTIK_BASE_URL= +OAUTH_PRIVACYPORTAL_ID= +OAUTH_PRIVACYPORTAL_SECRET= + +# If true, sign ins and sign ups will only be possible through the OAuth providers configured above +SSO_ONLY_MODE= + +# image exif cleaning options +# available value: none, sanitize, scrub +# can be set differently for user uploaded and external media +EXIF_CLEAN_MODE_UPLOADED=sanitize +EXIF_CLEAN_MODE_EXTERNAL=none +# path to exiftool binary, leave blank for auto PATH search +EXIF_EXIFTOOL_PATH= +# max execution time for exiftool in seconds, defaults to 10 seconds +EXIF_EXIFTOOL_TIMEOUT=10 + +###> symfony/framework-bundle ### +APP_ENV=dev +APP_SECRET=427f5e2940e5b2472c1b44b2d06e0525 +###< symfony/framework-bundle ### + +###> doctrine/doctrine-bundle ### +# Format described at https://www.doctrine-project.org/projects/doctrine-dbal/en/latest/reference/configuration.html#connecting-using-a-url +POSTGRES_HOST=db:5432 +POSTGRES_DB=mbin +POSTGRES_USER=mbin +POSTGRES_PASSWORD=!ChangeThisPostgresPass! +# IMPORTANT: You MUST configure your PostgreSQL server version! +POSTGRES_VERSION=13 +DATABASE_URL="postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}/${POSTGRES_DB}?serverVersion=${POSTGRES_VERSION}&charset=utf8" +###< doctrine/doctrine-bundle ### + +###> rabbitmq ### +RABBITMQ_DEFAULT_USER=mbin +RABBITMQ_DEFAULT_PASS=!ChangeThisRabbitPass! +###< rabbitmq ### + +###> symfony/messenger ### +# Choose one of the transports below +MESSENGER_TRANSPORT_DSN=amqp://mbin:${RABBITMQ_DEFAULT_PASS}@rabbitmq:5672/%2f/messages +#MESSENGER_TRANSPORT_DSN=doctrine://default +#MESSENGER_TRANSPORT_DSN=redis://${REDIS_PASSWORD}@${REDIS_HOST}/messages +###< symfony/messenger ### + + +###> symfony/mailer ### +# See https://symfony.com/doc/current/mailer.html#using-built-in-transports +# MAILER_DSN=sendmail://default # Use sendmail when you are using Postfix +MAILER_DSN=smtp://mailserver # Use a SMTP Docker service called 'mailserver' (see compose.yml) +# Explicitly url encode any character in username and password +# %40 = @ +# Gmail: +# MAILER_DSN=gmail+smtp://user%40domain.com:pass@default +# Our own SMTP server: +# MAILER_DSN=smtp://user%40domain.com:pass@smtp.example.com:port +###< symfony/mailer ### + +###> symfony/mailgun-mailer ### +# MAILER_DSN=mailgun://KEY:DOMAIN@default?region=us +# MAILER_DSN=mailgun+smtp://postmaster@sandboxxx.mailgun.org:key@default?region=us +###< symfony/mailgun-mailer ### + +###> symfony/mercure-bundle ### +# See https://symfony.com/doc/current/mercure.html#configuration +# The URL of the Mercure hub, used by the app to publish updates (can be a local URL) +# Assuming you are running Mercure Caddy on port 3000 +MERCURE_URL=http://www:80/.well-known/mercure +# The public URL of the Mercure hub, used by the browser to connect +MERCURE_PUBLIC_URL=https://${KBIN_DOMAIN}/.well-known/mercure +# The secret used to sign the JWTs +MERCURE_JWT_SECRET="!ChangeThisMercureHubJWTSecretKey!" +###< symfony/mercure-bundle ### + +###> nelmio/cors-bundle ### +CORS_ALLOW_ORIGIN="^https?://(${KBIN_DOMAIN}|127\.0\.0\.1)(:[0-9]+)?$" +###< nelmio/cors-bundle ### + +###> symfony/lock ### +# Choose one of the stores below +# postgresql+advisory://db_user:db_password@localhost/db_name +LOCK_DSN=flock +###< symfony/lock ### + +###> league/oauth2-server-bundle ### +OAUTH_PRIVATE_KEY= +OAUTH_PUBLIC_KEY= +OAUTH_PASSPHRASE= +OAUTH_ENCRYPTION_KEY= +###< league/oauth2-server-bundle ### diff --git a/docker/compose.yml b/docker/compose.yml index f46b1d9cc..3bf4d45db 100644 --- a/docker/compose.yml +++ b/docker/compose.yml @@ -75,7 +75,7 @@ services: restart: unless-stopped networks: - mbin_external_network - command: bin/console messenger:consume scheduler_default old async outbox deliver inbox resolve receive failed --time-limit=3600 + command: bin/console -vvvv messenger:consume --all --time-limit=3600 healthcheck: test: ["CMD-SHELL", "ps aux | grep 'messenger[:]consume' || exit 1"] volumes: @@ -85,9 +85,6 @@ services: # - ../config:/var/www/mbin/config env_file: - .env - deploy: - mode: replicated - replicas: 6 depends_on: - redis - db @@ -111,13 +108,12 @@ services: image: postgres:${POSTGRES_VERSION:-13}-alpine container_name: mbin-db restart: unless-stopped + ports: + - "127.0.0.1:5432:5432" networks: - mbin_external_network volumes: - ./storage/postgres:/var/lib/postgresql/data - environment: - - POSTGRES_DB=kbin - - POSTGRES_USER=kbin env_file: - .env @@ -127,8 +123,8 @@ services: restart: unless-stopped networks: - mbin_external_network - environment: - - RABBITMQ_DEFAULT_USER=kbin + env_file: + - .env volumes: - ./storage/rabbitmq:/var/lib/rabbitmq ports: diff --git a/notes.md b/notes.md index 5256cae48..fe110a53d 100644 --- a/notes.md +++ b/notes.md @@ -2,5 +2,11 @@ - [ ] mention how to activate debug logs for symfony (`bin/console -vvv some-command`) # Docker dev +- [ ] Make a separate `compose.dev.yml` - [ ] Make a separate `.env.dev` for development - [ ] Indicate that `.env.dev` should be copied into [/docker/](/docker/) +- [ ] mention that resetting means `sudo rm -rf docker/storage` + - [ ] move `.gitignore` out of storage otherwise resetting still creates a commit by deleting `.gitignore` +- [ ] `docker/storage` needs to be writable to `mbin` user! + - [ ] `sudo chmod 777 docker/storage/*` +- [ ] `MESSENGER_TRANSPORT_DSN=doctrine://default` doesn't work From e75466d7f7d0b725116db7aa4428c61d07958fa6 Mon Sep 17 00:00:00 2001 From: LoveIsGrief Date: Mon, 12 Aug 2024 17:40:53 +0200 Subject: [PATCH 03/43] rework docker for dev --- .dockerignore | 2 +- docker/compose.yml => compose.prod.yml | 38 ++-- compose.yml | 175 ++++++++++++++++++ docker/Dockerfile | 116 ------------ docker/caddy/Dockerfile | 16 ++ docker/node/build.sh | 10 + docker/php/Dockerfile | 41 ++++ docker/php/conf.d/app.dev.ini | 3 + .../{docker-entrypoint => php/entrypoint.sh} | 18 +- notes.md | 12 +- 10 files changed, 279 insertions(+), 152 deletions(-) rename docker/compose.yml => compose.prod.yml (82%) create mode 100644 compose.yml delete mode 100644 docker/Dockerfile create mode 100644 docker/caddy/Dockerfile create mode 100755 docker/node/build.sh create mode 100644 docker/php/Dockerfile rename docker/{docker-entrypoint => php/entrypoint.sh} (71%) diff --git a/.dockerignore b/.dockerignore index 6c20eadd1..25f76f046 100644 --- a/.dockerignore +++ b/.dockerignore @@ -14,7 +14,7 @@ **/compose.*.yaml **/compose.*.yml **/compose.yaml -**/compose.yml +compose.yml **/Dockerfile **/Thumbs.db .github/ diff --git a/docker/compose.yml b/compose.prod.yml similarity index 82% rename from docker/compose.yml rename to compose.prod.yml index 3bf4d45db..0d09ecd4f 100644 --- a/docker/compose.yml +++ b/compose.prod.yml @@ -4,8 +4,7 @@ services: www: # Builds the Docker image from scratch build: - context: ../ - dockerfile: docker/Dockerfile + dockerfile: docker/php/Dockerfile image: mbin # Or remove the build, context, dockerfile and image lines above and # use the pre-build image from ghcr.io (you can also a tag eg.: v1.0.0 instead of: latest): @@ -20,9 +19,9 @@ services: ports: - 8008:80 volumes: - - ./storage/caddy_config:/config - - ./storage/caddy_data:/data - - ./storage/media:/var/www/mbin/public/media + - ./docker/storage/caddy_config:/config + - ./docker/storage/caddy_data:/data + - ./:/var/www/mbin/ environment: - SERVER_NAME=:80 # the address that the web server binds - PHP_FASTCGI_HOST=php:9000 # caddy forward traffic to this host via fastcgi @@ -34,8 +33,7 @@ services: php: # Builds the Docker image from scratch build: - context: ../ - dockerfile: docker/Dockerfile + dockerfile: docker/php/Dockerfile image: mbin # Or remove the build, context, dockerfile and image lines above and # use the pre-build image from ghcr.io (you can also a tag eg.: v1.0.0 instead of: latest): @@ -45,6 +43,8 @@ services: networks: - mbin_external_network command: php-fpm + extra_hosts: + - "host.docker.internal:host-gateway" healthcheck: test: [ @@ -52,10 +52,11 @@ services: "REQUEST_METHOD=GET SCRIPT_NAME=/ping SCRIPT_FILENAME=/ping cgi-fcgi -bind -connect localhost:9000 || exit 1", ] volumes: - - ./storage/media:/var/www/mbin/public/media - - ./storage/logs:/var/www/mbin/var/log + - ./:/var/www/mbin/ + - ./docker/storage/logs:/var/www/mbin/var/log + - ./docker/php/conf.d/app.dev.ini:/usr/local/etc/php/conf.d/app.dev.ini # If you want to change configs locally, without rebuilding the image use: - # - ../config:/var/www/mbin/config + # - ../docker/config:/var/www/mbin/config env_file: - .env depends_on: @@ -66,8 +67,7 @@ services: messenger: # Builds the Docker image from scratch build: - context: ../ - dockerfile: docker/Dockerfile + dockerfile: docker/php/Dockerfile image: mbin # Or remove the build, context, dockerfile and image lines above and # use the pre-build image from ghcr.io (you can also a tag eg.: v1.0.0 instead of: latest): @@ -79,10 +79,10 @@ services: healthcheck: test: ["CMD-SHELL", "ps aux | grep 'messenger[:]consume' || exit 1"] volumes: - - ./storage/media:/var/www/mbin/public/media - - ./storage/logs:/var/www/mbin/var/log + - ./:/var/www/mbin/ + - ./docker/storage/logs:/var/www/mbin/var/log # If you want to change configs locally, without rebuilding the image use: - # - ../config:/var/www/mbin/config + # - ../docker/config:/var/www/mbin/config env_file: - .env depends_on: @@ -98,7 +98,7 @@ services: - mbin_external_network command: /bin/sh -c "redis-server --requirepass $${REDIS_PASSWORD}" volumes: - - ./storage/redis:/data + - ./docker/storage/redis:/data env_file: - .env healthcheck: @@ -113,7 +113,7 @@ services: networks: - mbin_external_network volumes: - - ./storage/postgres:/var/lib/postgresql/data + - ./docker/storage/postgres:/var/lib/postgresql/data env_file: - .env @@ -126,7 +126,7 @@ services: env_file: - .env volumes: - - ./storage/rabbitmq:/var/lib/rabbitmq + - ./docker/storage/rabbitmq:/var/lib/rabbitmq ports: - 15672:15672 @@ -140,7 +140,7 @@ services: # ports: # - 443:443 # volumes: - # - ./nginx.conf:/etc/nginx/nginx.conf + # - ./docker/nginx.conf:/etc/nginx/nginx.conf # Example of a SMTP docker service # More info: https://hub.docker.com/r/ixdotai/smtp diff --git a/compose.yml b/compose.yml new file mode 100644 index 000000000..0eed1f31a --- /dev/null +++ b/compose.yml @@ -0,0 +1,175 @@ +services: + www: + # Builds the Docker image from scratch + build: + dockerfile: docker/caddy/Dockerfile + container_name: mbin-www + restart: unless-stopped + networks: + - mbin_external_network + command: caddy run --config /etc/caddy/Caddyfile --adapter caddyfile + healthcheck: + test: ["CMD-SHELL", "curl -f http://localhost:2019/metrics || exit 1"] + ports: + - 8008:80 + volumes: + - caddy_config:/config + - caddy_data:/data + - ./:/var/www/mbin + environment: + - SERVER_NAME=:80 # the address that the web server binds + - PHP_FASTCGI_HOST=php:9000 # caddy forward traffic to this host via fastcgi + - MERCURE_PUBLISHER_JWT_KEY=$MERCURE_JWT_SECRET + - MERCURE_SUBSCRIBER_JWT_KEY=$MERCURE_JWT_SECRET + depends_on: + - php + + php: + # Builds the Docker image from scratch + build: + dockerfile: docker/php/Dockerfile + container_name: mbin-php + command: php-fpm + working_dir: /var/www/mbin + restart: unless-stopped + networks: + - mbin_external_network + extra_hosts: + - "host.docker.internal:host-gateway" + healthcheck: + test: + [ + "CMD-SHELL", + "REQUEST_METHOD=GET SCRIPT_NAME=/ping SCRIPT_FILENAME=/ping cgi-fcgi -bind -connect localhost:9000 || exit 1", + ] + volumes: + - ./docker/php/entrypoint.sh:/entrypoint.sh + - ./:/var/www/mbin/ + - ./docker/php/conf.d/app.dev.ini:/usr/local/etc/php/conf.d/app.dev.ini + - ./docker/php/php-fpm.d/zz-docker.conf:/usr/local/etc/php-fpm.d/zz-docker.conf + # If you want to change configs locally, without rebuilding the image use: + # - ../docker/config:/var/www/mbin/config + env_file: + - .env + depends_on: + - redis + - db + - rabbitmq + + node: + image: "node:22.5.1-alpine3.20" + command: docker/node/build.sh + container_name: node-builder + entrypoint: + - sh + - "-c" + working_dir: /var/www/mbin + depends_on: + php: + condition: service_healthy + networks: + - mbin_external_network + volumes: + - ./:/var/www/mbin/ + + messenger: + # Builds the Docker image from scratch + build: + dockerfile: docker/php/Dockerfile + restart: unless-stopped + networks: + - mbin_external_network + command: bin/console -vvvv messenger:consume --all --time-limit=3600 + healthcheck: + test: ["CMD-SHELL", "ps aux | grep 'messenger[:]consume' || exit 1"] + volumes: + - ./:/var/www/mbin/ + # If you want to change configs locally, without rebuilding the image use: + # - ../docker/config:/var/www/mbin/config + env_file: + - .env + depends_on: + - redis + - db + - rabbitmq + + redis: + image: redis:alpine + container_name: mbin-redis + restart: unless-stopped + networks: + - mbin_external_network + command: /bin/sh -c "redis-server --requirepass $${REDIS_PASSWORD}" + volumes: + - redis:/data + env_file: + - .env + healthcheck: + test: ["CMD", "redis-cli", "ping"] + + db: + image: postgres:${POSTGRES_VERSION:-13}-alpine + container_name: mbin-db + restart: unless-stopped + ports: + - "127.0.0.1:5432:5432" + networks: + - mbin_external_network + volumes: + - postgres:/var/lib/postgresql/data + env_file: + - .env + healthcheck: + test: [ "CMD-SHELL", "pg_isready" ] + interval: 10s + timeout: 5s + retries: 5 + + rabbitmq: + image: rabbitmq:3.13.6-management-alpine + container_name: mbin-rabbitmq + restart: unless-stopped + networks: + - mbin_external_network + env_file: + - .env + volumes: + - rabbitmq:/var/lib/rabbitmq + ports: + - 15672:15672 + + # Add your favorite reverse proxy (e.g nginx) which accept incoming HTTPS + # traffic and forward to http://www:80 + #nginx: + # image: nginx + # container_name: mbin-nginx + # networks: + # - mbin_external_network + # ports: + # - 443:443 + # volumes: + # - ./docker/nginx.conf:/etc/nginx/nginx.conf + + # Example of a SMTP docker service + # More info: https://hub.docker.com/r/ixdotai/smtp + #mailserver: + # image: ixdotai/smtp:latest + # networks: + # - mbin_external_network + # environment: + # - SMARTHOST_ADDRESS=mail.mysmtp.com + # - SMARTHOST_PORT=587 + # - SMARTHOST_USER=myuser + # - SMARTHOST_PASSWORD=secret + # - SMARTHOST_ALIASES=*.mysmtp.com + +networks: + mbin_external_network: + +volumes: + caddy_config: + caddy_data: + logs: + postgres: + rabbitmq: + redis: diff --git a/docker/Dockerfile b/docker/Dockerfile deleted file mode 100644 index b48dbaa63..000000000 --- a/docker/Dockerfile +++ /dev/null @@ -1,116 +0,0 @@ -FROM php:8.3-fpm-alpine as base - -# Install php extensions, by docker-php-extension-installer -COPY --from=mlocati/php-extension-installer /usr/bin/install-php-extensions /usr/local/bin/ -RUN install-php-extensions amqp bcmath pgsql pdo_pgsql gd curl simplexml dom xml redis intl opcache apcu pcntl exif - -# Install composer -COPY --from=composer:latest /usr/bin/composer /usr/bin/composer - -# Install caddy -RUN apk update && apk add caddy acl fcgi exiftool - -# **Environment variables** -ARG UID=1000 -ENV MBIN_HOME=/var/www/mbin \ - USER=mbin \ - GROUP=www-data - -# Create user -RUN adduser -u $UID -D -g "" $USER $GROUP - -# Create path -RUN mkdir -p $MBIN_HOME && \ - chown -R $USER:$GROUP $MBIN_HOME -WORKDIR $MBIN_HOME - -# PHP configuration (Requires these configuration before "composer install" ) -RUN cp "$PHP_INI_DIR/php.ini-production" "$PHP_INI_DIR/php.ini" -COPY --link docker/php/conf.d/app.ini $PHP_INI_DIR/conf.d/app.ini -COPY --link docker/php/conf.d/app.prod.ini $PHP_INI_DIR/conf.d/app.prod.or.dev.ini -COPY --link docker/php/conf.d/app.prod.ini $PHP_INI_DIR/conf.d/app.ini-production -COPY --link docker/php/conf.d/app.dev.ini $PHP_INI_DIR/conf.d/app.ini-development -COPY --link docker/php/php-fpm.d/zz-docker.conf /usr/local/etc/php-fpm.d/zz-docker.conf -RUN chown -R $USER:$GROUP $PHP_INI_DIR -RUN chown -R $USER:$GROUP /usr/local/etc/php-fpm.d - -#################### - -FROM caddy:2.8.4-builder-alpine AS builder-caddy - -# Build Caddy with the Mercure and Vulcain and brotil cache modules -RUN xcaddy build \ - --with github.com/dunglas/mercure/caddy \ - --with github.com/dunglas/vulcain/caddy \ - --with github.com/ueffel/caddy-brotli - -#################### - -FROM base as builder-composer - -# Composer: install package -COPY composer.* $MBIN_HOME -COPY symfony.lock $MBIN_HOME -ENV COMPOSER_ALLOW_SUPERUSER=1 -RUN composer install --prefer-dist --no-dev --no-autoloader --no-scripts --no-progress - -# Copy repository -COPY --link ./ $MBIN_HOME -RUN cp .env.example_docker .env - -# Dump-autoload and run post-install script -RUN composer dump-autoload --classmap-authoritative --no-dev -RUN composer run-script --no-dev post-install-cmd && \ - chmod +x bin/console && sync - -#################### - -FROM node:22.5.1-alpine3.20 as builder-nodejs - -# Set NodeJS as production by default -ARG NODE_ENV=production -ENV NODE_ENV=${NODE_ENV} - -# Setup environment -ENV MBIN_HOME=/var/www/mbin -RUN mkdir -p $MBIN_HOME -WORKDIR $MBIN_HOME - -# Copy required files -COPY package.json $MBIN_HOME -COPY package-lock.json $MBIN_HOME -COPY --from=builder-composer --link $MBIN_HOME/vendor $MBIN_HOME/vendor - -# NPM: install package -RUN npm ci --include=dev - -# NPM: build (production by default) -COPY --link ./ $MBIN_HOME -RUN npm run build - -#################### - -FROM base as runner - -COPY --chown=$USER:$GROUP --link ./ $MBIN_HOME -RUN cp .env.example_docker .env - -COPY --from=builder-caddy --link /usr/bin/caddy /usr/sbin/caddy -COPY --from=builder-composer --chown=$USER:$GROUP $MBIN_HOME/vendor $MBIN_HOME/vendor -COPY --from=builder-composer --chown=$USER:$GROUP $MBIN_HOME/public/bundles $MBIN_HOME/public/bundles -COPY --from=builder-nodejs --chown=$USER:$GROUP $MBIN_HOME/public $MBIN_HOME/public - -COPY --link docker/caddy/Caddyfile /etc/caddy/Caddyfile -COPY --chmod=755 --link docker/docker-entrypoint ./ - -RUN mkdir -p public/media var/log /data /config && \ - chown -R $USER:$GROUP public/media var /data /config .env && \ - chmod 777 public/media var - -# Switch user -USER $USER:$GROUP - -ENTRYPOINT ["./docker-entrypoint"] - -# Expose port 2019 for caddy metric -EXPOSE 2019/tcp diff --git a/docker/caddy/Dockerfile b/docker/caddy/Dockerfile new file mode 100644 index 000000000..c85471608 --- /dev/null +++ b/docker/caddy/Dockerfile @@ -0,0 +1,16 @@ +FROM caddy:2.8.4-builder-alpine AS builder + +# Build Caddy with the Mercure and Vulcain and brotil cache modules +RUN xcaddy build \ + --with github.com/dunglas/mercure/caddy \ + --with github.com/dunglas/vulcain/caddy \ + --with github.com/ueffel/caddy-brotli + + +FROM caddy:2.8.4 as final + +COPY --link docker/caddy/Caddyfile /etc/caddy/Caddyfile +COPY --from=builder /usr/bin/caddy /usr/bin/caddy + +# Expose port 2019 for caddy metric +EXPOSE 2019/tcp diff --git a/docker/node/build.sh b/docker/node/build.sh new file mode 100755 index 000000000..5d931ef4a --- /dev/null +++ b/docker/node/build.sh @@ -0,0 +1,10 @@ +#/bin/sh +set -e + +if [ -d node_modules ] || [ "$FORCE" != "y" ] ; then + echo "Not rebuilding. Pass env var FORCE=y to rebuild anyway" + exit +fi + +npm ci --include=dev +npm run build diff --git a/docker/php/Dockerfile b/docker/php/Dockerfile new file mode 100644 index 000000000..d4d4301c6 --- /dev/null +++ b/docker/php/Dockerfile @@ -0,0 +1,41 @@ +FROM php:8.3-fpm-alpine as base +ARG UID=1000 +ENV MBIN_HOME=/var/www/mbin +ENV USER=mbin +ENV GROUP=www-data + +# Create user +RUN adduser -u $UID -D -g "" $USER $GROUP +# Install deps +RUN apk update && apk add acl fcgi exiftool +# Install composer +COPY --from=composer:latest /usr/bin/composer /usr/bin/composer + +# Install php extensions, by docker-php-extension-installer +COPY --from=mlocati/php-extension-installer /usr/bin/install-php-extensions /usr/local/bin/ +RUN install-php-extensions \ + amqp \ + apcu \ + bcmath \ + curl \ + dom \ + exif \ + gd \ + intl \ + opcache \ + pcntl \ + pdo_pgsql \ + pgsql \ + redis \ + simplexml \ + xdebug \ + xml + + + +COPY ./docker/php/entrypoint.sh /entrypoint.sh +RUN chmod +x /entrypoint.sh +ENTRYPOINT ["/entrypoint.sh"] + +# Switch user +USER $USER:$GROUP diff --git a/docker/php/conf.d/app.dev.ini b/docker/php/conf.d/app.dev.ini index 2f486a94d..a9599ce73 100644 --- a/docker/php/conf.d/app.dev.ini +++ b/docker/php/conf.d/app.dev.ini @@ -2,6 +2,9 @@ ; See https://github.com/docker/for-linux/issues/264 ; The `client_host` below may optionally be replaced with `discover_client_host=yes` ; Add `start_with_request=yes` to start debug session on each request +xdebug.mode=debug xdebug.client_host = 'host.docker.internal' +xdebug.client_port=9000 +xdebug.start_with_request=yes max_execution_time = 120 diff --git a/docker/docker-entrypoint b/docker/php/entrypoint.sh similarity index 71% rename from docker/docker-entrypoint rename to docker/php/entrypoint.sh index 5db1ae18f..0c8896359 100755 --- a/docker/docker-entrypoint +++ b/docker/php/entrypoint.sh @@ -7,26 +7,14 @@ if [ "${1#-}" != "$1" ]; then fi if [ "$1" == "php-fpm" ] || [ "$1" == "php" ] || [ "$1" == "bin/console" ]; then + pwd # if running as a service install assets echo "Starting as service..." - # In production: dump the production PHP config files, - # validate the installed packages (no dev) and dump prod config - if [ "$APP_ENV" == "prod" ]; then - cp "$PHP_INI_DIR/php.ini-production" "$PHP_INI_DIR/php.ini" - cp "$PHP_INI_DIR/conf.d/app.ini-production" "$PHP_INI_DIR/conf.d/app.prod.or.dev.ini" - composer install --prefer-dist --no-dev --no-autoloader --no-scripts --no-progress - composer dump-env prod - fi - # In development: dump the development PHP config files, # validate the installed packages (including dev dependencies) and dump dev config - if [ "$APP_ENV" == "dev" ]; then - cp "$PHP_INI_DIR/php.ini-development" "$PHP_INI_DIR/php.ini" - cp "$PHP_INI_DIR/conf.d/app.ini-development" "$PHP_INI_DIR/conf.d/app.prod.or.dev.ini" - composer install --prefer-dist --no-scripts --no-progress - composer dump-env dev - fi + composer install --prefer-dist --no-scripts --no-progress + composer dump-env dev echo "Waiting for db to be ready..." ATTEMPTS_LEFT_TO_REACH_DATABASE=60 diff --git a/notes.md b/notes.md index fe110a53d..c47dd824d 100644 --- a/notes.md +++ b/notes.md @@ -3,10 +3,20 @@ # Docker dev - [ ] Make a separate `compose.dev.yml` + - [ ] rewrite `Dockerfile` to have targets for dev + - [ ] Use `Dockerfile` targets in `compose.yml` for dev and prod + - [ ] `host.docker.internal` doesn't exist on linux because it's not running docker in a VM + - [ ] conditionally add following for linux\ + ```angular2html + extra_hosts: + - "host.docker.internal:host-gateway" + ``` - [ ] Make a separate `.env.dev` for development -- [ ] Indicate that `.env.dev` should be copied into [/docker/](/docker/) + - [ ] use `.env.dev` in `compose.dev.yml` - [ ] mention that resetting means `sudo rm -rf docker/storage` - [ ] move `.gitignore` out of storage otherwise resetting still creates a commit by deleting `.gitignore` - [ ] `docker/storage` needs to be writable to `mbin` user! - [ ] `sudo chmod 777 docker/storage/*` - [ ] `MESSENGER_TRANSPORT_DSN=doctrine://default` doesn't work + +- [ ] nixos needs [iptables rules](https://discourse.nixos.org/t/docker-container-not-resolving-to-host/30259/8) for xdebug to access host From c83485b7fb418fafa6fa115b402d0b35b553a2b2 Mon Sep 17 00:00:00 2001 From: LoveIsGrief Date: Mon, 12 Aug 2024 20:23:40 +0200 Subject: [PATCH 04/43] wip: split compose.yml --- compose.dev.yml | 177 ++++++++++++++++++++++++++++++++++++++++++ compose.prod.yml | 34 ++------ docker/php/Dockerfile | 73 +++++++++++++++++ notes.md | 6 +- 4 files changed, 259 insertions(+), 31 deletions(-) create mode 100644 compose.dev.yml diff --git a/compose.dev.yml b/compose.dev.yml new file mode 100644 index 000000000..5eceaeee6 --- /dev/null +++ b/compose.dev.yml @@ -0,0 +1,177 @@ +services: + www: + # Builds the Docker image from scratch + build: + dockerfile: docker/caddy/Dockerfile + target: dev + container_name: mbin-www + restart: unless-stopped + networks: + - mbin_external_network + command: caddy run --config /etc/caddy/Caddyfile --adapter caddyfile + healthcheck: + test: ["CMD-SHELL", "curl -f http://localhost:2019/metrics || exit 1"] + ports: + - 8008:80 + volumes: + - caddy_config:/config + - caddy_data:/data + - ./:/var/www/mbin + environment: + - SERVER_NAME=:80 # the address that the web server binds + - PHP_FASTCGI_HOST=php:9000 # caddy forward traffic to this host via fastcgi + - MERCURE_PUBLISHER_JWT_KEY=$MERCURE_JWT_SECRET + - MERCURE_SUBSCRIBER_JWT_KEY=$MERCURE_JWT_SECRET + depends_on: + - php + + php: + # Builds the Docker image from scratch + build: + dockerfile: docker/php/Dockerfile + target: dev + container_name: mbin-php + command: php-fpm + working_dir: /var/www/mbin + restart: unless-stopped + networks: + - mbin_external_network + extra_hosts: + - "host.docker.internal:host-gateway" + healthcheck: + test: + [ + "CMD-SHELL", + "REQUEST_METHOD=GET SCRIPT_NAME=/ping SCRIPT_FILENAME=/ping cgi-fcgi -bind -connect localhost:9000 || exit 1", + ] + volumes: + - ./docker/php/entrypoint.sh:/entrypoint.sh + - ./:/var/www/mbin/ + - ./docker/php/conf.d/app.dev.ini:/usr/local/etc/php/conf.d/app.dev.ini + - ./docker/php/php-fpm.d/zz-docker.conf:/usr/local/etc/php-fpm.d/zz-docker.conf + # If you want to change configs locally, without rebuilding the image use: + # - ../docker/config:/var/www/mbin/config + env_file: + - .env + depends_on: + - redis + - db + - rabbitmq + + node: + image: "node:22.5.1-alpine3.20" + command: docker/node/build.sh + container_name: node-builder + entrypoint: + - sh + - "-c" + working_dir: /var/www/mbin + depends_on: + php: + condition: service_healthy + networks: + - mbin_external_network + volumes: + - ./:/var/www/mbin/ + + messenger: + # Builds the Docker image from scratch + build: + dockerfile: docker/php/Dockerfile + restart: unless-stopped + networks: + - mbin_external_network + command: bin/console -vvvv messenger:consume --all --time-limit=3600 + healthcheck: + test: ["CMD-SHELL", "ps aux | grep 'messenger[:]consume' || exit 1"] + volumes: + - ./:/var/www/mbin/ + # If you want to change configs locally, without rebuilding the image use: + # - ../docker/config:/var/www/mbin/config + env_file: + - .env + depends_on: + - redis + - db + - rabbitmq + + redis: + image: redis:alpine + container_name: mbin-redis + restart: unless-stopped + networks: + - mbin_external_network + command: /bin/sh -c "redis-server --requirepass $${REDIS_PASSWORD}" + volumes: + - redis:/data + env_file: + - .env + healthcheck: + test: ["CMD", "redis-cli", "ping"] + + db: + image: postgres:${POSTGRES_VERSION:-13}-alpine + container_name: mbin-db + restart: unless-stopped + ports: + - "127.0.0.1:5432:5432" + networks: + - mbin_external_network + volumes: + - postgres:/var/lib/postgresql/data + env_file: + - .env + healthcheck: + test: [ "CMD-SHELL", "pg_isready" ] + interval: 10s + timeout: 5s + retries: 5 + + rabbitmq: + image: rabbitmq:3.13.6-management-alpine + container_name: mbin-rabbitmq + restart: unless-stopped + networks: + - mbin_external_network + env_file: + - .env + volumes: + - rabbitmq:/var/lib/rabbitmq + ports: + - 15672:15672 + + # Add your favorite reverse proxy (e.g nginx) which accept incoming HTTPS + # traffic and forward to http://www:80 + #nginx: + # image: nginx + # container_name: mbin-nginx + # networks: + # - mbin_external_network + # ports: + # - 443:443 + # volumes: + # - ./docker/nginx.conf:/etc/nginx/nginx.conf + + # Example of a SMTP docker service + # More info: https://hub.docker.com/r/ixdotai/smtp + #mailserver: + # image: ixdotai/smtp:latest + # networks: + # - mbin_external_network + # environment: + # - SMARTHOST_ADDRESS=mail.mysmtp.com + # - SMARTHOST_PORT=587 + # - SMARTHOST_USER=myuser + # - SMARTHOST_PASSWORD=secret + # - SMARTHOST_ALIASES=*.mysmtp.com + +networks: + mbin_external_network: + +volumes: + caddy_config: + caddy_data: + logs: + postgres: + rabbitmq: + redis: diff --git a/compose.prod.yml b/compose.prod.yml index 0d09ecd4f..a47f7ab95 100644 --- a/compose.prod.yml +++ b/compose.prod.yml @@ -4,24 +4,15 @@ services: www: # Builds the Docker image from scratch build: - dockerfile: docker/php/Dockerfile - image: mbin - # Or remove the build, context, dockerfile and image lines above and + target: prod + # Or remove the build, dockerfile and image lines above and # use the pre-build image from ghcr.io (you can also a tag eg.: v1.0.0 instead of: latest): - #image: "ghcr.io/mbinorg/mbin:latest" - container_name: mbin-www - restart: unless-stopped - networks: - - mbin_external_network - command: caddy run --config /etc/caddy/Caddyfile --adapter caddyfile - healthcheck: - test: ["CMD-SHELL", "curl -f http://localhost:2019/metrics || exit 1"] + #image: "ghcr.io/mbinorg/mbin:caddy-latest" ports: - - 8008:80 + - 80:80 volumes: - ./docker/storage/caddy_config:/config - ./docker/storage/caddy_data:/data - - ./:/var/www/mbin/ environment: - SERVER_NAME=:80 # the address that the web server binds - PHP_FASTCGI_HOST=php:9000 # caddy forward traffic to this host via fastcgi @@ -34,27 +25,12 @@ services: # Builds the Docker image from scratch build: dockerfile: docker/php/Dockerfile - image: mbin + target: prod # Or remove the build, context, dockerfile and image lines above and # use the pre-build image from ghcr.io (you can also a tag eg.: v1.0.0 instead of: latest): #image: "ghcr.io/mbinorg/mbin:latest" - container_name: mbin-php - restart: unless-stopped - networks: - - mbin_external_network - command: php-fpm - extra_hosts: - - "host.docker.internal:host-gateway" - healthcheck: - test: - [ - "CMD-SHELL", - "REQUEST_METHOD=GET SCRIPT_NAME=/ping SCRIPT_FILENAME=/ping cgi-fcgi -bind -connect localhost:9000 || exit 1", - ] volumes: - - ./:/var/www/mbin/ - ./docker/storage/logs:/var/www/mbin/var/log - - ./docker/php/conf.d/app.dev.ini:/usr/local/etc/php/conf.d/app.dev.ini # If you want to change configs locally, without rebuilding the image use: # - ../docker/config:/var/www/mbin/config env_file: diff --git a/docker/php/Dockerfile b/docker/php/Dockerfile index d4d4301c6..9c2cac2fc 100644 --- a/docker/php/Dockerfile +++ b/docker/php/Dockerfile @@ -37,5 +37,78 @@ COPY ./docker/php/entrypoint.sh /entrypoint.sh RUN chmod +x /entrypoint.sh ENTRYPOINT ["/entrypoint.sh"] +################# + +FROM base as dev + +# Switch user +USER $USER:$GROUP + + +################# + + +#################### + +FROM base as builder-composer + +# Composer: install package +COPY composer.* $MBIN_HOME +COPY symfony.lock $MBIN_HOME +ENV COMPOSER_ALLOW_SUPERUSER=1 +RUN composer install --prefer-dist --no-dev --no-autoloader --no-scripts --no-progress + +# Copy repository +COPY --link ./ $MBIN_HOME + +# Dump-autoload and run post-install script +RUN composer dump-autoload --classmap-authoritative --no-dev +RUN composer run-script --no-dev post-install-cmd && \ + chmod +x bin/console && sync + +#################### + +FROM node:22.5.1-alpine3.20 as builder-nodejs + +# Set NodeJS as production by default +ARG NODE_ENV=production +ENV NODE_ENV=${NODE_ENV} + +# Setup environment +ENV MBIN_HOME=/var/www/mbin +RUN mkdir -p $MBIN_HOME +WORKDIR $MBIN_HOME + +# Copy required files +COPY package.json $MBIN_HOME +COPY package-lock.json $MBIN_HOME +COPY --from=builder-composer --link $MBIN_HOME/vendor $MBIN_HOME/vendor + +# NPM: install package +RUN npm ci --include=dev + +# NPM: build (production by default) +COPY --link ./ $MBIN_HOME +RUN npm run build + + +################# + +FROM base as prod + +COPY --chown=$USER:$GROUP --link ./ $MBIN_HOME + +COPY --from=builder-composer --chown=$USER:$GROUP $MBIN_HOME/vendor $MBIN_HOME/vendor +COPY --from=builder-composer --chown=$USER:$GROUP $MBIN_HOME/public/bundles $MBIN_HOME/public/bundles +COPY --from=builder-nodejs --chown=$USER:$GROUP $MBIN_HOME/public $MBIN_HOME/public + + +RUN mkdir -p public/media var/log /data /config && \ + chown -R $USER:$GROUP public/media var /data /config .env && \ + chmod 777 public/media var + +# TODO copy app.prod.ini + # Switch user USER $USER:$GROUP + diff --git a/notes.md b/notes.md index c47dd824d..a65878805 100644 --- a/notes.md +++ b/notes.md @@ -3,16 +3,18 @@ # Docker dev - [ ] Make a separate `compose.dev.yml` + - [ ] Doc: `ln -s compose.dev.yml compose.override.yml` - [ ] rewrite `Dockerfile` to have targets for dev - [ ] Use `Dockerfile` targets in `compose.yml` for dev and prod - [ ] `host.docker.internal` doesn't exist on linux because it's not running docker in a VM - [ ] conditionally add following for linux\ - ```angular2html + ```yaml extra_hosts: - "host.docker.internal:host-gateway" ``` -- [ ] Make a separate `.env.dev` for development + - [ ] Make a separate `.env.dev` for development - [ ] use `.env.dev` in `compose.dev.yml` + - [ ] mention that resetting means `sudo rm -rf docker/storage` - [ ] move `.gitignore` out of storage otherwise resetting still creates a commit by deleting `.gitignore` - [ ] `docker/storage` needs to be writable to `mbin` user! From d0053cdc8aa6144eea4ae6756fb8b64560aae6d4 Mon Sep 17 00:00:00 2001 From: LoveIsGrief Date: Mon, 12 Aug 2024 23:53:15 +0200 Subject: [PATCH 05/43] Add TODO for prod entrypoint --- docker/php/Dockerfile | 2 +- notes.md | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/docker/php/Dockerfile b/docker/php/Dockerfile index 9c2cac2fc..f0bafacb6 100644 --- a/docker/php/Dockerfile +++ b/docker/php/Dockerfile @@ -32,7 +32,7 @@ RUN install-php-extensions \ xml - +# TODO Use a different entrypoint for prod? COPY ./docker/php/entrypoint.sh /entrypoint.sh RUN chmod +x /entrypoint.sh ENTRYPOINT ["/entrypoint.sh"] diff --git a/notes.md b/notes.md index a65878805..ac8890efc 100644 --- a/notes.md +++ b/notes.md @@ -14,7 +14,8 @@ ``` - [ ] Make a separate `.env.dev` for development - [ ] use `.env.dev` in `compose.dev.yml` - +- [ ] Update `compose.prod.yml` + - [ ] Use separate `entrypoint.sh` for prod image? - [ ] mention that resetting means `sudo rm -rf docker/storage` - [ ] move `.gitignore` out of storage otherwise resetting still creates a commit by deleting `.gitignore` - [ ] `docker/storage` needs to be writable to `mbin` user! From 3094794fa142a914ec3cc8dcfd9fbfe7afb4e012 Mon Sep 17 00:00:00 2001 From: LoveIsGrief Date: Mon, 12 Aug 2024 23:54:57 +0200 Subject: [PATCH 06/43] Use .env.dev_docker in compose.dev.yml --- compose.dev.yml | 10 +++++----- notes.md | 6 +++--- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/compose.dev.yml b/compose.dev.yml index 5eceaeee6..7a6ccdabf 100644 --- a/compose.dev.yml +++ b/compose.dev.yml @@ -52,7 +52,7 @@ services: # If you want to change configs locally, without rebuilding the image use: # - ../docker/config:/var/www/mbin/config env_file: - - .env + - .env.dev_docker depends_on: - redis - db @@ -89,7 +89,7 @@ services: # If you want to change configs locally, without rebuilding the image use: # - ../docker/config:/var/www/mbin/config env_file: - - .env + - .env.dev_docker depends_on: - redis - db @@ -105,7 +105,7 @@ services: volumes: - redis:/data env_file: - - .env + - .env.dev_docker healthcheck: test: ["CMD", "redis-cli", "ping"] @@ -120,7 +120,7 @@ services: volumes: - postgres:/var/lib/postgresql/data env_file: - - .env + - .env.dev_docker healthcheck: test: [ "CMD-SHELL", "pg_isready" ] interval: 10s @@ -134,7 +134,7 @@ services: networks: - mbin_external_network env_file: - - .env + - .env.dev_docker volumes: - rabbitmq:/var/lib/rabbitmq ports: diff --git a/notes.md b/notes.md index ac8890efc..5e101baf4 100644 --- a/notes.md +++ b/notes.md @@ -2,7 +2,7 @@ - [ ] mention how to activate debug logs for symfony (`bin/console -vvv some-command`) # Docker dev -- [ ] Make a separate `compose.dev.yml` +- [x] Make a separate `compose.dev.yml` - [ ] Doc: `ln -s compose.dev.yml compose.override.yml` - [ ] rewrite `Dockerfile` to have targets for dev - [ ] Use `Dockerfile` targets in `compose.yml` for dev and prod @@ -12,8 +12,8 @@ extra_hosts: - "host.docker.internal:host-gateway" ``` - - [ ] Make a separate `.env.dev` for development - - [ ] use `.env.dev` in `compose.dev.yml` + - [x] Make a separate `.env.dev` for development + - [x] use `.env.dev` in `compose.dev.yml` - [ ] Update `compose.prod.yml` - [ ] Use separate `entrypoint.sh` for prod image? - [ ] mention that resetting means `sudo rm -rf docker/storage` From e39052baafe3ccd1af98d9cb6f0ac58c486a58eb Mon Sep 17 00:00:00 2001 From: LoveIsGrief Date: Tue, 13 Aug 2024 00:28:34 +0200 Subject: [PATCH 07/43] Simplify dev docker compose --- .gitignore | 2 +- compose.dev.yml | 138 ++++++---------------------------- compose.yml | 91 ++++------------------ docker/php/conf.d/app.dev.ini | 4 +- notes.md | 2 + 5 files changed, 41 insertions(+), 196 deletions(-) diff --git a/.gitignore b/.gitignore index 0bce5491f..681adf52b 100644 --- a/.gitignore +++ b/.gitignore @@ -18,7 +18,7 @@ tools/vendor/ .env /public/media/* /public/media -docker/compose.override.yml +compose.override.yml yarn.lock /metal/ diff --git a/compose.dev.yml b/compose.dev.yml index 7a6ccdabf..3f1a48897 100644 --- a/compose.dev.yml +++ b/compose.dev.yml @@ -1,49 +1,36 @@ services: www: - # Builds the Docker image from scratch - build: - dockerfile: docker/caddy/Dockerfile - target: dev - container_name: mbin-www - restart: unless-stopped - networks: - - mbin_external_network - command: caddy run --config /etc/caddy/Caddyfile --adapter caddyfile - healthcheck: - test: ["CMD-SHELL", "curl -f http://localhost:2019/metrics || exit 1"] - ports: - - 8008:80 volumes: - caddy_config:/config - caddy_data:/data - ./:/var/www/mbin - environment: - - SERVER_NAME=:80 # the address that the web server binds - - PHP_FASTCGI_HOST=php:9000 # caddy forward traffic to this host via fastcgi - - MERCURE_PUBLISHER_JWT_KEY=$MERCURE_JWT_SECRET - - MERCURE_SUBSCRIBER_JWT_KEY=$MERCURE_JWT_SECRET depends_on: - - php + node: + condition: service_completed_successfully + + # Builds the nodejs application + node: + image: "node:22.5.1-alpine3.20" + command: docker/node/build.sh + container_name: node-builder + entrypoint: + - sh + - "-c" + working_dir: /var/www/mbin + depends_on: + php: + condition: service_healthy + networks: + - mbin_external_network + volumes: + - ./:/var/www/mbin/ php: # Builds the Docker image from scratch build: - dockerfile: docker/php/Dockerfile target: dev - container_name: mbin-php - command: php-fpm - working_dir: /var/www/mbin - restart: unless-stopped - networks: - - mbin_external_network extra_hosts: - "host.docker.internal:host-gateway" - healthcheck: - test: - [ - "CMD-SHELL", - "REQUEST_METHOD=GET SCRIPT_NAME=/ping SCRIPT_FILENAME=/ping cgi-fcgi -bind -connect localhost:9000 || exit 1", - ] volumes: - ./docker/php/entrypoint.sh:/entrypoint.sh - ./:/var/www/mbin/ @@ -53,125 +40,42 @@ services: # - ../docker/config:/var/www/mbin/config env_file: - .env.dev_docker - depends_on: - - redis - - db - - rabbitmq - node: - image: "node:22.5.1-alpine3.20" - command: docker/node/build.sh - container_name: node-builder - entrypoint: - - sh - - "-c" - working_dir: /var/www/mbin - depends_on: - php: - condition: service_healthy - networks: - - mbin_external_network - volumes: - - ./:/var/www/mbin/ messenger: # Builds the Docker image from scratch build: - dockerfile: docker/php/Dockerfile - restart: unless-stopped - networks: - - mbin_external_network + target: dev command: bin/console -vvvv messenger:consume --all --time-limit=3600 - healthcheck: - test: ["CMD-SHELL", "ps aux | grep 'messenger[:]consume' || exit 1"] volumes: - ./:/var/www/mbin/ - # If you want to change configs locally, without rebuilding the image use: - # - ../docker/config:/var/www/mbin/config env_file: - .env.dev_docker - depends_on: - - redis - - db - - rabbitmq redis: - image: redis:alpine - container_name: mbin-redis - restart: unless-stopped - networks: - - mbin_external_network - command: /bin/sh -c "redis-server --requirepass $${REDIS_PASSWORD}" volumes: - redis:/data env_file: - .env.dev_docker - healthcheck: - test: ["CMD", "redis-cli", "ping"] db: - image: postgres:${POSTGRES_VERSION:-13}-alpine - container_name: mbin-db - restart: unless-stopped + # Allow connecting to the db from the localhost for debugging ports: - "127.0.0.1:5432:5432" - networks: - - mbin_external_network volumes: - postgres:/var/lib/postgresql/data env_file: - .env.dev_docker - healthcheck: - test: [ "CMD-SHELL", "pg_isready" ] - interval: 10s - timeout: 5s - retries: 5 rabbitmq: - image: rabbitmq:3.13.6-management-alpine - container_name: mbin-rabbitmq - restart: unless-stopped - networks: - - mbin_external_network env_file: - .env.dev_docker volumes: - rabbitmq:/var/lib/rabbitmq - ports: - - 15672:15672 - - # Add your favorite reverse proxy (e.g nginx) which accept incoming HTTPS - # traffic and forward to http://www:80 - #nginx: - # image: nginx - # container_name: mbin-nginx - # networks: - # - mbin_external_network - # ports: - # - 443:443 - # volumes: - # - ./docker/nginx.conf:/etc/nginx/nginx.conf - - # Example of a SMTP docker service - # More info: https://hub.docker.com/r/ixdotai/smtp - #mailserver: - # image: ixdotai/smtp:latest - # networks: - # - mbin_external_network - # environment: - # - SMARTHOST_ADDRESS=mail.mysmtp.com - # - SMARTHOST_PORT=587 - # - SMARTHOST_USER=myuser - # - SMARTHOST_PASSWORD=secret - # - SMARTHOST_ALIASES=*.mysmtp.com - -networks: - mbin_external_network: volumes: caddy_config: caddy_data: - logs: postgres: rabbitmq: redis: diff --git a/compose.yml b/compose.yml index 0eed1f31a..5d960ee5d 100644 --- a/compose.yml +++ b/compose.yml @@ -12,10 +12,6 @@ services: test: ["CMD-SHELL", "curl -f http://localhost:2019/metrics || exit 1"] ports: - 8008:80 - volumes: - - caddy_config:/config - - caddy_data:/data - - ./:/var/www/mbin environment: - SERVER_NAME=:80 # the address that the web server binds - PHP_FASTCGI_HOST=php:9000 # caddy forward traffic to this host via fastcgi @@ -34,43 +30,24 @@ services: restart: unless-stopped networks: - mbin_external_network - extra_hosts: - - "host.docker.internal:host-gateway" healthcheck: test: [ "CMD-SHELL", "REQUEST_METHOD=GET SCRIPT_NAME=/ping SCRIPT_FILENAME=/ping cgi-fcgi -bind -connect localhost:9000 || exit 1", ] - volumes: - - ./docker/php/entrypoint.sh:/entrypoint.sh - - ./:/var/www/mbin/ - - ./docker/php/conf.d/app.dev.ini:/usr/local/etc/php/conf.d/app.dev.ini - - ./docker/php/php-fpm.d/zz-docker.conf:/usr/local/etc/php-fpm.d/zz-docker.conf - # If you want to change configs locally, without rebuilding the image use: - # - ../docker/config:/var/www/mbin/config env_file: - .env depends_on: - - redis - - db - - rabbitmq - - node: - image: "node:22.5.1-alpine3.20" - command: docker/node/build.sh - container_name: node-builder - entrypoint: - - sh - - "-c" - working_dir: /var/www/mbin - depends_on: - php: + redis: + condition: service_healthy + db: + condition: service_healthy + rabbitmq: + condition: service_healthy + messenger: + # The main application depends on being able to send messages condition: service_healthy - networks: - - mbin_external_network - volumes: - - ./:/var/www/mbin/ messenger: # Builds the Docker image from scratch @@ -79,13 +56,10 @@ services: restart: unless-stopped networks: - mbin_external_network - command: bin/console -vvvv messenger:consume --all --time-limit=3600 + command: bin/console messenger:consume --all --time-limit=3600 + working_dir: /var/www/mbin healthcheck: test: ["CMD-SHELL", "ps aux | grep 'messenger[:]consume' || exit 1"] - volumes: - - ./:/var/www/mbin/ - # If you want to change configs locally, without rebuilding the image use: - # - ../docker/config:/var/www/mbin/config env_file: - .env depends_on: @@ -100,8 +74,6 @@ services: networks: - mbin_external_network command: /bin/sh -c "redis-server --requirepass $${REDIS_PASSWORD}" - volumes: - - redis:/data env_file: - .env healthcheck: @@ -111,12 +83,8 @@ services: image: postgres:${POSTGRES_VERSION:-13}-alpine container_name: mbin-db restart: unless-stopped - ports: - - "127.0.0.1:5432:5432" networks: - mbin_external_network - volumes: - - postgres:/var/lib/postgresql/data env_file: - .env healthcheck: @@ -133,43 +101,14 @@ services: - mbin_external_network env_file: - .env - volumes: - - rabbitmq:/var/lib/rabbitmq ports: - 15672:15672 - - # Add your favorite reverse proxy (e.g nginx) which accept incoming HTTPS - # traffic and forward to http://www:80 - #nginx: - # image: nginx - # container_name: mbin-nginx - # networks: - # - mbin_external_network - # ports: - # - 443:443 - # volumes: - # - ./docker/nginx.conf:/etc/nginx/nginx.conf - - # Example of a SMTP docker service - # More info: https://hub.docker.com/r/ixdotai/smtp - #mailserver: - # image: ixdotai/smtp:latest - # networks: - # - mbin_external_network - # environment: - # - SMARTHOST_ADDRESS=mail.mysmtp.com - # - SMARTHOST_PORT=587 - # - SMARTHOST_USER=myuser - # - SMARTHOST_PASSWORD=secret - # - SMARTHOST_ALIASES=*.mysmtp.com + healthcheck: + test: rabbitmq-diagnostics -q ping + interval: 30s + timeout: 30s + retries: 3 networks: mbin_external_network: -volumes: - caddy_config: - caddy_data: - logs: - postgres: - rabbitmq: - redis: diff --git a/docker/php/conf.d/app.dev.ini b/docker/php/conf.d/app.dev.ini index a9599ce73..652cf6b06 100644 --- a/docker/php/conf.d/app.dev.ini +++ b/docker/php/conf.d/app.dev.ini @@ -1,10 +1,10 @@ ; See https://docs.docker.com/desktop/networking/#i-want-to-connect-from-a-container-to-a-service-on-the-host ; See https://github.com/docker/for-linux/issues/264 ; The `client_host` below may optionally be replaced with `discover_client_host=yes` -; Add `start_with_request=yes` to start debug session on each request xdebug.mode=debug xdebug.client_host = 'host.docker.internal' xdebug.client_port=9000 -xdebug.start_with_request=yes +; Add start debug session on each request +;xdebug.start_with_request=yes max_execution_time = 120 diff --git a/notes.md b/notes.md index 5e101baf4..518600186 100644 --- a/notes.md +++ b/notes.md @@ -22,4 +22,6 @@ - [ ] `sudo chmod 777 docker/storage/*` - [ ] `MESSENGER_TRANSPORT_DSN=doctrine://default` doesn't work +- [ ] Doc: inform how to enable xdebug (enabling it by default slows everything down) + - [ ] nixos needs [iptables rules](https://discourse.nixos.org/t/docker-container-not-resolving-to-host/30259/8) for xdebug to access host From 6f41aeca30221e28acc5e1d2b6332e46f30c7eb1 Mon Sep 17 00:00:00 2001 From: LoveIsGrief Date: Tue, 13 Aug 2024 00:29:23 +0200 Subject: [PATCH 08/43] tick off docker internal host linux --- notes.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/notes.md b/notes.md index 518600186..8f780fe53 100644 --- a/notes.md +++ b/notes.md @@ -7,7 +7,7 @@ - [ ] rewrite `Dockerfile` to have targets for dev - [ ] Use `Dockerfile` targets in `compose.yml` for dev and prod - [ ] `host.docker.internal` doesn't exist on linux because it's not running docker in a VM - - [ ] conditionally add following for linux\ + - [x] conditionally add following for linux\ ```yaml extra_hosts: - "host.docker.internal:host-gateway" From 4596cb22325241ee2469bc905a44423614bd1d72 Mon Sep 17 00:00:00 2001 From: LoveIsGrief Date: Tue, 13 Aug 2024 23:18:27 +0200 Subject: [PATCH 09/43] chore(docker): add app.ini to compose.dev.yml --- compose.dev.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/compose.dev.yml b/compose.dev.yml index 3f1a48897..3df70add3 100644 --- a/compose.dev.yml +++ b/compose.dev.yml @@ -34,6 +34,7 @@ services: volumes: - ./docker/php/entrypoint.sh:/entrypoint.sh - ./:/var/www/mbin/ + - ./docker/php/conf.d/app.ini:/usr/local/etc/php/conf.d/app.ini - ./docker/php/conf.d/app.dev.ini:/usr/local/etc/php/conf.d/app.dev.ini - ./docker/php/php-fpm.d/zz-docker.conf:/usr/local/etc/php-fpm.d/zz-docker.conf # If you want to change configs locally, without rebuilding the image use: From 9ba6bef93669d40456652606d66fe8b76acd2225 Mon Sep 17 00:00:00 2001 From: LoveIsGrief Date: Tue, 13 Aug 2024 23:18:53 +0200 Subject: [PATCH 10/43] chore(docker): remove unnecessary comment --- compose.dev.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/compose.dev.yml b/compose.dev.yml index 3df70add3..f4616d343 100644 --- a/compose.dev.yml +++ b/compose.dev.yml @@ -37,8 +37,6 @@ services: - ./docker/php/conf.d/app.ini:/usr/local/etc/php/conf.d/app.ini - ./docker/php/conf.d/app.dev.ini:/usr/local/etc/php/conf.d/app.dev.ini - ./docker/php/php-fpm.d/zz-docker.conf:/usr/local/etc/php-fpm.d/zz-docker.conf - # If you want to change configs locally, without rebuilding the image use: - # - ../docker/config:/var/www/mbin/config env_file: - .env.dev_docker From f352c2facac2eb03c367b23edf11d4a92b9ea173 Mon Sep 17 00:00:00 2001 From: LoveIsGrief Date: Tue, 13 Aug 2024 23:23:26 +0200 Subject: [PATCH 11/43] chore: add replicas to messenger service in compose.prod.yml --- compose.prod.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/compose.prod.yml b/compose.prod.yml index a47f7ab95..51972c649 100644 --- a/compose.prod.yml +++ b/compose.prod.yml @@ -49,11 +49,14 @@ services: # use the pre-build image from ghcr.io (you can also a tag eg.: v1.0.0 instead of: latest): #image: "ghcr.io/mbinorg/mbin:latest" restart: unless-stopped - networks: - - mbin_external_network command: bin/console -vvvv messenger:consume --all --time-limit=3600 + deploy: + mode: replicated + replicas: 6 healthcheck: test: ["CMD-SHELL", "ps aux | grep 'messenger[:]consume' || exit 1"] + networks: + - mbin_external_network volumes: - ./:/var/www/mbin/ - ./docker/storage/logs:/var/www/mbin/var/log From bf23ce680a8a1e3bc1c98f5f6958a8ef627fca98 Mon Sep 17 00:00:00 2001 From: LoveIsGrief Date: Tue, 13 Aug 2024 23:24:01 +0200 Subject: [PATCH 12/43] chore: upgrade postgres to v16 It's the latest at the time of this commit --- compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose.yml b/compose.yml index 5d960ee5d..57934f3d2 100644 --- a/compose.yml +++ b/compose.yml @@ -80,7 +80,7 @@ services: test: ["CMD", "redis-cli", "ping"] db: - image: postgres:${POSTGRES_VERSION:-13}-alpine + image: postgres:${POSTGRES_VERSION:-16}-alpine container_name: mbin-db restart: unless-stopped networks: From 1d279e0153e9bbfbaa90b75793575d83eb02a08c Mon Sep 17 00:00:00 2001 From: LoveIsGrief Date: Wed, 14 Aug 2024 01:10:50 +0200 Subject: [PATCH 13/43] wip: semi-function production images --- .dockerignore | 9 ++++----- .env.dev_docker | 1 - .env.example_docker | 14 +++++++++----- compose.prod.yml | 12 ++++-------- compose.yml | 1 + docker/php/Dockerfile | 22 ++++++++++++++-------- docker/php/entrypoint.sh | 12 +++++++----- 7 files changed, 39 insertions(+), 32 deletions(-) diff --git a/.dockerignore b/.dockerignore index 25f76f046..b3b0c5a66 100644 --- a/.dockerignore +++ b/.dockerignore @@ -11,14 +11,13 @@ **/.gitattributes **/.gitignore **/.gitmodules -**/compose.*.yaml -**/compose.*.yml -**/compose.yaml -compose.yml **/Dockerfile **/Thumbs.db +compose.*.yaml +compose.*.yml +compose.yaml +compose.yml .github/ -docker/compose.override.yml docker/storage/ docs/ public/bundles/ diff --git a/.env.dev_docker b/.env.dev_docker index a72276e3c..f89d82399 100644 --- a/.env.dev_docker +++ b/.env.dev_docker @@ -135,7 +135,6 @@ MESSENGER_TRANSPORT_DSN=amqp://mbin:${RABBITMQ_DEFAULT_PASS}@rabbitmq:5672/%2f/m #MESSENGER_TRANSPORT_DSN=redis://${REDIS_PASSWORD}@${REDIS_HOST}/messages ###< symfony/messenger ### - ###> symfony/mailer ### # See https://symfony.com/doc/current/mailer.html#using-built-in-transports # MAILER_DSN=sendmail://default # Use sendmail when you are using Postfix diff --git a/.env.example_docker b/.env.example_docker index 94d692b03..9949888ad 100644 --- a/.env.example_docker +++ b/.env.example_docker @@ -115,18 +115,22 @@ APP_SECRET=!CHANGE_SECRET! ###> doctrine/doctrine-bundle ### # Format described at https://www.doctrine-project.org/projects/doctrine-dbal/en/latest/reference/configuration.html#connecting-using-a-url POSTGRES_HOST=db:5432 -POSTGRES_DB=kbin -POSTGRES_USER=kbin +POSTGRES_DB=mbin +POSTGRES_USER=mbin POSTGRES_PASSWORD=!ChangeThisPostgresPass! # IMPORTANT: You MUST configure your PostgreSQL server version! -POSTGRES_VERSION=13 +POSTGRES_VERSION=16 DATABASE_URL="postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}/${POSTGRES_DB}?serverVersion=${POSTGRES_VERSION}&charset=utf8" ###< doctrine/doctrine-bundle ### +###> rabbitmq ### +RABBITMQ_DEFAULT_USER=mbin +RABBITMQ_DEFAULT_PASS=!ChangeThisRabbitPass! +###< rabbitmq ### + ###> symfony/messenger ### # Choose one of the transports below -RABBITMQ_PASSWORD=!ChangeThisRabbitPass! -MESSENGER_TRANSPORT_DSN=amqp://kbin:${RABBITMQ_PASSWORD}@rabbitmq:5672/%2f/messages +MESSENGER_TRANSPORT_DSN=amqp://mbin:${RABBITMQ_DEFAULT_PASS}@rabbitmq:5672/%2f/messages #MESSENGER_TRANSPORT_DSN=doctrine://default #MESSENGER_TRANSPORT_DSN=redis://${REDIS_PASSWORD}@${REDIS_HOST}/messages ###< symfony/messenger ### diff --git a/compose.prod.yml b/compose.prod.yml index 51972c649..87fd3676f 100644 --- a/compose.prod.yml +++ b/compose.prod.yml @@ -1,10 +1,5 @@ -version: "3.9" - services: www: - # Builds the Docker image from scratch - build: - target: prod # Or remove the build, dockerfile and image lines above and # use the pre-build image from ghcr.io (you can also a tag eg.: v1.0.0 instead of: latest): #image: "ghcr.io/mbinorg/mbin:caddy-latest" @@ -31,6 +26,7 @@ services: #image: "ghcr.io/mbinorg/mbin:latest" volumes: - ./docker/storage/logs:/var/www/mbin/var/log + - ./docker/php/entrypoint.sh:/entrypoint.sh # If you want to change configs locally, without rebuilding the image use: # - ../docker/config:/var/www/mbin/config env_file: @@ -52,13 +48,13 @@ services: command: bin/console -vvvv messenger:consume --all --time-limit=3600 deploy: mode: replicated - replicas: 6 + replicas: 1 healthcheck: test: ["CMD-SHELL", "ps aux | grep 'messenger[:]consume' || exit 1"] networks: - mbin_external_network volumes: - - ./:/var/www/mbin/ + - ./docker/php/entrypoint.sh:/entrypoint.sh - ./docker/storage/logs:/var/www/mbin/var/log # If you want to change configs locally, without rebuilding the image use: # - ../docker/config:/var/www/mbin/config @@ -84,7 +80,7 @@ services: test: ["CMD", "redis-cli", "ping"] db: - image: postgres:${POSTGRES_VERSION:-13}-alpine + image: postgres:${POSTGRES_VERSION:-16}-alpine container_name: mbin-db restart: unless-stopped ports: diff --git a/compose.yml b/compose.yml index 57934f3d2..f0a6168dc 100644 --- a/compose.yml +++ b/compose.yml @@ -9,6 +9,7 @@ services: - mbin_external_network command: caddy run --config /etc/caddy/Caddyfile --adapter caddyfile healthcheck: + # TODO no curl in prod??? test: ["CMD-SHELL", "curl -f http://localhost:2019/metrics || exit 1"] ports: - 8008:80 diff --git a/docker/php/Dockerfile b/docker/php/Dockerfile index f0bafacb6..ffb9e6c26 100644 --- a/docker/php/Dockerfile +++ b/docker/php/Dockerfile @@ -8,8 +8,6 @@ ENV GROUP=www-data RUN adduser -u $UID -D -g "" $USER $GROUP # Install deps RUN apk update && apk add acl fcgi exiftool -# Install composer -COPY --from=composer:latest /usr/bin/composer /usr/bin/composer # Install php extensions, by docker-php-extension-installer COPY --from=mlocati/php-extension-installer /usr/bin/install-php-extensions /usr/local/bin/ @@ -40,17 +38,19 @@ ENTRYPOINT ["/entrypoint.sh"] ################# FROM base as dev +# Install composer +COPY --from=composer:latest /usr/bin/composer /usr/bin/composer # Switch user USER $USER:$GROUP - - ################# #################### - FROM base as builder-composer +COPY --from=composer:latest /usr/bin/composer /usr/bin/composer + +WORKDIR $MBIN_HOME # Composer: install package COPY composer.* $MBIN_HOME @@ -60,6 +60,10 @@ RUN composer install --prefer-dist --no-dev --no-autoloader --no-scripts --no-pr # Copy repository COPY --link ./ $MBIN_HOME +RUN cp .env.example_docker .env + +# Increase memory limit for following steps +RUN echo "memory_limit = 512M" > /usr/local/etc/php/conf.d/memory_limit.ini # Dump-autoload and run post-install script RUN composer dump-autoload --classmap-authoritative --no-dev @@ -102,13 +106,15 @@ COPY --from=builder-composer --chown=$USER:$GROUP $MBIN_HOME/vendor $MBIN_HOME/v COPY --from=builder-composer --chown=$USER:$GROUP $MBIN_HOME/public/bundles $MBIN_HOME/public/bundles COPY --from=builder-nodejs --chown=$USER:$GROUP $MBIN_HOME/public $MBIN_HOME/public - +WORKDIR $MBIN_HOME +RUN cp .env.example_docker .env # TODO why? RUN mkdir -p public/media var/log /data /config && \ chown -R $USER:$GROUP public/media var /data /config .env && \ chmod 777 public/media var -# TODO copy app.prod.ini +# Configure PHP for production +COPY docker/php/conf.d/app.ini /usr/local/etc/php/conf.d/ +COPY docker/php/conf.d/app.prod.ini /usr/local/etc/php/conf.d/ # Switch user USER $USER:$GROUP - diff --git a/docker/php/entrypoint.sh b/docker/php/entrypoint.sh index 0c8896359..67ca7bd28 100755 --- a/docker/php/entrypoint.sh +++ b/docker/php/entrypoint.sh @@ -1,5 +1,5 @@ #!/bin/sh -set -e +set -ex # first arg is `-f` or `--some-option` if [ "${1#-}" != "$1" ]; then @@ -11,10 +11,12 @@ if [ "$1" == "php-fpm" ] || [ "$1" == "php" ] || [ "$1" == "bin/console" ]; then # if running as a service install assets echo "Starting as service..." - # In development: dump the development PHP config files, - # validate the installed packages (including dev dependencies) and dump dev config - composer install --prefer-dist --no-scripts --no-progress - composer dump-env dev + if [ "$APP_ENV" == "dev" ] ; then + # In development: dump the development PHP config files, + # validate the installed packages (including dev dependencies) and dump dev config + composer install --prefer-dist --no-scripts --no-progress + composer dump-env dev + fi echo "Waiting for db to be ready..." ATTEMPTS_LEFT_TO_REACH_DATABASE=60 From f9d4a17ee02817b4a74937194794750e3aaf1f10 Mon Sep 17 00:00:00 2001 From: LoveIsGrief Date: Wed, 14 Aug 2024 22:37:48 +0200 Subject: [PATCH 14/43] refactor: remove `set -x` in docker entrypoint --- docker/php/entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/php/entrypoint.sh b/docker/php/entrypoint.sh index 67ca7bd28..94efc6e3b 100755 --- a/docker/php/entrypoint.sh +++ b/docker/php/entrypoint.sh @@ -1,5 +1,5 @@ #!/bin/sh -set -ex +set -e # first arg is `-f` or `--some-option` if [ "${1#-}" != "$1" ]; then From 8e9f9c8a0796e04ea73f430e77aae556023c642e Mon Sep 17 00:00:00 2001 From: LoveIsGrief Date: Wed, 14 Aug 2024 22:47:02 +0200 Subject: [PATCH 15/43] refactor(compose): don't mount entrypoint into prod It should be using the entrypoint from the image --- compose.prod.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/compose.prod.yml b/compose.prod.yml index 87fd3676f..6401df604 100644 --- a/compose.prod.yml +++ b/compose.prod.yml @@ -26,7 +26,6 @@ services: #image: "ghcr.io/mbinorg/mbin:latest" volumes: - ./docker/storage/logs:/var/www/mbin/var/log - - ./docker/php/entrypoint.sh:/entrypoint.sh # If you want to change configs locally, without rebuilding the image use: # - ../docker/config:/var/www/mbin/config env_file: @@ -54,7 +53,6 @@ services: networks: - mbin_external_network volumes: - - ./docker/php/entrypoint.sh:/entrypoint.sh - ./docker/storage/logs:/var/www/mbin/var/log # If you want to change configs locally, without rebuilding the image use: # - ../docker/config:/var/www/mbin/config From 43377ff303c1bfed3942bb50858852b2c85ad137 Mon Sep 17 00:00:00 2001 From: LoveIsGrief Date: Wed, 14 Aug 2024 22:48:47 +0200 Subject: [PATCH 16/43] refactor(docker): use dummy .env for prod Dockerfile https://github.com/symfony/symfony/issues/34660 Symfony devs always force the use of a `.env` file even if the env is set properly. The cleanest is thus to have an empty `.env` and pass the real env to the container. --- docker/php/Dockerfile | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/docker/php/Dockerfile b/docker/php/Dockerfile index ffb9e6c26..66e4244d9 100644 --- a/docker/php/Dockerfile +++ b/docker/php/Dockerfile @@ -60,10 +60,11 @@ RUN composer install --prefer-dist --no-dev --no-autoloader --no-scripts --no-pr # Copy repository COPY --link ./ $MBIN_HOME -RUN cp .env.example_docker .env # Increase memory limit for following steps RUN echo "memory_limit = 512M" > /usr/local/etc/php/conf.d/memory_limit.ini +# Create a dummy .env: https://github.com/symfony/symfony/issues/34660 +RUN touch .env # Dump-autoload and run post-install script RUN composer dump-autoload --classmap-authoritative --no-dev @@ -107,7 +108,10 @@ COPY --from=builder-composer --chown=$USER:$GROUP $MBIN_HOME/public/bundles $MBI COPY --from=builder-nodejs --chown=$USER:$GROUP $MBIN_HOME/public $MBIN_HOME/public WORKDIR $MBIN_HOME -RUN cp .env.example_docker .env # TODO why? + +# Create a dummy .env: https://github.com/symfony/symfony/issues/34660 +RUN touch .env + RUN mkdir -p public/media var/log /data /config && \ chown -R $USER:$GROUP public/media var /data /config .env && \ chmod 777 public/media var From b0034969fa913d13fa94c7d319438689a94c9b03 Mon Sep 17 00:00:00 2001 From: LoveIsGrief Date: Thu, 15 Aug 2024 19:35:24 +0200 Subject: [PATCH 17/43] fix(docker): use wget to check www health curl isn't installed in the image and the health check was constantly failing --- compose.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/compose.yml b/compose.yml index f0a6168dc..9b8e96618 100644 --- a/compose.yml +++ b/compose.yml @@ -9,8 +9,7 @@ services: - mbin_external_network command: caddy run --config /etc/caddy/Caddyfile --adapter caddyfile healthcheck: - # TODO no curl in prod??? - test: ["CMD-SHELL", "curl -f http://localhost:2019/metrics || exit 1"] + test: ["CMD-SHELL", "wget -O /dev/null http://localhost:2019/metrics || exit 1"] ports: - 8008:80 environment: From 2fccae4b14044781285744858f8649ddf485bbae Mon Sep 17 00:00:00 2001 From: LoveIsGrief Date: Fri, 16 Aug 2024 15:25:19 +0200 Subject: [PATCH 18/43] Support running mbin behind any reverse proxy /public files are copied into a common volume shared between reverse proxy and PHP-FPM service. --- .dockerignore | 2 + .env.dev_docker | 11 ++++- .env.example_docker | 9 ++++ compose.dev.yml | 2 + compose.prod.yml | 9 +++- compose.yml | 9 +--- docker/caddy/Caddyfile | 4 +- docker/php/Dockerfile | 101 +++++++++++++++++++++++++-------------- docker/php/entrypoint.sh | 31 +++++++++++- notes.md | 19 +++++--- 10 files changed, 142 insertions(+), 55 deletions(-) diff --git a/.dockerignore b/.dockerignore index b3b0c5a66..03d991aab 100644 --- a/.dockerignore +++ b/.dockerignore @@ -18,6 +18,7 @@ compose.*.yml compose.yaml compose.yml .github/ +node_modules/ docker/storage/ docs/ public/bundles/ @@ -25,6 +26,7 @@ tests/ tools/ var/ vendor/ +.idea/ .editorconfig .env.*.local .env.local diff --git a/.env.dev_docker b/.env.dev_docker index f89d82399..29106a2af 100644 --- a/.env.dev_docker +++ b/.env.dev_docker @@ -31,6 +31,9 @@ KBIN_META_KEYWORDS="mbin, content aggregator, open source, fediverse" KBIN_HEADER_LOGO=false KBIN_FEDERATION_PAGE_ENABLED=true MBIN_DEFAULT_THEME=default +MBIN_HOME=/var/www/mbin +MBIN_USER=mbin +MBIN_GROUP=www-data # If you are running Mbin behind a reverse proxy, uncomment the line below and adjust the proxy address/range below # to your server's IP address if it does not already fall within the private IP spaces specified. @@ -107,6 +110,10 @@ EXIF_EXIFTOOL_PATH= # max execution time for exiftool in seconds, defaults to 10 seconds EXIF_EXIFTOOL_TIMEOUT=10 +###> caddy ### +PHP_FASTCGI_HOST=php:9000 +###< caddy ### + ###> symfony/framework-bundle ### APP_ENV=dev APP_SECRET=427f5e2940e5b2472c1b44b2d06e0525 @@ -119,7 +126,7 @@ POSTGRES_DB=mbin POSTGRES_USER=mbin POSTGRES_PASSWORD=!ChangeThisPostgresPass! # IMPORTANT: You MUST configure your PostgreSQL server version! -POSTGRES_VERSION=13 +POSTGRES_VERSION=16 DATABASE_URL="postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}/${POSTGRES_DB}?serverVersion=${POSTGRES_VERSION}&charset=utf8" ###< doctrine/doctrine-bundle ### @@ -161,6 +168,8 @@ MERCURE_URL=http://www:80/.well-known/mercure MERCURE_PUBLIC_URL=https://${KBIN_DOMAIN}/.well-known/mercure # The secret used to sign the JWTs MERCURE_JWT_SECRET="!ChangeThisMercureHubJWTSecretKey!" +MERCURE_PUBLISHER_JWT_KEY=${MERCURE_JWT_SECRET} +MERCURE_SUBSCRIBER_JWT_KEY=${MERCURE_JWT_SECRET} ###< symfony/mercure-bundle ### ###> nelmio/cors-bundle ### diff --git a/.env.example_docker b/.env.example_docker index 9949888ad..eba234f6e 100644 --- a/.env.example_docker +++ b/.env.example_docker @@ -31,6 +31,9 @@ KBIN_META_KEYWORDS="mbin, content aggregator, open source, fediverse" KBIN_HEADER_LOGO=false KBIN_FEDERATION_PAGE_ENABLED=true MBIN_DEFAULT_THEME=default +MBIN_HOME=/var/www/mbin +MBIN_USER=mbin +MBIN_GROUP=www-data # If you are running Mbin behind a reverse proxy, uncomment the line below and adjust the proxy address/range below # to your server's IP address if it does not already fall within the private IP spaces specified. @@ -107,6 +110,10 @@ EXIF_EXIFTOOL_PATH= # max execution time for exiftool in seconds, defaults to 10 seconds EXIF_EXIFTOOL_TIMEOUT=10 +###> caddy ### +PHP_FASTCGI_HOST=php:9000 +###< caddy ### + ###> symfony/framework-bundle ### APP_ENV=prod APP_SECRET=!CHANGE_SECRET! @@ -161,6 +168,8 @@ MERCURE_URL=http://www:80/.well-known/mercure MERCURE_PUBLIC_URL=https://${KBIN_DOMAIN}/.well-known/mercure # The secret used to sign the JWTs MERCURE_JWT_SECRET="!ChangeThisMercureHubJWTSecretKey!" +MERCURE_PUBLISHER_JWT_KEY=${MERCURE_JWT_SECRET} +MERCURE_SUBSCRIBER_JWT_KEY=${MERCURE_JWT_SECRET} ###< symfony/mercure-bundle ### ###> nelmio/cors-bundle ### diff --git a/compose.dev.yml b/compose.dev.yml index f4616d343..16ced3824 100644 --- a/compose.dev.yml +++ b/compose.dev.yml @@ -7,6 +7,8 @@ services: depends_on: node: condition: service_completed_successfully + env_file: + - .env.dev_docker # Builds the nodejs application node: diff --git a/compose.prod.yml b/compose.prod.yml index 6401df604..801e25040 100644 --- a/compose.prod.yml +++ b/compose.prod.yml @@ -8,6 +8,8 @@ services: volumes: - ./docker/storage/caddy_config:/config - ./docker/storage/caddy_data:/data + - ./docker/caddy/Caddyfile:/etc/caddy/Caddyfile + - ./docker/storage/mbin/public:${MBIN_HOME}/public:ro environment: - SERVER_NAME=:80 # the address that the web server binds - PHP_FASTCGI_HOST=php:9000 # caddy forward traffic to this host via fastcgi @@ -26,8 +28,13 @@ services: #image: "ghcr.io/mbinorg/mbin:latest" volumes: - ./docker/storage/logs:/var/www/mbin/var/log + - ./docker/storage/mbin/:/opt/mbin/ + - ./docker/php/entrypoint.sh:/entrypoint.sh # If you want to change configs locally, without rebuilding the image use: # - ../docker/config:/var/www/mbin/config + environment: + # Where the public/ folder will be copied to for the reverse proxy to use + MBIN_PUBLIC_COPY_DIR: /opt/mbin/ env_file: - .env depends_on: @@ -39,7 +46,6 @@ services: # Builds the Docker image from scratch build: dockerfile: docker/php/Dockerfile - image: mbin # Or remove the build, context, dockerfile and image lines above and # use the pre-build image from ghcr.io (you can also a tag eg.: v1.0.0 instead of: latest): #image: "ghcr.io/mbinorg/mbin:latest" @@ -54,6 +60,7 @@ services: - mbin_external_network volumes: - ./docker/storage/logs:/var/www/mbin/var/log + - ./docker/php/entrypoint.sh:/entrypoint.sh # If you want to change configs locally, without rebuilding the image use: # - ../docker/config:/var/www/mbin/config env_file: diff --git a/compose.yml b/compose.yml index 9b8e96618..333abf3a0 100644 --- a/compose.yml +++ b/compose.yml @@ -10,13 +10,8 @@ services: command: caddy run --config /etc/caddy/Caddyfile --adapter caddyfile healthcheck: test: ["CMD-SHELL", "wget -O /dev/null http://localhost:2019/metrics || exit 1"] - ports: - - 8008:80 - environment: - - SERVER_NAME=:80 # the address that the web server binds - - PHP_FASTCGI_HOST=php:9000 # caddy forward traffic to this host via fastcgi - - MERCURE_PUBLISHER_JWT_KEY=$MERCURE_JWT_SECRET - - MERCURE_SUBSCRIBER_JWT_KEY=$MERCURE_JWT_SECRET + env_file: + - .env depends_on: - php diff --git a/docker/caddy/Caddyfile b/docker/caddy/Caddyfile index 958a2c631..c731841ce 100644 --- a/docker/caddy/Caddyfile +++ b/docker/caddy/Caddyfile @@ -5,7 +5,7 @@ auto_https off } -{$CADDY_EXTRA_CONFIG} +{$CADDY_EXTRA_CONFIG} {$SERVER_NAME} { log @@ -18,7 +18,7 @@ header @static_files Cache-Control max-age=259200 route { - root * /var/www/mbin/public + root * {$MBIN_HOME}/public @supports_webp { header_regexp Accept image/webp diff --git a/docker/php/Dockerfile b/docker/php/Dockerfile index 66e4244d9..91a63ecef 100644 --- a/docker/php/Dockerfile +++ b/docker/php/Dockerfile @@ -1,11 +1,14 @@ FROM php:8.3-fpm-alpine as base ARG UID=1000 -ENV MBIN_HOME=/var/www/mbin -ENV USER=mbin -ENV GROUP=www-data +ARG MBIN_HOME=/var/www/mbin +ENV MBIN_HOME=${MBIN_HOME} +ARG MBIN_USER=mbin +ENV MBIN_USER=${MBIN_USER} +ARG MBIN_GROUP=www-data +ENV MBIN_GROUP=${MBIN_GROUP} # Create user -RUN adduser -u $UID -D -g "" $USER $GROUP +RUN adduser -u $UID -D -g "" $MBIN_USER $MBIN_GROUP # Install deps RUN apk update && apk add acl fcgi exiftool @@ -29,24 +32,21 @@ RUN install-php-extensions \ xdebug \ xml - -# TODO Use a different entrypoint for prod? -COPY ./docker/php/entrypoint.sh /entrypoint.sh -RUN chmod +x /entrypoint.sh +# Dev will mount the entrypoint +# Prod will embed it ENTRYPOINT ["/entrypoint.sh"] -################# +################################################### FROM base as dev # Install composer COPY --from=composer:latest /usr/bin/composer /usr/bin/composer -# Switch user -USER $USER:$GROUP -################# +# Switch user to write as non-root into mounted local directories +USER $MBIN_USER:$MBIN_GROUP -#################### +###################################################### FROM base as builder-composer COPY --from=composer:latest /usr/bin/composer /usr/bin/composer @@ -58,67 +58,94 @@ COPY symfony.lock $MBIN_HOME ENV COMPOSER_ALLOW_SUPERUSER=1 RUN composer install --prefer-dist --no-dev --no-autoloader --no-scripts --no-progress -# Copy repository -COPY --link ./ $MBIN_HOME +# Copy only required resources +# TODO Use one line once COPY --parents is supported +COPY --link assets $MBIN_HOME/assets +COPY --link bin $MBIN_HOME/bin +COPY --link config $MBIN_HOME/config +COPY --link migrations $MBIN_HOME/migrations +COPY --link public $MBIN_HOME/public +COPY --link src $MBIN_HOME/src +COPY --link templates $MBIN_HOME/templates +COPY --link translations $MBIN_HOME/translations # Increase memory limit for following steps RUN echo "memory_limit = 512M" > /usr/local/etc/php/conf.d/memory_limit.ini -# Create a dummy .env: https://github.com/symfony/symfony/issues/34660 -RUN touch .env +# Must have an env file: https://github.com/symfony/symfony/issues/34660 +# Must have all env vars in it https://github.com/symfony/flex/issues/346 +COPY --link .env.example_docker .env # Dump-autoload and run post-install script RUN composer dump-autoload --classmap-authoritative --no-dev RUN composer run-script --no-dev post-install-cmd && \ chmod +x bin/console && sync -#################### +###################################################### FROM node:22.5.1-alpine3.20 as builder-nodejs +ARG MBIN_HOME=/var/www/mbin +ENV MBIN_HOME=${MBIN_HOME} # Set NodeJS as production by default ARG NODE_ENV=production ENV NODE_ENV=${NODE_ENV} # Setup environment -ENV MBIN_HOME=/var/www/mbin RUN mkdir -p $MBIN_HOME WORKDIR $MBIN_HOME -# Copy required files +# NPM: install package dependencies COPY package.json $MBIN_HOME COPY package-lock.json $MBIN_HOME COPY --from=builder-composer --link $MBIN_HOME/vendor $MBIN_HOME/vendor - -# NPM: install package RUN npm ci --include=dev # NPM: build (production by default) -COPY --link ./ $MBIN_HOME +# TODO Use one line once COPY --parents is supported +COPY --link assets/ $MBIN_HOME/assets/ +COPY --link config/ $MBIN_HOME/config/ +COPY --link public/ $MBIN_HOME/public/ +COPY --link src/ $MBIN_HOME/src/ +COPY --link webpack* $MBIN_HOME/ RUN npm run build -################# +################################################### FROM base as prod -COPY --chown=$USER:$GROUP --link ./ $MBIN_HOME - -COPY --from=builder-composer --chown=$USER:$GROUP $MBIN_HOME/vendor $MBIN_HOME/vendor -COPY --from=builder-composer --chown=$USER:$GROUP $MBIN_HOME/public/bundles $MBIN_HOME/public/bundles -COPY --from=builder-nodejs --chown=$USER:$GROUP $MBIN_HOME/public $MBIN_HOME/public +# Copy only necessary resources +# TODO Use one line once COPY --parents is supported +## Folders +COPY --chown=$MBIN_USER:$MBIN_GROUP --link assets/ $MBIN_HOME/assets +COPY --chown=$MBIN_USER:$MBIN_GROUP --link bin/ $MBIN_HOME/bin +COPY --chown=$MBIN_USER:$MBIN_GROUP --link config/ $MBIN_HOME/config +COPY --chown=$MBIN_USER:$MBIN_GROUP --link migrations/ $MBIN_HOME/migrations +COPY --chown=$MBIN_USER:$MBIN_GROUP --link public/ $MBIN_HOME/public +COPY --chown=$MBIN_USER:$MBIN_GROUP --link src/ $MBIN_HOME/src +COPY --chown=$MBIN_USER:$MBIN_GROUP --link templates/ $MBIN_HOME/templates +COPY --chown=$MBIN_USER:$MBIN_GROUP --link translations/ $MBIN_HOME/translations + +## Files +COPY --chown=$MBIN_USER:$MBIN_GROUP --link composer* $MBIN_HOME/ +COPY --chown=$MBIN_USER:$MBIN_GROUP --link symfony* $MBIN_HOME/ + +## Build output +COPY --from=builder-composer --chown=$MBIN_USER:$MBIN_GROUP $MBIN_HOME/vendor $MBIN_HOME/vendor +COPY --from=builder-composer --chown=$MBIN_USER:$MBIN_GROUP $MBIN_HOME/public/bundles $MBIN_HOME/public/bundles +COPY --from=builder-nodejs --chown=$MBIN_USER:$MBIN_GROUP $MBIN_HOME/public $MBIN_HOME/public WORKDIR $MBIN_HOME -# Create a dummy .env: https://github.com/symfony/symfony/issues/34660 -RUN touch .env - -RUN mkdir -p public/media var/log /data /config && \ - chown -R $USER:$GROUP public/media var /data /config .env && \ - chmod 777 public/media var +# Must have an env file: https://github.com/symfony/symfony/issues/34660 +# Must have all env vars in it https://github.com/symfony/flex/issues/346 +COPY --link .env.example_docker .env # Configure PHP for production COPY docker/php/conf.d/app.ini /usr/local/etc/php/conf.d/ COPY docker/php/conf.d/app.prod.ini /usr/local/etc/php/conf.d/ +COPY docker/php/php-fpm.d/zz-docker.conf /usr/local/etc/php-fpm.d/zz-docker.conf -# Switch user -USER $USER:$GROUP +# Embed entrypoint declared in base image +COPY ./docker/php/entrypoint.sh /entrypoint.sh +RUN chmod +x /entrypoint.sh diff --git a/docker/php/entrypoint.sh b/docker/php/entrypoint.sh index 94efc6e3b..0be09a589 100755 --- a/docker/php/entrypoint.sh +++ b/docker/php/entrypoint.sh @@ -12,12 +12,22 @@ if [ "$1" == "php-fpm" ] || [ "$1" == "php" ] || [ "$1" == "bin/console" ]; then echo "Starting as service..." if [ "$APP_ENV" == "dev" ] ; then + echo "Installing PHP dependencies" # In development: dump the development PHP config files, # validate the installed packages (including dev dependencies) and dump dev config composer install --prefer-dist --no-scripts --no-progress composer dump-env dev fi + if [ -d "$MBIN_PUBLIC_COPY_DIR" ] ; then + echo "Copying mbin to shared volume" + # Used in production to make the public folder available to the reverse proxy + # -f Overwrite + # -r - recursive + # -u - update older folder + cp -fru "$MBIN_HOME/public" -t "$MBIN_PUBLIC_COPY_DIR" + fi + echo "Waiting for db to be ready..." ATTEMPTS_LEFT_TO_REACH_DATABASE=60 until [ $ATTEMPTS_LEFT_TO_REACH_DATABASE -eq 0 ] || DATABASE_ERROR=$(bin/console dbal:run-sql "SELECT 1" 2>&1); do @@ -55,6 +65,25 @@ if [ "$1" == "php-fpm" ] || [ "$1" == "php" ] || [ "$1" == "bin/console" ]; then echo "ENABLE_ACL is not set!" fi fi + + if [ "$APP_ENV" == "prod" ] ; then + echo "Creating directories and setting ownership" + # Create necessary directories for php-fpm process run by mbin user + mkdir -p public/media var/log /data /config + chown -R $MBIN_USER:$MBIN_GROUP public/media var /data /config .env + chmod 777 public/media var + fi fi -exec "$@" +USER=$(whoami) +if [ "$USER" == "$MBIN_USER" ] ; then + # Probably dev + exec "$@" +else + # Most likely prod + # Run command as non-root user + # Workaround: Allow php-fpm to write to stderr + chown "$MBIN_USER" /proc/self/fd/2 + + exec su "$MBIN_USER" -c "$*" +fi diff --git a/notes.md b/notes.md index 8f780fe53..c13ad16fd 100644 --- a/notes.md +++ b/notes.md @@ -2,10 +2,11 @@ - [ ] mention how to activate debug logs for symfony (`bin/console -vvv some-command`) # Docker dev +- [ ] Use whitelist approach in .dockerignore instead of blacklist - [x] Make a separate `compose.dev.yml` - [ ] Doc: `ln -s compose.dev.yml compose.override.yml` - - [ ] rewrite `Dockerfile` to have targets for dev - - [ ] Use `Dockerfile` targets in `compose.yml` for dev and prod + - [x] rewrite `Dockerfile` to have targets for dev + - [x] Use `Dockerfile` targets in `compose.yml` for dev and prod - [ ] `host.docker.internal` doesn't exist on linux because it's not running docker in a VM - [x] conditionally add following for linux\ ```yaml @@ -14,14 +15,20 @@ ``` - [x] Make a separate `.env.dev` for development - [x] use `.env.dev` in `compose.dev.yml` -- [ ] Update `compose.prod.yml` - - [ ] Use separate `entrypoint.sh` for prod image? +- [x] Update `compose.prod.yml` + - [x] Use separate `entrypoint.sh` for prod image? **NOPE** - [ ] mention that resetting means `sudo rm -rf docker/storage` - [ ] move `.gitignore` out of storage otherwise resetting still creates a commit by deleting `.gitignore` -- [ ] `docker/storage` needs to be writable to `mbin` user! - - [ ] `sudo chmod 777 docker/storage/*` +- [x] `docker/storage` needs to be writable to `mbin` user! + - [x] `sudo chmod 777 docker/storage/*` - [ ] `MESSENGER_TRANSPORT_DSN=doctrine://default` doesn't work - [ ] Doc: inform how to enable xdebug (enabling it by default slows everything down) - [ ] nixos needs [iptables rules](https://discourse.nixos.org/t/docker-container-not-resolving-to-host/30259/8) for xdebug to access host + +# Prod + +Production now allows running a separate reverse proxy than caddy. +This is achieved by copying the necessary files into a common volume at startup. +`php` copies to `commonVolume` which can be accessed by `reverseProxy` From 5b12728bea303888e0a18f0af9de0eb147fa8c4c Mon Sep 17 00:00:00 2001 From: LoveIsGrief Date: Fri, 16 Aug 2024 18:35:07 +0200 Subject: [PATCH 19/43] Fix docker dev after making it work for prod --- .env.dev_docker | 4 ++-- compose.dev.yml | 8 ++++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/.env.dev_docker b/.env.dev_docker index 29106a2af..5799ffe58 100644 --- a/.env.dev_docker +++ b/.env.dev_docker @@ -14,8 +14,8 @@ # https://symfony.com/doc/current/best_practices.html#use-environment-variables-for-infrastructure-configuration # Mbin variables -SERVER_NAME=127.0.0.1:8000 -KBIN_DOMAIN=127.0.0.1:8000 +SERVER_NAME=:80 +KBIN_DOMAIN=127.0.0.1:8008 KBIN_TITLE=Mbin KBIN_DEFAULT_LANG=en KBIN_FEDERATION_ENABLED=true diff --git a/compose.dev.yml b/compose.dev.yml index 16ced3824..c719d5fd2 100644 --- a/compose.dev.yml +++ b/compose.dev.yml @@ -4,11 +4,14 @@ services: - caddy_config:/config - caddy_data:/data - ./:/var/www/mbin + - ./docker/caddy/Caddyfile:/etc/caddy/Caddyfile depends_on: node: condition: service_completed_successfully env_file: - .env.dev_docker + ports: + - 8008:80 # Builds the nodejs application node: @@ -34,10 +37,10 @@ services: extra_hosts: - "host.docker.internal:host-gateway" volumes: - - ./docker/php/entrypoint.sh:/entrypoint.sh - ./:/var/www/mbin/ - - ./docker/php/conf.d/app.ini:/usr/local/etc/php/conf.d/app.ini - ./docker/php/conf.d/app.dev.ini:/usr/local/etc/php/conf.d/app.dev.ini + - ./docker/php/conf.d/app.ini:/usr/local/etc/php/conf.d/app.ini + - ./docker/php/entrypoint.sh:/entrypoint.sh - ./docker/php/php-fpm.d/zz-docker.conf:/usr/local/etc/php-fpm.d/zz-docker.conf env_file: - .env.dev_docker @@ -50,6 +53,7 @@ services: command: bin/console -vvvv messenger:consume --all --time-limit=3600 volumes: - ./:/var/www/mbin/ + - ./docker/php/entrypoint.sh:/entrypoint.sh env_file: - .env.dev_docker From 968381f9005304360c1bd3167571c07ad3e58592 Mon Sep 17 00:00:00 2001 From: LoveIsGrief Date: Fri, 16 Aug 2024 20:09:37 +0200 Subject: [PATCH 20/43] Add documentation docker env --- CONTRIBUTING.md | 2 + .../development_environment.md | 237 ++++++++++++++++++ docs/04-contributing/development_server.md | 82 ------ notes.md | 10 +- 4 files changed, 244 insertions(+), 87 deletions(-) create mode 100644 docs/04-contributing/development_environment.md delete mode 100644 docs/04-contributing/development_server.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e4f770b2b..1e8ce6c0c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -28,6 +28,8 @@ With an account on [GitHub](https://github.com) you will be able to [fork this r > [!Note] > If you are a Maintainer with GitHub org admin rights, you do NOT need to fork the project, instead you are allowed to use git branches. See also [C4](C4.md). +To get started with development, follow the [guide](docs/04-contributing/development_environment). + ### Coding Style Guide We use [php-cs-fixer](https://cs.symfony.com/) to automatically fix code style issues according to [Symfony coding standard](https://symfony.com/doc/current/contributing/code/standards.html). diff --git a/docs/04-contributing/development_environment.md b/docs/04-contributing/development_environment.md new file mode 100644 index 000000000..1c00578f3 --- /dev/null +++ b/docs/04-contributing/development_environment.md @@ -0,0 +1,237 @@ +There are multiple ways to get started. + +If you can run `docker compose` follow the steps for [Docker](#docker), otherwise you can set up your +own [development server](#development-server) + +# Docker + +Using docker can get you up and running quickly as it spins up containers of all the required components. + +**Supported operating systems** + + - Linux + +**Requirements** + + - [Docker](https://docs.docker.com/get-docker/) + +## Quickstart + +The development environment is run with [docker compose]. + +```shell +# Create a symbolic link to the dev setup that merges and overrides part of compose.yml +ln -nfs compose.dev.yml compose.override.yml +# Bring up all the services and detach from the console +docker compose up -d +``` + +Once everything has started, you can navigate to http://localhost:8008. Here's a [docker compose cheatsheet] + +### Alternatives for getting started + +Should you want to edit the `compose.override.yml` file without making changes to `compose.dev.yml`, read further. +Your changes will be ignored by git. + +**Use the `include` directive** + +For docker versions `>= 2.20.0`, you can [include][docker compose include] other YAML files into a compose file. +This has the benefit of keeping up to date with changes to the `compose.dev.yml` without modifying it. + +```yaml +include: + - compose.dev.yml +``` + +**Copy the compose.dev.yml** + +Instead of linking, copy the `compose.dev.yml` to `compose.override.yml` and make your changes. + +The downside of this approach is that changes and fixes to `compose.dev.yml` from version control will not reflect +in your compose setup automatically. + +## How it works + +MBin depends on multiple services (postgreSQL, redis, a reverse proxy, PHP, ...). Instead of having one monolithic +docker image that includes these services and all their configs, the services each run in their own containers. + +There's minimal overlap between most services and their `Dockerfile`s. Here are a few things to know. + +### Split compose file + +Instead of having one large compose file to cater to every need, there are multiple - the main one being `compose.yml`. +This takes advantage of [merging][docker compose merging]. Merging is done automatically when a `compose.override.yml` +is present, otherwise the list of files has to be passed with `docker compose -f compose.yml -f file1.yml -f file2.yml`. + +### php/Dockerfile + +[This][php-dockerfile] is the biggest and most complex `Dockerfile` in the repo. +It's a multi-stage file that attempts to make the final image minimal. You can read it, but the most important things +to keep in mind are + +- the `base` target only has the minimal common items for the next steps +- for production there are builder targets to build the frontend and backend of mbin +- for development, the `dev` target is minimal as the frontend and backend are built when the services are first run +- the final target is `prod` which unites the outputs of the builder targets + +### Building the frontend and backend + +As mentioned [above](#phpdockerfile), the front and backend are built in the `Dockerfile` for production. + +In development, this is taken care of at runtime in [compose.dev.yml]. The `messenger` service is the first PHP +service to run, so it installs PHP dependencies and such. The `node` service is only in the dev compose not production +and requires files from the PHP dependencies to successfully build the frontend. It is thus run afterward. + + +## Frequently executed tasks + +### Modifying behavior with a quick turn-around + +The entire repository is mounted into the `caddy`, `php`, and `messenger` services. + +**PHP code** + +Most PHP code changes are picked up immediately after refreshing the page in the browser. + +**Other important files** + +| file | services | how to refresh | +|---------------|----------------|------------------------------------------------------------------------| +| Caddyfile | caddy | `docker compose exec caddy caddy reload --config /etc/caddy/Caddyfile` | +| entrypoint.sh | php, messenger | `docker compose up -d --no-deps $service` | + + +### Debugging PHP (XDebug) + +PHP supports remote debugging using [XDebug](https://xdebug.org/). This works by hosting an Xdebug server the php +process can call. + +```mermaid +sequenceDiagram + participant Browser + participant rp as Reverse Proxy + participant php as PHP-FPM + participant xdebug as XDebug Host + + Browser->>+rp: http://localhost + rp->>+php: index.php + php->>+xdebug: :9003 + xdebug->>php: break + php-->>xdebug: + xdebug->>php: continue + php-->>xdebug: + xdebug-->>-php: done + php-->>-rp: response + rp-->>-Browser: response +``` + +**Requirement** + +An XDebug server. The XDebug server is often hosted on port 9000 or 9003. Some IDEs (like [PHPStorm]) have it builtin. + +**Enabling** + +Open [app.dev.ini] and uncomment `;xdebug.start_with_request=yes` by removing the `;`, +then restart the `php` service using `docker compose restart php`. + +Once you navigate to a page, your IDE/editor should get called from `php-fpm` within docker. + +#### Special cases + +Firewalls can sometimes get in the way of communication between the docker container and the docker host. + +**NixOS** + +Nixos needs [iptables rules](https://discourse.nixos.org/t/docker-container-not-resolving-to-host/30259/8). + +# Development Server + +Requirements: + +- PHP v8.2 +- NodeJS +- Redis +- PostgreSQL +- _Optionally:_ Mercure + +--- + +- Increase execution time in PHP config file: `/etc/php/8.2/fpm/php.ini`: + +```ini +max_execution_time = 120 +``` + +- Restart the PHP-FPM service: `sudo systemctl restart php8.2-fpm.service` +- Connect to PostgreSQL using the postgres user: + +```bash +sudo -u postgres psql +``` + +- Create new mbin database user: + +```sql +sudo -u postgres createuser --createdb --createrole --pwprompt mbin +``` + +- Correctly configured `.env` file (`cp .env.example .env`), these are only the changes you need to pay attention to: + +```env +# Set domain to 127.0.0.1:8000 +SERVER_NAME=127.0.0.1:8000 +KBIN_DOMAIN=127.0.0.1:8000 +KBIN_STORAGE_URL=http://127.0.0.1:8000/media + +#Redis (without password) +REDIS_DNS=redis://127.0.0.1:6379 + +# Set App configs +APP_ENV=dev +APP_SECRET=427f5e2940e5b2472c1b44b2d06e0525 + +# Configure PostgreSQL +POSTGRES_DB=mbin +POSTGRES_USER=mbin +POSTGRES_PASSWORD= + +# Set messenger to Doctrine (= PostgresQL DB) +MESSENGER_TRANSPORT_DSN=doctrine://default +``` + +- If you are using `127.0.0.1` to connect to the PostgreSQL server, edit the following file: `/etc/postgresql//main/pg_hba.conf` and add: + +```conf +local mbin mbin md5 +``` + +- Restart the PostgreSQL server: `sudo systemctl restart postgresql` +- Create database: `php bin/console doctrine:database:create` +- Create tables and database structure: `php bin/console doctrine:migrations:migrate` +- Build frontend assets: `npm install && npm run dev` + +Starting the server: + +1. Install Symfony CLI: `wget https://get.symfony.com/cli/installer -O - | bash` +2. Check the requirements: `symfony check:requirements` +3. Install dependencies: `composer install` +4. Dump `.env` into `.env.local.php` via: `composer dump-env dev` +5. _Optionally:_ Increase verbosity log level in: `config/packages/monolog.yaml` in the `when@dev` section: `level: debug` (instead of `level: info`), +6. Clear cache: `APP_ENV=dev APP_DEBUG=1 php bin/console cache:clear -n` +7. Start Mbin: `symfony server:start` +8. Go to: [http://127.0.0.1:8000](http://127.0.0.1:8000/) + +This will give you a minimal working frontend with PostgreSQL setup. Keep in mind: this will _not_ start federating, for that you also need to setup Mercure to test the full Mbin setup. + +_Optionally:_ you could also setup RabbitMQ, but the Doctrine messenger configuration will be sufficient for local development. + +More info: [Contributing guide](./README.md), [Admin guide](../02-admin/README.md) and [Symfony Local Web Server](https://symfony.com/doc/current/setup/symfony_server.html) + +[app.dev.ini]: ../../docker/php/conf.d/app.dev.ini +[compose.dev.yml]: ../../compose.dev.yml +[docker compose]: https://docs.docker.com/compose/reference/ +[docker compose cheatsheet]: https://devhints.io/docker-compose +[docker compose include]: https://docs.docker.com/compose/compose-file/14-include/ +[docker compose merging]: https://docs.docker.com/compose/compose-file/13-merge/ +[php-dockerfile]: ../../docker/php/Dockerfile +[PHPStorm]: https://www.jetbrains.com/phpstorm/ diff --git a/docs/04-contributing/development_server.md b/docs/04-contributing/development_server.md deleted file mode 100644 index 71debea75..000000000 --- a/docs/04-contributing/development_server.md +++ /dev/null @@ -1,82 +0,0 @@ -# Development Server - -Requirements: - -- PHP v8.2 -- NodeJS -- Redis -- PostgreSQL -- _Optionally:_ Mercure - ---- - -- Increase execution time in PHP config file: `/etc/php/8.2/fpm/php.ini`: - -```ini -max_execution_time = 120 -``` - -- Restart the PHP-FPM service: `sudo systemctl restart php8.2-fpm.service` -- Connect to PostgreSQL using the postgres user: - -```bash -sudo -u postgres psql -``` - -- Create new mbin database user: - -```sql -sudo -u postgres createuser --createdb --createrole --pwprompt mbin -``` - -- Correctly configured `.env` file (`cp .env.example .env`), these are only the changes you need to pay attention to: - -```env -# Set domain to 127.0.0.1:8000 -SERVER_NAME=127.0.0.1:8000 -KBIN_DOMAIN=127.0.0.1:8000 -KBIN_STORAGE_URL=http://127.0.0.1:8000/media - -#Redis (without password) -REDIS_DNS=redis://127.0.0.1:6379 - -# Set App configs -APP_ENV=dev -APP_SECRET=427f5e2940e5b2472c1b44b2d06e0525 - -# Configure PostgreSQL -POSTGRES_DB=mbin -POSTGRES_USER=mbin -POSTGRES_PASSWORD= - -# Set messenger to Doctrine (= PostgresQL DB) -MESSENGER_TRANSPORT_DSN=doctrine://default -``` - -- If you are using `127.0.0.1` to connect to the PostgreSQL server, edit the following file: `/etc/postgresql//main/pg_hba.conf` and add: - -```conf -local mbin mbin md5 -``` - -- Restart the PostgreSQL server: `sudo systemctl restart postgresql` -- Create database: `php bin/console doctrine:database:create` -- Create tables and database structure: `php bin/console doctrine:migrations:migrate` -- Build frontend assets: `npm install && npm run dev` - -Starting the server: - -1. Install Symfony CLI: `wget https://get.symfony.com/cli/installer -O - | bash` -2. Check the requirements: `symfony check:requirements` -3. Install dependencies: `composer install` -4. Dump `.env` into `.env.local.php` via: `composer dump-env dev` -5. _Optionally:_ Increase verbosity log level in: `config/packages/monolog.yaml` in the `when@dev` section: `level: debug` (instead of `level: info`), -6. Clear cache: `APP_ENV=dev APP_DEBUG=1 php bin/console cache:clear -n` -7. Start Mbin: `symfony server:start` -8. Go to: [http://127.0.0.1:8000](http://127.0.0.1:8000/) - -This will give you a minimal working frontend with PostgreSQL setup. Keep in mind: this will _not_ start federating, for that you also need to setup Mercure to test the full Mbin setup. - -_Optionally:_ you could also setup RabbitMQ, but the Doctrine messenger configuration will be sufficient for local development. - -More info: [Contributing guide](./README.md), [Admin guide](../02-admin/README.md) and [Symfony Local Web Server](https://symfony.com/doc/current/setup/symfony_server.html) diff --git a/notes.md b/notes.md index c13ad16fd..71210b876 100644 --- a/notes.md +++ b/notes.md @@ -1,13 +1,13 @@ -- [ ] add link to [dev server docs](docs/04-contributing/development_server.md) in `CONTRIBUTING.md` +- [x] add link to [dev server docs](docs/04-contributing/development_environment) in `CONTRIBUTING.md` - [ ] mention how to activate debug logs for symfony (`bin/console -vvv some-command`) # Docker dev - [ ] Use whitelist approach in .dockerignore instead of blacklist - [x] Make a separate `compose.dev.yml` - - [ ] Doc: `ln -s compose.dev.yml compose.override.yml` + - [x] Doc: `ln -s compose.dev.yml compose.override.yml` - [x] rewrite `Dockerfile` to have targets for dev - [x] Use `Dockerfile` targets in `compose.yml` for dev and prod - - [ ] `host.docker.internal` doesn't exist on linux because it's not running docker in a VM + - [x] `host.docker.internal` doesn't exist on linux because it's not running docker in a VM - [x] conditionally add following for linux\ ```yaml extra_hosts: @@ -23,9 +23,9 @@ - [x] `sudo chmod 777 docker/storage/*` - [ ] `MESSENGER_TRANSPORT_DSN=doctrine://default` doesn't work -- [ ] Doc: inform how to enable xdebug (enabling it by default slows everything down) +- [x] Doc: inform how to enable xdebug (enabling it by default slows everything down) -- [ ] nixos needs [iptables rules](https://discourse.nixos.org/t/docker-container-not-resolving-to-host/30259/8) for xdebug to access host +- [x] nixos needs [iptables rules](https://discourse.nixos.org/t/docker-container-not-resolving-to-host/30259/8) for xdebug to access host # Prod From 96dc91dd1addd6e8db7dc76ef595f6fb0f2d7d4c Mon Sep 17 00:00:00 2001 From: LoveIsGrief Date: Fri, 16 Aug 2024 22:39:41 +0200 Subject: [PATCH 21/43] Comments + remove unnecessary mounts from prod compose --- compose.prod.yml | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/compose.prod.yml b/compose.prod.yml index 801e25040..ad6f6960b 100644 --- a/compose.prod.yml +++ b/compose.prod.yml @@ -8,8 +8,7 @@ services: volumes: - ./docker/storage/caddy_config:/config - ./docker/storage/caddy_data:/data - - ./docker/caddy/Caddyfile:/etc/caddy/Caddyfile - - ./docker/storage/mbin/public:${MBIN_HOME}/public:ro + - ./docker/storage/mbin/public:${MBIN_HOME}/public:ro # Shared volume for php container to write into environment: - SERVER_NAME=:80 # the address that the web server binds - PHP_FASTCGI_HOST=php:9000 # caddy forward traffic to this host via fastcgi @@ -28,8 +27,7 @@ services: #image: "ghcr.io/mbinorg/mbin:latest" volumes: - ./docker/storage/logs:/var/www/mbin/var/log - - ./docker/storage/mbin/:/opt/mbin/ - - ./docker/php/entrypoint.sh:/entrypoint.sh + - ./docker/storage/mbin/:/opt/mbin/ # Shared folder with `caddy` to copy static resources into # If you want to change configs locally, without rebuilding the image use: # - ../docker/config:/var/www/mbin/config environment: @@ -53,14 +51,13 @@ services: command: bin/console -vvvv messenger:consume --all --time-limit=3600 deploy: mode: replicated - replicas: 1 + replicas: 1 # Increase this for better parallelism healthcheck: test: ["CMD-SHELL", "ps aux | grep 'messenger[:]consume' || exit 1"] networks: - mbin_external_network volumes: - ./docker/storage/logs:/var/www/mbin/var/log - - ./docker/php/entrypoint.sh:/entrypoint.sh # If you want to change configs locally, without rebuilding the image use: # - ../docker/config:/var/www/mbin/config env_file: @@ -88,8 +85,6 @@ services: image: postgres:${POSTGRES_VERSION:-16}-alpine container_name: mbin-db restart: unless-stopped - ports: - - "127.0.0.1:5432:5432" networks: - mbin_external_network volumes: From b59f67c9a2f15a9d7b170f246bac870cc2cbe048 Mon Sep 17 00:00:00 2001 From: LoveIsGrief Date: Fri, 16 Aug 2024 23:17:35 +0200 Subject: [PATCH 22/43] fix: call pg_isready with user variable Otherwise the logs get filled with 'FATAL: role "root" does not exist' everytime the health is checked --- compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose.yml b/compose.yml index 333abf3a0..c995d548a 100644 --- a/compose.yml +++ b/compose.yml @@ -83,7 +83,7 @@ services: env_file: - .env healthcheck: - test: [ "CMD-SHELL", "pg_isready" ] + test: [ "CMD-SHELL", "pg_isready -U $POSTGRES_USER" ] interval: 10s timeout: 5s retries: 5 From 2cd5b58a87402ea28189eaa1bedbbbff76c4b842 Mon Sep 17 00:00:00 2001 From: LoveIsGrief Date: Fri, 16 Aug 2024 23:20:23 +0200 Subject: [PATCH 23/43] rm notes.md --- notes.md | 34 ---------------------------------- 1 file changed, 34 deletions(-) delete mode 100644 notes.md diff --git a/notes.md b/notes.md deleted file mode 100644 index 71210b876..000000000 --- a/notes.md +++ /dev/null @@ -1,34 +0,0 @@ -- [x] add link to [dev server docs](docs/04-contributing/development_environment) in `CONTRIBUTING.md` -- [ ] mention how to activate debug logs for symfony (`bin/console -vvv some-command`) -# Docker dev - -- [ ] Use whitelist approach in .dockerignore instead of blacklist -- [x] Make a separate `compose.dev.yml` - - [x] Doc: `ln -s compose.dev.yml compose.override.yml` - - [x] rewrite `Dockerfile` to have targets for dev - - [x] Use `Dockerfile` targets in `compose.yml` for dev and prod - - [x] `host.docker.internal` doesn't exist on linux because it's not running docker in a VM - - [x] conditionally add following for linux\ - ```yaml - extra_hosts: - - "host.docker.internal:host-gateway" - ``` - - [x] Make a separate `.env.dev` for development - - [x] use `.env.dev` in `compose.dev.yml` -- [x] Update `compose.prod.yml` - - [x] Use separate `entrypoint.sh` for prod image? **NOPE** -- [ ] mention that resetting means `sudo rm -rf docker/storage` - - [ ] move `.gitignore` out of storage otherwise resetting still creates a commit by deleting `.gitignore` -- [x] `docker/storage` needs to be writable to `mbin` user! - - [x] `sudo chmod 777 docker/storage/*` -- [ ] `MESSENGER_TRANSPORT_DSN=doctrine://default` doesn't work - -- [x] Doc: inform how to enable xdebug (enabling it by default slows everything down) - -- [x] nixos needs [iptables rules](https://discourse.nixos.org/t/docker-container-not-resolving-to-host/30259/8) for xdebug to access host - -# Prod - -Production now allows running a separate reverse proxy than caddy. -This is achieved by copying the necessary files into a common volume at startup. -`php` copies to `commonVolume` which can be accessed by `reverseProxy` From 86d6ae7965c982cbcff1bfd876172f8347bf9fc1 Mon Sep 17 00:00:00 2001 From: LoveIsGrief Date: Sun, 18 Aug 2024 14:37:02 +0200 Subject: [PATCH 24/43] fix(docker): support uploading media After supporting arbitrary reverse proxies, media uploads were broken. They are uploaded to public/media/ by `php` service, but the previous solution just provided a copy of public/media/ at first run. New uploads weren't shared with the `www` service. Now, instead of just providing a copy, a mount is provided. To do so the strategy was changed and a lead taken out of nextcloud's book: https://github.com/nextcloud/docker/blob/65138b6d22bec1ac15e2f0f125426290640bb97a/docker-entrypoint.sh **The problem** The image had the mbin sources in /var/www/mbin. Mounting docker/storage/public into /var/www/mbin/public would replace the image's public/ with a (hitherto) empty directory. **The solution** The image has its sources in /usr/src/mbin and on startup, syncs that into the new mount /var/www/mbin using `rsync`. In this manner, previous uploads are preserved and source changes update /var/www/mbin. --- .dockerignore | 2 + .env.example_docker | 1 + compose.prod.yml | 8 +--- docker/.gitignore | 2 +- docker/php/Dockerfile | 86 +++++++++++++++++++++------------------ docker/php/entrypoint.sh | 17 ++++---- docker/storage/.gitignore | 2 - 7 files changed, 63 insertions(+), 55 deletions(-) delete mode 100644 docker/storage/.gitignore diff --git a/.dockerignore b/.dockerignore index 03d991aab..60dc31a9b 100644 --- a/.dockerignore +++ b/.dockerignore @@ -21,7 +21,9 @@ compose.yml node_modules/ docker/storage/ docs/ +public/build/ public/bundles/ +public/media/ tests/ tools/ var/ diff --git a/.env.example_docker b/.env.example_docker index eba234f6e..beac392e5 100644 --- a/.env.example_docker +++ b/.env.example_docker @@ -32,6 +32,7 @@ KBIN_HEADER_LOGO=false KBIN_FEDERATION_PAGE_ENABLED=true MBIN_DEFAULT_THEME=default MBIN_HOME=/var/www/mbin +MBIN_SRC=/usr/src/mbin MBIN_USER=mbin MBIN_GROUP=www-data diff --git a/compose.prod.yml b/compose.prod.yml index ad6f6960b..bd3251d64 100644 --- a/compose.prod.yml +++ b/compose.prod.yml @@ -26,13 +26,9 @@ services: # use the pre-build image from ghcr.io (you can also a tag eg.: v1.0.0 instead of: latest): #image: "ghcr.io/mbinorg/mbin:latest" volumes: - - ./docker/storage/logs:/var/www/mbin/var/log - - ./docker/storage/mbin/:/opt/mbin/ # Shared folder with `caddy` to copy static resources into + - ./docker/storage/mbin:$MBIN_HOME # If you want to change configs locally, without rebuilding the image use: # - ../docker/config:/var/www/mbin/config - environment: - # Where the public/ folder will be copied to for the reverse proxy to use - MBIN_PUBLIC_COPY_DIR: /opt/mbin/ env_file: - .env depends_on: @@ -57,7 +53,7 @@ services: networks: - mbin_external_network volumes: - - ./docker/storage/logs:/var/www/mbin/var/log + - ./docker/storage/mbin:$MBIN_HOME # If you want to change configs locally, without rebuilding the image use: # - ../docker/config:/var/www/mbin/config env_file: diff --git a/docker/.gitignore b/docker/.gitignore index 7376571d1..34ff768e3 100644 --- a/docker/.gitignore +++ b/docker/.gitignore @@ -1 +1 @@ -docker-compose.override.yml +storage/ diff --git a/docker/php/Dockerfile b/docker/php/Dockerfile index 91a63ecef..9371779a8 100644 --- a/docker/php/Dockerfile +++ b/docker/php/Dockerfile @@ -48,26 +48,27 @@ USER $MBIN_USER:$MBIN_GROUP ###################################################### FROM base as builder-composer +ARG MBIN_SRC=/usr/src/mbin COPY --from=composer:latest /usr/bin/composer /usr/bin/composer -WORKDIR $MBIN_HOME +WORKDIR $MBIN_SRC # Composer: install package -COPY composer.* $MBIN_HOME -COPY symfony.lock $MBIN_HOME +COPY composer.* ./ +COPY symfony.lock ./ ENV COMPOSER_ALLOW_SUPERUSER=1 RUN composer install --prefer-dist --no-dev --no-autoloader --no-scripts --no-progress # Copy only required resources # TODO Use one line once COPY --parents is supported -COPY --link assets $MBIN_HOME/assets -COPY --link bin $MBIN_HOME/bin -COPY --link config $MBIN_HOME/config -COPY --link migrations $MBIN_HOME/migrations -COPY --link public $MBIN_HOME/public -COPY --link src $MBIN_HOME/src -COPY --link templates $MBIN_HOME/templates -COPY --link translations $MBIN_HOME/translations +COPY --link assets ./assets +COPY --link bin ./bin +COPY --link config ./config +COPY --link migrations ./migrations +COPY --link public ./public +COPY --link src ./src +COPY --link templates ./templates +COPY --link translations ./translations # Increase memory limit for following steps RUN echo "memory_limit = 512M" > /usr/local/etc/php/conf.d/memory_limit.ini @@ -83,69 +84,76 @@ RUN composer run-script --no-dev post-install-cmd && \ ###################################################### FROM node:22.5.1-alpine3.20 as builder-nodejs -ARG MBIN_HOME=/var/www/mbin -ENV MBIN_HOME=${MBIN_HOME} +ARG MBIN_SRC=/usr/src/mbin # Set NodeJS as production by default ARG NODE_ENV=production ENV NODE_ENV=${NODE_ENV} # Setup environment -RUN mkdir -p $MBIN_HOME -WORKDIR $MBIN_HOME +WORKDIR $MBIN_SRC # NPM: install package dependencies -COPY package.json $MBIN_HOME -COPY package-lock.json $MBIN_HOME -COPY --from=builder-composer --link $MBIN_HOME/vendor $MBIN_HOME/vendor +COPY package.json . +COPY package-lock.json . +COPY --from=builder-composer --link ${MBIN_SRC}/vendor ./vendor RUN npm ci --include=dev # NPM: build (production by default) # TODO Use one line once COPY --parents is supported -COPY --link assets/ $MBIN_HOME/assets/ -COPY --link config/ $MBIN_HOME/config/ -COPY --link public/ $MBIN_HOME/public/ -COPY --link src/ $MBIN_HOME/src/ -COPY --link webpack* $MBIN_HOME/ +COPY --link assets/ ./assets/ +COPY --link config/ ./config/ +COPY --link public/ ./public/ +COPY --link src/ ./src/ +COPY --link webpack* ./ RUN npm run build ################################################### FROM base as prod +ARG MBIN_SRC=/usr/src/mbin +ENV MBIN_SRC=${MBIN_SRC} + +# rsync for syncing the public/ folder +RUN apk update && apk add rsync + +WORKDIR $MBIN_SRC # Copy only necessary resources # TODO Use one line once COPY --parents is supported ## Folders -COPY --chown=$MBIN_USER:$MBIN_GROUP --link assets/ $MBIN_HOME/assets -COPY --chown=$MBIN_USER:$MBIN_GROUP --link bin/ $MBIN_HOME/bin -COPY --chown=$MBIN_USER:$MBIN_GROUP --link config/ $MBIN_HOME/config -COPY --chown=$MBIN_USER:$MBIN_GROUP --link migrations/ $MBIN_HOME/migrations -COPY --chown=$MBIN_USER:$MBIN_GROUP --link public/ $MBIN_HOME/public -COPY --chown=$MBIN_USER:$MBIN_GROUP --link src/ $MBIN_HOME/src -COPY --chown=$MBIN_USER:$MBIN_GROUP --link templates/ $MBIN_HOME/templates -COPY --chown=$MBIN_USER:$MBIN_GROUP --link translations/ $MBIN_HOME/translations +COPY --chown=$MBIN_USER:$MBIN_GROUP --link assets/ ./assets +COPY --chown=$MBIN_USER:$MBIN_GROUP --link bin/ ./bin +COPY --chown=$MBIN_USER:$MBIN_GROUP --link config/ ./config +COPY --chown=$MBIN_USER:$MBIN_GROUP --link migrations/ ./migrations +COPY --chown=$MBIN_USER:$MBIN_GROUP --link public/ ./public +COPY --chown=$MBIN_USER:$MBIN_GROUP --link src/ ./src +COPY --chown=$MBIN_USER:$MBIN_GROUP --link templates/ ./templates +COPY --chown=$MBIN_USER:$MBIN_GROUP --link translations/ ./translations ## Files -COPY --chown=$MBIN_USER:$MBIN_GROUP --link composer* $MBIN_HOME/ -COPY --chown=$MBIN_USER:$MBIN_GROUP --link symfony* $MBIN_HOME/ +COPY --chown=$MBIN_USER:$MBIN_GROUP --link composer* $MBIN_SRC/ +COPY --chown=$MBIN_USER:$MBIN_GROUP --link symfony* $MBIN_SRC/ ## Build output -COPY --from=builder-composer --chown=$MBIN_USER:$MBIN_GROUP $MBIN_HOME/vendor $MBIN_HOME/vendor -COPY --from=builder-composer --chown=$MBIN_USER:$MBIN_GROUP $MBIN_HOME/public/bundles $MBIN_HOME/public/bundles -COPY --from=builder-nodejs --chown=$MBIN_USER:$MBIN_GROUP $MBIN_HOME/public $MBIN_HOME/public - -WORKDIR $MBIN_HOME +COPY --from=builder-composer --chown=$MBIN_USER:$MBIN_GROUP $MBIN_SRC/vendor $MBIN_SRC/vendor +COPY --from=builder-composer --chown=$MBIN_USER:$MBIN_GROUP $MBIN_SRC/public/bundles $MBIN_SRC/public/bundles +COPY --from=builder-nodejs --chown=$MBIN_USER:$MBIN_GROUP $MBIN_SRC/public $MBIN_SRC/public # Must have an env file: https://github.com/symfony/symfony/issues/34660 # Must have all env vars in it https://github.com/symfony/flex/issues/346 -COPY --link .env.example_docker .env +COPY --link .env.example_docker $MBIN_SRC/.env # Configure PHP for production COPY docker/php/conf.d/app.ini /usr/local/etc/php/conf.d/ COPY docker/php/conf.d/app.prod.ini /usr/local/etc/php/conf.d/ COPY docker/php/php-fpm.d/zz-docker.conf /usr/local/etc/php-fpm.d/zz-docker.conf +# Empty on first start +# sources from $MBIN_SRC have to be copied into it +WORKDIR $MBIN_HOME + # Embed entrypoint declared in base image COPY ./docker/php/entrypoint.sh /entrypoint.sh RUN chmod +x /entrypoint.sh diff --git a/docker/php/entrypoint.sh b/docker/php/entrypoint.sh index 0be09a589..1da5176c1 100755 --- a/docker/php/entrypoint.sh +++ b/docker/php/entrypoint.sh @@ -19,13 +19,16 @@ if [ "$1" == "php-fpm" ] || [ "$1" == "php" ] || [ "$1" == "bin/console" ]; then composer dump-env dev fi - if [ -d "$MBIN_PUBLIC_COPY_DIR" ] ; then - echo "Copying mbin to shared volume" - # Used in production to make the public folder available to the reverse proxy - # -f Overwrite - # -r - recursive - # -u - update older folder - cp -fru "$MBIN_HOME/public" -t "$MBIN_PUBLIC_COPY_DIR" + if [ "$APP_ENV" == "prod" ] ; then + # Parts of mbin are served directly by the webserver without calling php-fpm (public/ folder) + # User uploads and other dynamically created content are inserted into public/ by this container + # The webserver needs access to those newly uploaded files + echo "Syncing mbin src" + rsync \ + --links \ + --recursive \ + --chown $MBIN_USER:$MBIN_GROUP \ + $MBIN_SRC/ $MBIN_HOME fi echo "Waiting for db to be ready..." diff --git a/docker/storage/.gitignore b/docker/storage/.gitignore deleted file mode 100644 index d6b7ef32c..000000000 --- a/docker/storage/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -* -!.gitignore From e4d27ce0df2fa758c0494fe9d4ff5cbed372ea0d Mon Sep 17 00:00:00 2001 From: LoveIsGrief Date: Sun, 18 Aug 2024 14:42:43 +0200 Subject: [PATCH 25/43] chore: remove unused file --- docker/compose.prod.yml | 35 ----------------------------------- 1 file changed, 35 deletions(-) delete mode 100644 docker/compose.prod.yml diff --git a/docker/compose.prod.yml b/docker/compose.prod.yml deleted file mode 100644 index 210887fa5..000000000 --- a/docker/compose.prod.yml +++ /dev/null @@ -1,35 +0,0 @@ -version: "3.9" - -services: - www: - environment: - - MERCURE_JWT_SECRET=!ChangeThisMercureHubJWTSecretKey! - - redis: - environment: - - REDIS_PASSWORD=!ChangeThisRedisPass! - - db: - environment: - - POSTGRES_PASSWORD=!ChangeThisPostgresPass! - # For backwards compatibility the default PostgreSQL image version is set to v13. - # Feel free to select a newer version of PostgreSQL when starting new. - # Don't forget to also update the .env file accordingly - #- POSTGRES_VERSION=16 - - rabbitmq: - environment: - - RABBITMQ_DEFAULT_PASS=!ChangeThisRabbitPass! - - # Set the following HTTPS variables to TRUE if your environment is using a - # valid certificate behind a reverse proxy. This is likely true for most - # production environments and is required for proper federation, that is, this - # will ensure the webfinger responses include `https:` in the URLs generated. - - php: - environment: - - HTTPS=TRUE - - messenger: - environment: - - HTTPS=TRUE From 427a86c533cba4dc9fe2c1fc466548fb0a9b639b Mon Sep 17 00:00:00 2001 From: LoveIsGrief Date: Sun, 18 Aug 2024 16:05:48 +0200 Subject: [PATCH 26/43] chore(doc): document overriding in dev docker env --- .gitignore | 1 + docs/04-contributing/development_environment.md | 2 ++ 2 files changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 681adf52b..ff3636b0b 100644 --- a/.gitignore +++ b/.gitignore @@ -19,6 +19,7 @@ tools/vendor/ /public/media/* /public/media compose.override.yml +compose.*.override.yml yarn.lock /metal/ diff --git a/docs/04-contributing/development_environment.md b/docs/04-contributing/development_environment.md index 1c00578f3..ca722f00d 100644 --- a/docs/04-contributing/development_environment.md +++ b/docs/04-contributing/development_environment.md @@ -38,9 +38,11 @@ Your changes will be ignored by git. For docker versions `>= 2.20.0`, you can [include][docker compose include] other YAML files into a compose file. This has the benefit of keeping up to date with changes to the `compose.dev.yml` without modifying it. +_compose.override.yml_ ```yaml include: - compose.dev.yml + - compose.dev.override.yml # Modifications go into that file ``` **Copy the compose.dev.yml** From ff25214f9827903ee26d6602b44c88d4f88bedc8 Mon Sep 17 00:00:00 2001 From: LoveIsGrief Date: Sun, 18 Aug 2024 16:48:25 +0200 Subject: [PATCH 27/43] chore(gh-actions): Publish PHP and caddy images Separate images are now available because mbin isn't solely bound to caddy and can be used with any reverse proxy. --- .github/workflows/action.yaml | 32 +++++++++++++++++++++++++------- compose.prod.yml | 24 ++++++++++-------------- 2 files changed, 35 insertions(+), 21 deletions(-) diff --git a/.github/workflows/action.yaml b/.github/workflows/action.yaml index fe2fd6454..832bac5ee 100644 --- a/.github/workflows/action.yaml +++ b/.github/workflows/action.yaml @@ -132,7 +132,7 @@ jobs: - name: php-cs-fixer dry-run run: tools/vendor/bin/php-cs-fixer fix --dry-run -v --show-progress=none #--format=checkstyle #would be nice if codeberg did something with this like github does. - build-and-publish-docker-image: + build-and-publish-docker-images: runs-on: ubuntu-latest # Let's only run this on branches and tagged releases only # Because the Docker build takes quite some time. @@ -151,18 +151,36 @@ jobs: username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Docker meta data - id: meta + - name: Docker meta data for php image + id: meta_php uses: docker/metadata-action@v5 with: images: ghcr.io/mbinorg/mbin + flavor: suffix=php - - name: Build and push Docker image + - name: Docker meta data for caddy image + id: meta_caddy + uses: docker/metadata-action@v5 + with: + images: ghcr.io/mbinorg/mbin + flavor: suffix=caddy + + - name: Build and push php image + uses: docker/build-push-action@v5 + with: + context: . + file: ./docker/php/Dockerfile + push: ${{ github.event_name != 'pull_request' }} + tags: ${{ steps.meta_php.outputs.tags }} + labels: ${{ steps.meta_php.outputs.labels }} + + - name: Build and push caddy image uses: docker/build-push-action@v5 with: context: . - file: ./docker/Dockerfile + file: ./docker/caddy/Dockerfile push: ${{ github.event_name != 'pull_request' }} - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} + tags: ${{ steps.meta_caddy.outputs.tags }} + labels: ${{ steps.meta_caddy.outputs.labels }} + # TODO: Integration tests diff --git a/compose.prod.yml b/compose.prod.yml index bd3251d64..6daa1b891 100644 --- a/compose.prod.yml +++ b/compose.prod.yml @@ -1,8 +1,9 @@ services: www: - # Or remove the build, dockerfile and image lines above and - # use the pre-build image from ghcr.io (you can also a tag eg.: v1.0.0 instead of: latest): - #image: "ghcr.io/mbinorg/mbin:caddy-latest" + # To use a prebuilt image from ghcr.io comment out the "image" and "pull_policy" lines + # You can also a tag eg.: "v1.0.0" instead of "latest" + #image: "ghcr.io/mbinorg/mbin:latest-caddy" + #pull_policy: never ports: - 80:80 volumes: @@ -18,13 +19,10 @@ services: - php php: - # Builds the Docker image from scratch - build: - dockerfile: docker/php/Dockerfile - target: prod - # Or remove the build, context, dockerfile and image lines above and - # use the pre-build image from ghcr.io (you can also a tag eg.: v1.0.0 instead of: latest): + # To use a prebuilt image from ghcr.io comment out the "image" and "pull_policy" lines + # You can also a tag eg.: "v1.0.0" instead of "latest" #image: "ghcr.io/mbinorg/mbin:latest" + #pull_policy: never volumes: - ./docker/storage/mbin:$MBIN_HOME # If you want to change configs locally, without rebuilding the image use: @@ -37,12 +35,10 @@ services: - rabbitmq messenger: - # Builds the Docker image from scratch - build: - dockerfile: docker/php/Dockerfile - # Or remove the build, context, dockerfile and image lines above and - # use the pre-build image from ghcr.io (you can also a tag eg.: v1.0.0 instead of: latest): + # To use a prebuilt image from ghcr.io comment out the "image" and "pull_policy" lines + # You can also a tag eg.: "v1.0.0" instead of "latest" #image: "ghcr.io/mbinorg/mbin:latest" + #pull_policy: never restart: unless-stopped command: bin/console -vvvv messenger:consume --all --time-limit=3600 deploy: From 1999588d57dbc052755edd25e22ea615c4bea129 Mon Sep 17 00:00:00 2001 From: LoveIsGrief Date: Mon, 19 Aug 2024 20:44:15 +0200 Subject: [PATCH 28/43] chore(docker): Update docs for production These come due to the refactoring: - `docker/compose.yml` --> `/compose.yml` - support for arbitrary reverse proxies without running caddy - smaller docker image - allow overrides without changing versioned files --- docs/02-admin/01-installation/docker.md | 231 +++++++++++++++--------- 1 file changed, 146 insertions(+), 85 deletions(-) diff --git a/docs/02-admin/01-installation/docker.md b/docs/02-admin/01-installation/docker.md index d92aabc66..2d63847d0 100644 --- a/docs/02-admin/01-installation/docker.md +++ b/docs/02-admin/01-installation/docker.md @@ -1,18 +1,24 @@ # Docker Installation > [!NOTE] -> Docker installation is currently not advised for production use. Try the [Bare Metal installation](./bare_metal.md) instead. +> Docker installation is currently not advised for production use. Try the [Bare Metal installation](./bare_metal.md) +> instead. + +> [!IMPORTANT] +> If you were already using docker in production, please see the [migration guide](#migration-guide) for updates ## System Requirements - Docker Engine - Docker Compose V2 - > If you are using Compose V1, replace `docker compose` with `docker-compose` in those commands below. +> [!WARNING] +> `docker-compose` is **not** supported! ### Docker Install -The most convenient way to install docker is using an official [convenience script](https://docs.docker.com/engine/install/ubuntu/#install-using-the-convenience-script) +The most convenient way to install docker is using an +official [convenience script](https://docs.docker.com/engine/install/ubuntu/#install-using-the-convenience-script) provided at [get.docker.com](https://get.docker.com/): ```bash @@ -20,7 +26,8 @@ curl -fsSL https://get.docker.com -o get-docker.sh sudo sh get-docker.sh ``` -Alternatively, you can follow the official [Docker install documentation](https://docs.docker.com/engine/install/) for your platform. +Alternatively, you can follow the official [Docker install documentation](https://docs.docker.com/engine/install/) for +your platform. Once Docker is installed on your system, it is recommended to create a `docker` group and add it to your user: @@ -31,82 +38,68 @@ sudo usermod -aG docker $USER ## Mbin Installation -### Preparation - -Clone git repository: +First clone the git repository: ```bash git clone https://github.com/MbinOrg/mbin.git cd mbin ``` -### Docker image preparation - -> [!NOTE] -> If you're using a version of Docker Engine earlier than 23.0, run `export DOCKER_BUILDKIT=1`, prior to building the image. This does not apply to users running Docker Desktop. More info can be found [here](https://docs.docker.com/build/buildkit/#getting-started) - -1. First go to the _docker directory_: +For a test "production" instance, all you have to do is -```bash -cd docker -``` +- [Copy example configuration](#copy-the-example-configuration) + - Replace `mbin.domain.tdl` with localhost in the `.env` file +- [Make compose override files](#make-override-files) +- [Start the instance](#start-the-instance) -2. Use the existing Docker image _OR_ build the docker image. Select one of the two options. +For a full production instance, you'll have to follow the steps below. -#### Build our own Docker image +### Configuration -If you want to build our own image, run (_no_ need to update the `compose.yml` file): +The configuration is held in a `.env` file. It contains information to connect the components to each other (passwords, +service names, ...), where certain things should end up (uploaded files and such), the domain name of your instance, +and more. -```bash -docker build --no-cache -t mbin -f Dockerfile .. -``` +`docker compose` references it in nearly every service. -#### Use Mbin pre-build image +#### Copy the example configuration -_OR_ use our pre-build images from [ghcr.io](https://ghcr.io). In this case you need to update the `compose.yml` file: +`.env.example_docker` contains an example configuration that with very a few tweaks can have you up and running. -```bash -nano compose.yml +```shell +cp .env.example_docker .env ``` -Find and replace or comment-out the following 4 lines: +#### Configure `.env` -```yml -build: - context: ../ - dockerfile: docker/Dockerfile -image: mbin -``` +Below is a table with env variables that are **mandatory** to update -And instead use the following line on all places (`www`, `php`, and `messenger` services): +| Variable | Purpose | +|-----------------------|-------------------------------------------------------------------------------------------------| +| APP_SECRET | Salt passwords and other secrets. Generate a random text at least 16 characters long | +| KBIN_DOMAIN | The postgres user's password | +| KBIN_STORAGE_URL | Where the uploaded media will reachable from externally. See [doc below](#uploaded-media-files) | +| POSTGRES_PASSWORD | The postgres user's password | +| RABBITMQ_DEFAULT_PASS | Used to connect to RabbitMQ | +| REDIS_PASSWORD | Used to connect to Redis | +| SERVER_NAME | Forces server to accept requests only to this domain. **Must have `www:80`** | -```yml -image: "ghcr.io/mbinorg/mbin:latest" -``` +These are optional but recommended to update -**Important:** Do _NOT_ forget to change **ALL LINES** in that matches `image: mbin` to: `image: "ghcr.io/mbinorg/mbin:latest"` in the `compose.yml` file (should be 4 matches in total). +| Variable | Purpose | +|--------------------|---------------------------------------------------| +| MERCURE_JWT_SECRET | Used to connect to the optional mercure service | +| POSTGRES_VERSION | Ensure you're running the latest postgres version | -3. Create config files and storage directories: +> [!IMPORTANT] +> Ensure the `HTTPS` environmental variable is set to `TRUE` in `compose.override.yml` for the `php`, `messenger`, +> and `messenger_ap` containers **if your environment is using a valid certificate behind a reverse proxy**. This is +> likely true for most production environments and is required for proper federation, that is, this will ensure the +> webfinger responses include `https:` in the URLs generated. -```bash -cp ../.env.example_docker .env -cp compose.prod.yml compose.override.yml -mkdir -p storage/media storage/caddy_config storage/caddy_data storage/logs -sudo chown $USER:$USER storage/media storage/caddy_config storage/caddy_data storage/logs -``` - -### Configure `.env` and `compose.override.yml` - -1. Choose your Redis password, PostgreSQL password, RabbitMQ password, and Mercure password. -2. Place the passwords in the corresponding variables in both `.env` and `compose.override.yml`. -3. Update the `SERVER_NAME`, `KBIN_DOMAIN` and `KBIN_STORAGE_URL` in `.env`. -4. Update `APP_SECRET` in `.env`, generate a new one via: `node -e "console.log(require('crypto').randomBytes(16).toString('hex'))"` -5. _Optionally_: Use a newer PostgreSQL version (current fallback is v13). Update/set the `POSTGRES_VERSION` variable in your `.env` and `compose.override.yml` under `db`. - -> [!NOTE] -> Ensure the `HTTPS` environmental variable is set to `TRUE` in `compose.override.yml` for the `php`, `messenger`, and `messenger_ap` containers **if your environment is using a valid certificate behind a reverse proxy**. This is likely true for most production environments and is required for proper federation, that is, this will ensure the webfinger responses include `https:` in the URLs generated. +#### Configure OAuth2 keys (optional) -### Configure OAuth2 keys +OAuth is used by 3rd party app developers. Without these, they will not be able to connect to the server. 1. Create an RSA key pair using OpenSSL: @@ -134,60 +127,110 @@ OAUTH_PASSPHRASE= OAUTH_ENCRYPTION_KEY= ``` -### Running the containers +### Compose override files -By default `docker compose` will execute the `compose.yml` and `compose.override.yml` files. +`docker compose` allows overriding a compose configuration by merging `compose.yml` and `compose.override.yml`. +The latter is ignored by git, which allows you to make modifications to the services without making changes to version +controlled files. -Run the container in the background (`-d` means detach, but this can also be omitted for testing or debugging purposes): +Create a **compose.override.yml** with these contents -```bash -# Go to the docker directory within the git repo -cd docker - -# Starts the containers -docker compose up -d +```yaml +include: + - compose.prod.yml + - compose.prod.override.yml ``` -See your running containers via: `docker ps`. +And an empty `compose.prod.override.yml`. -Then, you should be able to access the new instance via [http://localhost:8008](http://localhost:8008). -You can also access RabbitMQ management UI via [http://localhost:15672](http://localhost:15672). +docker compose will load these files in order and [merge][docker compose merging] sequentially +(last file is most significant): -### Add auxiliary containers to `compose.yml` +- `docker.compose.yml` +- `docker.prod.yml` +- `docker.prod.override.yml` (ignored by git) -Add any auxiliary container as you want. For example, add a Nginx container as reverse proxy to provide HTTPS encryption. +### Docker image preparation -> [!NOTE] -> If you are building the docker images yourself, you might get merge conflicts when changing the `compose.yml` +You have two options for the docker images: -### Uploaded media files +- [Build your own](#build-our-own-docker-image) +- [Use prebuilt images](#use-mbin-prebuilt-images) -Uploaded media files (e.g. photos uploaded by users) will be stored on the host directory `storage/media`. They will be served by the Caddy web server in the `www` container as static files. +#### Build our own Docker image -Make sure `KBIN_STORAGE_URL` in your `.env` configuration file is set to be `https://yourdomain.tld/media` (assuming you setup Nginx with SSL certificate by now). +> ![WARNING] +> Building your own image will use the code you currently checked out! +> Beware that updates to a running instance might break it. Read the release notes first! -You can also serve those media files on another server by mirroring the files at `storage/media` and changing `KBIN_STORAGE_URL` correspondingly. +If you want to build our own image, run (_no_ need to update the compose files): -### Filesystem ACL support +```bash +docker compose build --no-cache +``` + +#### Use Mbin prebuilt images + +There are prebuilt images from [ghcr.io](https://ghcr.io) which can speed up deployment. Should you want to use them +In this case you need to update the `compose.prod.override.yml` file with: + +```yaml +services: + www: + image: "ghcr.io/mbinorg/mbin:latest-caddy" + pull_policy: never + php: + image: "ghcr.io/mbinorg/mbin:latest" + pull_policy: never + messenger: + image: "ghcr.io/mbinorg/mbin:latest" + pull_policy: never +``` -The filesystem ACL is disabled by default, in the `mbin` image. You can set the environment variable `ENABLE_ACL=1` to enable it. Remember that not all filesystems support ACL. This will cause an error if you enable filesystem ACL for such filesystems. +> ![NOTE] +> You can replace `latest` with a version number e.g `1.0.0` -## Run Production +### Running the containers -If you created the file `compose.override.yml` with your configs (`cp compose.prod.yml compose.override.yml`), running production would be the same command: +Run the services in the background (`-d` means detach, but this can also be omitted for testing or debugging purposes): ```bash +# Go to the docker directory within the git repo +cd docker + +# Starts the containers docker compose up -d ``` -**Important:** The docker instance is can be reached at [http://127.0.0.1:8008](http://127.0.0.1:8008), we strongly advise you to put a reverse proxy (like Nginx) in front of the docker instance. Nginx can could listen on ports 80 and 443 and Nginx should handle SSL/TLS offloading. See also Nginx example below. +See your running services via: `docker compose ps`. + +Then, you should be able to access the new instance via [http://localhost](http://localhost:8008). +You can also access RabbitMQ management UI via [http://localhost:15672](http://localhost:15672). + +## Notes -If you want to deploy your app on a cluster of machines, you can -use [Docker Swarm](https://docs.docker.com/engine/swarm/stack-deploy/), which is compatible with the provided Compose -files. +### Uploaded media files + +Uploaded media files (e.g. photos uploaded by users) will be stored on the host directory `storage/mbin/public/media` +by the `php` container and served by the Caddy web server in the `www` container as static files. + +Make sure `KBIN_STORAGE_URL` in your `.env` configuration file is set to be `https://yourdomain.tld/media` +(assuming you setup Nginx with SSL certificate by now). + +You can also serve those media files on another server by mirroring the files at `storage/mbin/public/media` and +changing `KBIN_STORAGE_URL` correspondingly. + +### Filesystem ACL support + +The filesystem ACL is disabled by default, in the `mbin` image. You can set the environment variable `ENABLE_ACL=1` to +enable it. Remember that not all filesystems support ACL. This will cause an error if you enable filesystem ACL for such +filesystems. ### Mbin NGINX Server Block +> ![WARNING] +> This is not up to date! PRs for a valid nginx service and config welcome! + NGINX reverse proxy example for the Mbin Docker instance: ```nginx @@ -248,3 +291,21 @@ server { } } ``` + +## Migration Guide + +For admins using docker already, as of 2024-08-19, the docker configuration has changed. Read this guide to be sure +you're up to date. The major change is where media files are stored. + +Previously media files were stored at `docker/storage/media`. They will now be stored in `docker/storage/mbin/public/media`. +The easiest way to migrate them is by running these commands as root (`sudo`) + +```shell +mkdir -p docker/storage/www/public +# Copy to have the previous files as a backup +cp -r docker/storage/media docker/storage/www/public +``` + +You should then be set all set. + +[docker compose merging]: https://docs.docker.com/compose/compose-file/13-merge/ From 37b9705deda4f5adc5e88802a7c5ca2c210077ad Mon Sep 17 00:00:00 2001 From: LoveIsGrief Date: Mon, 19 Aug 2024 20:44:41 +0200 Subject: [PATCH 29/43] fix(docker): do not expose rabbitmq port to the world in production --- compose.prod.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose.prod.yml b/compose.prod.yml index 6daa1b891..9cbc7e2ab 100644 --- a/compose.prod.yml +++ b/compose.prod.yml @@ -95,7 +95,7 @@ services: volumes: - ./docker/storage/rabbitmq:/var/lib/rabbitmq ports: - - 15672:15672 + - 127.0.0.1:15672:15672 # Add your favorite reverse proxy (e.g nginx) which accept incoming HTTPS # traffic and forward to http://www:80 From 1e03b8307dcd4b52ad0efd28e2d235a03fc55ac6 Mon Sep 17 00:00:00 2001 From: LoveIsGrief Date: Mon, 19 Aug 2024 21:03:26 +0200 Subject: [PATCH 30/43] chore(doc): Add link to development environment in CONTRIBUTING.md --- CONTRIBUTING.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1e8ce6c0c..52044ad87 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -28,7 +28,9 @@ With an account on [GitHub](https://github.com) you will be able to [fork this r > [!Note] > If you are a Maintainer with GitHub org admin rights, you do NOT need to fork the project, instead you are allowed to use git branches. See also [C4](C4.md). -To get started with development, follow the [guide](docs/04-contributing/development_environment). +### Development + +To get started with development, follow the [guide](docs/04-contributing/development_environment.md). ### Coding Style Guide From fa8616f0a333c26842e71ec5733cc019551f168b Mon Sep 17 00:00:00 2001 From: LoveIsGrief Date: Tue, 27 Aug 2024 20:35:28 +0200 Subject: [PATCH 31/43] refactor(docker): remove unnecessary networks from compose.prod.yml --- compose.prod.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/compose.prod.yml b/compose.prod.yml index 9cbc7e2ab..bba4dd22d 100644 --- a/compose.prod.yml +++ b/compose.prod.yml @@ -121,6 +121,3 @@ services: # - SMARTHOST_USER=myuser # - SMARTHOST_PASSWORD=secret # - SMARTHOST_ALIASES=*.mysmtp.com - -networks: - mbin_external_network: From 26127f0424f70d26fcde503e0e3c6a06626968a7 Mon Sep 17 00:00:00 2001 From: LoveIsGrief Date: Tue, 27 Aug 2024 20:36:10 +0200 Subject: [PATCH 32/43] chore(docker): Update nodejs image to 22.7.0 --- docker/php/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/php/Dockerfile b/docker/php/Dockerfile index 9371779a8..dcc4adf4e 100644 --- a/docker/php/Dockerfile +++ b/docker/php/Dockerfile @@ -83,7 +83,7 @@ RUN composer run-script --no-dev post-install-cmd && \ ###################################################### -FROM node:22.5.1-alpine3.20 as builder-nodejs +FROM node:22.7.0-alpine3.20 as builder-nodejs ARG MBIN_SRC=/usr/src/mbin # Set NodeJS as production by default From 9da2e85be5a02365026ba8e3b0c0b991ee49bb0e Mon Sep 17 00:00:00 2001 From: LoveIsGrief Date: Sun, 1 Sep 2024 23:29:20 +0200 Subject: [PATCH 33/43] chore(docker): docker-compose does actually work :O --- docs/02-admin/01-installation/docker.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/02-admin/01-installation/docker.md b/docs/02-admin/01-installation/docker.md index 2d63847d0..1d25952c0 100644 --- a/docs/02-admin/01-installation/docker.md +++ b/docs/02-admin/01-installation/docker.md @@ -12,8 +12,7 @@ - Docker Engine - Docker Compose V2 -> [!WARNING] -> `docker-compose` is **not** supported! +> If you are using Compose V1, replace `docker compose` with `docker-compose` in those commands below. ### Docker Install From 58014b19671531063ffec1a9b9509ca7e5488393 Mon Sep 17 00:00:00 2001 From: LoveIsGrief Date: Sun, 1 Sep 2024 21:38:05 +0000 Subject: [PATCH 34/43] Fix typos in docs/04-contributing/development_environment.md Co-authored-by: Melroy van den Berg --- docs/04-contributing/development_environment.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/04-contributing/development_environment.md b/docs/04-contributing/development_environment.md index ca722f00d..872101c6e 100644 --- a/docs/04-contributing/development_environment.md +++ b/docs/04-contributing/development_environment.md @@ -69,7 +69,7 @@ is present, otherwise the list of files has to be passed with `docker compose -f [This][php-dockerfile] is the biggest and most complex `Dockerfile` in the repo. It's a multi-stage file that attempts to make the final image minimal. You can read it, but the most important things -to keep in mind are +to keep in mind are: - the `base` target only has the minimal common items for the next steps - for production there are builder targets to build the frontend and backend of mbin From 25578c7ca6f3f7091bf8c4fe74c65905a3485ec5 Mon Sep 17 00:00:00 2001 From: LoveIsGrief Date: Sun, 1 Sep 2024 21:38:27 +0000 Subject: [PATCH 35/43] Fix more typos in docs/04-contributing/development_environment.md Co-authored-by: Melroy van den Berg --- docs/04-contributing/development_environment.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/04-contributing/development_environment.md b/docs/04-contributing/development_environment.md index 872101c6e..35d9e0aaf 100644 --- a/docs/04-contributing/development_environment.md +++ b/docs/04-contributing/development_environment.md @@ -54,7 +54,7 @@ in your compose setup automatically. ## How it works -MBin depends on multiple services (postgreSQL, redis, a reverse proxy, PHP, ...). Instead of having one monolithic +Mbin depends on multiple services (PostgreSQL, Redis, a reverse proxy, PHP, ...). Instead of having one monolithic docker image that includes these services and all their configs, the services each run in their own containers. There's minimal overlap between most services and their `Dockerfile`s. Here are a few things to know. From ca668e0835d5e719c428e5c21652ed5d40159a03 Mon Sep 17 00:00:00 2001 From: LoveIsGrief Date: Sun, 1 Sep 2024 23:40:24 +0200 Subject: [PATCH 36/43] chore(docker): Reference PHP8.3 in development_environment.md --- docs/04-contributing/development_environment.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/04-contributing/development_environment.md b/docs/04-contributing/development_environment.md index 35d9e0aaf..9e0e8c767 100644 --- a/docs/04-contributing/development_environment.md +++ b/docs/04-contributing/development_environment.md @@ -150,7 +150,7 @@ Nixos needs [iptables rules](https://discourse.nixos.org/t/docker-container-not- Requirements: -- PHP v8.2 +- PHP v8.3 - NodeJS - Redis - PostgreSQL @@ -158,13 +158,13 @@ Requirements: --- -- Increase execution time in PHP config file: `/etc/php/8.2/fpm/php.ini`: +- Increase execution time in PHP config file: `/etc/php/8.3/fpm/php.ini`: ```ini max_execution_time = 120 ``` -- Restart the PHP-FPM service: `sudo systemctl restart php8.2-fpm.service` +- Restart the PHP-FPM service: `sudo systemctl restart php8.3-fpm.service` - Connect to PostgreSQL using the postgres user: ```bash From 585ae55c6a354251e1fded7c9be07818324ab50c Mon Sep 17 00:00:00 2001 From: LoveIsGrief Date: Sun, 1 Sep 2024 23:47:25 +0200 Subject: [PATCH 37/43] refactor(docker): remove unnecessary `pwd` from entrypoint.sh --- docker/php/entrypoint.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/docker/php/entrypoint.sh b/docker/php/entrypoint.sh index 1da5176c1..48f36b98a 100755 --- a/docker/php/entrypoint.sh +++ b/docker/php/entrypoint.sh @@ -7,7 +7,6 @@ if [ "${1#-}" != "$1" ]; then fi if [ "$1" == "php-fpm" ] || [ "$1" == "php" ] || [ "$1" == "bin/console" ]; then - pwd # if running as a service install assets echo "Starting as service..." From 3e5e1efb8c6f6e1bb8b38368b251ebc07c8601aa Mon Sep 17 00:00:00 2001 From: LoveIsGrief Date: Mon, 2 Sep 2024 00:00:29 +0200 Subject: [PATCH 38/43] chore(docker): Remove warning about nginx config --- docs/02-admin/01-installation/docker.md | 3 --- 1 file changed, 3 deletions(-) diff --git a/docs/02-admin/01-installation/docker.md b/docs/02-admin/01-installation/docker.md index 1d25952c0..f60026ac5 100644 --- a/docs/02-admin/01-installation/docker.md +++ b/docs/02-admin/01-installation/docker.md @@ -227,9 +227,6 @@ filesystems. ### Mbin NGINX Server Block -> ![WARNING] -> This is not up to date! PRs for a valid nginx service and config welcome! - NGINX reverse proxy example for the Mbin Docker instance: ```nginx From 7466547f873d60f3a308bbeed690264927e41aef Mon Sep 17 00:00:00 2001 From: LoveIsGrief Date: Mon, 2 Sep 2024 00:34:33 +0200 Subject: [PATCH 39/43] fix(docker): Depend on /.env and add instructions It wouldn't start without the file and I realised it had been there the entire time when testing the documentation. @melroy tested it without and discovered the bug --- compose.dev.yml | 14 ++------------ compose.prod.yml | 10 ---------- compose.yml | 2 -- docs/04-contributing/development_environment.md | 2 ++ 4 files changed, 4 insertions(+), 24 deletions(-) diff --git a/compose.dev.yml b/compose.dev.yml index c719d5fd2..5ada11640 100644 --- a/compose.dev.yml +++ b/compose.dev.yml @@ -8,8 +8,6 @@ services: depends_on: node: condition: service_completed_successfully - env_file: - - .env.dev_docker ports: - 8008:80 @@ -42,8 +40,6 @@ services: - ./docker/php/conf.d/app.ini:/usr/local/etc/php/conf.d/app.ini - ./docker/php/entrypoint.sh:/entrypoint.sh - ./docker/php/php-fpm.d/zz-docker.conf:/usr/local/etc/php-fpm.d/zz-docker.conf - env_file: - - .env.dev_docker messenger: @@ -54,14 +50,10 @@ services: volumes: - ./:/var/www/mbin/ - ./docker/php/entrypoint.sh:/entrypoint.sh - env_file: - - .env.dev_docker redis: volumes: - redis:/data - env_file: - - .env.dev_docker db: # Allow connecting to the db from the localhost for debugging @@ -69,12 +61,10 @@ services: - "127.0.0.1:5432:5432" volumes: - postgres:/var/lib/postgresql/data - env_file: - - .env.dev_docker rabbitmq: - env_file: - - .env.dev_docker + ports: + - 15672:15672 volumes: - rabbitmq:/var/lib/rabbitmq diff --git a/compose.prod.yml b/compose.prod.yml index bba4dd22d..6aa969a9b 100644 --- a/compose.prod.yml +++ b/compose.prod.yml @@ -27,8 +27,6 @@ services: - ./docker/storage/mbin:$MBIN_HOME # If you want to change configs locally, without rebuilding the image use: # - ../docker/config:/var/www/mbin/config - env_file: - - .env depends_on: - redis - db @@ -52,8 +50,6 @@ services: - ./docker/storage/mbin:$MBIN_HOME # If you want to change configs locally, without rebuilding the image use: # - ../docker/config:/var/www/mbin/config - env_file: - - .env depends_on: - redis - db @@ -68,8 +64,6 @@ services: command: /bin/sh -c "redis-server --requirepass $${REDIS_PASSWORD}" volumes: - ./docker/storage/redis:/data - env_file: - - .env healthcheck: test: ["CMD", "redis-cli", "ping"] @@ -81,8 +75,6 @@ services: - mbin_external_network volumes: - ./docker/storage/postgres:/var/lib/postgresql/data - env_file: - - .env rabbitmq: image: rabbitmq:3.13.6-management-alpine @@ -90,8 +82,6 @@ services: restart: unless-stopped networks: - mbin_external_network - env_file: - - .env volumes: - ./docker/storage/rabbitmq:/var/lib/rabbitmq ports: diff --git a/compose.yml b/compose.yml index c995d548a..ec70c7d62 100644 --- a/compose.yml +++ b/compose.yml @@ -96,8 +96,6 @@ services: - mbin_external_network env_file: - .env - ports: - - 15672:15672 healthcheck: test: rabbitmq-diagnostics -q ping interval: 30s diff --git a/docs/04-contributing/development_environment.md b/docs/04-contributing/development_environment.md index 9e0e8c767..c18afd170 100644 --- a/docs/04-contributing/development_environment.md +++ b/docs/04-contributing/development_environment.md @@ -22,6 +22,8 @@ The development environment is run with [docker compose]. ```shell # Create a symbolic link to the dev setup that merges and overrides part of compose.yml ln -nfs compose.dev.yml compose.override.yml +# Create a symbolic link to the configuration for the dev setup +ln -nfs .env.dev_docker .env # Bring up all the services and detach from the console docker compose up -d ``` From bea9c074765bda895422613ceaeb31c18dac5f5f Mon Sep 17 00:00:00 2001 From: LoveIsGrief Date: Mon, 2 Sep 2024 14:05:55 +0200 Subject: [PATCH 40/43] chore(docs): Split development_environment.md in two This follows what is done for the production guide, which has 2 guides/files as well. --- CONTRIBUTING.md | 3 +- docs/04-contributing/bare_metal.md | 82 +++++++++++++++++ .../{development_environment.md => docker.md} | 88 ------------------- 3 files changed, 84 insertions(+), 89 deletions(-) create mode 100644 docs/04-contributing/bare_metal.md rename docs/04-contributing/{development_environment.md => docker.md} (67%) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 52044ad87..7150d5c0c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -30,7 +30,8 @@ With an account on [GitHub](https://github.com) you will be able to [fork this r ### Development -To get started with development, follow the [guide](docs/04-contributing/development_environment.md). +To get started with development, follow the [bare metal](docs/04-contributing/bare_metal.md) or +[docker](docs/04-contributing/docker.md) guide. ### Coding Style Guide diff --git a/docs/04-contributing/bare_metal.md b/docs/04-contributing/bare_metal.md new file mode 100644 index 000000000..b489a08db --- /dev/null +++ b/docs/04-contributing/bare_metal.md @@ -0,0 +1,82 @@ +# Bare Metal Development Server + +Requirements: + +- PHP v8.3 +- NodeJS +- Redis +- PostgreSQL +- _Optionally:_ Mercure + +--- + +- Increase execution time in PHP config file: `/etc/php/8.3/fpm/php.ini`: + +```ini +max_execution_time = 120 +``` + +- Restart the PHP-FPM service: `sudo systemctl restart php8.3-fpm.service` +- Connect to PostgreSQL using the postgres user: + +```bash +sudo -u postgres psql +``` + +- Create new mbin database user: + +```sql +sudo -u postgres createuser --createdb --createrole --pwprompt mbin +``` + +- Correctly configured `.env` file (`cp .env.example .env`), these are only the changes you need to pay attention to: + +```env +# Set domain to 127.0.0.1:8000 +SERVER_NAME=127.0.0.1:8000 +KBIN_DOMAIN=127.0.0.1:8000 +KBIN_STORAGE_URL=http://127.0.0.1:8000/media + +#Redis (without password) +REDIS_DNS=redis://127.0.0.1:6379 + +# Set App configs +APP_ENV=dev +APP_SECRET=427f5e2940e5b2472c1b44b2d06e0525 + +# Configure PostgreSQL +POSTGRES_DB=mbin +POSTGRES_USER=mbin +POSTGRES_PASSWORD= + +# Set messenger to Doctrine (= PostgresQL DB) +MESSENGER_TRANSPORT_DSN=doctrine://default +``` + +- If you are using `127.0.0.1` to connect to the PostgreSQL server, edit the following file: `/etc/postgresql//main/pg_hba.conf` and add: + +```conf +local mbin mbin md5 +``` + +- Restart the PostgreSQL server: `sudo systemctl restart postgresql` +- Create database: `php bin/console doctrine:database:create` +- Create tables and database structure: `php bin/console doctrine:migrations:migrate` +- Build frontend assets: `npm install && npm run dev` + +Starting the server: + +1. Install Symfony CLI: `wget https://get.symfony.com/cli/installer -O - | bash` +2. Check the requirements: `symfony check:requirements` +3. Install dependencies: `composer install` +4. Dump `.env` into `.env.local.php` via: `composer dump-env dev` +5. _Optionally:_ Increase verbosity log level in: `config/packages/monolog.yaml` in the `when@dev` section: `level: debug` (instead of `level: info`), +6. Clear cache: `APP_ENV=dev APP_DEBUG=1 php bin/console cache:clear -n` +7. Start Mbin: `symfony server:start` +8. Go to: [http://127.0.0.1:8000](http://127.0.0.1:8000/) + +This will give you a minimal working frontend with PostgreSQL setup. Keep in mind: this will _not_ start federating, for that you also need to setup Mercure to test the full Mbin setup. + +_Optionally:_ you could also setup RabbitMQ, but the Doctrine messenger configuration will be sufficient for local development. + +More info: [Contributing guide](../../README.md), [Admin guide](../02-admin/README.md) and [Symfony Local Web Server](https://symfony.com/doc/current/setup/symfony_server.html) diff --git a/docs/04-contributing/development_environment.md b/docs/04-contributing/docker.md similarity index 67% rename from docs/04-contributing/development_environment.md rename to docs/04-contributing/docker.md index c18afd170..9fbf42178 100644 --- a/docs/04-contributing/development_environment.md +++ b/docs/04-contributing/docker.md @@ -1,8 +1,3 @@ -There are multiple ways to get started. - -If you can run `docker compose` follow the steps for [Docker](#docker), otherwise you can set up your -own [development server](#development-server) - # Docker Using docker can get you up and running quickly as it spins up containers of all the required components. @@ -148,89 +143,6 @@ Firewalls can sometimes get in the way of communication between the docker conta Nixos needs [iptables rules](https://discourse.nixos.org/t/docker-container-not-resolving-to-host/30259/8). -# Development Server - -Requirements: - -- PHP v8.3 -- NodeJS -- Redis -- PostgreSQL -- _Optionally:_ Mercure - ---- - -- Increase execution time in PHP config file: `/etc/php/8.3/fpm/php.ini`: - -```ini -max_execution_time = 120 -``` - -- Restart the PHP-FPM service: `sudo systemctl restart php8.3-fpm.service` -- Connect to PostgreSQL using the postgres user: - -```bash -sudo -u postgres psql -``` - -- Create new mbin database user: - -```sql -sudo -u postgres createuser --createdb --createrole --pwprompt mbin -``` - -- Correctly configured `.env` file (`cp .env.example .env`), these are only the changes you need to pay attention to: - -```env -# Set domain to 127.0.0.1:8000 -SERVER_NAME=127.0.0.1:8000 -KBIN_DOMAIN=127.0.0.1:8000 -KBIN_STORAGE_URL=http://127.0.0.1:8000/media - -#Redis (without password) -REDIS_DNS=redis://127.0.0.1:6379 - -# Set App configs -APP_ENV=dev -APP_SECRET=427f5e2940e5b2472c1b44b2d06e0525 - -# Configure PostgreSQL -POSTGRES_DB=mbin -POSTGRES_USER=mbin -POSTGRES_PASSWORD= - -# Set messenger to Doctrine (= PostgresQL DB) -MESSENGER_TRANSPORT_DSN=doctrine://default -``` - -- If you are using `127.0.0.1` to connect to the PostgreSQL server, edit the following file: `/etc/postgresql//main/pg_hba.conf` and add: - -```conf -local mbin mbin md5 -``` - -- Restart the PostgreSQL server: `sudo systemctl restart postgresql` -- Create database: `php bin/console doctrine:database:create` -- Create tables and database structure: `php bin/console doctrine:migrations:migrate` -- Build frontend assets: `npm install && npm run dev` - -Starting the server: - -1. Install Symfony CLI: `wget https://get.symfony.com/cli/installer -O - | bash` -2. Check the requirements: `symfony check:requirements` -3. Install dependencies: `composer install` -4. Dump `.env` into `.env.local.php` via: `composer dump-env dev` -5. _Optionally:_ Increase verbosity log level in: `config/packages/monolog.yaml` in the `when@dev` section: `level: debug` (instead of `level: info`), -6. Clear cache: `APP_ENV=dev APP_DEBUG=1 php bin/console cache:clear -n` -7. Start Mbin: `symfony server:start` -8. Go to: [http://127.0.0.1:8000](http://127.0.0.1:8000/) - -This will give you a minimal working frontend with PostgreSQL setup. Keep in mind: this will _not_ start federating, for that you also need to setup Mercure to test the full Mbin setup. - -_Optionally:_ you could also setup RabbitMQ, but the Doctrine messenger configuration will be sufficient for local development. - -More info: [Contributing guide](./README.md), [Admin guide](../02-admin/README.md) and [Symfony Local Web Server](https://symfony.com/doc/current/setup/symfony_server.html) - [app.dev.ini]: ../../docker/php/conf.d/app.dev.ini [compose.dev.yml]: ../../compose.dev.yml [docker compose]: https://docs.docker.com/compose/reference/ From 2d79a3bd94980a32fa46eb751e1e71cf3c5100be Mon Sep 17 00:00:00 2001 From: LoveIsGrief Date: Mon, 2 Sep 2024 16:11:38 +0200 Subject: [PATCH 41/43] chore(docs): Replace bold single lines with headers --- docs/04-contributing/docker.md | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/docs/04-contributing/docker.md b/docs/04-contributing/docker.md index 9fbf42178..20c309ceb 100644 --- a/docs/04-contributing/docker.md +++ b/docs/04-contributing/docker.md @@ -2,11 +2,11 @@ Using docker can get you up and running quickly as it spins up containers of all the required components. -**Supported operating systems** +## Supported operating systems - Linux -**Requirements** +## Requirements - [Docker](https://docs.docker.com/get-docker/) @@ -30,7 +30,7 @@ Once everything has started, you can navigate to http://localhost:8008. Here's a Should you want to edit the `compose.override.yml` file without making changes to `compose.dev.yml`, read further. Your changes will be ignored by git. -**Use the `include` directive** +#### Use the `include` directive For docker versions `>= 2.20.0`, you can [include][docker compose include] other YAML files into a compose file. This has the benefit of keeping up to date with changes to the `compose.dev.yml` without modifying it. @@ -42,7 +42,7 @@ include: - compose.dev.override.yml # Modifications go into that file ``` -**Copy the compose.dev.yml** +#### Copy the compose.dev.yml Instead of linking, copy the `compose.dev.yml` to `compose.override.yml` and make your changes. @@ -88,11 +88,11 @@ and requires files from the PHP dependencies to successfully build the frontend. The entire repository is mounted into the `caddy`, `php`, and `messenger` services. -**PHP code** +#### PHP code Most PHP code changes are picked up immediately after refreshing the page in the browser. -**Other important files** +#### Other important files | file | services | how to refresh | |---------------|----------------|------------------------------------------------------------------------| @@ -124,11 +124,11 @@ sequenceDiagram rp-->>-Browser: response ``` -**Requirement** +#### Requirement An XDebug server. The XDebug server is often hosted on port 9000 or 9003. Some IDEs (like [PHPStorm]) have it builtin. -**Enabling** +#### Enabling Open [app.dev.ini] and uncomment `;xdebug.start_with_request=yes` by removing the `;`, then restart the `php` service using `docker compose restart php`. @@ -139,7 +139,7 @@ Once you navigate to a page, your IDE/editor should get called from `php-fpm` wi Firewalls can sometimes get in the way of communication between the docker container and the docker host. -**NixOS** +##### NixOS Nixos needs [iptables rules](https://discourse.nixos.org/t/docker-container-not-resolving-to-host/30259/8). From 6c10103ce8c2c63a835de8e1ec0de08c6fb4577d Mon Sep 17 00:00:00 2001 From: LoveIsGrief Date: Mon, 2 Sep 2024 19:04:26 +0200 Subject: [PATCH 42/43] chore(docs): update docs/04-contributing/docker.md title --- docs/04-contributing/docker.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/04-contributing/docker.md b/docs/04-contributing/docker.md index 20c309ceb..c2759ac2a 100644 --- a/docs/04-contributing/docker.md +++ b/docs/04-contributing/docker.md @@ -1,4 +1,4 @@ -# Docker +# Docker Development Server Using docker can get you up and running quickly as it spins up containers of all the required components. From 97ec77d822fc3ed9d739a95354a46f5d53f54b4a Mon Sep 17 00:00:00 2001 From: LoveIsGrief Date: Mon, 2 Sep 2024 19:05:30 +0200 Subject: [PATCH 43/43] refactor(docker): Use 8008 for www service in prod too --- compose.prod.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose.prod.yml b/compose.prod.yml index 6aa969a9b..39d7e16c2 100644 --- a/compose.prod.yml +++ b/compose.prod.yml @@ -5,7 +5,7 @@ services: #image: "ghcr.io/mbinorg/mbin:latest-caddy" #pull_policy: never ports: - - 80:80 + - 8008:80 volumes: - ./docker/storage/caddy_config:/config - ./docker/storage/caddy_data:/data