Merge pull request #347 from kobotoolbox/kobocat-as-django-app

Remove KoboCAT container, move KPI services to their own containers

docker-compose.backend.primary.override.yml.sample → docker-compose.backend.override.yml.sample

@@ -1,11 +1,9 @@
-# Override for primary back-end server
-version: '2.2'
+# Override for back-end server
+version: '3'
 
 services:
 
   postgres:
-    #environment:
-    #  - POSTGRES_BACKUP_FROM_SECONDARY=True
     # Uncomment `ports` section if you want to expose ports (e.g. use as separated servers)
     # Before uncommenting, be sure you have a firewall configured to prevent these ports
     # from being publicly accessible
@@ -32,9 +30,6 @@ services:
   # Comment out section (`redis_main` and `redis_cache`) below if you want to
   # start redis containers from the secondary back-end server
   redis_main:
-    extends:
-      file: docker-compose.backend.template.yml
-      service: redis_main
     # Uncomment `ports` section if you want to expose ports (e.g. use as separated servers)
     # Before uncommenting, be sure you have a firewall configured to prevent these ports
     # from being publicly accessible
@@ -47,9 +42,6 @@ services:
           - redis-main.kobo.private
 
   redis_cache:
-    extends:
-      file: docker-compose.backend.template.yml
-      service: redis_cache
     # Uncomment `ports` section if you want to expose ports (e.g. use as separated servers)
     # Before uncommenting, be sure you have a firewall configured to prevent these ports
     # from being publicly accessible

docker-compose.backend.template.yml → docker-compose.backend.yml

@@ -1,14 +1,12 @@
-# Back-end containers template.
-# The primary and secondary compose files extend this file
-
-version: '2.2'
+# Back-end containers
+version: '3'
 
 services:
   postgres:
     image: postgis/postgis:14-3.2
     hostname: postgres
     env_file:
-      - ../kobo-env/envfile.txt
+      - ../kobo-env/envfiles/domains.txt
       - ../kobo-env/envfiles/databases.txt
       - ../kobo-env/envfiles/aws.txt
     volumes:
@@ -26,7 +24,7 @@ services:
     environment:
       - MONGO_DATA=/data/db
     env_file:
-      - ../kobo-env/envfile.txt
+      - ../kobo-env/envfiles/domains.txt
       - ../kobo-env/envfiles/databases.txt
       - ../kobo-env/envfiles/aws.txt
     volumes:
@@ -39,9 +37,9 @@ services:
     stop_grace_period: 5m
 
   redis_main:
-    image: redis:6.2
+    image: redis:7.2
     env_file:
-      - ../kobo-env/envfile.txt
+      - ../kobo-env/envfiles/domains.txt
       - ../kobo-env/envfiles/databases.txt
       - ../kobo-env/envfiles/aws.txt
     environment:
@@ -60,7 +58,7 @@ services:
     command: "bash /tmp/redis/entrypoint.sh"
 
   redis_cache:
-    image: redis:6.2
+    image: redis:7.2
     env_file:
       - ../kobo-env/envfiles/databases.txt
     environment:

docker-compose.frontend.override.yml.sample

@@ -2,29 +2,8 @@
 version: '3'
 
 services:
-  kobocat:
-    environment:
-      # change `ENKETO_PROTOCOL` to http if HTTPS is not used
-      - ENKETO_PROTOCOL=https
-      # `NGINX_PUBLIC_PORT` is the port used to access KoboToolbox (e.g. `https://kc.kobotoolbox.org:<NGINX_PUBLIC_PORT>`)
-      - NGINX_PUBLIC_PORT=80
-      # Uncomment the lines below to tweak uWSGI
-      #- KC_UWSGI_WORKERS_COUNT=2
-      #- KC_UWSGI_CHEAPER_WORKERS_COUNT=1
-      #- KC_UWSGI_MAX_REQUESTS=512
-      #- KC_UWSGI_CHEAPER_RSS_LIMIT_SOFT=134217728
-      #- KC_UWSGI_HARAKIRI=120
-      #- KC_UWSGI_WORKER_RELOAD_MERCY=120
-    networks:
-      kobo-be-network:
-        aliases:
-          - kobocat
-          - kobocat.docker.container
-
   kpi:
     environment:
-      # `NGINX_PUBLIC_PORT` is the port used to access KoboToolbox (e.g. `https://kf.kobotoolbox.org:<NGINX_PUBLIC_PORT>`)
-      - NGINX_PUBLIC_PORT=80
       # Uncomment the lines below to tweak uWSGI
       #- KPI_UWSGI_WORKERS_COUNT=2
       #- KPI_UWSGI_CHEAPER_WORKERS_COUNT=1
@@ -37,13 +16,34 @@ services:
     networks:
       kobo-be-network:
         aliases:
-          - kpi
           - kpi.docker.container
 
+  worker_kobocat:
+    networks:
+      kobo-be-network:
+        aliases:
+          - worker_kobocat
+          - worker_kobocat.docker.container
+
+  worker_low_priority:
+    networks:
+      kobo-be-network:
+        aliases:
+          - worker_kobocat
+          - worker_kobocat.docker.container
+
+  beat:
+    networks:
+      kobo-be-network:
+        aliases:
+          - beat
+          - beat.docker.container
+
   nginx:
     environment:
       # `NGINX_PUBLIC_PORT` is the port used to access KoboToolbox (e.g. `https://kf.kobotoolbox.org:<NGINX_PUBLIC_PORT>`)
-      - NGINX_PUBLIC_PORT=80
+      # Include the colons with the port
+      - NGINX_PUBLIC_PORT=:8080
       #- UWSGI_PASS_TIMEOUT=130
     ports:
       # <proxy_port>:80 . If no proxies, `proxy_port` should be the same as `NGINX_PUBLIC_PORT`
@@ -70,3 +70,4 @@ networks:
       # name: <prefix>_kobo-be-network`, where `prefix` is usually the parent
       # folder name
       name: kobodocker_kobo-be-network
+      external: true

docker-compose.frontend.yml

@@ -4,92 +4,67 @@
 version: '3'
 
 services:
-  kobocat:
-    image: kobotoolbox/kobocat:2.024.19e
-    hostname: kobocat
-    env_file:
-      - ../kobo-env/envfile.txt
-      - ../kobo-env/envfiles/databases.txt
-      - ../kobo-env/envfiles/aws.txt
-      - ../kobo-env/envfiles/external_services.txt
-      - ../kobo-env/envfiles/kobocat.txt
-      - ../kobo-env/envfiles/nginx.txt
-      - ../kobo-env/envfiles/smtp.txt
-    sysctls:
-      - net.core.somaxconn=2048
-    environment:
-      - ENKETO_PROTOCOL=https
-      - KPI_PORT=8000
-      - KC_UWSGI_MAX_REQUESTS=512
-      - KC_UWSGI_WORKERS_COUNT=2
-      - KC_UWSGI_CHEAPER_RSS_LIMIT_SOFT=134217728
-      - KC_UWSGI_CHEAPER_WORKERS_COUNT=1
-      - KC_UWSGI_HARAKIRI=120
-      - KC_UWSGI_WORKER_RELOAD_MERCY=120
-    volumes:
-      - ./.vols/static/kobocat:/srv/static
-      - ./.vols/kobocat_media_uploads:/srv/src/kobocat/media
-      - ./backups/kobocat:/srv/backups
-      - ./log/kobocat:/srv/logs
-      - ./scripts/wait_for_mongo.bash:/srv/init/wait_for_mongo.bash:ro
-      - ./scripts/wait_for_postgres.bash:/srv/init/wait_for_postgres.bash:ro
-      - ./scripts/runtime_variables_kobocat.source.bash:/etc/profile.d/runtime_variables_kobocat.source.bash.sh:ro
-    restart: always
-    networks:
-      kobo-fe-network:
-        aliases:
-          - kobocat
-          - kobocat.docker.container
-          - kobocat.internal
-
-  kpi:
-    image: kobotoolbox/kpi:2.024.19e
+  kpi: &django
+    image: kobotoolbox/kpi:2.024.25a
+    init: true
     hostname: kpi
     env_file:
-      - ../kobo-env/envfile.txt
-      - ../kobo-env/envfiles/databases.txt
       - ../kobo-env/envfiles/aws.txt
+      - ../kobo-env/envfiles/databases.txt
+      - ../kobo-env/envfiles/django.txt
       - ../kobo-env/envfiles/external_services.txt
-      - ../kobo-env/envfiles/kpi.txt
-      - ../kobo-env/envfiles/nginx.txt
+      - ../kobo-env/envfiles/domains.txt
       - ../kobo-env/envfiles/smtp.txt
     sysctls:
       - net.core.somaxconn=2048
     environment:
-      - SYNC_KOBOCAT_XFORMS=False # Should be True on at least one frontend environment
-      - KPI_UWSGI_MAX_REQUESTS=512
-      - KPI_UWSGI_WORKERS_COUNT=2
-      - KPI_UWSGI_CHEAPER_RSS_LIMIT_SOFT=134217728
-      - KPI_UWSGI_CHEAPER_WORKERS_COUNT=1
-      - KPI_UWSGI_HARAKIRI=120
-      - KPI_UWSGI_WORKER_RELOAD_MERCY=120
+      - UWSGI_MAX_REQUESTS=512
+      - UWSGI_WORKERS_COUNT=2
+      - UWSGI_CHEAPER_RSS_LIMIT_SOFT=134217728
+      - UWSGI_CHEAPER_WORKERS_COUNT=1
+      - UWSGI_HARAKIRI=120
+      - UWSGI_WORKER_RELOAD_MERCY=120
     volumes:
       - ./.vols/static/kpi:/srv/static
       - ./log/kpi:/srv/logs
       - ./scripts/wait_for_mongo.bash:/srv/init/wait_for_mongo.bash:ro
       - ./scripts/wait_for_postgres.bash:/srv/init/wait_for_postgres.bash:ro
-      - ./scripts/runtime_variables_kpi.source.bash:/etc/profile.d/runtime_variables_kpi.source.bash.sh:ro
       # Persistent storage for FileFields when S3 not used (e.g. exports, uploaded map layers)
       - ./.vols/kpi_media:/srv/src/kpi/media
-      # Allow access to KoBoCAT media uploads within KPI
+      # Allow access to KoboCAT media uploads within KPI
       - ./.vols/kobocat_media_uploads:/srv/src/kobocat/media
-    restart: always
+    restart: unless-stopped
     networks:
       kobo-fe-network:
         aliases:
-          - kpi
           - kpi.docker.container
           - kpi.internal
+          - kobocat.docker.container
+          - kobocat.internal
+
+  worker:
+    <<: *django
+    command: ['bash', '/srv/src/kpi/docker/entrypoint_celery_kpi_worker.bash']
+
+  worker_low_priority:
+    <<: *django
+    command: ['bash', '/srv/src/kpi/docker/entrypoint_celery_kpi_low_priority_worker.bash']
+
+  worker_kobocat:
+    <<: *django
+    command: ['bash', '/srv/src/kpi/docker/entrypoint_celery_kobocat_worker.bash']
+
+  beat:
+    <<: *django
+    command: ['bash', '/srv/src/kpi/docker/entrypoint_celery_kobocat_worker.bash']
 
   nginx:
-    image: nginx:1.21
+    image: nginx:1.26
     hostname: nginx
     env_file:
-      - ../kobo-env/envfile.txt
+      - ../kobo-env/envfiles/domains.txt
       - ../kobo-env/envfiles/databases.txt
-      - ../kobo-env/envfiles/nginx.txt
-      - ../kobo-env/envfiles/kobocat.txt
-      - ../kobo-env/envfiles/kpi.txt
+      - ../kobo-env/envfiles/django.txt
     environment:
       - UWSGI_PASS_TIMEOUT=130
       - TEMPLATED_VAR_REFS=$${PUBLIC_REQUEST_SCHEME} $${INTERNAL_DOMAIN_NAME} $${PUBLIC_DOMAIN_NAME} $${KOBOFORM_PUBLIC_SUBDOMAIN} $${KOBOCAT_PUBLIC_SUBDOMAIN} $${ENKETO_EXPRESS_PUBLIC_SUBDOMAIN}
@@ -100,25 +75,27 @@ services:
         - ./.vols/kpi_media/__public:/srv/kpi_media/__public:ro
         - ./nginx/docker-entrypoint.d/30-init-kobo-nginx.sh:/docker-entrypoint.d/30-init-kobo-nginx.sh
         - ./nginx/kobo-docker-scripts/:/kobo-docker-scripts
-    restart: always
+    restart: unless-stopped
     networks:
       kobo-fe-network:
         aliases:
-          - nginx
           - nginx.internal
+          - nginx.docker.container
 
   enketo_express:
     image: kobotoolbox/enketo-express-extra-widgets:6.2.2
+    init: true
+    hostname: enketo_express
     env_file:
-      - ../kobo-env/envfile.txt
-    restart: always
+      - ../kobo-env/envfiles/domains.txt
+    restart: unless-stopped
     volumes:
       - ../kobo-env/enketo_express/config.json:/srv/src/enketo_express/config/config.json
     networks:
       kobo-fe-network:
         aliases:
-          - enketo_express
           - enketo_express.internal
+          - enketo_express.docker.container
 
 networks:
   kobo-fe-network:

docker-compose.maintenance.yml

@@ -5,10 +5,10 @@ services:
 
   maintenance:
     # set to match `docker-compose.frontend.yml` and avoid pulling different image
-    image: nginx:1.21
+    image: nginx:1.23
     hostname: maintenance
     env_file:
-      - ../kobo-env/envfile.txt
+      - ../kobo-env/envfiles/domains.txt
     volumes:
         - ./log/nginx:/var/log/nginx
         - ./nginx/docker-entrypoint.d/30-init-maintenance-nginx.sh:/docker-entrypoint.d/30-init-kobo-nginx.sh