From 96e9bc85f762b01a3b88c31057c7a7d2ede46489 Mon Sep 17 00:00:00 2001 From: Nick Satterly Date: Sun, 7 Apr 2024 20:14:09 +0200 Subject: [PATCH 1/6] fix: switch to gunicorn WSGI server --- Dockerfile | 10 +++---- README.md | 4 +-- config/templates/app/gunicorn.conf.py.j2 | 14 ++++++++++ config/templates/app/nginx.conf.j2 | 11 ++++---- config/templates/app/supervisord.conf.j2 | 6 +++-- config/templates/app/uwsgi.ini.j2 | 34 ------------------------ contrib/kubernetes/backend/Dockerfile | 2 +- docker-entrypoint.sh | 13 ++++----- requirements-docker.txt | 2 +- 9 files changed, 37 insertions(+), 59 deletions(-) create mode 100644 config/templates/app/gunicorn.conf.py.j2 delete mode 100644 config/templates/app/uwsgi.ini.j2 diff --git a/Dockerfile b/Dockerfile index b1f31495..1a96061a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.9-slim-buster +FROM python:3.10-slim-buster ENV PYTHONUNBUFFERED 1 ENV PIP_DISABLE_PIP_VERSION_CHECK=1 @@ -15,11 +15,7 @@ ENV WEBUI_VERSION=8.7.1 ENV NGINX_WORKER_PROCESSES=1 ENV NGINX_WORKER_CONNECTIONS=1024 -ENV UWSGI_PROCESSES=5 -ENV UWSGI_LISTEN=100 -ENV UWSGI_BUFFER_SIZE=8192 -ENV UWSGI_MAX_WORKER_LIFETIME=30 -ENV UWSGI_WORKER_LIFETIME_DELTA=3 +ENV GUNICORN_WORKERS=5 ENV HEARTBEAT_SEVERITY=major ENV HK_EXPIRED_DELETE_HRS=2 @@ -77,7 +73,7 @@ COPY requirements*.txt /app/ # hadolint ignore=DL3013 RUN pip install --no-cache-dir pip virtualenv jinja2 && \ python3 -m venv /venv && \ - /venv/bin/pip install --no-cache-dir --upgrade setuptools && \ + /venv/bin/pip install --no-cache-dir --upgrade setuptools wheel && \ /venv/bin/pip install --no-cache-dir --requirement /app/requirements.txt && \ /venv/bin/pip install --no-cache-dir --requirement /app/requirements-docker.txt ENV PATH $PATH:/venv/bin diff --git a/README.md b/README.md index f6ceb7fa..232be71e 100644 --- a/README.md +++ b/README.md @@ -126,8 +126,8 @@ API to ease deployment more generally: `NGINX_WORKER_CONNECTIONS` - maximum number of simultaneous connections that can be opened by a worker process (default:`1024`) -`UWSGI_PROCESSES` - - number of processes for uWSGI (default:`5`) +`GUNICORN_WORKERS` + - number of worker processes for Gunicorn (default:`5`) `UWSGI_LISTEN` - max number of concurrent connections (default:`100`) diff --git a/config/templates/app/gunicorn.conf.py.j2 b/config/templates/app/gunicorn.conf.py.j2 new file mode 100644 index 00000000..8d35639c --- /dev/null +++ b/config/templates/app/gunicorn.conf.py.j2 @@ -0,0 +1,14 @@ + +bind = '127.0.0.1:29000' + +chdir = '/app' +daemon = False +raw_env = [ + 'SCRIPT_NAME=/api', +] +worker_tmp_dir = '/dev/shm' +workers = {{ env.GUNICORN_WORKERS }} + +{%- if env.DEBUG %} +loglevel = 'debug' +{%- endif %} diff --git a/config/templates/app/nginx.conf.j2 b/config/templates/app/nginx.conf.j2 index 76772ab7..4327eb2f 100644 --- a/config/templates/app/nginx.conf.j2 +++ b/config/templates/app/nginx.conf.j2 @@ -47,13 +47,12 @@ http { access_log /dev/stdout main; location /api { - include /etc/nginx/uwsgi_params; - uwsgi_pass backend; + proxy_pass http://backend; - uwsgi_param Host $host; - uwsgi_param X-Real-IP $remote_addr; - uwsgi_param X-Forwarded-For $proxy_add_x_forwarded_for; - uwsgi_param X-Forwarded-Proto $http_x_forwarded_proto; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $http_x_forwarded_proto; } root /web; diff --git a/config/templates/app/supervisord.conf.j2 b/config/templates/app/supervisord.conf.j2 index b5d4919e..e01f8dea 100644 --- a/config/templates/app/supervisord.conf.j2 +++ b/config/templates/app/supervisord.conf.j2 @@ -4,8 +4,10 @@ logfile=/tmp/supervisord.log loglevel={{ env.SUPERVISORD_LOG_LEVEL|lower or 'debug' }} pidfile=/tmp/supervisord.pid -[program:uwsgi] -command=/venv/bin/uwsgi --ini /app/uwsgi.ini +[program:gunicorn] +command=/venv/bin/gunicorn wsgi:app -c /app/gunicorn.conf.py +autostart=true +autorestart=true redirect_stderr=true [program:nginx] diff --git a/config/templates/app/uwsgi.ini.j2 b/config/templates/app/uwsgi.ini.j2 deleted file mode 100644 index f40eb73e..00000000 --- a/config/templates/app/uwsgi.ini.j2 +++ /dev/null @@ -1,34 +0,0 @@ -[uwsgi] -chdir = /app -module = wsgi -manage-script-name = true -mount = /api=wsgi:app -master = true -processes = {{ env.UWSGI_PROCESSES }} -listen = {{ env.UWSGI_LISTEN }} -{%- if env.UWSGI_MAX_WORKER_LIFETIME %} -max-worker-lifetime = {{ env.UWSGI_MAX_WORKER_LIFETIME }} -max-worker-lifetime-delta = {{ env.UWSGI_WORKER_LIFETIME_DELTA }} -{%- endif %} - -{%- if env.UWSGI_THREADS %} -threads = {{ env.UWSGI_THREADS }} -enable-threads = True -{%- endif %} - -socket = 127.0.0.1:29000 -buffer-size = {{ env.UWSGI_BUFFER_SIZE }} -chmod-socket = 664 -uid = alerta -gid = root -vacuum = true - -die-on-term = true - -{%- if env.DEBUG %} -show-config -stats = :1717 -stats-http -{%- else %} -disable-logging = True -{%- endif %} diff --git a/contrib/kubernetes/backend/Dockerfile b/contrib/kubernetes/backend/Dockerfile index 7298f808..e11baa56 100644 --- a/contrib/kubernetes/backend/Dockerfile +++ b/contrib/kubernetes/backend/Dockerfile @@ -10,7 +10,7 @@ RUN apt-get update && apt-get install -y \ RUN pip install --no-cache-dir virtualenv && \ virtualenv --python=python3 /venv && \ - /venv/bin/pip install uwsgi alerta alerta-server==$VERSION + /venv/bin/pip install gunicorn alerta alerta-server==$VERSION ENV PATH $PATH:/venv/bin diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh index c77a24ef..cbd6deeb 100755 --- a/docker-entrypoint.sh +++ b/docker-entrypoint.sh @@ -7,7 +7,7 @@ ALERTA_CONF_FILE=${ALERTA_CONF_FILE:-/app/alerta.conf} ALERTA_SVR_CONF_FILE=${ALERTA_SVR_CONF_FILE:-/app/alertad.conf} ALERTA_WEB_CONF_FILE=${ALERTA_WEB_CONF_FILE:-/web/config.json} NGINX_CONF_FILE=/app/nginx.conf -UWSGI_CONF_FILE=/app/uwsgi.ini +GUNICORN_CONF_FILE=/app/gunicorn.conf.py SUPERVISORD_CONF_FILE=/app/supervisord.conf ADMIN_USER=${ADMIN_USERS%%,*} @@ -67,12 +67,13 @@ if [ ! -f "${NGINX_CONF_FILE}" ]; then echo "# Create nginx configuration file." python3 -c "${JINJA2}" < ${NGINX_CONF_FILE}.j2 >${NGINX_CONF_FILE} fi +cat ${NGINX_CONF_FILE} nginx -t -c ${NGINX_CONF_FILE} -# Generate uWSGI config, if not supplied. -if [ ! -f "${UWSGI_CONF_FILE}" ]; then - echo "# Create uWSGI configuration file." - python3 -c "${JINJA2}" < ${UWSGI_CONF_FILE}.j2 >${UWSGI_CONF_FILE} +# Generate Gunicorn config, if not supplied. +if [ ! -f "${GUNICORN_CONF_FILE}" ]; then + echo "# Create Gunicorn configuration file." + python3 -c "${JINJA2}" < ${GUNICORN_CONF_FILE}.j2 >${GUNICORN_CONF_FILE} fi # Generate web config, if not supplied. @@ -88,7 +89,7 @@ echo Alerta Client ${CLIENT_VERSION} echo Alerta WebUI ${WEBUI_VERSION} nginx -v -echo uwsgi $(uwsgi --version) +gunicorn --version mongo --version | grep MongoDB psql --version python3 --version diff --git a/requirements-docker.txt b/requirements-docker.txt index f744ca74..82ce7a83 100644 --- a/requirements-docker.txt +++ b/requirements-docker.txt @@ -1,4 +1,4 @@ lxml==5.2.1 pysaml2==7.2.1 python-ldap==3.4.4 -uWSGI==2.0.21 +gunicorn==21.2.0 From ed4ebdaadb06621e6e3ffff7b01cf0a8dcc85179 Mon Sep 17 00:00:00 2001 From: Nick Satterly Date: Tue, 9 Apr 2024 13:01:34 +0200 Subject: [PATCH 2/6] Check for HTTP_X_FORWARDED_FOR envvar in tests --- tests/spec/api_spec.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/spec/api_spec.rb b/tests/spec/api_spec.rb index bcdbe2ef..fca0ee95 100644 --- a/tests/spec/api_spec.rb +++ b/tests/spec/api_spec.rb @@ -39,8 +39,8 @@ it "return 200" do expect(result.code).to eq(200) end - it "X-Forwarded-For header is set" do - expect(result.body).to include("X-Forwarded-For") + it "HTTP_X_FORWARDED_FOR envvar is set" do + expect(result.body).to include("HTTP_X_FORWARDED_FOR") end end context "get healthcheck" do From a78cd9ed2c24ecae94515341be8951162da114f8 Mon Sep 17 00:00:00 2001 From: Nick Satterly Date: Tue, 9 Apr 2024 13:58:33 +0200 Subject: [PATCH 3/6] nginx debug --- config/templates/app/nginx.conf.j2 | 2 ++ tests/docker-compose.test.mongodb.yml | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/config/templates/app/nginx.conf.j2 b/config/templates/app/nginx.conf.j2 index 4327eb2f..1a464578 100644 --- a/config/templates/app/nginx.conf.j2 +++ b/config/templates/app/nginx.conf.j2 @@ -49,10 +49,12 @@ http { location /api { proxy_pass http://backend; + proxy_http_version 1.1; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $http_x_forwarded_proto; + proxy_set_header Connection ""; } root /web; diff --git a/tests/docker-compose.test.mongodb.yml b/tests/docker-compose.test.mongodb.yml index d9ca5d62..9a3912cb 100644 --- a/tests/docker-compose.test.mongodb.yml +++ b/tests/docker-compose.test.mongodb.yml @@ -14,7 +14,7 @@ services: depends_on: - db environment: - # - DEBUG=1 # remove this line to turn DEBUG off + - DEBUG=1 # remove this line to turn DEBUG off - LOG_LEVEL=error # debug, info, warn (default), or error - SECRET_KEY=super-secret - DATABASE_URL=mongodb://db:27017/monitoring From 12c577a8ff1a673b0d59ce9a8ceda26c60767384 Mon Sep 17 00:00:00 2001 From: Nick Satterly Date: Tue, 9 Apr 2024 15:06:42 +0200 Subject: [PATCH 4/6] Rollback debug changes --- Dockerfile | 4 ++-- README.md | 12 ------------ docker-entrypoint.sh | 1 - tests/docker-compose.test.mongodb.yml | 2 +- 4 files changed, 3 insertions(+), 16 deletions(-) diff --git a/Dockerfile b/Dockerfile index 1a96061a..e392d204 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.10-slim-buster +FROM python:3.9-slim-buster ENV PYTHONUNBUFFERED 1 ENV PIP_DISABLE_PIP_VERSION_CHECK=1 @@ -73,7 +73,7 @@ COPY requirements*.txt /app/ # hadolint ignore=DL3013 RUN pip install --no-cache-dir pip virtualenv jinja2 && \ python3 -m venv /venv && \ - /venv/bin/pip install --no-cache-dir --upgrade setuptools wheel && \ + /venv/bin/pip install --no-cache-dir --upgrade setuptools && \ /venv/bin/pip install --no-cache-dir --requirement /app/requirements.txt && \ /venv/bin/pip install --no-cache-dir --requirement /app/requirements-docker.txt ENV PATH $PATH:/venv/bin diff --git a/README.md b/README.md index 232be71e..83d5a640 100644 --- a/README.md +++ b/README.md @@ -129,18 +129,6 @@ API to ease deployment more generally: `GUNICORN_WORKERS` - number of worker processes for Gunicorn (default:`5`) -`UWSGI_LISTEN` - - max number of concurrent connections (default:`100`) - -`UWSGI_BUFFER_SIZE` - - size of the unix socket buffer (default:`8192`) - -`UWSGI_MAX_WORKER_LIFETIME` - - reload worker after this many seconds (default:`30`) - -`UWSGI_WORKER_LIFETIME_DELTA` - - time in seconds to stagger UWSGI worker respawns (default:`3`) - Configuration Files ------------------- diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh index cbd6deeb..6bad3610 100755 --- a/docker-entrypoint.sh +++ b/docker-entrypoint.sh @@ -67,7 +67,6 @@ if [ ! -f "${NGINX_CONF_FILE}" ]; then echo "# Create nginx configuration file." python3 -c "${JINJA2}" < ${NGINX_CONF_FILE}.j2 >${NGINX_CONF_FILE} fi -cat ${NGINX_CONF_FILE} nginx -t -c ${NGINX_CONF_FILE} # Generate Gunicorn config, if not supplied. diff --git a/tests/docker-compose.test.mongodb.yml b/tests/docker-compose.test.mongodb.yml index 9a3912cb..d9ca5d62 100644 --- a/tests/docker-compose.test.mongodb.yml +++ b/tests/docker-compose.test.mongodb.yml @@ -14,7 +14,7 @@ services: depends_on: - db environment: - - DEBUG=1 # remove this line to turn DEBUG off + # - DEBUG=1 # remove this line to turn DEBUG off - LOG_LEVEL=error # debug, info, warn (default), or error - SECRET_KEY=super-secret - DATABASE_URL=mongodb://db:27017/monitoring From ae771a979c4fd24409d5fc4ca27699d9c107102b Mon Sep 17 00:00:00 2001 From: Nick Satterly Date: Fri, 12 Apr 2024 06:23:10 +0200 Subject: [PATCH 5/6] Bump python requirements --- requirements.txt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/requirements.txt b/requirements.txt index d00b04cb..5f8e1f7d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ bcrypt==4.1.2 blinker==1.7.0 cryptography==42.0.5 -Flask==3.0.2 +Flask==3.0.3 Flask-Compress==1.14 Flask-Cors==4.0.0 mohawk==1.1.0 @@ -13,7 +13,7 @@ python-dateutil==2.9.0.post0 pytz==2024.1 PyYAML==6.0.1 requests==2.31.0 -requests_hawk==1.2.1 -sentry-sdk[flask]==1.43.0 +requests-hawk==1.2.1 +sentry-sdk[flask]==1.45.0 StrEnum==0.4.15 -werkzeug==3.0.1 +werkzeug==3.0.2 From 0664345d1ed2d70f343be2a11ca6a09408ce75f6 Mon Sep 17 00:00:00 2001 From: Nick Satterly Date: Fri, 12 Apr 2024 11:31:31 +0200 Subject: [PATCH 6/6] Set PLATFORM to gunicorn --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e96b0017..6aa13ae5 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -48,7 +48,7 @@ jobs: env: REPOSITORY_URL: registry.hub.docker.com IMAGE_NAME: alerta/alerta-web - PLATFORM: 3.9-buster-uwsgi + PLATFORM: 3.9-buster-gunicorn steps: - uses: actions/checkout@v4