From cbc6b397e8e06b0173be0912275acf2716199c24 Mon Sep 17 00:00:00 2001 From: Mohamed Abdel Wedoud Date: Mon, 7 Oct 2024 11:38:17 +0200 Subject: [PATCH 1/8] perf(scripts): improve load balancing, make the main process wait for background processes --- scripts/start.sh | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/scripts/start.sh b/scripts/start.sh index aedcca2202..e5875943eb 100755 --- a/scripts/start.sh +++ b/scripts/start.sh @@ -5,9 +5,24 @@ set -e CUR_DIR=$(cd "$(dirname "$0")" && pwd) BASE_DIR=$(dirname "$CUR_DIR") +min() { + echo $(( $1 < $2 ? $1 : $2 )) +} + +workers=$(min 30 ${NB_WORKERS_ANTARES:-17}) # default to 17 and max is 30 + if [ -z "$1" ] ; then sh $CUR_DIR/pre-start.sh - gunicorn --config $BASE_DIR/conf/gunicorn.py --worker-class=uvicorn.workers.UvicornWorker antarest.wsgi:app + pids=() # Initialize empty array to store background process IDs + for ((i=0; i Date: Mon, 7 Oct 2024 18:17:32 +0200 Subject: [PATCH 2/8] refactor(scripts): add the possibility of using `Gunicorn` when specified --- scripts/start.sh | 33 +++++++++++++++++++++++---------- 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/scripts/start.sh b/scripts/start.sh index e5875943eb..9e7bb63d95 100755 --- a/scripts/start.sh +++ b/scripts/start.sh @@ -11,18 +11,31 @@ min() { workers=$(min 30 ${NB_WORKERS_ANTARES:-17}) # default to 17 and max is 30 +# Check for --no-gunicorn or --multiple-ports argument +use_gunicorn=true +for arg in "$@" +do + if [[ $arg == "--no-gunicorn" || $arg == "--multiple-ports" ]]; then + use_gunicorn=false + break + fi +done + if [ -z "$1" ] ; then sh $CUR_DIR/pre-start.sh - pids=() # Initialize empty array to store background process IDs - for ((i=0; i Date: Mon, 7 Oct 2024 18:18:40 +0200 Subject: [PATCH 3/8] refactor(docker): pass the `docker command` to the `docker-compose.yml` --- docker-compose.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/docker-compose.yml b/docker-compose.yml index 9c0f877db2..890eb352f1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -16,6 +16,7 @@ services: - ./resources/deploy/logs:/logs - ./resources/deploy/gunicorn.py:/conf/gunicorn.py - ./antares-8.2.2-Ubuntu-20.04/bin:/antares_simulator + command: ./scripts/start.sh --no-gunicorn antares-antarest-watcher: image: antarest:latest container_name: antarest-watcher From 09f4067da8658e626d0a65391db36d1566ffe503 Mon Sep 17 00:00:00 2001 From: Mohamed Abdel Wedoud Date: Wed, 9 Oct 2024 11:28:39 +0200 Subject: [PATCH 4/8] feat(scripts): set default number of workers to `(2*nproc + 1)` --- scripts/start.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/start.sh b/scripts/start.sh index 9e7bb63d95..398a855ced 100755 --- a/scripts/start.sh +++ b/scripts/start.sh @@ -9,7 +9,7 @@ min() { echo $(( $1 < $2 ? $1 : $2 )) } -workers=$(min 30 ${NB_WORKERS_ANTARES:-17}) # default to 17 and max is 30 +workers=$(min 30 ${ANTARES_NB_WORKERS:-$((2*$(nproc) + 1))}) # default (2*nproc + 1) and max is 30 # Check for --no-gunicorn or --multiple-ports argument use_gunicorn=true From df0cfd5a655c51bf26bfcf7893ab2ed05733ace3 Mon Sep 17 00:00:00 2001 From: Mohamed Abdel Wedoud Date: Wed, 9 Oct 2024 11:59:30 +0200 Subject: [PATCH 5/8] fix(docker): rewrite Antares `docker` command --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 890eb352f1..3bcff9be5d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -16,7 +16,7 @@ services: - ./resources/deploy/logs:/logs - ./resources/deploy/gunicorn.py:/conf/gunicorn.py - ./antares-8.2.2-Ubuntu-20.04/bin:/antares_simulator - command: ./scripts/start.sh --no-gunicorn + command: ["./scripts/start.sh", "--no-gunicorn"] antares-antarest-watcher: image: antarest:latest container_name: antarest-watcher From 771f8ea6cf8a43812cb2a4b6eb48920ec7f6f3c9 Mon Sep 17 00:00:00 2001 From: Mohamed Abdel Wedoud Date: Wed, 9 Oct 2024 20:56:36 +0200 Subject: [PATCH 6/8] refactor(docker): use `gunicorn` for public repo --- docker-compose.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 3bcff9be5d..9c0f877db2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -16,7 +16,6 @@ services: - ./resources/deploy/logs:/logs - ./resources/deploy/gunicorn.py:/conf/gunicorn.py - ./antares-8.2.2-Ubuntu-20.04/bin:/antares_simulator - command: ["./scripts/start.sh", "--no-gunicorn"] antares-antarest-watcher: image: antarest:latest container_name: antarest-watcher From 5aae1a6740979ae1c5644cd78833bab3e2ec2e68 Mon Sep 17 00:00:00 2001 From: Mohamed Abdel Wedoud Date: Wed, 9 Oct 2024 20:56:59 +0200 Subject: [PATCH 7/8] fix(scripts): correct `bash` conditional statement --- scripts/start.sh | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/scripts/start.sh b/scripts/start.sh index 398a855ced..a963ddc7a3 100755 --- a/scripts/start.sh +++ b/scripts/start.sh @@ -12,30 +12,30 @@ min() { workers=$(min 30 ${ANTARES_NB_WORKERS:-$((2*$(nproc) + 1))}) # default (2*nproc + 1) and max is 30 # Check for --no-gunicorn or --multiple-ports argument -use_gunicorn=true +use_uvicorn=false for arg in "$@" do if [[ $arg == "--no-gunicorn" || $arg == "--multiple-ports" ]]; then - use_gunicorn=false + use_uvicorn=true break fi done if [ -z "$1" ] ; then sh $CUR_DIR/pre-start.sh - if [ "$use_gunicorn" = true ]; then - gunicorn --config $BASE_DIR/conf/gunicorn.py --worker-class=uvicorn.workers.UvicornWorker antarest.wsgi:app - else - pids=() # Initialize empty array to store background process IDs - for ((i=0; i Date: Mon, 14 Oct 2024 15:24:38 +0200 Subject: [PATCH 8/8] doc(script): add some documentation to script Signed-off-by: Sylvain Leclerc --- scripts/start.sh | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/scripts/start.sh b/scripts/start.sh index a963ddc7a3..5b4fefa1e5 100755 --- a/scripts/start.sh +++ b/scripts/start.sh @@ -1,4 +1,23 @@ #!/bin/bash +# +# Starter script used in docker image. +# +# Usages are: +# +# 1. ./start.sh +# Starts antares web server with gunicorn. +# Number of workers is defined by GUNICORN_WORKERS env variable, +# or equal to 2*cpu + 1. +# +# 2. ./start.sh +# Where module is for example "watcher". +# Starts one of the backend services (watcher, garbage collector, ...) +# +# 3. ./start.sh --no-gunicorn +# Starts multiple workers on different ports. You are expected to +# configure some load balancing upstream, in that case. +# Number of workers is defined by ANTARES_NB_WORKERS env variable, +# or equal to 2*cpu + 1. set -e @@ -39,4 +58,4 @@ elif [ "$use_uvicorn" = true ]; then else export PYTHONPATH=$BASE_DIR python3 $BASE_DIR/antarest/main.py -c $ANTAREST_CONF --module "$1" -fi \ No newline at end of file +fi