portal: put all env vars in supervisord conf

.gitignore

@@ -30,3 +30,4 @@ mpcontribs-api/mpcontribs/api/swagger*/
 mpcontribs-portal/mpcontribs/portal/templates/notebooks
 coverage.xml
 mpcontribs-api/supervisord.conf
+mpcontribs-portal/supervisord.conf

mpcontribs-api/supervisord/supervisord.conf.jinja

@@ -15,7 +15,7 @@ environment=
     MAIL_DEFAULT_SENDER="[email protected]",
     DD_PROFILING_ENABLED="false",
     DD_LOGS_INJECTION="true",
-    DD_PATCH_MODULES="aioredis:false,aredis:false,boto:false,bottle:false,cassandra:false,consul:false,django:false,algoliasearch:false,grpc:false,httpx:false,mysql:false,mysqldb:false,pymysql:false,mariadb:false,pylibmc:false,pymemcache:false",
+    DD_PATCH_MODULES="aioredis:false,aredis:false,boto:false,bottle:false,cassandra:false,consul:false,django:false,algoliasearch:false,grpc:false,httpx:false,mysql:false,mysqldb:false,pymysql:false,mariadb:false,pylibmc:false,pymemcache:false,rediscluster:false",
     FLASK_APP="mpcontribs.api",
     PYTHONUNBUFFERED=1,
     MAX_REQUESTS=100000,

mpcontribs-portal/Dockerfile

@@ -54,7 +54,6 @@ COPY --from=python-builds /root/.local/bin /root/.local/bin
 COPY --from=python-builds /app/static /app/static
 COPY --from=python-builds /app/mpcontribs /app/mpcontribs
 COPY --from=python-builds /app/supervisord /app/supervisord
-COPY --from=python-builds /app/envs /app/envs
 COPY --from=webpack /app/webpack-stats.json /app/webpack-stats.json
 
 RUN apt-get update && apt-get install -y --no-install-recommends supervisor && apt-get clean

mpcontribs-portal/start.sh

@@ -1,9 +1,5 @@
 #!/bin/bash
 
-set -a
-source $ENV_FILE
-env
-
 python manage.py migrate --noinput
 wait-for-it.sh $MPCONTRIBS_API_HOST -s -t 60 -- gunicorn -c gunicorn.conf.py \
     -b 0.0.0.0:$PORTAL_PORT -k gevent -w $NWORKERS \

mpcontribs-portal/supervisord/conf.py

@@ -2,13 +2,29 @@
 from jinja2 import Environment, FileSystemLoader
 
 DIR = os.path.abspath(os.path.dirname(__file__))
+PRODUCTION = int(os.environ.get("PRODUCTION", "1"))
 
-deployments = [
-    entry.name.split(".")[0]
-    for entry in os.scandir(os.environ["ENV_FILES"])
-    if entry.is_file()
-]
+deployments = {}
+
+for deployment in os.environ.get("DEPLOYMENTS", "ml:5002").split(","):
+    name, s3, tm, portal_port = deployment.split(":")
+    api_port = 5000 + int(portal_port) % 8080
+    deployments[name] = {
+        "api_port": api_port,
+        "portal_port": portal_port,
+        "s3": s3,
+        "tm": tm.upper()
+    }
+
+kwargs = {
+    "deployments": deployments,
+    "nworkers": 2 if PRODUCTION else 1,
+    "reload": "" if PRODUCTION else "--reload",
+    "node_env": "production" if PRODUCTION else "development",
+    "dd_agent_host": "localhost" if PRODUCTION else "datadog",
+    "mpcontribs_api_host": "localhost" if PRODUCTION else "contribs-apis",
+}
 
 env = Environment(loader=FileSystemLoader(DIR))
 template = env.get_template("supervisord.conf.jinja")
-template.stream(deployments=deployments).dump("supervisord.conf")
+template.stream(**kwargs).dump("supervisord.conf")

mpcontribs-portal/supervisord/supervisord.conf.jinja

@@ -5,7 +5,18 @@ logfile=/tmp/supervisord.log
 pidfile=/tmp/supervisord.pid
 environment=
     AWS_ACCESS_KEY_ID="%(ENV_AWS_ACCESS_KEY_ID)s",
-    AWS_SECRET_ACCESS_KEY="%(ENV_AWS_SECRET_ACCESS_KEY)s"
+    AWS_SECRET_ACCESS_KEY="%(ENV_AWS_SECRET_ACCESS_KEY)s",
+    DD_PROFILING_ENABLED="false",
+    DD_LOGS_INJECTION="true",
+    DD_PATCH_MODULES="aioredis:false,aredis:false,boto:false,bottle:false,cassandra:false,consul:false,django:false,algoliasearch:false,grpc:false,httpx:false,mysql:false,mysqldb:false,pymysql:false,mariadb:false,pylibmc:false,pymemcache:false",
+    PYTHONUNBUFFERED=1,
+    MAX_REQUESTS=100000,
+    MAX_REQUESTS_JITTER=10000,
+    NWORKERS={{ nworkers }},
+    RELOAD="{{ reload }}",
+    NODE_ENV="{{ node_env }}",
+    DD_AGENT_HOST="{{ dd_agent_host }}",
+    DJANGO_SETTINGS_FILE="settings.py"
 
 [unix_http_server]
 file=/var/run/supervisor.sock
@@ -16,10 +27,12 @@ supervisor.rpcinterface_factory=supervisor.rpcinterface:make_main_rpcinterface
 [supervisorctl]
 serverurl=unix:///var/run/supervisor.sock
 
+{% set names = deployments.keys() %}
+
 [group:portals]
-programs={{ deployments|join('-portal,') }}-portal
+programs={{ names|join('-portal,') }}-portal
 
-{% for deployment in deployments %}
+{% for deployment, cfg in deployments.items() %}
 [program:{{ deployment }}-portal]
 command=./start.sh
 directory=/app
@@ -32,5 +45,9 @@ stdout_logfile_maxbytes=0
 stderr_logfile=/dev/fd/2
 stderr_logfile_maxbytes=0
 environment=
-    ENV_FILE="%(ENV_ENV_FILES)s/{{ deployment }}.env",
+    PORTAL_PORT={{ cfg.portal_port }},
+    MPCONTRIBS_API_HOST="{{ mpcontribs_api_host }}:{{ cfg.api_port }}",
+    TRADEMARK="{{ cfg.tm }}",
+    S3_DOWNLOADS_BUCKET="mpcontribs-downloads-{{ cfg.s3 }}",
+    ADMIN_GROUP="admin_{{ deployment }}.materialsproject.org"
 {% endfor %}