Preloading Modules / Dockerization of FAME / Dockerception (combined PR of all changes, do not merge!)

Dockerfile.base

@@ -0,0 +1,20 @@
+FROM python:2-slim-stretch
+
+RUN apt-get update && apt-get install -y git libmagic1 && \
+    useradd -s /bin/false -U fame -m && \
+    pip install --no-cache-dir virtualenv && \
+    rm -rf /var/lib/apt/lists/*
+
+COPY . /fame
+COPY docker/fame/docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh
+
+WORKDIR /fame
+RUN utils/run.sh -m pip install --no-cache-dir -r requirements.txt
+
+ENV FAME_DOCKER=1
+ENV PYTHONUNBUFFERED=1
+
+VOLUME [ "/fame/conf" ]
+
+ENTRYPOINT [ "docker-entrypoint.sh" ]
+CMD [ "/bin/bash" ]

Dockerfile.web

@@ -0,0 +1,16 @@
+FROM fame-base
+
+ENV FAME_WORKER=0
+
+COPY docker/fame/run.web.sh /fame/run.sh
+
+RUN apt-get update && apt-get install -y libldap2-dev libsasl2-dev build-essential && \
+    utils/run.sh -m pip install --no-cache-dir -r requirements-web.txt uwsgi && \
+    apt-get purge -y libldap2-dev libsasl2-dev build-essential && \
+    rm -rf /var/lib/apt/lists/*
+
+EXPOSE 8080
+
+VOLUME [ "/fame/fame/modules", "/fame/storage", "/fame/web/static/img/avatars" ]
+
+CMD [ "/fame/run.sh" ]

Dockerfile.worker

@@ -0,0 +1,14 @@
+FROM fame-base
+
+ENV FAME_WORKER=1
+
+RUN apt-get update -y && apt-get install -y build-essential && \
+    rm -rf /var/lib/apt/lists/* && \
+    git config --global user.name "FAME Web" && \
+    git config --global user.email "[email protected]"
+
+COPY docker/fame/run.worker.sh /fame/run.sh
+
+RUN utils/run.sh -m pip install --no-cache-dir -r requirements-worker.txt
+
+CMD [ "/fame/run.sh" ]

Makefile

@@ -0,0 +1,15 @@
+.DEFAULT_GOAL := all
+
+mongo:
+	docker build --pull -t fame-mongo docker/mongo/
+
+base:
+	docker build --pull -t fame-base -f Dockerfile.base .
+
+web: base
+	docker build -t fame-web -f Dockerfile.web .
+
+worker: base
+	docker build -t fame-worker -f Dockerfile.worker .
+
+all: mongo base web worker

agent/agent.py

@@ -115,10 +115,11 @@ def run_each_with_type(self, target, target_type):
                 return self.each_with_type(target, target_type)
             except IsolatedExceptions.ModuleExecutionError, e:
                 self.log("error", "Could not run on %s: %s" % (target, e))
+                self.log("debug", traceback.format_exc())
                 return False
-            except:
-                tb = traceback.format_exc()
-                self.log("error", "Could not run on %s.\n %s" % (target, tb))
+            except Exception:
+                self.log("error", "Could not run on %s.\n" % (target,))
+                self.log("debug", traceback.format_exc())
                 return False
 
 
@@ -134,6 +135,7 @@ def fake_module(path, klass):
 
     sys.modules[path] = klass
 
+
 fake_module('fame.core.module', IsolatedModule)
 fake_module('fame.common.exceptions', IsolatedExceptions)
 
@@ -169,7 +171,7 @@ def set_module(self, name, config):
             module = import_module('module')
 
         for _, obj in inspect.getmembers(module, inspect.isclass):
-            if obj.name and obj.name == name:
+            if hasattr(obj, "name") and obj.name == name:
                 self.queue = Queue()
                 self.module = obj()
 
@@ -286,7 +288,8 @@ def get_file(task_id):
     with open(filepath, 'rb') as fd:
         response = make_response(fd.read())
 
-    response.headers["Content-Disposition"] = "attachment; filename='{0}'".format(os.path.basename(filepath))
+    response.headers["Content-Disposition"] = \
+        "attachment; filename={0}".format(os.path.basename(filepath))
 
     return response
 

celeryconfig.py

@@ -16,11 +16,16 @@
 CELERY_ACCEPT_CONTENT = ['json_util']
 CELERY_TASK_SERIALIZER = 'json_util'
 
-CELERY_IMPORTS = ('fame.core.analysis', 'fame.core.repository')
+CELERY_IMPORTS = ('fame.worker.analysis', 'fame.worker.repository')
 
 
 def connect_to_db(**kwargs):
     fame_init()
 
+    from fame.core.user import User
+    worker_user = User.get(email="worker@fame")
+    if worker_user:
+        fame_config.api_key = worker_user['api_key']
+
 
 signals.worker_process_init.connect(connect_to_db)

docker-compose.yml

@@ -0,0 +1,95 @@
+version: '3.4'
+
+networks:
+  fame_internal:
+    internal: true
+
+  gateway:
+
+  traefik_default:
+    external:
+      name: traefik_default
+
+x-mongo-env: &mongo_env
+  MONGO_HOST: "fame-mongo"
+  MONGO_PORT: "27017"
+  MONGO_DB: "fame"
+  MONGO_USERNAME: "fame"
+  MONGO_PASSWORD: "super-secret-password"
+
+secrets:
+  ssh_priv_key:
+    file: ./ssh/id_rsa
+
+volumes:
+  fame-share:
+
+services:
+  fame-mongo:
+    image: fame-mongo
+    environment:
+      MONGO_INITDB_DATABASE: "fame"
+    container_name: fame-mongo
+    restart: unless-stopped
+    volumes:
+      - /opt/fame-mongo:/data/db:z
+    command: --auth
+    networks:
+      - "fame_internal"
+
+  fame-web:
+    image: fame-web
+    container_name: fame-web
+    depends_on:
+      - fame-mongo
+    environment:
+      <<: *mongo_env
+      FAME_INSTALL_COMMUNITY_REPO: "1"
+      FAME_URL: "http://localhost/"
+      FAME_ADMIN_FULLNAME: "The Admin"
+      FAME_ADMIN_EMAIL: "[email protected]"
+      FAME_ADMIN_GROUPS: "cert"
+      FAME_ADMIN_DEFAULT_SHARING: "cert"
+      FAME_ADMIN_PERMISSIONS: "*"
+      FAME_ADMIN_PASSWORD: "secret"
+      FAME_PUBLIC_KEY: "ssh-rsa ..."
+      FAME_SECRET_KEY: "<ssh secret key>"
+      FAME_AUTHENTICATION_TYPE: "user_password"
+      # LDAP_URI: "ldap://example.com"
+      # LDAP_USER: "ldap-user"
+      # LDAP_PASSWORD: "ldap-password"
+      # LDAP_FILTER_EMAIL: "(&(objectCategory=Person)(sAMAccountName=*)(mail={})"
+      # LDAP_FILTER_DN: "OU=People,DC=example,DC=com"
+    volumes:
+      - /opt/fame-modules:/fame/fame/modules:z
+      - /opt/fame-storage:/fame/storage:z
+      - /opt/fame-avatars:/fame/web/static/img/avatars:z
+    labels:
+      - "traefik.enable=true"
+      - "traefik.docker.network=traefik_default"
+      - "traefik.port=8080"
+      - "traefik.frontend.rule=Host: fame.example.com"
+    networks:
+      - "fame_internal"
+      - "traefik_default"
+    restart: unless-stopped
+    hostname: fame-web
+
+  fame-worker:
+    image: fame-worker
+    container_name: fame-worker
+    depends_on:
+      - fame-mongo
+      - fame-web
+    environment:
+      <<: *mongo_env
+      FAME_URL: "http://fame-web:8080/"
+      DOCKER_HOST: "unix:///var/run/docker.sock"
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock:z
+      - fame-share:/fame/docker-storage:z
+    networks:
+      - "fame_internal"
+      - "gateway"
+    secrets:
+      - ssh_priv_key

docker/fame/docker-entrypoint.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+
+echo "[+] Ensuring empty __init__.py in modules directory"
+touch /fame/fame/modules/__init__.py
+
+echo "[+] Adjusting permissions"
+chown fame:fame /fame -R
+
+exec "$@"

docker/fame/run.web.sh

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+
+TIMEOUT=60
+
+echo "[+] Waiting $TIMEOUT seconds for MongoDB to come up"
+python docker/wait-for.py fame-mongo 27017 $TIMEOUT
+if [ "$?" -ne "0" ]; then
+    echo "[X] Could not connect to MongoDB instance - is it up and running?"
+    exit 1
+fi
+
+utils/run.sh utils/install_docker.py
+
+chown fame:fame /fame -R
+
+echo "[+] Running webserver"
+exec /fame/env/bin/uwsgi -H /fame/env --uid fame --http :8080 -w webserver --callable app

docker/fame/run.worker.sh

@@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+
+echo "[+] Ensuring presence of temp dir"
+mkdir -p temp && chown fame:fame temp/
+
+if [ -f /run/secrets/ssh_priv_key ]; then
+    echo "[+] Copying SSH private key"
+    mkdir -p conf
+    cp /run/secrets/ssh_priv_key conf/id_rsa
+    chown fame:fame conf -R
+    chmod 600 conf/id_rsa
+fi
+
+if [ -e /var/run/docker.sock ]; then
+    gid="$(stat -c %g /var/run/docker.sock)"
+    echo "[+] Creating docker_fame group with gid $gid and adding user 'fame' to it"
+    groupadd -g $gid docker_fame
+    usermod -aG docker_fame fame
+fi
+
+TIMEOUT=60
+
+echo "[+] Waiting $TIMEOUT seconds for MongoDB to come up"
+python docker/wait-for.py fame-mongo 27017 $TIMEOUT
+if [ "$?" -ne "0" ]; then
+    echo "[X] Could not connect to MongoDB instance - is it up and running?"
+    exit 1
+fi
+
+echo "[+] Waiting $TIMEOUT seconds for web server to come up"
+python docker/wait-for.py fame-web 8080 $TIMEOUT
+if [ "$?" -ne "0" ]; then
+    echo "[X] Could not connect to web server instance - is it up and running?"
+    exit 1
+fi
+
+exec utils/run.sh worker.py -r 5 -c '--uid fame --gid fame'

docker/mongo/Dockerfile

@@ -0,0 +1,3 @@
+FROM mongo
+
+COPY adduser.js /docker-entrypoint-initdb.d/adduser.js

docker/mongo/adduser.js

@@ -0,0 +1,9 @@
+"use fame";
+db.createUser({
+    user: "fame",
+    pwd: "super-secret-password",
+    roles: [
+        { role: "readWrite", db: "fame" },
+        { role: "dbOwner", db: "fame" }
+    ]
+});