Preloading Modules / Dockerization of FAME / Dockerception (combined PR of all changes, do not merge!)

Dockerfile.base

@@ -0,0 +1,19 @@
+FROM python:2-slim-stretch
+
+RUN apt-get update && apt-get install -y python python-dev python-pip git && \
+    useradd -s /bin/false -U fame -m && \
+    pip install --no-cache-dir virtualenv && \
+    rm -rf /var/lib/apt/lists/*
+
+COPY . /fame
+COPY docker/fame/docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh
+
+WORKDIR /fame
+RUN utils/run.sh -m pip install --no-cache-dir -r requirements.txt
+
+ENV FAME_DOCKER=1
+
+VOLUME [ "/fame/conf" ]
+
+ENTRYPOINT [ "docker-entrypoint.sh" ]
+CMD [ "/bin/bash" ]

Dockerfile.web

@@ -0,0 +1,13 @@
+FROM fame-base
+
+ENV FAME_WORKER=0
+
+COPY docker/fame/run.web.sh /fame/run.sh
+
+RUN utils/run.sh -m pip install --no-cache-dir -r requirements-web.txt
+
+EXPOSE 8080
+
+VOLUME [ "/fame/fame/modules", "/fame/storage", "/fame/web/static/img/avatars" ]
+
+CMD [ "/fame/run.sh" ]

Dockerfile.worker

@@ -0,0 +1,9 @@
+FROM fame-base
+
+ENV FAME_WORKER=1
+
+COPY docker/fame/run.worker.sh /fame/run.sh
+
+RUN utils/run.sh -m pip install --no-cache-dir -r requirements-worker.txt
+
+CMD [ "/fame/run.sh" ]

Makefile

@@ -0,0 +1,15 @@
+.DEFAULT_GOAL := all
+
+mongo:
+	docker build --pull -t fame-mongo docker/mongo/
+
+base:
+	docker build --pull -t fame-base -f Dockerfile.base .
+
+web: base
+	docker build -t fame-web -f Dockerfile.web .
+
+worker: base
+	docker build -t fame-worker -f Dockerfile.worker .
+
+all: mongo base web worker

agent/agent.py

@@ -115,10 +115,11 @@ def run_each_with_type(self, target, target_type):
                 return self.each_with_type(target, target_type)
             except IsolatedExceptions.ModuleExecutionError, e:
                 self.log("error", "Could not run on %s: %s" % (target, e))
+                self.log("debug", traceback.format_exc())
                 return False
-            except:
-                tb = traceback.format_exc()
-                self.log("error", "Could not run on %s.\n %s" % (target, tb))
+            except Exception:
+                self.log("error", "Could not run on %s.\n" % (target,))
+                self.log("debug", traceback.format_exc())
                 return False
 
 
@@ -134,6 +135,7 @@ def fake_module(path, klass):
 
     sys.modules[path] = klass
 
+
 fake_module('fame.core.module', IsolatedModule)
 fake_module('fame.common.exceptions', IsolatedExceptions)
 
@@ -169,7 +171,7 @@ def set_module(self, name, config):
             module = import_module('module')
 
         for _, obj in inspect.getmembers(module, inspect.isclass):
-            if obj.name and obj.name == name:
+            if hasattr(obj, "name") and obj.name == name:
                 self.queue = Queue()
                 self.module = obj()
 

celeryconfig.py

@@ -21,6 +21,17 @@
 
 def connect_to_db(**kwargs):
     fame_init()
-
+    if fame_config.is_worker:
+        from fame.core.user import User
+        worker_user = User.get(email="worker@fame")
+        if worker_user:
+            fame_config.api_key = worker_user['api_key']
+
+
+if fame_config.is_worker:
+    try:
+        from celeryconfig_worker import *
+    except ImportError:
+        pass
 
 signals.worker_process_init.connect(connect_to_db)

celeryconfig_worker.py

@@ -0,0 +1,3 @@
+
+
+CELERY_IMPORTS = ('fame.worker.analysis', 'fame.worker.repository')

docker-compose.yml

@@ -0,0 +1,84 @@
+version: '3.4'
+
+networks:
+  fame_internal:
+    internal: true
+
+  gateway:
+
+  traefik_default:
+    external: 
+      name: traefik_default
+
+x-mongo-env: &mongo_env
+  MONGO_HOST: "fame-mongo"
+  MONGO_PORT: "27017"
+  MONGO_DB: "fame"
+  MONGO_USERNAME: "fame" 
+  MONGO_PASSWORD: "super-secret-password"
+
+secrets:
+  ssh_priv_key:
+    file: ./ssh/id_rsa
+
+services:
+  fame-mongo:
+    image: fame-mongo
+    environment:
+      MONGO_INITDB_DATABASE: "fame"
+    container_name: fame-mongo
+    restart: unless-stopped
+    volumes:
+      - /opt/fame-mongo:/data/db:z
+    command: --auth
+    networks:
+      - "fame_internal"
+
+  fame-web:
+    image: fame-web
+    container_name: fame-web
+    depends_on: 
+      - fame-mongo
+    environment:
+      <<: *mongo_env
+      FAME_INSTALL_COMMUNITY_REPO: "1"
+      FAME_URL: "http://localhost/"
+      FAME_ADMIN_FULLNAME: "The Admin"
+      FAME_ADMIN_EMAIL: "[email protected]"
+      FAME_ADMIN_GROUPS: "cert"
+      FAME_ADMIN_DEFAULT_SHARING: ""
+      FAME_ADMIN_PERMISSIONS: ""
+      FAME_ADMIN_PASSWORD: "tercespot"
+      FAME_PUBLIC_KEY: "ssh-rsa [...]"
+      FAME_SECRET_KEY: ""  # cat /dev/urandom | head -c 256 | sha512sum
+    volumes:
+      - /opt/fame-modules:/fame/fame/modules:z
+      - /opt/fame-storage:/fame/storage:z
+      - /opt/fame-avatars:/fame/web/static/img/avatars:z
+    labels:
+      - "traefik.enable=true"
+      - "traefik.docker.network=traefik_default"
+      - "traefik.port=8080"
+      - "traefik.frontend.rule=Host: fame.example.com"
+    networks:
+      - "fame_internal"
+      - "traefik_default"
+    # ports:
+    #  - "80:8080"
+    restart: unless-stopped
+    hostname: fame-web
+
+  fame-worker:
+    image: fame-worker
+    container_name: fame-worker
+    depends_on: 
+      - fame-mongo
+      - fame-web
+    environment:
+      <<: *mongo_env
+      FAME_URL: "http://fame-web:8080/"
+    networks:
+      - "fame_internal"
+      - "gateway"
+    secrets:
+      - ssh_priv_key

docker/fame/docker-entrypoint.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+
+echo "[+] Ensuring empty __init__.py in modules directory"
+touch /fame/fame/modules/__init__.py
+
+echo "[+] Adjusting permissions"
+chown fame:fame /fame -R
+
+exec "$@"

docker/fame/run.web.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+
+utils/run.sh utils/install_docker.py
+
+echo "[+] Ensuring presence of uwsgi"
+utils/run.sh -m pip install uwsgi > /dev/null
+
+TIMEOUT=60
+
+echo "[+] Waiting $TIMEOUT seconds for MongoDB to come up"
+python docker/wait-for.py fame-mongo 27017 $TIMEOUT
+if [ "$?" -ne "0" ]; then
+    echo "[X] Could not connect to MongoDB instance - is it up and running?"
+    exit 1
+fi
+
+echo "[+] Running webserver"
+chown fame:fame /fame -R
+exec /fame/env/bin/uwsgi -H /fame/env --uid fame --http :8080 -w webserver --callable app

docker/fame/run.worker.sh

@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+
+echo "[+] Setting up git user and email"
+git config --global user.name "FAME Web"
+git config --global user.email "[email protected]"
+
+echo "[+] Ensuring presence of temp dir"
+mkdir -p temp && chown fame:fame temp/
+
+if [ -f /run/secrets/ssh_priv_key ]; then
+    echo "[+] Copying SSH private key"
+    mkdir -p config
+    cp /run/secrets/ssh_priv_key conf/id_rsa
+    chown fame:fame conf -R
+    chmod 600 conf/id_rsa
+fi
+
+TIMEOUT=60
+
+echo "[+] Waiting $TIMEOUT seconds for MongoDB to come up"
+python docker/wait-for.py fame-mongo 27017 $TIMEOUT
+if [ "$?" -ne "0" ]; then
+    echo "[X] Could not connect to MongoDB instance - is it up and running?"
+    exit 1
+fi
+
+echo "[+] Waiting $TIMEOUT seconds for web server to come up"
+python docker/wait-for.py fame-web 8080 $TIMEOUT
+if [ "$?" -ne "0" ]; then
+    echo "[X] Could not connect to web server instance - is it up and running?"
+    exit 1
+fi
+
+exec utils/run.sh worker.py -r 5 -c -- '--uid fame --gid fame'

docker/mongo/Dockerfile

@@ -0,0 +1,3 @@
+FROM mongo
+
+COPY adduser.js /docker-entrypoint-initdb.d/adduser.js

docker/mongo/adduser.js

@@ -0,0 +1,9 @@
+"use fame";
+db.createUser({
+    user: "fame",
+    pwd: "super-secret-password",
+    roles: [
+        { role: "readWrite", db: "fame" },
+        { role: "dbOwner", db: "fame" }
+    ]
+});

docker/wait-for.py

@@ -0,0 +1,58 @@
+#!/usr/bin/env python2
+"""
+Taken from: http://code.activestate.com/recipes/576655-wait-for-network-service-to-appear/
+"""
+import errno
+import socket
+import sys
+
+
+def wait_net_service(server, port, timeout=None):
+    """ Wait for network service to appear
+        @param timeout: in seconds, if None or 0 wait forever
+        @return: True of False, if timeout is None may return only True or
+                 throw unhandled network exception
+    """
+    s = socket.socket()
+    if timeout:
+        from time import time as now
+        # time module is needed to calc timeout shared between two exceptions
+        end = now() + timeout
+
+    while True:
+        try:
+            if timeout:
+                next_timeout = end - now()
+                if next_timeout < 0:
+                    return False
+                else:
+                    s.settimeout(next_timeout)
+
+            s.connect((server, port))
+
+        except socket.timeout:
+            # this exception occurs only if timeout is set
+            if timeout:
+                return False
+
+        except socket.error:
+            # just ignore anything else until we run into timeout
+            pass
+        else:
+            s.close()
+            return True
+
+
+if __name__ == "__main__":
+    if len(sys.argv) not in [3, 4]:
+        print "Usage: %s <host> <port> [<timeout>]" % (sys.argv[0])
+        sys.exit(1)
+
+    timeout = 60
+    if len(sys.argv) == 4:
+        timeout = int(sys.argv[3])
+
+    if wait_net_service(sys.argv[1], int(sys.argv[2]), timeout):
+        sys.exit(0)
+    else:
+        sys.exit(1)

docs/concept.rst

@@ -59,6 +59,7 @@ FAME relies on modules to add functionality. Modules are actually Python classes
 
 Several kind of modules can be created:
 
+* ``PreloadingModule``: these modules can be used to preload the information provided to FAME for analysis. This, for example, includes downloading a sample file when only a hash could was provided to FAME.
 * ``ProcessingModule``: this is where FAME's magic is. A ``ProcessingModule`` should define some automated analysis that can be performed on some types of files / analysis information.
 * ``ReportingModule``: this kind of module enables reporting options, such as send analysis results by email, or post a Slack notification when the analysis is finished.
 * ``ThreatIntelligenceModule``: this kind of modules acts on IOCs. a ``ThreatIntelligenceModule`` has two roles:
@@ -79,8 +80,8 @@ FAME relies on three components:
 
 .. image:: /images/concept-architecture.png
 
-Components can all be on the same server, or split across multiple servers.
+Components can all be on the same server, or split across multiple servers. A dockerized version of FAME is also available.
 
 The web server is where antivirus modules and threat intelligence modules are executed.
 
-Everything else (processing modules, reporting modules and intelligence modules lookups) is executed on workers.
+Everything else (preloading modules, processing modules, reporting modules and intelligence modules lookups) is executed on workers.