docker-compose.yml

###### MODELS CONTAINERS #############
models0: # PORT 8020
    image: tp33/django
    external_links:
      -  mysql:db
    volumes:
      - ./models/:/app
    ports:
      - "8020:8000"
    command: bash -c "pip install -r requirements.txt && python manage.py makemigrations && python manage.py migrate && python manage.py loaddata db.json && mod_wsgi-express start-server --reload-on-changes ./models/wsgi.py"
    container_name: models0

models1: # PORT 8021
    image: tp33/django
    external_links:
      -  mysql:db
    volumes:
      - ./models/:/app
    ports:
      - "8021:8000"
    command: bash -c "sleep 35 && pip install -r requirements.txt && python manage.py loaddata db.json && mod_wsgi-express start-server --reload-on-changes ./models/wsgi.py"
    container_name: models1
    # sleep to wait for models0 to spin up and migrate to db
    # could use `wait-for-it` instead...

models2: # PORT 8022
    image: tp33/django
    external_links:
      -  mysql:db
    volumes:
      - ./models/:/app
    ports:
      - "8022:8000"
    command: bash -c "sleep 35 && pip install -r requirements.txt && python manage.py loaddata db.json && mod_wsgi-express start-server --reload-on-changes ./models/wsgi.py"
    container_name: models2
    # sleep to wait for models0 to spin up and migrate to db
    # could use `wait-for-it` instead...

###### EXP CONTAINERS #############
exp0: # PORT 8010
    image: tp33/django
    links:
      - modelsbal:models-api
      - kafka:kafka
      - es:es
    volumes:
      - ./exp/:/app
    ports:
      - "8010:8000"
    command: bash -c "pip install -r requirements.txt && mod_wsgi-express start-server --reload-on-changes ./exp/wsgi.py"
    container_name: exp0

exp1: # PORT 8011
    image: tp33/django
    links:
      - modelsbal:models-api
      - kafka:kafka
      - es:es
    volumes:
      - ./exp/:/app
    ports:
      - "8011:8000"
    command: bash -c "pip install -r requirements.txt && mod_wsgi-express start-server --reload-on-changes ./exp/wsgi.py"
    container_name: exp1

exp2: # PORT 8012
    image: tp33/django
    links:
      - modelsbal:models-api
      - kafka:kafka
      - es:es
    volumes:
      - ./exp/:/app
    ports:
      - "8012:8000"
    command: bash -c "pip install -r requirements.txt && mod_wsgi-express start-server --reload-on-changes ./exp/wsgi.py"
    container_name: exp2


###### WEB CONTAINERS #############
web0: # PORT 8030
    image: tp33/django
    links:
      -  expbal:exp-api
      -  selenium
    volumes:
      - ./web/:/app
    ports:
      - "8030:8000"
    command: bash -c "pip install -r requirements.txt && python manage.py collectstatic --noinput && mod_wsgi-express start-server --reload-on-changes ./web/wsgi.py"
    container_name: web0
    environment:
      - SELENIUM_HOST=http://selenium:4444/wd/hub
      - TEST_SELENIUM='yest'

web1: # PORT 8031
    image: tp33/django
    links:
      -  expbal:exp-api
      -  selenium
    volumes:
      - ./web/:/app
    ports:
      - "8031:8000"
    command: bash -c "pip install -r requirements.txt && python manage.py collectstatic --noinput && mod_wsgi-express start-server --reload-on-changes ./web/wsgi.py"
    container_name: web1
    environment:
      - SELENIUM_HOST=http://selenium:4444/wd/hub
      - TEST_SELENIUM='yest'

web2: # PORT 8032
    image: tp33/django
    links:
      -  expbal:exp-api
      -  selenium
    volumes:
      - ./web/:/app
    ports:
      - "8032:8000"
    command: bash -c "pip install -r requirements.txt && python manage.py collectstatic --noinput && mod_wsgi-express start-server --reload-on-changes ./web/wsgi.py"
    container_name: web2
    environment:
      - SELENIUM_HOST=http://selenium:4444/wd/hub
      - TEST_SELENIUM='yest'

############## LOAD BALANCERS ###############
# yes, docker swarm would probably make more sense here, 
# in conjunction with compose's scale features.
# configuring HAProxy is a great learning experience, though :-)
loadwatcher: # logs all load balancing activity to stdout! 
             # ...and really anything else. not a great privacy-preserving solution.
   image: tp33/django
   container_name: loadwatcher
   command: bash -c "tail -f /var/log/syslog"
   volumes:
     - /dev/log:/dev/log
     - /var/log:/var/log

webbal:
   build: ./load/web
   links:
      -  web0:web0
      -  web1:web1
      -  web2:web2
   ports:
      -  "8000:8000"
      -  "1936:1936" # watch stats
   container_name: webbal
   volumes:
     - /dev/log:/dev/log

expbal:
   build: ./load/exp
   links:
      -  exp0:exp0
      -  exp1:exp1
      -  exp2:exp2
   ports:
      -  "8001:8000"
      -  "1937:1937" # watch stats
   container_name: expbal
   volumes:
     - /dev/log:/dev/log

modelsbal:
   build: ./load/models
   links:
      -  models0:models0
      -  models1:models1
      -  models2:models2
   ports:
      -  "8002:8000"
      -  "1938:1938" # watch stats
   container_name: modelsbal
   volumes:
     - /dev/log:/dev/log


###### SEARCH CONTAINERS ########
kafka:
   image: spotify/kafka
   container_name: kafka
   environment:
      ADVERTISED_HOST: kafka
      ADVERTISED_PORT: 9092
   hostname: kafka

es:
   image: elasticsearch:2.0
   container_name: es
   ports:
      - "9200:9200"

batch:
   image: tp33/django
   container_name: batch
   links:
      - kafka:kafka
      - es:es
   command: bash -c "sleep 75 && echo 'kafka queueing' && python es_index.py"
   volumes:
      - ./batch/:/app

# Integration/end-to-end testing
selenium:
    image: selenium/standalone-chrome:2.48.2
    container_name: selenium
    ports:
      -  "5900:5900"


###### SPARK CONTAINERS ######
spark-master:
  image: gettyimages/spark
  command: bin/spark-class org.apache.spark.deploy.master.Master -h spark-master
  container_name: spark-master
  hostname: spark-master
  environment:
    MASTER: spark://spark-master:7077
    SPARK_CONF_DIR: /conf
  expose:
    - 7001
    - 7002
    - 7003
    - 7004
    - 7005
    - 7006
    - 7077
    - 6066
  ports:
    - 4040:4040
    - 6066:6066
    - 7077:7077
    - 8080:8080
  volumes:
    - ./recs:/tmp/data

spark-worker:
  image: gettyimages/spark
  command: bin/spark-class org.apache.spark.deploy.worker.Worker spark://spark-master:7077
  container_name: spark-worker
  hostname: spark-worker
  environment:
    SPARK_CONF_DIR: /conf
    SPARK_WORKER_CORES: 2
    SPARK_WORKER_MEMORY: 512m
    SPARK_WORKER_PORT: 8881
    SPARK_WORKER_WEBUI_PORT: 8081
  links:
    - spark-master
  expose:
    - 7012
    - 7013
    - 7014
    - 7015
    - 7016
    - 8881
  ports:
    - 8081:8081
  volumes:
    - ./recs:/tmp/data