finished load balancers and syslogging

havron · Dec 2, 2016 · 570cd53 · 570cd53
1 parent 176e566
commit 570cd53
Show file tree

Hide file tree

Showing 10 changed files with 107 additions and 21 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -57,4 +57,4 @@ after_script:
   - docker stop mysql
   - docker rm `docker ps -aq`
   - docker rmi `docker images -q`
-  - rm -rf *
+  - sudo rm -rf *
diff --git a/Makefile b/Makefile
@@ -2,6 +2,7 @@
 MSG=small edit
 dev:
 	sudo docker-compose down
+	sudo docker-compose build
 	sudo docker-compose up
 
 github:

diff --git a/README.md b/README.md
@@ -80,9 +80,9 @@ targets of the
 ### HAProxy load balancers (round-robin)
 Load balancers for each app tier powered by [HAProxy](http://www.haproxy.org/)'s
 [docker build](http://hub.docker.com/_/haproxy/). Cookie-based policies 
-(e.g. return users to same servers for caching purposes) is currently 
+(e.g. return users to same servers for caching purposes) are currently 
 not enabled; load balancing is purely round-robin style. 
 In actual production, Docker Swarm would (should) be
-used, and could leverage `docker-compose`'s `scale` feature (currently
-not used, app servers are fully enumerated and handled by respective load
+used, and could leverage `docker-compose`'s `scale` feature (which is 
+currently not used, web/api servers are fully enumerated and handled by respective load
 balancers).
diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml
@@ -1,3 +1,4 @@
+# not entirely sure if this builds successfully in production...? exposing ports is tricky with HAProxy
 ###### MODELS CONTAINERS #############
 models0: # PORT 8020
     image: tp33/django
@@ -69,6 +70,8 @@ web0: # PORT 8030
       -  selenium
     volumes:
       - ./web/:/app
+    ports:
+      - "8030:8000"
     command: bash -c "pip install -r requirements.txt && python manage.py collectstatic --noinput && mod_wsgi-express start-server --reload-on-changes ./web/wsgi.py && python manage.py test"
     container_name: web0
     environment:
@@ -82,6 +85,8 @@ web1: # PORT 8031
       -  selenium
     volumes:
       - ./web/:/app
+    ports:
+      - "8031:8000"
     command: bash -c "pip install -r requirements.txt && python manage.py collectstatic --noinput && mod_wsgi-express start-server --reload-on-changes ./web/wsgi.py && python manage.py test"
     container_name: web1
     environment:
@@ -95,13 +100,23 @@ web2: # PORT 8032
       -  selenium
     volumes:
       - ./web/:/app
+    ports:
+      - "8032:8000"
     command: bash -c "pip install -r requirements.txt && python manage.py collectstatic --noinput && mod_wsgi-express start-server --reload-on-changes ./web/wsgi.py && python manage.py test"
     container_name: web2
     environment:
       - SELENIUM_HOST=http://selenium:4444/wd/hub
       - TEST_SELENIUM='yest'
 
 ############## LOAD BALANCERS ###############
+loadwatcher: # logs all load balancing activity to stdout!
+   image: tp33/django
+   container_name: loadwatcher
+   command: bash -c "tail -f /var/log/syslog"
+   volumes:
+     - /dev/log:/dev/log
+     - /var/log:/var/log
+
 webbal:
    build: ./load/web
    links:
@@ -110,23 +125,34 @@ webbal:
       -  web2:web2
    ports:
       -  "8000:8000"
+      -  "1936:1936" # watch stats
    container_name: webbal
+   volumes:
+     - /dev/log:/dev/log
 
 expbal:
    build: ./load/exp
    links:
       -  exp0:exp0
       -  exp1:exp1
       -  exp2:exp2
+   ports:
+      -  "1937:1937" # watch stats
    container_name: expbal
+   volumes:
+     - /dev/log:/dev/log
 
 modelsbal:
    build: ./load/models
    links:
       -  models0:models0
       -  models1:models1
       -  models2:models2
+   ports:
+      -  "1938:1938" # watch stats
    container_name: modelsbal
+   volumes:
+     - /dev/log:/dev/log
 
 
 ###### SEARCH CONTAINERS ########

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -120,6 +120,14 @@ web2: # PORT 8032
       - TEST_SELENIUM='yest'
 
 ############## LOAD BALANCERS ###############
+loadwatcher: # logs all load balancing activity to stdout!
+   image: tp33/django
+   container_name: loadwatcher
+   command: bash -c "tail -f /var/log/syslog"
+   volumes:
+     - /dev/log:/dev/log
+     - /var/log:/var/log
+
 webbal:
    build: ./load/web
    links:
@@ -128,7 +136,10 @@ webbal:
       -  web2:web2
    ports:
       -  "8000:8000"
+      -  "1936:1936" # watch stats
    container_name: webbal
+   volumes:
+     - /dev/log:/dev/log
 
 expbal:
    build: ./load/exp
@@ -138,7 +149,10 @@ expbal:
       -  exp2:exp2
    ports:
       -  "8001:8000"
+      -  "1937:1937" # watch stats
    container_name: expbal
+   volumes:
+     - /dev/log:/dev/log
 
 modelsbal:
    build: ./load/models
@@ -148,7 +162,10 @@ modelsbal:
       -  models2:models2
    ports:
       -  "8002:8000"
+      -  "1938:1938" # watch stats
    container_name: modelsbal
+   volumes:
+     - /dev/log:/dev/log
 
 
 ###### SEARCH CONTAINERS ########

diff --git a/load/README.md b/load/README.md
@@ -1,9 +1,9 @@
-# HAProxy load balancers (round-robin)
+### HAProxy load balancers (round-robin)
 Load balancers for each app tier powered by [HAProxy](http://www.haproxy.org/)'s
 [docker build](http://hub.docker.com/_/haproxy/). Cookie-based policies 
-(e.g. return users to same servers for caching purposes) is currently 
+(e.g. return users to same servers for caching purposes) are currently 
 not enabled; load balancing is purely round-robin style. 
 In actual production, Docker Swarm would (should) be
-used, and could leverage `docker-compose`'s `scale` feature (currently
-not used, app servers are fully enumerated and handled by respective load
+used, and could leverage `docker-compose`'s `scale` feature (which is 
+currently not used, web/api servers are fully enumerated and handled by respective load
 balancers).
diff --git a/load/exp/haproxy.cfg b/load/exp/haproxy.cfg
@@ -1,17 +1,31 @@
 global
     maxconn 2048
+    log /dev/log local2
+    log-send-hostname
 
 defaults
+    log global
     mode http
+    option httplog
+    option  dontlognull
     timeout connect 5000ms
     timeout client 50000ms
     timeout server 50000ms
 
-frontend http-in
+frontend http-in 
     bind *:8000
+    capture request header Host len 50
     default_backend servers
 
 backend servers
-    server server1 ${EXP0_PORT_8000_TCP_ADDR}:${EXP0_PORT_8000_TCP_PORT}
-    server server2 ${EXP1_PORT_8000_TCP_ADDR}:${EXP1_PORT_8000_TCP_PORT}
-    server server3 ${EXP2_PORT_8000_TCP_ADDR}:${EXP2_PORT_8000_TCP_PORT}
+    server exp0 ${EXP0_PORT_8000_TCP_ADDR}:${EXP0_PORT_8000_TCP_PORT}
+    server exp1 ${EXP1_PORT_8000_TCP_ADDR}:${EXP1_PORT_8000_TCP_PORT}
+    server exp2 ${EXP2_PORT_8000_TCP_ADDR}:${EXP2_PORT_8000_TCP_PORT}
+
+listen stats
+    bind *:1937
+    mode http
+    stats enable
+    stats uri /
+    stats hide-version
+    stats auth username:password
diff --git a/load/models/haproxy.cfg b/load/models/haproxy.cfg
@@ -1,17 +1,31 @@
 global
     maxconn 2048
+    log /dev/log local2
+    log-send-hostname
 
 defaults
+    log global
     mode http
+    option httplog
+    option  dontlognull
     timeout connect 5000ms
     timeout client 50000ms
     timeout server 50000ms
 
-frontend http-in
+frontend http-in 
     bind *:8000
+    capture request header Host len 50
     default_backend servers
 
 backend servers
-    server server1 ${MODELS0_PORT_8000_TCP_ADDR}:${MODELS0_PORT_8000_TCP_PORT}
-    server server2 ${MODELS1_PORT_8000_TCP_ADDR}:${MODELS1_PORT_8000_TCP_PORT}
-    server server3 ${MODELS2_PORT_8000_TCP_ADDR}:${MODELS2_PORT_8000_TCP_PORT}
+    server models0 ${MODELS0_PORT_8000_TCP_ADDR}:${MODELS0_PORT_8000_TCP_PORT}
+    server models1 ${MODELS1_PORT_8000_TCP_ADDR}:${MODELS1_PORT_8000_TCP_PORT}
+    server models2 ${MODELS2_PORT_8000_TCP_ADDR}:${MODELS2_PORT_8000_TCP_PORT}
+
+listen stats
+    bind *:1938
+    mode http
+    stats enable
+    stats uri /
+    stats hide-version
+    stats auth username:password
diff --git a/load/web/haproxy.cfg b/load/web/haproxy.cfg
@@ -1,17 +1,31 @@
 global
     maxconn 2048
+    log /dev/log local2
+    log-send-hostname
 
 defaults
+    log global
     mode http
+    option httplog
+    option  dontlognull
     timeout connect 5000ms
     timeout client 50000ms
     timeout server 50000ms
 
-frontend http-in
+frontend http-in 
     bind *:8000
+    capture request header Host len 50
     default_backend servers
 
 backend servers
-    server server1 ${WEB0_PORT_8000_TCP_ADDR}:${WEB0_PORT_8000_TCP_PORT}
-    server server2 ${WEB1_PORT_8000_TCP_ADDR}:${WEB1_PORT_8000_TCP_PORT}
-    server server3 ${WEB2_PORT_8000_TCP_ADDR}:${WEB2_PORT_8000_TCP_PORT}
+    server web0 ${WEB0_PORT_8000_TCP_ADDR}:${WEB0_PORT_8000_TCP_PORT}
+    server web1 ${WEB1_PORT_8000_TCP_ADDR}:${WEB1_PORT_8000_TCP_PORT}
+    server web2 ${WEB2_PORT_8000_TCP_ADDR}:${WEB2_PORT_8000_TCP_PORT}
+
+listen stats
+    bind *:1936 
+    mode http
+    stats enable
+    stats uri /
+    stats hide-version
+    stats auth username:password
diff --git a/web/daasapp/templates/base.html b/web/daasapp/templates/base.html
@@ -195,7 +195,7 @@
 						</div>
 					</div>
 					<div class="col-sm-3">
-              <span style="color: green; font-size:1.35em;">Served by {{ web_machine }} {{ exp_machine }} {{ model_machine }}!</span>
+              <span style="color: green; font-size:1.35em;">Pinged {{ web_machine }} {{ exp_machine }} {{ model_machine }}!</span>
 						<div class="search_box pull-right">
              {% comment %}   <form name="qForm"action="/search/" method="get">