Merge pull request #57 from totem/develop

0.5.0 Release

.dockerignore

@@ -1,10 +1,14 @@
 .idea
 tests
 docs
-.git
+.git*
 *.md
 .travis.yml
 dev-requirements.txt
 *.iml
 LICENSE
-local-*
+local-*
+docker-compose.yml
+local-server.py
+local-celery.py
+local-celery-beat.py

Dockerfile

@@ -1,54 +1,78 @@
-FROM totem/python-base:2.7-trusty-b3
+FROM python:2.7.11-alpine
 
-RUN apt-get update --fix-missing && \
-    apt-get install -y gettext && \
-    apt-get clean && rm -rf /var/cache/apt/archives/* /var/lib/apt/lists/*
+ENV DEBIAN_FRONTEND noninteractive
+ENV ETCDCTL_VERSION v2.2.5
+ENV DUMB_INIT_VERSION 1.0.1
+ENV CONFD_VERSION 0.12.0-alpha3
 
-##SSH Key for fleet
-RUN mkdir /root/.ssh && \
-    chmod  500 /root/.ssh && \
-    chown -R root:root /root/.ssh
+# Native dependencies
+RUN apk add --no-cache --update \
+        pcre \
+        bash \
+        gettext \
+        curl \
+        openssl \
+        openssh-client \
 
-#Confd
-ENV CONFD_VERSION 0.11.0
-RUN curl -L https://github.com/kelseyhightower/confd/releases/download/v$CONFD_VERSION/confd-${CONFD_VERSION}-linux-amd64 -o /usr/local/bin/confd && \
-    chmod 555 /usr/local/bin/confd
+    # Etcdctl
+    && curl -L https://github.com/coreos/etcd/releases/download/$ETCDCTL_VERSION/etcd-$ETCDCTL_VERSION-linux-amd64.tar.gz -o /tmp/etcd-$ETCDCTL_VERSION-linux-amd64.tar.gz \
+    && cd /tmp && gzip -dc etcd-$ETCDCTL_VERSION-linux-amd64.tar.gz | tar -xof -  \
+    && cp -f /tmp/etcd-$ETCDCTL_VERSION-linux-amd64/etcdctl /usr/local/bin \
 
-#Etcdctl
-ENV ETCDCTL_VERSION v2.2.5
-RUN curl -L https://github.com/coreos/etcd/releases/download/$ETCDCTL_VERSION/etcd-$ETCDCTL_VERSION-linux-amd64.tar.gz -o /tmp/etcd-$ETCDCTL_VERSION-linux-amd64.tar.gz && \
-    cd /tmp && gzip -dc etcd-$ETCDCTL_VERSION-linux-amd64.tar.gz | tar -xof - && \
-    cp -f /tmp/etcd-$ETCDCTL_VERSION-linux-amd64/etcdctl /usr/local/bin && \
-    rm -rf /tmp/etcd-$ETCDCTL_VERSION-linux-amd64.tar.gz && \
-    rm -rf /tmp/etcd-$ETCDCTL_VERSION-linux-amd64
-
-# Supervisor and App dependencies
-RUN pip install supervisor==3.1.2 supervisor-stdout
-ADD requirements.txt /opt/requirements.txt
-RUN pip install -r /opt/requirements.txt
-
-#Supervisor Config
-RUN mkdir -p /var/log/supervisor
-ADD bin/supervisord-wrapper.sh /usr/sbin/supervisord-wrapper.sh
-RUN chmod +x /usr/sbin/supervisord-wrapper.sh && \
-    ln -sf /etc/supervisor/supervisord.conf /etc/supervisord.conf
-
-#Confd Defaults
-ADD bin/confd-wrapper.sh /usr/sbin/confd-wrapper.sh
-RUN chmod +x /usr/sbin/confd-wrapper.sh
-
-#SSH Keys
-ADD bin/decrypt-ssh-keys.sh /usr/local/bin/
-RUN chmod +x /usr/local/bin/decrypt-ssh-keys.sh
-
-#Etc Config
+    # Confd
+    && curl -L https://github.com/kelseyhightower/confd/releases/download/v$CONFD_VERSION/confd-${CONFD_VERSION}-linux-amd64 -o /usr/local/bin/confd \
+    && chmod 555 /usr/local/bin/confd \
+
+    # Dumb Init
+    && wget -O /usr/bin/dumb-init https://github.com/Yelp/dumb-init/releases/download/v${DUMB_INIT_VERSION}/dumb-init_${DUMB_INIT_VERSION}_amd64 \
+    && chmod +x /usr/bin/dumb-init \
+
+    # SSH Key for fleet
+    && mkdir /root/.ssh  \
+    && chmod  500 /root/.ssh  \
+    && chown -R root:root /root/.ssh \
+
+    # Cleanup
+    && rm -rf /tmp/*
+
+# Application dependencies
+ADD requirements.txt /opt/cluster-deployer/requirements.txt
+RUN apk add --no-cache --update --virtual build-dependencies \
+      musl-dev \
+      linux-headers \
+      build-base \
+      pcre-dev \
+      libffi-dev \
+      openssl-dev \
+
+    # Python dependencies
+    && pip install --ignore-installed  --no-cache-dir \
+      supervisor==3.2.3 \
+      supervisor-stdout  \
+      -r /opt/cluster-deployer/requirements.txt \
+
+    # Supervisor (Post Setup)
+    && mkdir -p /var/log/supervisor \
+    && ln -sf /etc/supervisor/supervisord.conf /etc/supervisord.conf \
+
+    # Cleanup
+    && apk del build-dependencies \
+    && find /usr/local \
+        \( -type d -a -name test -o -name tests \) -exec echo rm -rf '{}' + \
+        -o \( -type f -a -name '*.pyc' -o -name '*.pyo' \) -exec echo rm -f '{}' +
+
+
+# Custom Scripts
+ADD bin/*.sh /usr/sbin/
+RUN chmod +x /usr/sbin/*.sh
+
+# Etc Config
 ADD etc /etc
 
 ADD . /opt/cluster-deployer
-RUN pip install -r /opt/cluster-deployer/requirements.txt
 
 EXPOSE 9000
 
 WORKDIR /opt/cluster-deployer
 
-ENTRYPOINT ["/usr/sbin/supervisord-wrapper.sh"]
+CMD ["/usr/bin/dumb-init", "/usr/sbin/supervisord-wrapper.sh"]

README.md

@@ -105,6 +105,8 @@ sudo docker run -it --rm -h cluster-deployer-${USER} --name cluster-deployer -v
 | MONGODB_PASSWORD | Mongo db password | | |
 | MONGODB_DB | Mongo db database | totem-{TOTEM_ENV} | totem-{TOTEM_ENV} |
 | LOG_ROOT_LEVEL | Root log level | INFO | INFO |
+| DISCOVER_RABBIMQ | Use static discovery for rabbitmq (true / false) | false | false |
+| DISCOVER_MONGO | Use static discovery for mongo (true / false) | false | false |
 
 ## Coding Standards and Guidelines
 

bin/confd-wrapper.sh

@@ -1,8 +1,8 @@
-#!/bin/bash -le
+#!/bin/sh -e
 
-export ETCD_URL="$ETCD_HOST:$ETCD_PORT"
-export ETCDCTL="etcdctl --peers $ETCD_URL"
+. /usr/sbin/deployer-env.sh
 
+sed -i -e "s|http[:]//172.17.42.1[:]4001|$ETCD_URL|g" -e "s|/totem|$ETCD_TOTEM_BASE|g" /etc/confd/confd.toml
 
-sed -i -e "s/172.17.42.1[:]4001/$ETCD_URL/g" -e "s|/totem|$ETCD_TOTEM_BASE|g" /etc/confd/confd.toml
+$ETCDCTL ls $ETCD_TOTEM_BASE/ssh  || $ETCDCTL mkdir $ETCD_TOTEM_BASE/ssh
 confd

bin/decrypt-ssh-keys.sh

@@ -1,4 +1,6 @@
-#!/bin/bash -el
+#!/bin/sh -e
+
+. /usr/sbin/deployer-env.sh
 
 cp /root/.ssh/id_rsa.encrypted /root/.ssh/id_rsa.new
 ssh-keygen -P "$SSH_PASSPHRASE" -N '' -p -f /root/.ssh/id_rsa.new

bin/deployer-env.sh

@@ -0,0 +1,45 @@
+#!/bin/sh -e
+
+export HOST_IP="${HOST_IP:-$(/sbin/ip route|awk '/default/ { print $3 }')}"
+export ETCD_HOST="${ETCD_HOST:-$HOST_IP}"
+export ETCD_PORT="${ETCD_PORT:-4001}"
+export ETCD_URL="${ETCD_URL:-http://$ETCD_HOST:$ETCD_PORT}"
+export ETCDCTL="${ETCDCTL:-etcdctl --peers $ETCD_URL}"
+export TOTEM_ENV="${TOTEM_ENV:-local}"
+export ETCD_TOTEM_BASE="${ETCD_TOTEM_BASE:-/totem}"
+export ETCD_YODA_BASE="${ETCD_YODA_BASE:-/yoda}"
+export CELERY_GEVENT_EXECUTORS="${CELERY_GEVENT_EXECUTORS:-1}"
+export CELERY_GEVENT_CONCURRENCY="${CELERY_GEVENT_CONCURRENCY:-50}"
+export CELERY_PREFORK_CONCURRENCY="${CELERY_PREFORK_CONCURRENCY:-2}"
+export API_EXECUTORS="${API_EXECUTORS:-2}"
+export C_FORCE_ROOT="${C_FORCE_ROOT:-true}"
+export SSH_HOST_KEY="${SSH_HOST_KEY:-/root/.ssh/id_rsa}"
+export AMQP_HOST="${AMQP_HOST:-$HOST_IP}"
+export AMQP_PORT="${AMQP_PORT:-5672}"
+export AMQP_USERNAME="${AMQP_USERNAME:-guest}"
+export AMQP_PASSWORD="${AMQP_PASSWORD:-guest}"
+export MONGODB_USERNAME="${MONGODB_USERNAME:-}"
+export MONGODB_PASSWORD="${MONGODB_PASSWORD:-}"
+export MONGODB_HOST="${MONGODB_HOST:-$HOST_IP}"
+export MONGODB_PORT="${MONGODB_PORT:-27017}"
+export MONGODB_DB="${MONGODB_DB}"
+export MONGODB_AUTH_DB="${MONGODB_AUTH_DB}"
+export BROKER_URL="${BROKER_URL}"
+export SSH_PASSPHRASE="${SSH_PASSPHRASE}"
+export GITHUB_TOKEN="${GITHUB_TOKEN}"
+export FLEET_HOST="${FLEET_HOST:-$HOST_IP}"
+export CLUSTER_NAME="${CLUSTER_NAME:-local}"
+export QUAY_ORGANIZATION="${QUAY_ORGANIZATION:-totem}"
+export QUAY_PREFIX="${QUAY_PREFIX:-totem-}"
+export C_FORCE_ROOT=true
+export ENCRYPTION_PASSPHRASE="${ENCRYPTION_PASSPHRASE:-changeit}"
+export ENCRYPTION_S3_BUCKET="${ENCRYPTION_S3_BUCKET:-not-set}"
+export ENCRYPTION_STORE="${ENCRYPTION_PROVIDER:-s3}"
+export HIPCHAT_TOKEN="${HIPCHAT_TOKEN}"
+export HIPCHAT_ENABLED="${HIPCHAT_ENABLED:-false}"
+export HIPCHAT_TOKEN="${HIPCHAT_TOKEN}"
+export HIPCHAT_ROOM="${HIPCHAT_ROOM:-not-set}"
+export GITHUB_NOTIFICATION_ENABLED="${GITHUB_NOTIFICATION_ENABLED:-false}"
+export BASE_URL="${BASE_URL:-http://$HOST_IP:9000}"
+export LOG_IDENTIFIER="${LOG_IDENTIFIER:-cluster-deployer}"
+export LOG_ROOT_LEVEL="${LOG_ROOT_LEVEL}"

bin/supervisord-wrapper.sh

@@ -1,60 +1,35 @@
-#!/bin/bash -le
+#!/bin/sh -e
 
-HOST_IP="${HOST_IP:-$(/sbin/ip route|awk '/default/ { print $3 }')}"
+. /usr/sbin/deployer-env.sh
 
-cat <<END>> /etc/profile.d/cluster-deployer-env.sh
-export ETCD_HOST='${ETCD_HOST:-$HOST_IP}'
-export ETCD_PORT='${ETCD_PORT:-4001}'
-export ETCD_TOTEM_BASE='${ETCD_TOTEM_BASE:-/totem}'
-export ETCD_YODA_BASE='${ETCD_YODA_BASE:-/yoda}'
-export CELERY_GEVENT_EXECUTORS='${CELERY_GEVENT_EXECUTORS:-1}'
-export CELERY_GEVENT_CONCURRENCY='${CELERY_GEVENT_CONCURRENCY:-50}'
-export CELERY_PREFORK_CONCURRENCY='${CELERY_PREFORK_CONCURRENCY:-2}'
-export API_EXECUTORS='${API_EXECUTORS:-2}'
-export C_FORCE_ROOT='${C_FORCE_ROOT:-true}'
-export SSH_HOST_KEY='${SSH_HOST_KEY:-/root/.ssh/id_rsa}'
-export AMQP_HOST='${AMQP_HOST:-$HOST_IP}'
-export AMQP_PORT='${AMQP_PORT:-5672}'
-export AMQP_USERNAME='${AMQP_USERNAME:-guest}'
-export AMQP_PASSWORD='${AMQP_PASSWORD:-guest}'
-export MONGODB_USERNAME='${MONGODB_USERNAME:-}'
-export MONGODB_PASSWORD='${MONGODB_PASSWORD:-}'
-export MONGODB_HOST='${MONGODB_HOST:-$HOST_IP}'
-export MONGODB_PORT='${MONGODB_PORT:-27017}'
-export MONGODB_DB='${MONGODB_DB}'
-export MONGODB_AUTH_DB='${MONGODB_AUTH_DB}'
-export BROKER_URL='${BROKER_URL}'
-export SSH_PASSPHRASE='${SSH_PASSPHRASE}'
-export GITHUB_TOKEN='${GITHUB_TOKEN}'
-export FLEET_HOST='${FLEET_HOST:-$HOST_IP}'
-export CLUSTER_NAME='${CLUSTER_NAME:-local}'
-export TOTEM_ENV='${TOTEM_ENV:-local}'
-export QUAY_ORGANIZATION='${QUAY_ORGANIZATION:-totem}'
-export QUAY_PREFIX='${QUAY_PREFIX:-totem-}'
-export C_FORCE_ROOT=true
-export ENCRYPTION_PASSPHRASE='${ENCRYPTION_PASSPHRASE:-changeit}'
-export ENCRYPTION_S3_BUCKET='${ENCRYPTION_S3_BUCKET:-not-set}'
-export ENCRYPTION_STORE='${ENCRYPTION_PROVIDER:-s3}'
-export HIPCHAT_TOKEN='${HIPCHAT_TOKEN}'
-export HIPCHAT_ENABLED='${HIPCHAT_ENABLED:-false}'
-export HIPCHAT_TOKEN='${HIPCHAT_TOKEN}'
-export HIPCHAT_ROOM='${HIPCHAT_ROOM:-not-set}'
-export GITHUB_NOTIFICATION_ENABLED='${GITHUB_NOTIFICATION_ENABLED:-false}'
-export BASE_URL='${BASE_URL:-http://$HOST_IP:9000}'
-export LOG_IDENTIFIER='${LOG_IDENTIFIER:-cluster-deployer}'
-export LOG_ROOT_LEVEL='${LOG_ROOT_LEVEL}'
-END
+until $ETCDCTL cluster-health; do
+  >&2 echo "Etcdctl cluster not healthy - sleeping"
+  sleep 5
+done
 
-echo "Registering shutdown hook prior to shutdown"
-function shutdown() {
-    set +e;
-    echo Stopping supervisor;
-    kill -s SIGTERM "$(cat /var/run/supervisord.pid)";
-    exit 0
-}
-trap 'shutdown' EXIT
+# Cleanup for local testing
+if ls celerybeat* 2>/dev/null; then
+  rm celerybeat*
+fi
 
-/bin/bash -le -c " envsubst  < /etc/supervisor/conf.d/supervisord.conf.template  > /etc/supervisor/conf.d/supervisord.conf; \
-                    /usr/local/bin/supervisord -c /etc/supervisor/supervisord.conf"
+if [ "$DISCOVER_RABBITMQ" == "true" ]; then
+  export AMQP_HOST="$($ETCDCTL ls $ETCD_TOTEM_BASE/rabbitmq/nodes | xargs -n 1  $ETCDCTL get | xargs echo -n | tr ' ' ',')"
+  until [ ! -z "$AMQP_HOST" ]; do
+    >&2 echo "Rabbitmq could not be discovered - sleeping"
+    sleep 10
+    export AMQP_HOST="$($ETCDCTL ls $ETCD_TOTEM_BASE/rabbitmq/nodes | xargs -n 1  $ETCDCTL get | xargs echo -n | tr ' ' ',')"
+  done
+fi
 
+if [ "$DISCOVER_MONGO" == "true" ]; then
+  export MONGODB_SERVERS="$($ETCDCTL ls $ETCD_TOTEM_BASE/mongo/nodes | xargs -n 1  $ETCDCTL get | xargs echo -n | tr ' ' ',')"
+  until [ ! -z "$MONGODB_SERVERS" ]; do
+    >&2 echo "Mongo servers could not be discovered - sleeping"
+    sleep 10
+    export MONGODB_SERVERS="$($ETCDCTL ls $ETCD_TOTEM_BASE/mongo/nodes | xargs -n 1  $ETCDCTL get | xargs echo -n | tr ' ' ',')"
+  done
+fi
+
+envsubst  < /etc/supervisor/conf.d/supervisord.conf.template  > /etc/supervisor/conf.d/supervisord.conf
+/usr/local/bin/supervisord -c /etc/supervisor/supervisord.conf
 

conf/appconfig.py

@@ -279,12 +279,15 @@
 MONGODB_PASSWORD = os.getenv('MONGODB_PASSWORD', '')
 MONGODB_HOST = os.getenv('MONGODB_HOST', '127.0.0.1')
 MONGODB_PORT = int(os.getenv('MONGODB_PORT', '27017'))
+MONGODB_SERVERS = os.getenv('MONGODB_SERVERS') \
+                  or '{}:{}'.format(MONGODB_HOST, MONGODB_PORT)
 MONGODB_DB = os.getenv('MONGODB_DB') or 'totem-{}'.format(TOTEM_ENV)
 MONGODB_AUTH_DB = os.getenv('MONGODB_AUTH_DB') or 'admin'
 MONGODB_AUTH = '{0}:{1}@'.format(MONGODB_USERNAME, MONGODB_PASSWORD) \
     if MONGODB_USERNAME else ''
-MONGODB_DEFAULT_URL = 'mongodb://{0}{1}:{2}/{3}'.format(
-    MONGODB_AUTH, MONGODB_HOST, MONGODB_PORT, MONGODB_AUTH_DB)
+
+MONGODB_DEFAULT_URL = 'mongodb://{0}{1}/{2}'.format(
+    MONGODB_AUTH, MONGODB_SERVERS, MONGODB_AUTH_DB)
 MONGODB_URL = os.getenv('MONGODB_URL') or MONGODB_DEFAULT_URL
 MONGODB_DEPLOYMENT_COLLECTION = os.getenv('MONGODB_DEPLOYMENT_COLLECTION') or \
     'deployments'

conf/celeryconfig.py

@@ -17,8 +17,11 @@
 AMQP_PASSWORD = os.getenv('AMQP_PASSWORD', 'guest')
 AMQP_HOST = os.getenv('AMQP_HOST', 'localhost')
 AMQP_PORT = int(os.getenv('AMQP_PORT', '5672'))
-DEFAULT_BROKER_URL = 'amqp://%s:%s@%s:%s' % (AMQP_USERNAME, AMQP_PASSWORD,
-                                             AMQP_HOST, AMQP_PORT)
+DEFAULT_BROKER_URL = [
+    'amqp://{}:{}@{}:{}'.format(
+        AMQP_USERNAME, AMQP_PASSWORD, host.strip(), AMQP_PORT
+    ) for host in AMQP_HOST.split(',') if host.strip()
+]
 BROKER_URL = os.getenv('BROKER_URL') or DEFAULT_BROKER_URL
 
 BROKER_CONNECTION_TIMEOUT = int(os.getenv('BROKER_CONNECTION_TIMEOUT', '20'))
@@ -59,6 +62,7 @@
 }
 
 # Backend Settings
+print('Mongo URL %s', MONGODB_URL)
 CELERY_RESULT_BACKEND = MONGODB_URL
 CELERY_MONGODB_BACKEND_SETTINGS = {
     'database': MONGODB_DB,
@@ -112,12 +116,12 @@
     },
     'deployer.tasks.deployment.sync_promoted_units': {
         'task': 'deployer.tasks.deployment.sync_promoted_units',
-        'schedule': crontab(minute='*/5'),
+        'schedule': crontab(minute='*/15'),
         'args': ()
     },
     'deployer.tasks.deployment.sync_promoted_upstreams': {
         'task': 'deployer.tasks.deployment.sync_promoted_upstreams',
-        'schedule': crontab(minute='*/2'),
+        'schedule': crontab(minute='*/15'),
         'args': ()
     }
 }

deployer/__init__.py

@@ -3,7 +3,7 @@
 from celery.signals import setup_logging
 
 
-__version__ = '0.4.8'
+__version__ = '0.5.0'
 __author__ = 'sukrit'
 
 deployer.logger.init_logging()