Merge pull request #44 from totem/develop

0.4.0 Release
totem · Jan 21, 2016 · 4148abd · 4148abd
2 parents 13311db + 0a86257
commit 4148abd
Show file tree

Hide file tree

Showing 10 changed files with 646 additions and 606 deletions.
diff --git a/conf/appconfig.py b/conf/appconfig.py
@@ -156,7 +156,8 @@
         'security': {
             'profile': 'default'
         },
-        'notifications': NOTIFICATIONS_DEFAULTS
+        'notifications': NOTIFICATIONS_DEFAULTS,
+        'environment': {}
     }
 }
 

diff --git a/deployer/__init__.py b/deployer/__init__.py
@@ -3,7 +3,7 @@
 from celery.signals import setup_logging
 
 
-__version__ = '0.3.11'
+__version__ = '0.4.0'
 __author__ = 'sukrit'
 
 deployer.logger.init_logging()

diff --git a/deployer/services/deployment.py b/deployer/services/deployment.py
@@ -1,10 +1,16 @@
+import copy
+import json
 import logging
+import time
+import datetime
 from fleet.deploy.deployer import filter_units
-from conf.appconfig import CLUSTER_NAME
+from conf.appconfig import CLUSTER_NAME, DEPLOYMENT_TYPE_GIT_QUAY, \
+    DEPLOYMENT_DEFAULTS, TEMPLATE_DEFAULTS, \
+    DEPLOYMENT_STATE_STARTED, UPSTREAM_DEFAULTS, DEPLOYMENT_TYPE_DEFAULT
 from deployer.fleet import get_fleet_provider
 from deployer.services.proxy import get_discovered_nodes
 from deployer.services.storage.factory import get_store
-from deployer.util import dict_merge
+from deployer.util import dict_merge, to_milliseconds
 
 __author__ = 'sukrit'
 
@@ -108,3 +114,140 @@ def generate_deployment_id(app_name, app_version):
     :rtype: str
     """
     return '{}-{}-{}'.format(CLUSTER_NAME, app_name, app_version)
+
+
+def _git_quay_defaults(deployment):
+    """
+    Applies defaults for git-quay deployment
+
+    :param deployment: Deployment that needs to be updated
+    :type deployment: dict
+    :return: Updated deployment
+    :rtype: dict
+    """
+    deploy_args = deployment['templates']['app']['args']
+    git_meta = deployment['meta-info']['git']
+    deploy_args['image'] = deploy_args['image'] \
+        .format(GIT_OWNER=git_meta['owner'],
+                GIT_REPO=git_meta['repo'],
+                GIT_COMMIT=git_meta['commit'])
+    deployment['deployment']['name'] = deployment['deployment']['name']\
+        .format(GIT_OWNER=git_meta['owner'],
+                GIT_REPO=git_meta['repo'],
+                GIT_REF=git_meta['ref'])
+    return deployment
+
+
+def _create_discover_check(deployment):
+    """
+    Creates the dictionary to be used by discover for health check
+
+    :param deployment:
+    :return:
+    """
+    return {
+        port: upstream.get('health', {}) for port, upstream in
+        deployment.get('proxy', {}).get('upstreams', {}).iteritems()
+    }
+
+
+def _get_app_environment(deployment, exposed_ports):
+    app_template = deployment.get('templates').get('app')
+    env = copy.deepcopy(app_template['args']['environment'])
+    env['DISCOVER_PORTS'] = ','.join(
+        [str(port) for port in exposed_ports])
+    env['DISCOVER_MODE'] = deployment['deployment']['mode']
+    env['DISCOVER_HEALTH'] = json.dumps(
+        _create_discover_check(deployment))
+    env = dict_merge(env, deployment.get('environment'))
+    return env
+
+
+def apply_defaults(deployment):
+    """
+    Applies the defaults for the deployment
+
+    :param deployment: Dictionary representing deployment
+    :type deployment: dict
+    :return: Deployment with defaults applied
+    :rtype: dict
+    """
+
+    # Set the default deployment type.
+    deployment_upd = dict_merge(deployment, {
+        'deployment': {
+            'type': DEPLOYMENT_TYPE_GIT_QUAY
+        }
+    })
+    deployment_type = deployment_upd['deployment']['type']
+
+    # Apply defaults
+    if deployment_type in DEPLOYMENT_DEFAULTS:
+        deployment_upd = dict_merge(deployment_upd,
+                                    DEPLOYMENT_DEFAULTS[deployment_type])
+    deployment_upd = dict_merge(deployment_upd,
+                                DEPLOYMENT_DEFAULTS['default'])
+
+    if deployment_type == DEPLOYMENT_TYPE_GIT_QUAY:
+        deployment_upd = _git_quay_defaults(deployment_upd)
+
+    for template_name, template in deployment_upd['templates'].iteritems():
+        deployment_upd['templates'][template_name] = \
+            dict_merge(template, TEMPLATE_DEFAULTS)
+
+    deployment_upd['deployment']['version'] = \
+        deployment_upd['deployment']['version'] or \
+        int(round(time.time() * 1000))
+
+    deployment_upd['deployment']['version'] = \
+        str(deployment_upd['deployment']['version'])
+
+    # Set the deployment identifier.  We will use deployment name and version
+    # as unique identifier as there can only be one deployment with same name
+    # and version
+    app_name = deployment_upd['deployment']['name']
+    app_version = deployment_upd['deployment']['version']
+    deployment_upd['id'] = generate_deployment_id(app_name, app_version)
+    deployment_upd['state'] = DEPLOYMENT_STATE_STARTED
+    deployment_upd['started-at'] = datetime.datetime.utcnow()
+
+    app_template = deployment_upd.get('templates').get('app')
+    exposed_ports = get_exposed_ports(deployment_upd)
+
+    # Create default upstreams (using exposed ports)
+    for port in exposed_ports:
+        deployment_upd['proxy']['upstreams'].setdefault(str(port), {})
+
+    # Apply proxy defaults
+    for upstream_name, upstream in deployment_upd['proxy']['upstreams'] \
+            .iteritems():
+        upd_upstream = dict_merge(upstream, UPSTREAM_DEFAULTS)
+        if upd_upstream.get('mode') != 'http' and \
+                upd_upstream['health'].get('uri'):
+            del(upd_upstream['health']['uri'])
+        deployment_upd['proxy']['upstreams'][upstream_name] = upd_upstream
+
+    if app_template:
+        app_template['args']['environment'] = _get_app_environment(
+            deployment_upd, exposed_ports)
+        sidekicks = [service_type for service_type, template in
+                     deployment_upd['templates'].iteritems()
+                     if template['enabled'] and service_type != 'app']
+        app_template['args']['sidekicks'] = sidekicks
+        timeout_stop = deployment_upd['deployment']['stop']['timeout'] or \
+            DEPLOYMENT_DEFAULTS[DEPLOYMENT_TYPE_DEFAULT]['deployment']['stop']
+        ['timeout']
+        timeout_stop_sec = to_milliseconds(timeout_stop) / 1000
+        app_template['args']['service'] = dict_merge(
+            app_template['args'].get('service') or {},
+            {
+                'container-stop-sec': timeout_stop_sec
+            }
+        )
+
+    # Override/Set Clustername
+    deployment_upd['cluster'] = CLUSTER_NAME
+
+    # Reset runtime if it exists
+    deployment_upd['runtime'] = {}
+    return deployment_upd
diff --git a/deployer/tasks/deployment.py b/deployer/tasks/deployment.py
@@ -2,12 +2,9 @@
 Defines celery tasks for deployment (e.g.: create, undeploy, wire, unwire)
 """
 import copy
-import datetime
 from httplib import HTTPException
-import json
 import socket
 import logging
-import time
 import urllib2
 from fabric.exceptions import NetworkError
 from fleet.client.fleet_fabric import FleetExecutionException
@@ -20,7 +17,7 @@
 from deployer.services.storage.factory import get_store
 from deployer.services.util import create_notify_ctx
 from deployer.services.deployment import fetch_runtime_units, \
-    get_exposed_ports, sync_upstreams, sync_units, generate_deployment_id
+    sync_upstreams, sync_units, apply_defaults
 from deployer.tasks import notification
 from deployer.tasks.exceptions import NodeNotUndeployed, MinNodesNotRunning, \
     NodeCheckFailed, MinNodesNotDiscovered, NodeNotStopped
@@ -39,10 +36,10 @@
 from fleet.deploy.deployer import Deployment, undeploy, stop
 
 from deployer.celery import app
-from conf.appconfig import DEPLOYMENT_DEFAULTS, DEPLOYMENT_TYPE_GIT_QUAY, \
-    TEMPLATE_DEFAULTS, TASK_SETTINGS, DEPLOYMENT_MODE_BLUEGREEN, \
-    DEPLOYMENT_MODE_REDGREEN, DEPLOYMENT_STATE_STARTED, \
-    DEPLOYMENT_STATE_FAILED, DEPLOYMENT_STATE_PROMOTED, UPSTREAM_DEFAULTS, \
+from conf.appconfig import DEPLOYMENT_DEFAULTS, \
+    TASK_SETTINGS, DEPLOYMENT_MODE_BLUEGREEN, \
+    DEPLOYMENT_MODE_REDGREEN, \
+    DEPLOYMENT_STATE_FAILED, DEPLOYMENT_STATE_PROMOTED, \
     LEVEL_STARTED, LEVEL_FAILED, LEVEL_SUCCESS, CLUSTER_NAME, \
     DEPLOYMENT_STATE_DECOMMISSIONED, LOCK_JOB_BASE, DEPLOYMENT_TYPE_DEFAULT
 
@@ -148,7 +145,7 @@ def create(deployment):
     :rtype: dict
     """
     # Step1: Apply defaults
-    deployment = _deployment_defaults(deployment)
+    deployment = apply_defaults(deployment)
 
     # Step2: Apply Lock
     # Step3: Un-deploy existing versions
@@ -354,137 +351,6 @@ def _deploy_all(deployment, search_params, next_task=None):
     )()
 
 
-def _git_quay_defaults(deployment):
-    """
-    Applies defaults for git-quay deployment
-
-    :param deployment: Deployment that needs to be updated
-    :type deployment: dict
-    :return: Updated deployment
-    :rtype: dict
-    """
-    deploy_args = deployment['templates']['app']['args']
-    git_meta = deployment['meta-info']['git']
-    deploy_args['image'] = deploy_args['image'] \
-        .format(GIT_OWNER=git_meta['owner'],
-                GIT_REPO=git_meta['repo'],
-                GIT_COMMIT=git_meta['commit'])
-    deployment['deployment']['name'] = \
-        deployment['deployment']['name'] \
-        .format(GIT_OWNER=git_meta['owner'],
-                GIT_REPO=git_meta['repo'],
-                GIT_REF=git_meta['ref'])
-    return deployment
-
-
-def _create_discover_check(deployment):
-    """
-    Creates the dictionary to be used by discover for health check
-
-    :param deployment:
-    :return:
-    """
-    return {
-        port: upstream.get('health', {}) for port, upstream in
-        deployment.get('proxy', {}).get('upstreams', {}).iteritems()
-    }
-
-
-def _deployment_defaults(deployment):
-    """
-    Applies the defaults for the deployment
-
-    :param deployment: Dictionary representing deployment
-    :type deployment: dict
-    :return: Deployment with defaults applied
-    :rtype: dict
-    """
-
-    # Set the default deployment type.
-    deployment_upd = dict_merge(deployment, {
-        'deployment': {
-            'type': DEPLOYMENT_TYPE_GIT_QUAY
-        }
-    })
-    deployment_type = deployment_upd['deployment']['type']
-
-    # Apply defaults
-    if deployment_type in DEPLOYMENT_DEFAULTS:
-        deployment_upd = dict_merge(deployment_upd,
-                                    DEPLOYMENT_DEFAULTS[deployment_type])
-    deployment_upd = dict_merge(deployment_upd,
-                                DEPLOYMENT_DEFAULTS['default'])
-
-    if deployment_type == DEPLOYMENT_TYPE_GIT_QUAY:
-        deployment_upd = _git_quay_defaults(deployment_upd)
-
-    for template_name, template in deployment_upd['templates'].iteritems():
-        deployment_upd['templates'][template_name] = \
-            dict_merge(template, TEMPLATE_DEFAULTS)
-
-    deployment_upd['deployment']['version'] = \
-        deployment_upd['deployment']['version'] or \
-        int(round(time.time() * 1000))
-
-    deployment_upd['deployment']['version'] = \
-        str(deployment_upd['deployment']['version'])
-
-    # Set the deployment identifier.  We will use deployment name and version
-    # as unique identifier as there can only be one deployment with same name
-    # and version
-    app_name = deployment_upd['deployment']['name']
-    app_version = deployment_upd['deployment']['version']
-    deployment_upd['id'] = generate_deployment_id(app_name, app_version)
-    deployment_upd['state'] = DEPLOYMENT_STATE_STARTED
-    deployment_upd['started-at'] = datetime.datetime.utcnow()
-
-    app_template = deployment_upd.get('templates').get('app')
-    exposed_ports = get_exposed_ports(deployment_upd)
-
-    # Create default upstreams (using exposed ports)
-    for port in exposed_ports:
-        deployment_upd['proxy']['upstreams'].setdefault(str(port), {})
-
-    # Apply proxy defaults
-    for upstream_name, upstream in deployment_upd['proxy']['upstreams']\
-            .iteritems():
-        upd_upstream = dict_merge(upstream, UPSTREAM_DEFAULTS)
-        if upd_upstream.get('mode') != 'http' and \
-                upd_upstream['health'].get('uri'):
-            del(upd_upstream['health']['uri'])
-        deployment_upd['proxy']['upstreams'][upstream_name] = upd_upstream
-
-    if app_template:
-        env = app_template['args']['environment']
-        env['DISCOVER_PORTS'] = ','.join(
-            [str(port) for port in exposed_ports])
-        env['DISCOVER_MODE'] = deployment_upd['deployment']['mode']
-        env['DISCOVER_HEALTH'] = json.dumps(
-            _create_discover_check(deployment_upd))
-
-        sidekicks = [service_type for service_type, template in
-                     deployment_upd['templates'].iteritems()
-                     if template['enabled'] and service_type != 'app']
-        app_template['args']['sidekicks'] = sidekicks
-        timeout_stop = deployment_upd['deployment']['stop']['timeout'] or \
-            DEPLOYMENT_DEFAULTS[DEPLOYMENT_TYPE_DEFAULT]['deployment']['stop'][
-                'timeout']
-        timeout_stop_sec = to_milliseconds(timeout_stop) / 1000
-        app_template['args']['service'] = dict_merge(
-            app_template['args'].get('service') or {},
-            {
-                'container-stop-sec': timeout_stop_sec
-            }
-        )
-
-    # Override/Set Clustername
-    deployment_upd['cluster'] = CLUSTER_NAME
-
-    # Reset runtime if it exists
-    deployment_upd['runtime'] = {}
-    return deployment_upd
-
-
 @app.task(bind=True)
 def _fleet_deploy(self, search_params, name, version, nodes, service_type,
                   template, security_profile):

diff --git a/dev-requirements.txt b/dev-requirements.txt
@@ -4,7 +4,7 @@ flake8==2.2.3
 Sphinx==1.2.3
 
 # For freezing time
-freezegun==0.2.8
+https://github.com/spulec/freezegun/archive/master.tar.gz
 
 # Code Coverage
 coveralls
diff --git a/requirements.txt b/requirements.txt
@@ -2,8 +2,8 @@ Flask==0.10.1
 Flask-Cors==1.9.0
 pymongo==3.0.3
 flower==0.7.2
-gevent==1.0.1
-uWSGI==2.0.7
+gevent==1.0.2
+uWSGI==2.0.12
 supervisor==3.1.2
 python-etcd==0.3.2
 elasticsearch==1.2.0

diff --git a/schemas/app-version-create-v1.json b/schemas/app-version-create-v1.json
@@ -21,6 +21,9 @@
     },
     "notifications": {
       "$ref": "${base_url}/schemas/app-version-v1#/properties/notifications"
+    },
+    "environment": {
+      "$ref": "${base_url}/schemas/app-version-v1#/properties/environment"
     }
   },
   "required": ["meta-info"],