Merge pull request #48 from totem/develop

0.4.4 Release

.dockerignore

@@ -0,0 +1,10 @@
+.idea
+tests
+docs
+.git
+*.md
+.travis.yml
+dev-requirements.txt
+*.iml
+LICENSE
+local-*

Dockerfile

@@ -10,12 +10,12 @@ RUN mkdir /root/.ssh && \
     chown -R root:root /root/.ssh
 
 #Confd
-ENV CONFD_VERSION 0.6.2
+ENV CONFD_VERSION 0.11.0
 RUN curl -L https://github.com/kelseyhightower/confd/releases/download/v$CONFD_VERSION/confd-${CONFD_VERSION}-linux-amd64 -o /usr/local/bin/confd && \
     chmod 555 /usr/local/bin/confd
 
 #Etcdctl
-ENV ETCDCTL_VERSION v0.4.6
+ENV ETCDCTL_VERSION v2.2.5
 RUN curl -L https://github.com/coreos/etcd/releases/download/$ETCDCTL_VERSION/etcd-$ETCDCTL_VERSION-linux-amd64.tar.gz -o /tmp/etcd-$ETCDCTL_VERSION-linux-amd64.tar.gz && \
     cd /tmp && gzip -dc etcd-$ETCDCTL_VERSION-linux-amd64.tar.gz | tar -xof - && \
     cp -f /tmp/etcd-$ETCDCTL_VERSION-linux-amd64/etcdctl /usr/local/bin && \

conf/appconfig.py

@@ -65,7 +65,7 @@
     'CHECK_NODE_RETRY_DELAY': 10,
     'DEPLOYMENT_STOP_MIN_CHECK_RETRY_DELAY': 2,
     'DEFAULT_DEPLOYMENT_STOP_CHECK_RETRIES': 10,
-    'START_CONCURRENCY': os.getenv('START_CONCURRENCY', 4),
+    'START_CONCURRENCY': os.getenv('START_CONCURRENCY', 3),
     'START_CONCURRENCY_RETRIES': 60,
     'START_CONCURRENCY_RETRY_DELAY': 60,
 
@@ -201,6 +201,8 @@
     }
 }
 
+FLEET_STARTED_STATES = ('running', 'waiting')
+
 DISCOVER_UPSTREAM_TTL_DEFAULT = '86400'
 
 DEFAULT_LOCK_TTL = 3600

deployer/__init__.py

@@ -3,7 +3,7 @@
 from celery.signals import setup_logging
 
 
-__version__ = '0.4.3'
+__version__ = '0.4.4'
 __author__ = 'sukrit'
 
 deployer.logger.init_logging()

deployer/services/deployment.py

@@ -195,11 +195,51 @@ def _get_app_environment(deployment, exposed_ports):
         'DISCOVER_HEALTH': json.dumps(
                 _create_discover_check(deployment)),
         'DISCOVER_UPSTREAM_TTL': DISCOVER_UPSTREAM_TTL_DEFAULT
-    }
+    } if not deployment['schedule'] else {}
     return dict_merge(app_template['args']['environment'],
                       deployment.get('environment'), discover)
 
 
+def check_and_apply_schedule(deployment):
+    deployment_upd = dict_merge(deployment, {
+        'schedule': None
+    })
+    if deployment_upd['schedule']:
+        deployment_upd['proxy'].update({
+            'upstreams': {},
+            'hosts': {},
+            'listeners': {}
+        })
+        deployment_upd['deployment']['check'].update({
+            'port': None,
+            'path': ''
+        })
+        app_template = deployment_upd.get('templates').get('app')
+        timer_template = dict_merge(
+            deployment_upd.get('templates').get('timer'), {
+                'name': app_template['name'],
+                'args': {}
+            })
+        deployment_upd['templates'] = {}
+        if app_template and app_template['enabled']:
+            deployment_upd['templates']['app'] = app_template
+            deployment_upd['templates']['timer'] = timer_template
+            service_params = {
+                'service': {
+                    'schedule': deployment_upd['schedule']
+                }
+            }
+            app_template['args'] = dict_merge(service_params,
+                                              app_template['args'])
+            timer_template['args'] = dict_merge(service_params,
+                                                timer_template['args'])
+            timer_template.update({
+                'enabled': True,
+            })
+
+    return deployment_upd
+
+
 def apply_defaults(deployment):
     """
     Applies the defaults for the deployment
@@ -228,10 +268,12 @@ def apply_defaults(deployment):
     if deployment_type == DEPLOYMENT_TYPE_GIT_QUAY:
         deployment_upd = _git_quay_defaults(deployment_upd)
 
-    for template_name, template in deployment_upd['templates'].iteritems():
+    for template_name, template in deployment_upd['templates'].items():
         deployment_upd['templates'][template_name] = \
             dict_merge(template, TEMPLATE_DEFAULTS)
 
+    deployment_upd = check_and_apply_schedule(deployment_upd)
+
     deployment_upd['deployment']['version'] = \
         deployment_upd['deployment']['version'] or \
         int(round(time.time() * 1000))
@@ -268,8 +310,9 @@ def apply_defaults(deployment):
         app_template['args']['environment'] = _get_app_environment(
             deployment_upd, exposed_ports)
         sidekicks = [service_type for service_type, template in
-                     deployment_upd['templates'].iteritems()
-                     if template['enabled'] and service_type != 'app']
+                     deployment_upd['templates'].items()
+                     if template['enabled'] and service_type
+                     not in ('app', 'timer')]
         app_template['args']['sidekicks'] = sidekicks
         timeout_stop = deployment_upd['deployment']['stop']['timeout'] or \
             DEPLOYMENT_DEFAULTS[DEPLOYMENT_TYPE_DEFAULT]['deployment']['stop']

deployer/tasks/deployment.py

@@ -43,7 +43,7 @@
     DEPLOYMENT_STATE_FAILED, DEPLOYMENT_STATE_PROMOTED, \
     LEVEL_STARTED, LEVEL_FAILED, LEVEL_SUCCESS, CLUSTER_NAME, \
     DEPLOYMENT_STATE_DECOMMISSIONED, LOCK_JOB_BASE, DEPLOYMENT_TYPE_DEFAULT, \
-    DEFAULT_CHORD_OPTIONS, DEPLOYMENT_STATE_STARTED
+    DEFAULT_CHORD_OPTIONS, DEPLOYMENT_STATE_STARTED, FLEET_STARTED_STATES
 
 from deployer.tasks.common import async_wait
 from deployer.services.proxy import wire_proxy, register_upstreams, \
@@ -346,7 +346,7 @@ def _deploy_all(deployment, search_params, next_task=None):
         group(
             _fleet_deploy.si(search_params, name, version, nodes, service_type,
                              template, security_profile)
-            for service_type, template in deployment['templates'].iteritems()
+            for service_type, template in deployment['templates'].items()
             if template['enabled']
         ),
         _fleet_start_and_wait.si(deployment, search_params,
@@ -373,10 +373,12 @@ def _fleet_deploy(self, search_params, name, version, nodes, service_type,
                 nodes, template)
     template_args = decrypt_config(template.get('args', {}),
                                    profile=security_profile)
+    is_timer = (service_type == 'timer')
     fleet_deployment = Deployment(
         fleet_provider=get_fleet_provider(), jinja_env=jinja_env, name=name,
-        version=version, template=template['name'] + '.service', nodes=nodes,
-        template_args=template_args, service_type=service_type)
+        version=version, template=template['name'], nodes=nodes,
+        template_args=template_args, service_type=service_type,
+        timer=is_timer)
     try:
         fleet_deployment.deploy(start=False)
     except RETRYABLE_FLEET_EXCEPTIONS as exc:
@@ -407,10 +409,12 @@ def _fleet_start(self, search_params, name, version, nodes, service_type,
     """
     logger.info('Starting %s:%s:%s nodes:%d %r', name, version, service_type,
                 nodes, template)
+    is_timer = (service_type == 'timer')
     fleet_deployment = Deployment(
         fleet_provider=get_fleet_provider(), jinja_env=jinja_env, name=name,
-        version=version, template=template['name'] + '.service', nodes=nodes,
-        template_args=template['args'], service_type=service_type)
+        version=version, template=template['name'], nodes=nodes,
+        template_args=template['args'], service_type=service_type,
+        timer=is_timer)
     try:
         fleet_deployment.start_units()
     except RETRYABLE_FLEET_EXCEPTIONS as exc:
@@ -442,17 +446,20 @@ def _fleet_start_and_wait(deployment, search_params, next_task=None):
     name, version, nodes = deployment['deployment']['name'], \
         deployment['deployment']['version'], \
         deployment['deployment']['nodes']
-    service_types = {service_type for service_type, template in
-                     deployment['templates'].iteritems()
-                     if template['enabled']}
+    if not deployment['schedule']:
+        service_types = {service_type for service_type, template in
+                         deployment['templates'].items()
+                         if template['enabled']}
+    else:
+        service_types = {'timer'}
     min_nodes = deployment['deployment'].get('check', {}).get(
         'min-nodes', nodes)
+    templates = deployment['templates']
     return chord(
         group(
             _fleet_start.si(search_params, name, version, nodes, service_type,
-                            template)
-            for service_type, template in deployment['templates'].iteritems()
-            if template['enabled']
+                            templates[service_type])
+            for service_type in service_types
         ),
         _fleet_check_deploy.si(name, version, len(service_types), min_nodes,
                                search_params, next_task=next_task),
@@ -639,7 +646,7 @@ def _fleet_check_deploy(self, name, version, service_cnt, min_nodes,
                          countdown=TASK_SETTINGS['SSH_RETRY_DELAY'])
 
     running_units = [unit for unit in units
-                     if unit['sub'].lower() == 'running']
+                     if unit['sub'].lower() in FLEET_STARTED_STATES]
     if len(running_units) < expected_cnt:
         raise self.retry(exc=MinNodesNotRunning(
             name, version, expected_cnt, units))

requirements.txt

@@ -11,9 +11,9 @@ future==0.15.0
 pytz==2015.4
 requests[security]==2.7.0
 urllib3==1.11
+celery[mongodb]==3.1.20
 
-https://github.com/totem/fleet-py/archive/master.tar.gz
-https://github.com/sukrit007/celery/archive/3.1.tar.gz
+https://github.com/totem/fleet-py/archive/0.1.6.tar.gz
 https://github.com/totem/yoda-py/archive/v0.1.8b2.tar.gz
 https://github.com/totem/flask-hyperschema/archive/v0.1.1.tar.gz
 https://github.com/totem/totem-encrypt/archive/master.tar.gz

schemas/app-version-create-v1.json

@@ -24,6 +24,9 @@
     },
     "environment": {
       "$ref": "${base_url}/schemas/app-version-v1#/properties/environment"
+    },
+    "schedule": {
+      "$ref": "${base_url}/schemas/app-version-v1#/properties/schedule"
     }
   },
   "required": ["meta-info"],

schemas/app-version-v1.json

@@ -28,6 +28,10 @@
     "environment": {
       "$ref": "#/definitions/environment",
       "description": "Defines top level environment (key, value) support"
+    },
+    "schedule": {
+      "type": "string",
+      "description": "Run this deployment as scheduled job. See: https://www.freedesktop.org/software/systemd/man/systemd.time.html#Calendar%20Events"
     }
   },
   "required": ["meta-info"],