Merge branch 'release/1.3.5'

CHANGELOG.md

@@ -2,6 +2,10 @@
 All notable changes to this project will be documented in this file.
 This project adheres to [WebDevOps.io Dockerfile](https://github.com/webdevops/Dockerfile).
 
+## [1.3.5] - 2017-02-12
+- Improve and fix parallel run of serverspec
+- Reduced python subprocess wait loop cpu time
+
 ## [1.3.4] - 2017-02-03
 - Add serverspec wrapper (do not run tests if Dockerfile isn't set or available)
 

Makefile

@@ -36,7 +36,7 @@ requirements:
  cd tests/serverspec && bundle install --path=vendor
 
 test:
- python bin/console test:serverspec --threads=auto/2 -v
+ python bin/console test:serverspec --threads=auto -v
 
 baselayout:
  python bin/console generate:provision --baselayout

bin/webdevops/Command.py

@@ -18,7 +18,7 @@
 # OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
 # OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
-import os, subprocess, tempfile
+import os, subprocess, tempfile, copy, time
 
 def execute(cmd, cwd=False, env=None):
  """
@@ -27,9 +27,20 @@ def execute(cmd, cwd=False, env=None):
 
  print 'Execute: %s' % ' '.join(cmd)
 
- # remove _ from env (prevent errors)
- if env is not None and '_' in env:
- del env['_']
+ if env is not None:
+ env = copy.deepcopy(env)
+ env['PWD'] = os.environ['PWD']
+ env['LC_CTYPE'] = os.environ['LC_CTYPE']
+ env['SHELL'] = os.environ['SHELL']
+
+ if '_' in env:
+ del env['_']
+
+ # add system env vars
+ system_env_vars = ['PWD', 'PATH', 'LC_CTYPE', 'SHELL', 'DOCKER_HOST', 'DOCKER_CERT_PATH', 'DOCKER_MACHINE_NAME', 'DOCKER_TLS_VERIFY']
+ for var_name in system_env_vars:
+ if not var_name in env and var_name in os.environ:
+ env[var_name] = os.environ[var_name]
 
  # set current working directory
  path_current = os.getcwd()
@@ -51,7 +62,7 @@ def execute(cmd, cwd=False, env=None):
 
  # wait for process end
  while proc.poll() is None:
- pass
+ time.sleep(1)
 
  # output stdout
  with open(file_stdout.name, 'r') as f:

bin/webdevops/command/DoitCommand.py

@@ -28,7 +28,7 @@ def run_doit(self, task_loader, configuration):
  extra_configuration = {}
 
  if 'threads' in configuration and configuration.get('threads') > 1:
- arguments.extend(['-n', str(configuration.get('threads'))])
+ arguments.extend(['-n', str(configuration.get('threads')), '--parallel-type', 'process'])
 
  if 'doitConfig' in configuration:
  extra_configuration = configuration.get('doitConfig')

bin/webdevops/taskloader/DockerBuildTaskLoader.py

@@ -39,7 +39,7 @@ def generate_task_list(self, dockerfileList):
  }
 
  if dockerfile['dependency']:
- task['task_dep'].append('DockerBuild|%s' % dockerfile['dependency']);
+ task['task_dep'].append('DockerBuild|%s' % dockerfile['dependency'])
 
  tasklist.append(task)
 

bin/webdevops/taskloader/DockerTestServerspecTaskLoader.py

@@ -18,7 +18,7 @@
 # OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
 # OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
-import os, re, tempfile, json
+import os, re, tempfile, json, base64
 from webdevops import Command
 from .BaseDockerTaskLoader import BaseDockerTaskLoader
 from .BaseTaskLoader import BaseTaskLoader
@@ -40,6 +40,10 @@ def generate_task_list(self, dockerfile_list):
  'actions': [(BaseTaskLoader.task_runner, [DockerTestServerspecTaskLoader.task_run, [dockerfile, self.configuration]])],
  'task_dep': []
  }
+
+ if dockerfile['dependency']:
+ task['task_dep'].append('DockerTestServerspec|%s' % dockerfile['dependency'])
+
  tasklist.append(task)
 
  # task = {
@@ -71,18 +75,18 @@ def task_run(dockerfile, configuration, task):
  tmp_suffix = '.%s_%s_%s.tmp' % (dockerfile['image']['repository'], dockerfile['image']['imageName'], dockerfile['image']['tag'])
  test_dockerfile = tempfile.NamedTemporaryFile(prefix='Dockerfile.', suffix=tmp_suffix, dir=configuration.get('serverspecPath'), bufsize=0, delete=False)
 
- # serverspec options
- serverspec_opts = []
- serverspec_opts.extend(['--pattern', spec_path])
-
- # serverspec env
- serverspec_env = DockerTestServerspecTaskLoader.generate_serverspec_environment(
+ # serverspec conf
+ serverspec_conf = DockerTestServerspecTaskLoader.generate_serverspec_configuration(
  path=os.path.basename(test_dockerfile.name),
  dockerfile=dockerfile,
  configuration=configuration,
  is_toolimage=is_toolimage
  )
 
+ # serverspec options
+ serverspec_opts = []
+ serverspec_opts.extend([spec_path, dockerfile['image']['fullname'], base64.b64encode(json.dumps(serverspec_conf)), os.path.basename(test_dockerfile.name)])
+
  # dockerfile content
  dockerfile_content = DockerTestServerspecTaskLoader.generate_dockerfile(
  dockerfile=dockerfile,
@@ -99,15 +103,13 @@ def task_run(dockerfile, configuration, task):
  print ' path: %s' % (spec_path)
  print ' args: %s' % (' '.join(serverspec_opts))
  print ''
- print 'environment:'
- print '------------'
- print json.dumps(serverspec_env, indent=4, sort_keys=True)
+ print 'spec configuration:'
+ print '-------------------'
+ print json.dumps(serverspec_conf, indent=4, sort_keys=True)
  print ''
  print 'Dockerfile:'
  print '-----------'
  print dockerfile_content
-
- os.remove(test_dockerfile.name)
  return True
 
  # check if we have any tests
@@ -119,10 +121,6 @@ def task_run(dockerfile, configuration, task):
  cmd = ['bash', 'serverspec.sh']
  cmd.extend(serverspec_opts)
 
- # Set environment variables
- env = os.environ.copy()
- env.update(serverspec_env)
-
  # create Dockerfile
  with open(test_dockerfile.name, mode='w', buffering=0) as f:
  f.write(dockerfile_content)
@@ -133,7 +131,7 @@ def task_run(dockerfile, configuration, task):
  test_status = False
  for retry_count in range(0, configuration.get('retry')):
  try:
- test_status = Command.execute(cmd, cwd=configuration.get('serverspecPath'), env=env)
+ test_status = Command.execute(cmd, cwd=configuration.get('serverspecPath'))
  except Exception as e:
  print e
  pass
@@ -145,25 +143,24 @@ def task_run(dockerfile, configuration, task):
  else:
  print ' failed, giving up'
 
- os.remove(test_dockerfile.name)
  return test_status
 
  @staticmethod
- def generate_serverspec_environment(path, dockerfile, configuration, is_toolimage=False):
+ def generate_serverspec_configuration(path, dockerfile, configuration, is_toolimage=False):
  """
- Generate serverspec environment dict
+ Generate serverspec configuration dict
  """
  ret = {}
 
  # add default vars
- default_env_list = configuration.get('dockerTest.environment.default', False)
+ default_env_list = configuration.get('dockerTest.configuration.default', False)
  if default_env_list:
- ret = default_env_list.to_dict()
+ ret = default_env_list.to_dict().copy()
 
  # add docker image specific vars
- image_env_list = configuration.get('dockerTest.environment.image')
+ image_env_list = configuration.get('dockerTest.configuration.image')
  if image_env_list:
- image_env_list = image_env_list.to_dict()
+ image_env_list = image_env_list.to_dict().copy()
  for term in image_env_list:
  if term in dockerfile['image']['fullname']:
  for key in image_env_list[term]:
@@ -187,6 +184,7 @@ def generate_dockerfile(dockerfile, configuration, is_toolimage=False):
 
  ret.append('FROM %s' % dockerfile['image']['fullname'])
  ret.append('COPY conf/ /')
+ ret.append('RUN echo "%s" > /DOCKER.IMAGENAME' % dockerfile['image']['fullname'])
 
  if is_toolimage:
  ret.append('RUN chmod +x /loop-entrypoint.sh')

bin/webdevops/taskloader/DockerTestTestinfraTaskLoader.py

@@ -40,6 +40,10 @@ def generate_task_list(self, dockerfile_list):
  'actions': [(BaseTaskLoader.task_runner, [DockerTestTestinfraTaskLoader.task_run, [dockerfile, self.configuration]])],
  'task_dep': []
  }
+
+ if dockerfile['dependency']:
+ task['task_dep'].append('DockerTestTestinfra|%s' % dockerfile['dependency'])
+
  tasklist.append(task)
 
  # task = {

conf/console.yml

@@ -7,6 +7,7 @@ imagePath: 'documentation/docs/resources/images/'
 baselayoutPath: 'baselayout'
 testinfraPath: 'tests/testinfra'
 serverspecPath: 'tests/serverspec'
+testDockerfilePath: 'tests/dockerfile'
 blacklistFile: 'BLACKLIST'
 
 docker:
@@ -35,7 +36,7 @@ dockerTest:
  '/varnish':
  - 'ENV VARNISH_BACKEND_HOST webdevops.io'
 
- environment:
+ configuration:
  default:
  OS_FAMILY: 'ubuntu'
  OS_VERSION: '16.04'

docker/ansible/alpine-3/Dockerfile

@@ -8,4 +8,4 @@ FROM webdevops/bootstrap:alpine-3
 MAINTAINER [email protected]
 LABEL vendor=WebDevOps.io
 LABEL io.webdevops.layout=8
-LABEL io.webdevops.version=1.3.4
+LABEL io.webdevops.version=1.3.5

docker/ansible/centos-7/Dockerfile

@@ -8,4 +8,4 @@ FROM webdevops/bootstrap:centos-7
 MAINTAINER [email protected]
 LABEL vendor=WebDevOps.io
 LABEL io.webdevops.layout=8
-LABEL io.webdevops.version=1.3.4
+LABEL io.webdevops.version=1.3.5

docker/ansible/debian-7/Dockerfile

@@ -8,4 +8,4 @@ FROM webdevops/bootstrap:debian-7
 MAINTAINER [email protected]
 LABEL vendor=WebDevOps.io
 LABEL io.webdevops.layout=8
-LABEL io.webdevops.version=1.3.4
+LABEL io.webdevops.version=1.3.5

docker/ansible/debian-8/Dockerfile

@@ -8,4 +8,4 @@ FROM webdevops/bootstrap:debian-8
 MAINTAINER [email protected]
 LABEL vendor=WebDevOps.io
 LABEL io.webdevops.layout=8
-LABEL io.webdevops.version=1.3.4
+LABEL io.webdevops.version=1.3.5

docker/ansible/debian-9/Dockerfile

@@ -8,4 +8,4 @@ FROM webdevops/bootstrap:debian-9
 MAINTAINER [email protected]
 LABEL vendor=WebDevOps.io
 LABEL io.webdevops.layout=8
-LABEL io.webdevops.version=1.3.4
+LABEL io.webdevops.version=1.3.5

docker/ansible/ubuntu-12.04/Dockerfile

@@ -8,4 +8,4 @@ FROM webdevops/bootstrap:ubuntu-12.04
 MAINTAINER [email protected]
 LABEL vendor=WebDevOps.io
 LABEL io.webdevops.layout=8
-LABEL io.webdevops.version=1.3.4
+LABEL io.webdevops.version=1.3.5

docker/ansible/ubuntu-14.04/Dockerfile

@@ -8,4 +8,4 @@ FROM webdevops/bootstrap:ubuntu-14.04
 MAINTAINER [email protected]
 LABEL vendor=WebDevOps.io
 LABEL io.webdevops.layout=8
-LABEL io.webdevops.version=1.3.4
+LABEL io.webdevops.version=1.3.5

docker/ansible/ubuntu-15.04/Dockerfile

@@ -8,4 +8,4 @@ FROM webdevops/bootstrap:ubuntu-15.04
 MAINTAINER [email protected]
 LABEL vendor=WebDevOps.io
 LABEL io.webdevops.layout=8
-LABEL io.webdevops.version=1.3.4
+LABEL io.webdevops.version=1.3.5

docker/ansible/ubuntu-15.10/Dockerfile

@@ -8,4 +8,4 @@ FROM webdevops/bootstrap:ubuntu-15.10
 MAINTAINER [email protected]
 LABEL vendor=WebDevOps.io
 LABEL io.webdevops.layout=8
-LABEL io.webdevops.version=1.3.4
+LABEL io.webdevops.version=1.3.5

docker/ansible/ubuntu-16.04/Dockerfile

@@ -8,4 +8,4 @@ FROM webdevops/bootstrap:ubuntu-16.04
 MAINTAINER [email protected]
 LABEL vendor=WebDevOps.io
 LABEL io.webdevops.layout=8
-LABEL io.webdevops.version=1.3.4
+LABEL io.webdevops.version=1.3.5

docker/apache-dev/alpine-3/Dockerfile

@@ -8,7 +8,7 @@ FROM webdevops/apache:alpine-3
 MAINTAINER [email protected]
 LABEL vendor=WebDevOps.io
 LABEL io.webdevops.layout=8
-LABEL io.webdevops.version=1.3.4
+LABEL io.webdevops.version=1.3.5
 
 ENV WEB_DOCUMENT_ROOT /app
 ENV WEB_DOCUMENT_INDEX index.php

docker/apache-dev/centos-7/Dockerfile

@@ -8,7 +8,7 @@ FROM webdevops/apache:centos-7
 MAINTAINER [email protected]
 LABEL vendor=WebDevOps.io
 LABEL io.webdevops.layout=8
-LABEL io.webdevops.version=1.3.4
+LABEL io.webdevops.version=1.3.5
 
 ENV WEB_DOCUMENT_ROOT /app
 ENV WEB_DOCUMENT_INDEX index.php

docker/apache-dev/debian-7/Dockerfile

@@ -8,7 +8,7 @@ FROM webdevops/apache:debian-7
 MAINTAINER [email protected]
 LABEL vendor=WebDevOps.io
 LABEL io.webdevops.layout=8
-LABEL io.webdevops.version=1.3.4
+LABEL io.webdevops.version=1.3.5
 
 ENV WEB_DOCUMENT_ROOT /app
 ENV WEB_DOCUMENT_INDEX index.php

docker/apache-dev/debian-8/Dockerfile

@@ -8,7 +8,7 @@ FROM webdevops/apache:debian-8
 MAINTAINER [email protected]
 LABEL vendor=WebDevOps.io
 LABEL io.webdevops.layout=8
-LABEL io.webdevops.version=1.3.4
+LABEL io.webdevops.version=1.3.5
 
 ENV WEB_DOCUMENT_ROOT /app
 ENV WEB_DOCUMENT_INDEX index.php

docker/apache-dev/debian-9/Dockerfile

@@ -8,7 +8,7 @@ FROM webdevops/apache:debian-9
 MAINTAINER [email protected]
 LABEL vendor=WebDevOps.io
 LABEL io.webdevops.layout=8
-LABEL io.webdevops.version=1.3.4
+LABEL io.webdevops.version=1.3.5
 
 ENV WEB_DOCUMENT_ROOT /app
 ENV WEB_DOCUMENT_INDEX index.php

docker/apache-dev/ubuntu-12.04/Dockerfile

@@ -8,7 +8,7 @@ FROM webdevops/apache:ubuntu-12.04
 MAINTAINER [email protected]
 LABEL vendor=WebDevOps.io
 LABEL io.webdevops.layout=8
-LABEL io.webdevops.version=1.3.4
+LABEL io.webdevops.version=1.3.5
 
 ENV WEB_DOCUMENT_ROOT /app
 ENV WEB_DOCUMENT_INDEX index.php

docker/apache-dev/ubuntu-14.04/Dockerfile

@@ -8,7 +8,7 @@ FROM webdevops/apache:ubuntu-14.04
 MAINTAINER [email protected]
 LABEL vendor=WebDevOps.io
 LABEL io.webdevops.layout=8
-LABEL io.webdevops.version=1.3.4
+LABEL io.webdevops.version=1.3.5
 
 ENV WEB_DOCUMENT_ROOT /app
 ENV WEB_DOCUMENT_INDEX index.php

docker/apache-dev/ubuntu-15.04/Dockerfile

@@ -8,7 +8,7 @@ FROM webdevops/apache:ubuntu-15.04
 MAINTAINER [email protected]
 LABEL vendor=WebDevOps.io
 LABEL io.webdevops.layout=8
-LABEL io.webdevops.version=1.3.4
+LABEL io.webdevops.version=1.3.5
 
 ENV WEB_DOCUMENT_ROOT /app
 ENV WEB_DOCUMENT_INDEX index.php

docker/apache-dev/ubuntu-15.10/Dockerfile

@@ -8,7 +8,7 @@ FROM webdevops/apache:ubuntu-15.10
 MAINTAINER [email protected]
 LABEL vendor=WebDevOps.io
 LABEL io.webdevops.layout=8
-LABEL io.webdevops.version=1.3.4
+LABEL io.webdevops.version=1.3.5
 
 ENV WEB_DOCUMENT_ROOT /app
 ENV WEB_DOCUMENT_INDEX index.php

docker/apache-dev/ubuntu-16.04/Dockerfile

@@ -8,7 +8,7 @@ FROM webdevops/apache:ubuntu-16.04
 MAINTAINER [email protected]
 LABEL vendor=WebDevOps.io
 LABEL io.webdevops.layout=8
-LABEL io.webdevops.version=1.3.4
+LABEL io.webdevops.version=1.3.5
 
 ENV WEB_DOCUMENT_ROOT /app
 ENV WEB_DOCUMENT_INDEX index.php

docker/apache/alpine-3/Dockerfile

@@ -8,7 +8,7 @@ FROM webdevops/base:alpine-3
 MAINTAINER [email protected]
 LABEL vendor=WebDevOps.io
 LABEL io.webdevops.layout=8
-LABEL io.webdevops.version=1.3.4
+LABEL io.webdevops.version=1.3.5
 
 ENV WEB_DOCUMENT_ROOT /app
 ENV WEB_DOCUMENT_INDEX index.php

docker/apache/centos-7/Dockerfile

@@ -8,7 +8,7 @@ FROM webdevops/base:centos-7
 MAINTAINER [email protected]
 LABEL vendor=WebDevOps.io
 LABEL io.webdevops.layout=8
-LABEL io.webdevops.version=1.3.4
+LABEL io.webdevops.version=1.3.5
 
 ENV WEB_DOCUMENT_ROOT /app
 ENV WEB_DOCUMENT_INDEX index.php