Merge branch 'release/1.1.0'

CHANGELOG.md

@@ -2,6 +2,15 @@
 All notable changes to this project will be documented in this file.
 This project adheres to [WebDevOps.io Dockerfile](https://github.com/webdevops/Dockerfile).
 
+## [1.1.0] - 2016-12-14
+- Fixed dnsmasq startup
+- Removed all logfiles inside containers (using stdout)
+- Fixed syslog-ng setup (was complaining about version)
+- Fixed some php/hhvm tests
+- Improve bin/console
+- Add cleanup after container installation
+- Add multiple vhost support for dns lookup (VIRTUAL_HOST)
+
 ## [1.0.0] - 2016-11-28
 - Introduced python based processing script
 - Introduced testinfra test suite

bin/command/docker_build_command.py

@@ -28,6 +28,7 @@ class DockerBuildCommand(DoitCommand):
  Build images
 
  docker:build
+ {docker images?* : Docker images (whitelist)}
  {--dry-run : show only which images will be build}
  {--no-cache : build without caching}
  {--t|threads=0 : threads}

bin/command/docker_exec_command.py

@@ -0,0 +1,93 @@
+#!/usr/bin/env/python
+# -*- coding: utf-8 -*-
+#
+# (c) 2016 WebDevOps.io
+#
+# This file is part of Dockerfile Repository.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+# documentation files (the "Software"), to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and
+# to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all copies or substantial portions
+# of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO
+# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS
+# OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+from cleo import Output
+from termcolor import colored
+from webdevops.command import DoitCommand
+from webdevops.taskloader import DockerPushTaskLoader
+from webdevops import Command, DockerfileUtility
+
+class DockerExecCommand(DoitCommand):
+ """
+ Push images to registry/hub
+
+ docker:exec
+ {docker command* : Command}
+ {--dry-run : show only which images will be build}
+ {--whitelist=?* : image/tag whitelist }
+ {--blacklist=?* : image/tag blacklist }
+ """
+
+ def run_task(self, configuration):
+ dockerfile_list = DockerfileUtility.find_dockerfiles_in_path(
+ base_path=configuration.get('dockerPath'),
+ path_regex=configuration.get('docker.pathRegex'),
+ image_prefix=configuration.get('docker.imagePrefix'),
+ whitelist=configuration.get('whitelist'),
+ blacklist=configuration.get('blacklist'),
+ )
+
+ image_fail_list = []
+
+ docker_command = self.argument('docker command')
+
+ for dockerfile in dockerfile_list:
+ title = dockerfile['image']['fullname']
+
+ print title
+ print '~' * len(title)
+
+ if configuration['dryRun']:
+ print ' exec: %s' % (docker_command)
+ else:
+
+ cmd = [
+ 'docker',
+ 'run',
+ '-t',
+ '--rm',
+ dockerfile['image']['fullname'],
+ 'sh -c "%s"' % (' '.join(docker_command))
+ ]
+
+ status = Command.execute(cmd)
+
+ if status:
+ print colored(' -> successfull', 'green')
+ else:
+ print colored(' -> failed', 'red')
+ image_fail_list.append(dockerfile['image']['fullname'])
+ print ''
+ print ''
+
+ if len(image_fail_list) >= 1:
+ print ''
+ print colored(' => failed images (%s):' % (str(len(image_fail_list))), 'red')
+ for image in image_fail_list:
+ print ' - %s ' % image
+ print ''
+
+ return False
+ else:
+ return True
+
+
+
+

bin/command/docker_pull_command.py

@@ -27,6 +27,7 @@ class DockerPullCommand(DoitCommand):
  Pull all built images from registry/hub
 
  docker:pull
+ {docker images?* : Docker images (whitelist)}
  {--dry-run : show only which images will be build}
  {--t|threads=0 : threads}
  {--r|retry=0 : retry}

bin/command/docker_push_command.py

@@ -27,6 +27,7 @@ class DockerPushCommand(DoitCommand):
  Push images to registry/hub
 
  docker:push
+ {docker images?* : Docker images (whitelist)}
  {--dry-run : show only which images will be build}
  {--t|threads=0 : threads}
  {--r|retry=0 : retry}

bin/command/generate_dockerfile_command.py

@@ -29,6 +29,7 @@ class GenerateDockerfileCommand(BaseCommand):
  Build Dockerfile containers
 
  generate:dockerfile
+ {docker images?* : Docker images (whitelist)}
  {--whitelist=?* : image/tag whitelist }
  {--blacklist=?* : image/tag blacklist }
  """

bin/command/test_serverspec_command.py

@@ -29,6 +29,7 @@ class TestServerspecCommand(DoitCommand):
  Test docker images with Serverspec
 
  test:serverspec
+ {docker images?* : Docker images (whitelist)}
  {--dry-run : show only which images will be build}
  {--t|threads=0 : threads}
  {--r|retry=0 : retry}

bin/command/test_testinfra_command.py

@@ -27,6 +27,7 @@ class TestTestinfraCommand(DoitCommand):
  Test docker images with Testinfra
 
  test:testinfra
+ {docker images?* : Docker images (whitelist)}
  {--dry-run : show only which images will be build}
  {--t|threads=0 : threads}
  {--whitelist=?* : image/tag whitelist }

bin/console

@@ -20,6 +20,7 @@ from webdevops.docker.DockerCliClient import DockerCliClient
 from command.docker_build_command import DockerBuildCommand
 from command.docker_push_command import DockerPushCommand
 from command.docker_pull_command import DockerPullCommand
+from command.docker_exec_command import DockerExecCommand
 from command.test_testinfra_command import TestTestinfraCommand
 from command.test_serverspec_command import TestServerspecCommand
 from command.generate_dockerfile_command import GenerateDockerfileCommand
@@ -88,6 +89,7 @@ if __name__ == '__main__':
  application.add(DockerBuildCommand(configuration=configuration))
  application.add(DockerPushCommand(configuration=configuration))
  application.add(DockerPullCommand(configuration=configuration))
+ application.add(DockerExecCommand(configuration=configuration))
  application.add(TestTestinfraCommand(configuration=configuration))
  application.add(TestServerspecCommand(configuration=configuration))
  application.add(GenerateDockerfileCommand(configuration=configuration))

bin/webdevops/DockerfileUtility.py

@@ -53,8 +53,13 @@ def find_file_in_path(dockerfile_path, filename="Dockerfile", whitelist=False, b
 
  # filter by whitelist
  if whitelist:
- for term in whitelist:
- file_list = filter(lambda x: term in x, file_list)
+ tmp = []
+ for file in file_list:
+ for term in whitelist:
+ if term in file:
+ tmp.append(file)
+ break
+ file_list = tmp
 
  if blacklist:
  for term in blacklist:
@@ -108,12 +113,11 @@ def parse_docker_info_from_path(path):
  }
  ret.append(dockerfile)
 
- if whitelist or blacklist:
- ret = filter_dockerfile(
- dockerfile_list=ret,
- whitelist=whitelist,
- blacklist = blacklist
- )
+ ret = filter_dockerfile(
+ dockerfile_list=ret,
+ whitelist=whitelist,
+ blacklist = blacklist
+ )
 
  return ret
 
@@ -123,8 +127,13 @@ def filter_dockerfile(dockerfile_list, whitelist=False, blacklist=False):
  Filter Dockerfiles by white- and blacklist
  """
  if whitelist:
- for term in whitelist:
- dockerfile_list = filter(lambda x: term in x['image']['fullname'], dockerfile_list)
+ tmp = []
+ for dockerfile in dockerfile_list:
+ for term in whitelist:
+ if term in dockerfile['image']['fullname']:
+ tmp.append(dockerfile)
+ break
+ dockerfile_list = tmp
 
  if blacklist:
  for term in blacklist:

bin/webdevops/command/BaseCommand.py

@@ -183,7 +183,17 @@ def get_whitelist(self):
  """
  Get whitelist
  """
- return list(self.option('whitelist'))
+
+ # add whitelist from --whitelist
+ ret = list(self.option('whitelist'))
+
+ # add whitelist from argument list
+ try:
+ ret.extend(self.argument('docker images'))
+ except (Exception):
+ pass
+
+ return ret
 
  def get_blacklist(self):
  """
@@ -211,13 +221,13 @@ def get_threads(self):
  # use configuration value
  threads = self.configuration.get('threads')
 
- match = re.match('auto(([-*+/])([0-9]+))?', str(threads))
+ match = re.match('auto(([-*+/])([0-9]+\.?[0-9]?))?', str(threads))
  if match is not None:
  ret = multiprocessing.cpu_count()
 
  if match.group(2) and match.group(3):
  math_sign = match.group(2)
- math_value = int(match.group(3))
+ math_value = float(match.group(3))
 
  if math_sign == "*":
  ret = int(ret * math_value)

bin/webdevops/docker/DockerCliClient.py

@@ -18,13 +18,10 @@
 # OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
 # OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
-import subprocess
-import os
-import tempfile
+import os, subprocess, tempfile
 from .DockerBaseClient import DockerBaseClient
 from webdevops import Command
 
-
 class DockerCliClient(DockerBaseClient):
 
  def pull_image(self, name, tag):

bin/webdevops/docker/DockerPyClient.py

@@ -18,7 +18,6 @@
 # OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
 # OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
-
 import os
 from .DockerBaseClient import DockerBaseClient
 

bin/webdevops/doit/DoitReporter.py

@@ -18,13 +18,10 @@
 # OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
 # OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
-import sys
-import os
-import time
-import datetime
-import StringIO
+import os, sys, time, datetime, StringIO
 import termcolor
 from termcolor import colored
+from ..taskloader.BaseTaskLoader import BaseTaskLoader
 
 class TaskResult(object):
  """
@@ -141,7 +138,7 @@ def add_failure(self, task, exception):
 
  if task.actions and (task.name[0] != '_'):
  duration = self.duration(self.t_results[task.name].elapsed)
- self.write(colored('. %s FAILED (%s)\n' % (task.title(), duration), 'red'))
+ self.writeln(colored('. %s FAILED (%s)' % (BaseTaskLoader.human_task_name(task.title()), duration), 'red'))
  self.failures.append({'task': task, 'exception': exception})
 
  def add_success(self, task):
@@ -152,7 +149,7 @@ def add_success(self, task):
 
  if task.actions and (task.name[0] != '_'):
  duration = self.duration(self.t_results[task.name].elapsed)
- self.write(colored('. %s finished (%s)\n' % (task.title(), duration), 'green'))
+ self.writeln(colored('. %s finished (%s)' % (BaseTaskLoader.human_task_name(task.title()), duration), 'green'))
 
  def skip_uptodate(self, task):
  """
@@ -252,7 +249,7 @@ def task_stdout(self, title, duration=False, stdout=False, stderr=False, error=F
  if duration:
  text_duration = ' (%s)' % self.duration(duration)
 
- title_full = 'Task %s%s:' % (title, text_duration)
+ title_full = 'Task %s%s:' % (BaseTaskLoader.human_task_name(title), text_duration)
 
  self.writeln(title_full)
  self.writeln('~' * len(title_full))
@@ -277,7 +274,7 @@ def task_stdout(self, title, duration=False, stdout=False, stderr=False, error=F
  self.write('%s' % exception.get_msg())
 
  self.writeln()
- self.writeln(':: end of output "%s"' % title)
+ self.writeln(':: end of output "%s"' % (BaseTaskLoader.human_task_name(title)))
  self.writeln()
 
  def duration(self, duration):

bin/webdevops/taskloader/BaseDockerTaskLoader.py

@@ -18,10 +18,7 @@
 # OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
 # OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
-import os
-import sys
-import re
-import copy
+import os, sys, re, copy
 from .BaseTaskLoader import BaseTaskLoader
 from webdevops import DockerfileUtility
 from doit.task import dict_to_task

bin/webdevops/taskloader/BaseTaskLoader.py

@@ -18,10 +18,7 @@
 # OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
 # OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
-import sys
-import re
-import time
-import StringIO
+import sys, re, time, StringIO
 from webdevops import DockerfileUtility
 from doit.cmd_base import TaskLoader
 from doit.task import dict_to_task
@@ -44,6 +41,9 @@ def process_tasklist(self, tasklist):
  ret = []
  for task in tasklist:
  ret.append(dict_to_task(task))
+
+ print 'Starting execution of %s tasks...' % (len(ret))
+
  return ret
 
 
@@ -52,7 +52,12 @@ def human_task_name(name):
  """
  Translate internal task name to human readable name
  """
- return re.search('^.*\|(.*)', name).group(1)
+ res = re.search('^.*\|(.*)', name)
+
+ if res:
+ return re.search('^.*\|(.*)', name).group(1)
+ else:
+ return name
 
 
  @staticmethod

bin/webdevops/taskloader/DockerBuildTaskLoader.py

@@ -18,11 +18,7 @@
 # OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
 # OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
-import os
-import sys
-import re
-import copy
-import time
+import os, sys, re, copy, time
 from .BaseTaskLoader import BaseTaskLoader
 from .BaseDockerTaskLoader import BaseDockerTaskLoader
 from webdevops import DockerfileUtility