Release 1.6.0

- Refactored and reduced size of all images (on-demand installation, removed not needed packages, improved ENVs/LABELs to reduce numbers of layers)
- Cleanup images
- Add new on-demand installation and provisioning of services (eg. postfix, ssh...)
- Improved python console handling and output, also fixes dependency detection
- Renamed alpine-3 to alpine (symlink to keep backward compatibility)
- Add webdevops/php-official and also to webdevops/php and other images
- Introduce docker-image-info for gathering information about current used image (family, distribution, version ...)
- Introduce docker-service enable and docker-service disable to enable and disable services (eg. postfix, ssh ...)
- Fix supervisord message about passwordless http server (unix socket)
- Fix and improve tests (eg. in ipv6 environments)
- Add go-replace to replace sed/awk stuff
- Add webdevops/liquisoap
- Add LOG_STDOUT and LOG_STDERR (redirect docker output, also possible to prevent output using /dev/null)
- Set nginx log level to warn in production and info in nginx-dev images
- Move ansible to webdevops/base

Makefile

@@ -57,7 +57,7 @@ graph-full:
  python ./bin/console generate:graph --all --filename docker-image-full-layout.gv
 
 documentation:
- docker run -t -i --rm -p 8080:8000 -v "$$(pwd)/documentation/docs/:/opt/docs" -e "VIRTUAL_HOST=documentation.docker" -e "VIRTUAL_PORT=8000" webdevops/sphinx sphinx-autobuild --poll -H 0.0.0.0 /opt/docs html
+ docker run -t -i --rm -p 8000 -v "$$(pwd)/documentation/docs/:/opt/docs" -e "VIRTUAL_HOST=documentation.docker" -e "VIRTUAL_PORT=8000" webdevops/sphinx sphinx-autobuild --poll -H 0.0.0.0 /opt/docs html
 
 webdevops/bootstrap:
  python ./bin/console docker:build --threads=auto --whitelist=webdevops/bootstrap

baselayout/usr/local/bin/apt-add-repository

@@ -0,0 +1,8 @@
+#!/bin/sh
+
+set -o nounset ## set -u : exit the script if you try to use an uninitialised variable
+set -o errexit ## set -e : exit the script if any statement returns a non-true return value
+
+apt-install software-properties-common
+add-apt-repository $@
+apt-get purge -y -f software-properties-common

baselayout/usr/local/bin/apt-install

@@ -7,12 +7,18 @@ set -o errexit ## set -e : exit the script if any statement returns a non-true
 
 export DEBIAN_FRONTEND=noninteractive
 
-# Update apt cache
-apt-get update
+if [[ -f "/tmp/.apt-update" ]]; then
+ echo "Detected prefetched 'apt-get update'"
+ # Install packages
+ apt-get install -y -f --no-install-recommends $*
+else
+ # Update apt cache
+ apt-get update
 
-# Install packages
-apt-get install -y -f --no-install-recommends $*
+ # Install packages
+ apt-get install -y -f --no-install-recommends $*
 
-# Clear files (reduce snapshot size)
-rm -rf /var/lib/apt/lists/*
-apt-get clean -y
+ # Clear files (reduce snapshot size)
+ rm -rf /var/lib/apt/lists/*
+ apt-get clean -y
+fi

baselayout/usr/local/bin/apt-update

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+set -o pipefail # trace ERR through pipes
+set -o errtrace # trace ERR through 'time command' and other functions
+set -o nounset ## set -u : exit the script if you try to use an uninitialised variable
+set -o errexit ## set -e : exit the script if any statement returns a non-true return value
+
+apt-get update
+touch /tmp/.apt-update

baselayout/usr/local/bin/apt-upgrade

@@ -7,12 +7,18 @@ set -o errexit ## set -e : exit the script if any statement returns a non-true
 
 export DEBIAN_FRONTEND=noninteractive
 
-# Update apt cache
-apt-get update
+if [[ -f "/tmp/.apt-update" ]]; then
+ echo "Detected prefetched 'apt-get update'"
+ # Update packages
+ apt-get dist-upgrade -y -f
+else
+ # Update apt cache
+ apt-get update
 
-# Install packages
-apt-get dist-upgrade -y -f
+ # Update packages
+ apt-get dist-upgrade -y -f
 
-# Clear files (reduce snapshot size)
-rm -rf /var/lib/apt/lists/*
-apt-get clean -y
+ # Clear files (reduce snapshot size)
+ rm -rf /var/lib/apt/lists/*
+ apt-get clean -y
+fi

baselayout/usr/local/bin/docker-image-cleanup

@@ -0,0 +1,36 @@
+#!/bin/sh
+
+set -o nounset ## set -u : exit the script if you try to use an uninitialised variable
+set -o errexit ## set -e : exit the script if any statement returns a non-true return value
+
+LSB_FAMILY=$(docker-image-info family)
+
+case "$LSB_FAMILY" in
+ Debian)
+ rm -f /tmp/.apt-update
+ apt-get autoremove -y -f
+ apt-get clean -y
+ rm -rf /var/lib/apt/lists/*
+ ;;
+
+ RedHat)
+ yum autoremove --assumeyes
+ yum clean all
+ ;;
+
+ Alpine)
+ find /var/lib/apk/ -mindepth 1 -delete
+ ;;
+
+ Arch)
+ pacman -Sc
+ ;;
+
+ *)
+ echo "ERROR: Distribution $LSB_FAMILY not supported"
+ exit 1
+ ;;
+esac
+
+find /tmp/ /var/log/ -mindepth 1 -delete
+rm -rf /root/.cache

baselayout/usr/local/bin/docker-image-info

@@ -0,0 +1,87 @@
+#!/bin/sh
+
+set -o nounset ## set -u : exit the script if you try to use an uninitialised variable
+set -o errexit ## set -e : exit the script if any statement returns a non-true return value
+
+help() {
+ if [ -n "$1" ]; then
+ echo "$1"
+ echo ""
+ fi
+
+ echo "Usage: $0 <argument>"
+ echo ""
+ echo " Application arguments:"
+ echo " family Get distribution family"
+ echo " dist Get distribution name"
+ echo " dist-version Get distribution version"
+ echo " dist-release Get distribution release"
+ echo " dist-codename Get distribution codename"
+ echo " lsb Get lsb informations (if available)"
+ echo " lsb-desc Get lsb description (if available)"
+ echo " buildtime Get buildtime of docker image"
+ echo ""
+
+ exit $2
+
+}
+
+if [ "$#" -ne 1 ]; then
+ help "[ERROR] Invalid argument" 1
+fi
+
+INFO_FILE=""
+
+case "$1" in
+ dist-family|distribution-family|family)
+ INFO_FILE=/opt/docker/etc/.registry/image_info_distribution_family
+ ;;
+
+ dist|distribution)
+ INFO_FILE=/opt/docker/etc/.registry/image_info_distribution
+ ;;
+
+ dist-version|distribution-version)
+ INFO_FILE=/opt/docker/etc/.registry/image_info_distribution_version
+ ;;
+
+ dist-release|distribution-release)
+ INFO_FILE=/opt/docker/etc/.registry/image_info_lsb_release
+ ;;
+
+ dist-codename|distribution-codename)
+ INFO_FILE=/opt/docker/etc/.registry/image_info_lsb_codename
+ ;;
+
+ lsb)
+ INFO_FILE=/opt/docker/etc/.registry/image_info_lsb
+ ;;
+
+ lsb-desc|lsb-description)
+ INFO_FILE=/opt/docker/etc/.registry/image_info_lsb_description
+ ;;
+
+ buildtime)
+ INFO_FILE=/opt/docker/etc/.registry/image_info_buildtime
+ ;;
+
+ help)
+ help "" 0
+ ;;
+
+ *)
+ help "[ERROR] Invalid argument" 1
+ ;;
+esac
+
+if [ -n "$INFO_FILE" ]; then
+ if [ -f "$INFO_FILE" ]; then
+ cat -- "$INFO_FILE"
+ else
+ echo "[ERROR] Infomation file $INFO_FILE not found!"
+ echo " Please run generate-dockerimage-info on docker image creation!"
+ exit 2
+ fi
+else
+ help "" 1
+fi

baselayout/usr/local/bin/generate-dockerimage-info

@@ -0,0 +1,96 @@
+#!/bin/sh
+
+set -o nounset ## set -u : exit the script if you try to use an uninitialised variable
+set -o errexit ## set -e : exit the script if any statement returns a non-true return value
+
+LSB_FAMILY=""
+
+#############################
+# Distribution detection
+#############################
+
+if [ -x "/usr/bin/apt-get" ]; then
+ # Debian family
+ LSB_FAMILY="Debian"
+
+elif [ -x "/bin/yum" ]; then
+ # RedHat family
+ LSB_FAMILY="RedHat"
+
+elif [ -x "/sbin/apk" ]; then
+ # Alpine family
+ LSB_FAMILY="Alpine"
+
+elif [ -f "/etc/arch-release" ]; then
+ # Alpine family
+ LSB_FAMILY="Arch"
+
+else
+ # Unknown
+ echo "ERROR: Distribution detection failed"
+ exit 1
+fi
+
+#############################
+# Install
+#############################
+
+case "$LSB_FAMILY" in
+ Debian)
+ apt-install lsb-release
+ ;;
+
+ RedHat)
+ yum-install redhat-lsb-core
+ ;;
+esac
+
+#############################
+# Set distribution information
+#############################
+
+echo "Detected $LSB_FAMILY"
+
+mkdir -p /opt/docker/etc/.registry/
+echo "$LSB_FAMILY" > /opt/docker/etc/.registry/image_info_distribution_family
+echo "$LSB_FAMILY" > /opt/docker/etc/.registry/image_info_distribution
+date +%s >/opt/docker/etc/.registry/image_info_buildtime
+
+# Create all files
+touch /opt/docker/etc/.registry/image_info_distribution_version
+touch /opt/docker/etc/.registry/image_info_lsb
+touch /opt/docker/etc/.registry/image_info_lsb_description
+touch /opt/docker/etc/.registry/image_info_lsb_release
+touch /opt/docker/etc/.registry/image_info_lsb_codename
+
+# Collect distribution specific informations
+case "$LSB_FAMILY" in
+ Debian|RedHat)
+ lsb_release -i -s > /opt/docker/etc/.registry/image_info_distribution
+ lsb_release -r -s > /opt/docker/etc/.registry/image_info_distribution_version
+ lsb_release -a > /opt/docker/etc/.registry/image_info_lsb
+ lsb_release -d -s > /opt/docker/etc/.registry/image_info_lsb_description
+ lsb_release -r -s > /opt/docker/etc/.registry/image_info_lsb_release
+ lsb_release -c -s > /opt/docker/etc/.registry/image_info_lsb_codename
+ ;;
+
+ Alpine)
+ cat /etc/alpine-release > /opt/docker/etc/.registry/image_info_distribution_version
+ ;;
+esac
+
+
+#############################
+# Uninstall
+#############################
+
+case "$LSB_FAMILY" in
+ Debian)
+ apt-get purge -y -f lsb-release
+ ;;
+
+ RedHat)
+ yum erase --assumeyes redhat-lsb-core
+ yum autoremove --assumeyes
+ ;;
+esac

baselayout/usr/local/bin/generate-locales

@@ -1,25 +1,28 @@
 #!/bin/bash
 
-set -o pipefail # trace ERR through pipes
-set -o errtrace # trace ERR through 'time command' and other functions
+set -o pipefail ## trace ERR through pipes
+set -o errtrace ## trace ERR through 'time command' and other functions
 set -o nounset ## set -u : exit the script if you try to use an uninitialised variable
 set -o errexit ## set -e : exit the script if any statement returns a non-true return value
 
+IMAGE_DISTRIBUTION_FAMILY=$(docker-image-info family)
+IMAGE_DISTRIBUTION=$(docker-image-info distribution)
+IMAGE_DISTRIBUTION_VERSION=$(docker-image-info distribution-version)
 
 #######################################
 ## Debian
 #######################################
 
-if [[ "$(lsb_release -i -s)" == "Debian" ]]; then
+if [[ "$IMAGE_DISTRIBUTION" == "Debian" ]]; then
  /usr/local/bin/apt-install locales-all
 fi
 
 #######################################
 ## Ubuntu
 #######################################
 
-if [[ "$(lsb_release -i -s)" == "Ubuntu" ]]; then
- if [[ "$(lsb_release -r -s | cut -f 1 -d .)" -ge "16" ]]; then
+if [[ "$IMAGE_DISTRIBUTION" == "Ubuntu" ]]; then
+ if [[ "$(echo $IMAGE_DISTRIBUTION_VERSION| cut -f 1 -d .)" -ge "16" ]]; then
  # Ubuntu 16.04 or later
  /usr/local/bin/apt-install locales-all
  else
@@ -33,12 +36,12 @@ fi
 ## RedHat family
 #######################################
 
-function localedefdebug() {
- echo $*
- localedef "$@"
-}
+if [[ "$IMAGE_DISTRIBUTION_FAMILY" == "RedHat" ]]; then
 
-if [[ -f /etc/redhat-release ]]; then
+ function localedefdebug() {
+ echo $*
+ localedef "$@"
+ }
 
  # Failing locales
  ## && localedefdebug -c -i bo_CN -f UTF-8 bo_CN.utf8 \

bin/webdevops/DockerfileUtility.py

@@ -79,6 +79,8 @@ def parse_docker_info_from_path(path):
  image_name = (image_name_info['image'] if 'image' in image_name_info else '')
  image_tag = (image_name_info['tag'] if 'tag' in image_name_info else '')
 
+ image_is_duplicate = False
+
  # check if path is linked
  if os.path.islink(os.path.dirname(path)):
  linked_image_name_info = ([m.groupdict() for m in path_regex.finditer(os.path.realpath(path))])[0]
@@ -88,6 +90,7 @@ def parse_docker_info_from_path(path):
  linked_image_tag = (linked_image_name_info['tag'] if 'tag' in linked_image_name_info else '')
 
  image_from = image_prefix + linked_image_repository + '/' + linked_image_name + ':' + linked_image_tag
+ image_is_duplicate = True
  else:
  image_from = parse_dockerfile_from_statement(path)
 
@@ -97,7 +100,8 @@ def parse_docker_info_from_path(path):
  'tag': image_tag,
  'repository': image_prefix + image_repository,
  'imageName': image_name,
- 'from': image_from
+ 'from': image_from,
+ 'duplicate': image_is_duplicate
  }
  return imageInfo
 

bin/webdevops/command/BaseCommand.py

@@ -23,6 +23,7 @@
 import multiprocessing
 from cleo import Command
 from webdevops import Configuration
+from ..doit.DoitReporter import DoitReporter
 
 class BaseCommand(Command):
  configuration = False
@@ -86,6 +87,8 @@ def startup(self):
 
  if 'dryRun' in self.configuration and self.configuration.get('dryRun'):
  options.append('dry-run')
+ DoitReporter.simulation_mode = True
+
 
  print 'Executing %s (%s)' % (self.name, ', '.join(options))
  print ''