Merge pull request #181 from qld-gov-au/develop

Develop to master

.ahoy.yml

@@ -10,14 +10,19 @@ commands:
       ahoy title "Building project"
       ahoy pre-flight
       ahoy clean
-      (docker network prune -f > /dev/null && docker network inspect amazeeio-network > /dev/null || docker network create amazeeio-network)
+      ahoy build-network
       ahoy up -- --build --force-recreate
-      ahoy install-dev
       ahoy install-site
       ahoy title "Build complete"
       ahoy doctor
       ahoy info 1
 
+  build-network:
+    usage: Ensure that the amazeeio network exists.
+    cmd: |
+      docker network prune -f > /dev/null
+      docker network inspect amazeeio-network > /dev/null || docker network create amazeeio-network
+
   info:
     usage: Print information about this project.
     cmd: |
@@ -33,15 +38,15 @@ commands:
       docker-compose up -d "$@"
       sleep 10
       docker-compose logs
-      ahoy cli "dockerize -wait tcp://ckan:3000 -timeout 1m || (docker-compose logs; exit 1)"
+      ahoy cli "dockerize -wait tcp://ckan:3000 -timeout 1m"
       if docker-compose logs | grep -q "\[Error\]"; then docker-compose logs; exit 1; fi
       if docker-compose logs | grep -q "Exception"; then docker-compose logs; exit 1; fi
       docker ps -a --filter name=^/${COMPOSE_PROJECT_NAME}_
       export DOCTOR_CHECK_CLI=0
 
   down:
     usage: Stop Docker containers and remove container, images, volumes and networks.
-    cmd: "if [ -f \"docker-compose.yml\" ]; then docker-compose down --volumes; fi"
+    cmd: 'if [ -f "docker-compose.yml" ]; then docker-compose down --volumes; fi'
 
   start:
     usage: Start existing Docker containers.
@@ -53,31 +58,29 @@ commands:
 
   restart:
     usage: Restart all stopped and running Docker containers.
-    cmd: docker-compose restart
+    cmd: docker-compose restart "$@"
 
   logs:
     usage: Show Docker logs.
     cmd: docker-compose logs "$@"
 
   pull:
     usage: Pull latest docker images.
-    cmd: if [ ! -z "$(docker image ls -q)" ]; then docker image ls --format \"{{.Repository}}:{{.Tag}}\" | grep 'amazeeio/|/ckan.*-dev' | grep -v none | xargs -n1 docker pull | cat; fi
+    cmd: if [ ! -z "$(docker image ls -q)" ]; then docker image ls --format \"{{.Repository}}:{{.Tag}}\" | grep amazeeio/ | grep -v none | xargs -n1 docker pull | cat; fi
 
   cli:
     usage: Start a shell inside CLI container or run a command.
-    cmd: if \[ "${#}" -ne 0 \]; then docker exec $(docker-compose ps -q ckan) sh -c ". /app/ckan/default/bin/activate; $*"; else docker exec $(docker-compose ps -q ckan) sh -c ". /app/ckan/default/bin/activate && sh"; fi
+    cmd: if \[ "${#}" -ne 0 \]; then docker exec $(docker-compose ps -q ckan) sh -c '. ${VENV_DIR}/bin/activate; cd $APP_DIR;'" $*"; else docker exec $(docker-compose ps -q ckan) sh -c '. ${VENV_DIR}/bin/activate && cd $APP_DIR && sh'; fi
 
   doctor:
     usage: Find problems with current project setup.
     cmd: .docker/scripts/doctor.sh "$@"
 
-
   install-site:
     usage: Install a site.
     cmd: |
       ahoy title "Installing a fresh site"
-      docker cp -L .docker/test.ini $(docker-compose ps -q ckan):/app/ckan/default/production.ini
-      ahoy cli "/app/scripts/init.sh"
+      ahoy cli "./scripts/init.sh"
 
   clean:
     usage: Remove containers and all build files.
@@ -96,36 +99,57 @@ commands:
       git ls-files --others -i --exclude-from=.git/info/exclude | xargs rm -Rf
       find . -type d -not -path "./.git/*" -empty -delete
 
-  install-dev:
-    usage: Install dependencies.
-    cmd: |
-      docker cp -L requirements-dev.txt $(docker-compose ps -q ckan):/app/.
-      docker cp -L .flake8 $(docker-compose ps -q ckan):/app/.
-      docker cp -L test $(docker-compose ps -q ckan):/app/.
-      ahoy cli "pip install -r /app/requirements-dev.txt"
-    hide: true
-
   flush-redis:
     usage: Flush Redis cache.
     cmd: docker exec -i $(docker-compose ps -q redis) redis-cli flushall > /dev/null
 
   lint:
     usage: Lint code.
     cmd: |
-      ahoy cli "flake8 ${@:-/app/ckanext}" || \
+      ahoy cli "flake8 ${@:-ckanext}" || \
       [ "${ALLOW_LINT_FAIL:-0}" -eq 1 ]
 
   test-unit:
     usage: Run unit tests.
     cmd: |
-      ahoy cli "nosetests" || \
+      ahoy cli 'nosetests --with-pylons=${CKAN_INI}' || \
       [ "${ALLOW_UNIT_FAIL:-0}" -eq 1 ]
 
   test-bdd:
     usage: Run BDD tests.
     cmd: |
-      ahoy cli "behave ${*:-/app/test/features}" || \
+      ahoy start-ckan-job-worker &
+      ahoy start-mailmock &
+      sleep 5 &&
+      ahoy cli "behave ${*:-test/features}" || \
       [ "${ALLOW_BDD_FAIL:-0}" -eq 1 ]
+      ahoy stop-mailmock
+      ahoy stop-ckan-job-worker
+
+  start-mailmock:
+    usage: Starts email mock server used for email BDD tests
+    cmd: |
+      ahoy title 'Starting mailmock'
+      ahoy cli 'mailmock -p 8025 -o ${APP_DIR}/test/emails --no-stdout' # for debugging mailmock email output remove --no-stdout
+
+  stop-mailmock:
+    usage: Stops email mock server used for email BDD tests
+    cmd: |
+      ahoy title 'Stopping mailmock'
+      ahoy cli "killall -2 mailmock"
+
+  start-ckan-job-worker:
+    usage: Starts CKAN background job worker
+    cmd: |
+      ahoy title 'Starting CKAN background job worker'
+      ahoy cli "ckan_cli jobs clear && \
+                ckan_cli jobs worker"
+
+  stop-ckan-job-worker:
+    usage: Stops CKAN background job worker
+    cmd: |
+      ahoy title 'Stopping CKAN background job worker'
+      ahoy cli "pkill -f 'jobs worker'"
 
   # Utilities.
   title:
@@ -161,5 +185,5 @@ entrypoint:
     export LAGOON_LOCALDEV_URL=http://ckanext-data-qld-theme.docker.amazee.io
     [ -f .env ] && [ -s .env ] && export $(grep -v '^#' .env | xargs) && if [ -f .env.local ] && [ -s .env.local ]; then export $(grep -v '^#' .env.local | xargs); fi
     bash -e -c "$0" "$@"
-  - '{{cmd}}'
-  - '{{name}}'
+  - "{{cmd}}"
+  - "{{name}}"

.circleci/process-artifacts.sh

@@ -6,8 +6,8 @@ set -e
 
 # Create screenshots directory in case it was not created before. This is to
 # avoid this script to fail when copying artifacts.
-ahoy cli "mkdir -p /app/test/screenshots"
+ahoy cli "mkdir -p test/screenshots"
 
 # Copy from the app container to the build host for storage.
-mkdir -p /tmp/artifacts/behave/screenshots
-docker cp "$(docker-compose ps -q ckan)":/app/test/screenshots /tmp/artifacts/behave
+mkdir -p /tmp/artifacts/behave
+docker cp "$(docker-compose ps -q ckan)":/app/test/screenshots /tmp/artifacts/behave/

.circleci/test.sh

@@ -5,10 +5,10 @@
 set -e
 
 echo "==> Lint code"
-ahoy lint || exit 1
+ahoy lint
 
 echo "==> Run Unit tests"
-ahoy test-unit || exit 1
+ahoy test-unit
 
 echo "==> Run BDD tests"
-ahoy test-bdd || exit 1
+ahoy test-bdd || (ahoy logs; exit 1)

.docker/Dockerfile.ckan

@@ -1,42 +1,46 @@
 FROM amazeeio/python:2.7-ckan-21.6.0
 
-ENV WORKDIR=/app
-
 ARG SITE_URL
 ENV SITE_URL="${SITE_URL}"
-ENV APP_DIR=/app/ckan/default
+ENV VENV_DIR=/app/ckan/default
+ENV APP_DIR=/app
 ENV CKAN_INI=/app/ckan/default/production.ini
 
+WORKDIR "${APP_DIR}"
+
 ENV DOCKERIZE_VERSION v0.6.1
-RUN apk add --no-cache curl && curl -s -L -O https://github.com/jwilder/dockerize/releases/download/${DOCKERIZE_VERSION}/dockerize-alpine-linux-amd64-${DOCKERIZE_VERSION}.tar.gz \
+RUN apk add --no-cache curl build-base \
+    && curl -s -L -O https://github.com/jwilder/dockerize/releases/download/${DOCKERIZE_VERSION}/dockerize-alpine-linux-amd64-${DOCKERIZE_VERSION}.tar.gz \
     && tar -C /usr/local/bin -xzvf dockerize-alpine-linux-amd64-${DOCKERIZE_VERSION}.tar.gz \
     && rm dockerize-alpine-linux-amd64-${DOCKERIZE_VERSION}.tar.gz
 
 # Install CKAN.
 ENV CKAN_VERSION 2.8.8
-RUN . ${APP_DIR}/bin/activate \
-    && cd ${APP_DIR} \
+
+RUN . ${VENV_DIR}/bin/activate \
     && pip install setuptools==36.1 \
     && pip install -e "git+https://github.com/ckan/ckan.git@ckan-${CKAN_VERSION}#egg=ckan" \
-    && sed -i "s/psycopg2==2.4.5/psycopg2==2.7.7/g" "${APP_DIR}/src/ckan/requirements.txt" \
-    && pip install -r "${APP_DIR}/src/ckan/requirements.txt" \
-    && ln -s "${APP_DIR}/src/ckan/who.ini" "${APP_DIR}/who.ini" \
+    && sed -i "s/psycopg2==2.4.5/psycopg2==2.7.7/g" "${VENV_DIR}/src/ckan/requirements.txt" \
+    && pip install -r "${VENV_DIR}/src/ckan/requirements.txt" \
+    && ln -s "${VENV_DIR}/src/ckan/who.ini" "${VENV_DIR}/who.ini" \
     && deactivate \
-    && ln -s /app/ckan /usr/lib/ckan
+    && ln -s ${APP_DIR}/ckan /usr/lib/ckan \
+    && fix-permissions ${APP_DIR}/ckan
 
-COPY .docker/test.ini /app/ckan/default/production.ini
+COPY .docker/test.ini $CKAN_INI
 
-COPY .docker/scripts /app/scripts
+# Add current extension and files.
+COPY . ${APP_DIR}/
 
-RUN fix-permissions /app/ckan \
-    && chmod +x /app/scripts/*.sh
+COPY .docker/scripts ${APP_DIR}/scripts
 
-# Add current extension and files.
-COPY ckanext /app/ckanext
-COPY requirements.txt requirements-dev.txt setup.cfg setup.py .flake8 /app/
+COPY .docker/scripts/ckan_cli ${VENV_DIR}/bin/
+
+RUN chmod +x ${APP_DIR}/scripts/*.sh\
+    && chmod +x ${VENV_DIR}/bin/ckan_cli
 
 # Init current extension.
-RUN /app/scripts/init-ext.sh
+RUN ${APP_DIR}/scripts/init-ext.sh
 
 ENTRYPOINT ["/sbin/tini", "--", "/lagoon/entrypoints.sh"]
 CMD ["/app/scripts/serve.sh"]

.docker/scripts/ckan_cli

@@ -0,0 +1,68 @@
+#!/bin/sh
+
+# Call either 'ckan' (from CKAN >= 2.9) or 'paster' (from CKAN <= 2.8)
+# with appropriate syntax, depending on what is present on the system.
+# This is intended to smooth the upgrade process from 2.8 to 2.9.
+# Eg:
+#   ckan_cli jobs list
+# could become either:
+#   paster --plugin=ckan jobs list -c /etc/ckan/default/production.ini
+# or:
+#   ckan -c /etc/ckan/default/production.ini jobs list
+
+# This script is aware of the VIRTUAL_ENV environment variable, and will
+# attempt to respect it with similar behaviour to commands like 'pip'.
+# Eg placing this script in a virtualenv 'bin' directory will cause it
+# to call the 'ckan' or 'paster' command in that directory, while
+# placing this script elsewhere will cause it to rely on the VIRTUAL_ENV
+# variable, or if that is not set, the system PATH.
+
+# Since the positioning of the CKAN configuration file is central to the
+# differences between 'paster' and 'ckan', this script needs to be aware
+# of the config file location. It will use the CKAN_INI environment
+# variable if it exists, or default to /etc/ckan/default/production.ini.
+
+# If 'paster' is being used, the default plugin is 'ckan'. A different
+# plugin can be specified by setting the PASTER_PLUGIN environment
+# variable. This variable is irrelevant if using the 'ckan' command.
+
+CKAN_INI="${CKAN_INI:-/etc/ckan/default/production.ini}"
+PASTER_PLUGIN="${PASTER_PLUGIN:-ckan}"
+# First, look for a command alongside this file
+ENV_DIR=$(dirname "$0")
+if [ -f "$ENV_DIR/ckan" ]; then
+    COMMAND=ckan
+elif [ -f "$ENV_DIR/paster" ]; then
+    COMMAND=paster
+elif [ "$VIRTUAL_ENV" != "" ]; then
+    # If command not found alongside this file, check the virtualenv
+    ENV_DIR="$VIRTUAL_ENV/bin"
+    if [ -f "$ENV_DIR/ckan" ]; then
+        COMMAND=ckan
+    elif [ -f "$ENV_DIR/paster" ]; then
+        COMMAND=paster
+    fi
+else
+    # if no virtualenv is active, try the system path
+    if (which ckan > /dev/null 2>&1); then
+        ENV_DIR=$(dirname $(which ckan))
+        COMMAND=ckan
+    elif (which paster > /dev/null 2>&1); then
+        ENV_DIR=$(dirname $(which paster))
+        COMMAND=paster
+    else
+        echo "Unable to locate 'ckan' or 'paster' command" >&2
+        exit 1
+    fi
+fi
+
+if [ "$COMMAND" = "ckan" ]; then
+    echo "Using 'ckan' command from $ENV_DIR with config ${CKAN_INI}..." >&2
+    exec $ENV_DIR/ckan -c ${CKAN_INI} "$@"
+elif [ "$COMMAND" = "paster" ]; then
+    echo "Using 'paster' command from $ENV_DIR with config ${CKAN_INI}..." >&2
+    exec $ENV_DIR/paster --plugin=$PASTER_PLUGIN "$@" -c ${CKAN_INI}
+else
+    echo "Unable to locate 'ckan' or 'paster' command in $ENV_DIR" >&2
+    exit 1
+fi