From 49c371888ce091d0e4091cc9eeed6bd442a9109f Mon Sep 17 00:00:00 2001
From: ThrawnCA <shell_layer-github@yahoo.com.au>
Date: Fri, 18 Jun 2021 15:36:35 +1000
Subject: [PATCH 1/7] [QOL-8104] tell bots not to index presigned S3 URLs

---
 ckanext/data_qld_theme/templates/robots.txt | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/ckanext/data_qld_theme/templates/robots.txt b/ckanext/data_qld_theme/templates/robots.txt
index 718fb4b8..b70c933f 100644
--- a/ckanext/data_qld_theme/templates/robots.txt
+++ b/ckanext/data_qld_theme/templates/robots.txt
@@ -71,4 +71,7 @@ Allow: /api/
 
 {% endraw %}
 {% endif %}
-{% endblock %}
\ No newline at end of file
+
+# presigned S3 URLs that are only temporary
+Disallow: /ckan-*-attachments-*/
+{% endblock %}

From 3f30b3b2ffc50afa7a8e5b39f665a8525a9e1dfb Mon Sep 17 00:00:00 2001
From: ThrawnCA <shell_layer-github@yahoo.com.au>
Date: Fri, 25 Jun 2021 12:04:43 +1000
Subject: [PATCH 2/7] [QOL-8104] sync CI config with other repos

- make directories consistent
- increase debugging on failures
- make test data more consistent
- use 'develop' branches of other QGOV extensions
---
 .ahoy.yml                           |  72 +++++++++++------
 .circleci/process-artifacts.sh      |   6 +-
 .circleci/test.sh                   |   6 +-
 .docker/Dockerfile.ckan             |  40 +++++-----
 .docker/scripts/ckan_cli            |  68 ++++++++++++++++
 .docker/scripts/create-test-data.sh | 119 ++++++++++++++++++++++++----
 .docker/scripts/init-ext.sh         |  17 ++--
 .docker/scripts/init.sh             |  22 ++---
 .docker/scripts/serve.sh            |   5 +-
 .docker/test.ini                    |   2 +-
 .env                                |   2 +-
 .github/workflows/test.yml          |  15 +++-
 docker-compose.yml                  |  10 +--
 requirements-dev.txt                |   6 +-
 setup.py                            |  18 ++---
 test/features/environment.py        |  53 +++++++++++--
 16 files changed, 352 insertions(+), 109 deletions(-)
 create mode 100644 .docker/scripts/ckan_cli

diff --git a/.ahoy.yml b/.ahoy.yml
index 4deb93e8..a649c4c2 100644
--- a/.ahoy.yml
+++ b/.ahoy.yml
@@ -10,14 +10,19 @@ commands:
       ahoy title "Building project"
       ahoy pre-flight
       ahoy clean
-      (docker network prune -f > /dev/null && docker network inspect amazeeio-network > /dev/null || docker network create amazeeio-network)
+      ahoy build-network
       ahoy up -- --build --force-recreate
-      ahoy install-dev
       ahoy install-site
       ahoy title "Build complete"
       ahoy doctor
       ahoy info 1
 
+  build-network:
+    usage: Ensure that the amazeeio network exists.
+    cmd: |
+      docker network prune -f > /dev/null
+      docker network inspect amazeeio-network > /dev/null || docker network create amazeeio-network
+
   info:
     usage: Print information about this project.
     cmd: |
@@ -33,7 +38,7 @@ commands:
       docker-compose up -d "$@"
       sleep 10
       docker-compose logs
-      ahoy cli "dockerize -wait tcp://ckan:3000 -timeout 1m || (docker-compose logs; exit 1)"
+      ahoy cli "dockerize -wait tcp://ckan:3000 -timeout 1m"
       if docker-compose logs | grep -q "\[Error\]"; then docker-compose logs; exit 1; fi
       if docker-compose logs | grep -q "Exception"; then docker-compose logs; exit 1; fi
       docker ps -a --filter name=^/${COMPOSE_PROJECT_NAME}_
@@ -41,7 +46,7 @@ commands:
 
   down:
     usage: Stop Docker containers and remove container, images, volumes and networks.
-    cmd: "if [ -f \"docker-compose.yml\" ]; then docker-compose down --volumes; fi"
+    cmd: 'if [ -f "docker-compose.yml" ]; then docker-compose down --volumes; fi'
 
   start:
     usage: Start existing Docker containers.
@@ -53,7 +58,7 @@ commands:
 
   restart:
     usage: Restart all stopped and running Docker containers.
-    cmd: docker-compose restart
+    cmd: docker-compose restart "$@"
 
   logs:
     usage: Show Docker logs.
@@ -61,23 +66,21 @@ commands:
 
   pull:
     usage: Pull latest docker images.
-    cmd: if [ ! -z "$(docker image ls -q)" ]; then docker image ls --format \"{{.Repository}}:{{.Tag}}\" | grep 'amazeeio/|/ckan.*-dev' | grep -v none | xargs -n1 docker pull | cat; fi
+    cmd: if [ ! -z "$(docker image ls -q)" ]; then docker image ls --format \"{{.Repository}}:{{.Tag}}\" | grep amazeeio/ | grep -v none | xargs -n1 docker pull | cat; fi
 
   cli:
     usage: Start a shell inside CLI container or run a command.
-    cmd: if \[ "${#}" -ne 0 \]; then docker exec $(docker-compose ps -q ckan) sh -c ". /app/ckan/default/bin/activate; $*"; else docker exec $(docker-compose ps -q ckan) sh -c ". /app/ckan/default/bin/activate && sh"; fi
+    cmd: if \[ "${#}" -ne 0 \]; then docker exec $(docker-compose ps -q ckan) sh -c '. ${VENV_DIR}/bin/activate; cd $APP_DIR;'" $*"; else docker exec $(docker-compose ps -q ckan) sh -c '. ${VENV_DIR}/bin/activate && cd $APP_DIR && sh'; fi
 
   doctor:
     usage: Find problems with current project setup.
     cmd: .docker/scripts/doctor.sh "$@"
 
-
   install-site:
     usage: Install a site.
     cmd: |
       ahoy title "Installing a fresh site"
-      docker cp -L .docker/test.ini $(docker-compose ps -q ckan):/app/ckan/default/production.ini
-      ahoy cli "/app/scripts/init.sh"
+      ahoy cli "./scripts/init.sh"
 
   clean:
     usage: Remove containers and all build files.
@@ -96,15 +99,6 @@ commands:
       git ls-files --others -i --exclude-from=.git/info/exclude | xargs rm -Rf
       find . -type d -not -path "./.git/*" -empty -delete
 
-  install-dev:
-    usage: Install dependencies.
-    cmd: |
-      docker cp -L requirements-dev.txt $(docker-compose ps -q ckan):/app/.
-      docker cp -L .flake8 $(docker-compose ps -q ckan):/app/.
-      docker cp -L test $(docker-compose ps -q ckan):/app/.
-      ahoy cli "pip install -r /app/requirements-dev.txt"
-    hide: true
-
   flush-redis:
     usage: Flush Redis cache.
     cmd: docker exec -i $(docker-compose ps -q redis) redis-cli flushall > /dev/null
@@ -112,20 +106,50 @@ commands:
   lint:
     usage: Lint code.
     cmd: |
-      ahoy cli "flake8 ${@:-/app/ckanext}" || \
+      ahoy cli "flake8 ${@:-ckanext}" || \
       [ "${ALLOW_LINT_FAIL:-0}" -eq 1 ]
 
   test-unit:
     usage: Run unit tests.
     cmd: |
-      ahoy cli "nosetests" || \
+      ahoy cli 'nosetests --with-pylons=${CKAN_INI}' || \
       [ "${ALLOW_UNIT_FAIL:-0}" -eq 1 ]
 
   test-bdd:
     usage: Run BDD tests.
     cmd: |
-      ahoy cli "behave ${*:-/app/test/features}" || \
+      ahoy start-ckan-job-worker &
+      ahoy start-mailmock &
+      sleep 5 &&
+      ahoy cli "behave ${*:-test/features}" || \
       [ "${ALLOW_BDD_FAIL:-0}" -eq 1 ]
+      ahoy stop-mailmock
+      ahoy stop-ckan-job-worker
+
+  start-mailmock:
+    usage: Starts email mock server used for email BDD tests
+    cmd: |
+      ahoy title 'Starting mailmock'
+      ahoy cli 'mailmock -p 8025 -o ${APP_DIR}/test/emails --no-stdout' # for debugging mailmock email output remove --no-stdout
+
+  stop-mailmock:
+    usage: Stops email mock server used for email BDD tests
+    cmd: |
+      ahoy title 'Stopping mailmock'
+      ahoy cli "killall -2 mailmock"
+
+  start-ckan-job-worker:
+    usage: Starts CKAN background job worker
+    cmd: |
+      ahoy title 'Starting CKAN background job worker'
+      ahoy cli "ckan_cli jobs clear && \
+                ckan_cli jobs worker"
+
+  stop-ckan-job-worker:
+    usage: Stops CKAN background job worker
+    cmd: |
+      ahoy title 'Stopping CKAN background job worker'
+      ahoy cli "pkill -f 'jobs worker'"
 
   # Utilities.
   title:
@@ -161,5 +185,5 @@ entrypoint:
     export LAGOON_LOCALDEV_URL=http://ckanext-data-qld-theme.docker.amazee.io
     [ -f .env ] && [ -s .env ] && export $(grep -v '^#' .env | xargs) && if [ -f .env.local ] && [ -s .env.local ]; then export $(grep -v '^#' .env.local | xargs); fi
     bash -e -c "$0" "$@"
-  - '{{cmd}}'
-  - '{{name}}'
+  - "{{cmd}}"
+  - "{{name}}"
diff --git a/.circleci/process-artifacts.sh b/.circleci/process-artifacts.sh
index 8fbc3d20..55bdbef9 100755
--- a/.circleci/process-artifacts.sh
+++ b/.circleci/process-artifacts.sh
@@ -6,8 +6,8 @@ set -e
 
 # Create screenshots directory in case it was not created before. This is to
 # avoid this script to fail when copying artifacts.
-ahoy cli "mkdir -p /app/test/screenshots"
+ahoy cli "mkdir -p test/screenshots"
 
 # Copy from the app container to the build host for storage.
-mkdir -p /tmp/artifacts/behave/screenshots
-docker cp "$(docker-compose ps -q ckan)":/app/test/screenshots /tmp/artifacts/behave
+mkdir -p /tmp/artifacts/behave
+docker cp "$(docker-compose ps -q ckan)":/app/test/screenshots /tmp/artifacts/behave/
diff --git a/.circleci/test.sh b/.circleci/test.sh
index 864b4b34..9c584d9b 100755
--- a/.circleci/test.sh
+++ b/.circleci/test.sh
@@ -5,10 +5,10 @@
 set -e
 
 echo "==> Lint code"
-ahoy lint || exit 1
+ahoy lint
 
 echo "==> Run Unit tests"
-ahoy test-unit || exit 1
+ahoy test-unit
 
 echo "==> Run BDD tests"
-ahoy test-bdd || exit 1
+ahoy test-bdd || (ahoy logs; exit 1)
diff --git a/.docker/Dockerfile.ckan b/.docker/Dockerfile.ckan
index bf8b7dfc..789bf9e1 100644
--- a/.docker/Dockerfile.ckan
+++ b/.docker/Dockerfile.ckan
@@ -1,42 +1,46 @@
 FROM amazeeio/python:2.7-ckan-21.6.0
 
-ENV WORKDIR=/app
-
 ARG SITE_URL
 ENV SITE_URL="${SITE_URL}"
-ENV APP_DIR=/app/ckan/default
+ENV VENV_DIR=/app/ckan/default
+ENV APP_DIR=/app
 ENV CKAN_INI=/app/ckan/default/production.ini
 
+WORKDIR "${APP_DIR}"
+
 ENV DOCKERIZE_VERSION v0.6.1
-RUN apk add --no-cache curl && curl -s -L -O https://github.com/jwilder/dockerize/releases/download/${DOCKERIZE_VERSION}/dockerize-alpine-linux-amd64-${DOCKERIZE_VERSION}.tar.gz \
+RUN apk add --no-cache curl build-base \
+    && curl -s -L -O https://github.com/jwilder/dockerize/releases/download/${DOCKERIZE_VERSION}/dockerize-alpine-linux-amd64-${DOCKERIZE_VERSION}.tar.gz \
     && tar -C /usr/local/bin -xzvf dockerize-alpine-linux-amd64-${DOCKERIZE_VERSION}.tar.gz \
     && rm dockerize-alpine-linux-amd64-${DOCKERIZE_VERSION}.tar.gz
 
 # Install CKAN.
 ENV CKAN_VERSION 2.8.8
-RUN . ${APP_DIR}/bin/activate \
-    && cd ${APP_DIR} \
+
+RUN . ${VENV_DIR}/bin/activate \
     && pip install setuptools==36.1 \
     && pip install -e "git+https://github.com/ckan/ckan.git@ckan-${CKAN_VERSION}#egg=ckan" \
-    && sed -i "s/psycopg2==2.4.5/psycopg2==2.7.7/g" "${APP_DIR}/src/ckan/requirements.txt" \
-    && pip install -r "${APP_DIR}/src/ckan/requirements.txt" \
-    && ln -s "${APP_DIR}/src/ckan/who.ini" "${APP_DIR}/who.ini" \
+    && sed -i "s/psycopg2==2.4.5/psycopg2==2.7.7/g" "${VENV_DIR}/src/ckan/requirements.txt" \
+    && pip install -r "${VENV_DIR}/src/ckan/requirements.txt" \
+    && ln -s "${VENV_DIR}/src/ckan/who.ini" "${VENV_DIR}/who.ini" \
     && deactivate \
-    && ln -s /app/ckan /usr/lib/ckan
+    && ln -s ${APP_DIR}/ckan /usr/lib/ckan \
+    && fix-permissions ${APP_DIR}/ckan
 
-COPY .docker/test.ini /app/ckan/default/production.ini
+COPY .docker/test.ini $CKAN_INI
 
-COPY .docker/scripts /app/scripts
+# Add current extension and files.
+COPY . ${APP_DIR}/
 
-RUN fix-permissions /app/ckan \
-    && chmod +x /app/scripts/*.sh
+COPY .docker/scripts ${APP_DIR}/scripts
 
-# Add current extension and files.
-COPY ckanext /app/ckanext
-COPY requirements.txt requirements-dev.txt setup.cfg setup.py .flake8 /app/
+COPY .docker/scripts/ckan_cli ${VENV_DIR}/bin/
+
+RUN chmod +x ${APP_DIR}/scripts/*.sh\
+    && chmod +x ${VENV_DIR}/bin/ckan_cli
 
 # Init current extension.
-RUN /app/scripts/init-ext.sh
+RUN ${APP_DIR}/scripts/init-ext.sh
 
 ENTRYPOINT ["/sbin/tini", "--", "/lagoon/entrypoints.sh"]
 CMD ["/app/scripts/serve.sh"]
diff --git a/.docker/scripts/ckan_cli b/.docker/scripts/ckan_cli
new file mode 100644
index 00000000..fcc045fd
--- /dev/null
+++ b/.docker/scripts/ckan_cli
@@ -0,0 +1,68 @@
+#!/bin/sh
+
+# Call either 'ckan' (from CKAN >= 2.9) or 'paster' (from CKAN <= 2.8)
+# with appropriate syntax, depending on what is present on the system.
+# This is intended to smooth the upgrade process from 2.8 to 2.9.
+# Eg:
+#   ckan_cli jobs list
+# could become either:
+#   paster --plugin=ckan jobs list -c /etc/ckan/default/production.ini
+# or:
+#   ckan -c /etc/ckan/default/production.ini jobs list
+
+# This script is aware of the VIRTUAL_ENV environment variable, and will
+# attempt to respect it with similar behaviour to commands like 'pip'.
+# Eg placing this script in a virtualenv 'bin' directory will cause it
+# to call the 'ckan' or 'paster' command in that directory, while
+# placing this script elsewhere will cause it to rely on the VIRTUAL_ENV
+# variable, or if that is not set, the system PATH.
+
+# Since the positioning of the CKAN configuration file is central to the
+# differences between 'paster' and 'ckan', this script needs to be aware
+# of the config file location. It will use the CKAN_INI environment
+# variable if it exists, or default to /etc/ckan/default/production.ini.
+
+# If 'paster' is being used, the default plugin is 'ckan'. A different
+# plugin can be specified by setting the PASTER_PLUGIN environment
+# variable. This variable is irrelevant if using the 'ckan' command.
+
+CKAN_INI="${CKAN_INI:-/etc/ckan/default/production.ini}"
+PASTER_PLUGIN="${PASTER_PLUGIN:-ckan}"
+# First, look for a command alongside this file
+ENV_DIR=$(dirname "$0")
+if [ -f "$ENV_DIR/ckan" ]; then
+    COMMAND=ckan
+elif [ -f "$ENV_DIR/paster" ]; then
+    COMMAND=paster
+elif [ "$VIRTUAL_ENV" != "" ]; then
+    # If command not found alongside this file, check the virtualenv
+    ENV_DIR="$VIRTUAL_ENV/bin"
+    if [ -f "$ENV_DIR/ckan" ]; then
+        COMMAND=ckan
+    elif [ -f "$ENV_DIR/paster" ]; then
+        COMMAND=paster
+    fi
+else
+    # if no virtualenv is active, try the system path
+    if (which ckan > /dev/null 2>&1); then
+        ENV_DIR=$(dirname $(which ckan))
+        COMMAND=ckan
+    elif (which paster > /dev/null 2>&1); then
+        ENV_DIR=$(dirname $(which paster))
+        COMMAND=paster
+    else
+        echo "Unable to locate 'ckan' or 'paster' command" >&2
+        exit 1
+    fi
+fi
+
+if [ "$COMMAND" = "ckan" ]; then
+    echo "Using 'ckan' command from $ENV_DIR with config ${CKAN_INI}..." >&2
+    exec $ENV_DIR/ckan -c ${CKAN_INI} "$@"
+elif [ "$COMMAND" = "paster" ]; then
+    echo "Using 'paster' command from $ENV_DIR with config ${CKAN_INI}..." >&2
+    exec $ENV_DIR/paster --plugin=$PASTER_PLUGIN "$@" -c ${CKAN_INI}
+else
+    echo "Unable to locate 'ckan' or 'paster' command in $ENV_DIR" >&2
+    exit 1
+fi
diff --git a/.docker/scripts/create-test-data.sh b/.docker/scripts/create-test-data.sh
index 6dc2e1c8..fd5df67b 100644
--- a/.docker/scripts/create-test-data.sh
+++ b/.docker/scripts/create-test-data.sh
@@ -1,4 +1,3 @@
-
 #!/usr/bin/env sh
 ##
 # Create some example content for extension BDD tests.
@@ -7,31 +6,123 @@ set -e
 
 CKAN_ACTION_URL=http://ckan:3000/api/action
 
-. ${APP_DIR}/bin/activate
-
-ckan_cli () {
-    if (which ckan > /dev/null); then
-        ckan -c ${CKAN_INI} "$@"
-    else
-        paster --plugin=ckan "$@" -c ${CKAN_INI}
-    fi
-}
+if [ "$VENV_DIR" != "" ]; then
+  . ${VENV_DIR}/bin/activate
+fi
 
 # We know the "admin" sysadmin account exists, so we'll use her API KEY to create further data
 API_KEY=$(ckan_cli user admin | tr -d '\n' | sed -r 's/^(.*)apikey=(\S*)(.*)/\2/')
 
+##
+# BEGIN: Create a test organisation with test users for admin, editor and member
+#
+TEST_ORG_NAME=test-organisation
+TEST_ORG_TITLE="Test Organisation"
+
+echo "Creating test users for ${TEST_ORG_TITLE} Organisation:"
+
+ckan_cli user add ckan_user email=ckan_user@localhost password=password
+ckan_cli user add test_org_admin email=test_org_admin@localhost password=password
+ckan_cli user add test_org_editor email=test_org_editor@localhost password=password
+ckan_cli user add test_org_member email=test_org_member@localhost password=password
+
+echo "Creating ${TEST_ORG_TITLE} Organisation:"
+
+TEST_ORG=$( \
+    curl -LsH "Authorization: ${API_KEY}" \
+    --data "name=${TEST_ORG_NAME}&title=${TEST_ORG_TITLE}" \
+    ${CKAN_ACTION_URL}/organization_create
+)
+
+TEST_ORG_ID=$(echo $TEST_ORG | sed -r 's/^(.*)"id": "(.*)",(.*)/\2/')
+
+echo "Assigning test users to ${TEST_ORG_TITLE} Organisation:"
+
+curl -LsH "Authorization: ${API_KEY}" \
+    --data "id=${TEST_ORG_ID}&object=test_org_admin&object_type=user&capacity=admin" \
+    ${CKAN_ACTION_URL}/member_create
+
+curl -LsH "Authorization: ${API_KEY}" \
+    --data "id=${TEST_ORG_ID}&object=test_org_editor&object_type=user&capacity=editor" \
+    ${CKAN_ACTION_URL}/member_create
+
+curl -LsH "Authorization: ${API_KEY}" \
+    --data "id=${TEST_ORG_ID}&object=test_org_member&object_type=user&capacity=member" \
+    ${CKAN_ACTION_URL}/member_create
+##
+# END.
+#
+
 # Creating test data hierarchy which creates organisations assigned to datasets
 ckan_cli create-test-data hierarchy
 
 # Creating basic test data which has datasets with resources
 ckan_cli create-test-data
 
-echo "Updating annakarenina to use department-of-health Organisation:"
+# Datasets need to be assigned to an organisation
+
+echo "Assigning test Datasets to Organisation..."
+
+echo "Updating annakarenina to use ${TEST_ORG_TITLE} organisation:"
+package_owner_org_update=$( \
+    curl -LsH "Authorization: ${API_KEY}" \
+    --data "id=annakarenina&organization_id=${TEST_ORG_NAME}" \
+    ${CKAN_ACTION_URL}/package_owner_org_update
+)
+echo ${package_owner_org_update}
+
+echo "Updating warandpeace to use ${TEST_ORG_TITLE} organisation:"
 package_owner_org_update=$( \
-    curl -L -s --header "Authorization: ${API_KEY}" \
-    --data "id=annakarenina&organization_id=department-of-health" \
+    curl -LsH "Authorization: ${API_KEY}" \
+    --data "id=warandpeace&organization_id=${TEST_ORG_NAME}" \
     ${CKAN_ACTION_URL}/package_owner_org_update
 )
 echo ${package_owner_org_update}
 
-deactivate
+##
+# BEGIN: Create a Data Request organisation with test users for admin, editor and member and default data requests
+#
+# Data Requests requires a specific organisation to exist in order to create DRs for Data.Qld
+DR_ORG_NAME=open-data-administration-data-requests
+DR_ORG_TITLE="Open Data Administration (data requests)"
+
+echo "Creating test users for ${DR_ORG_TITLE} Organisation:"
+
+ckan_cli user add dr_admin email=dr_admin@localhost password=password
+ckan_cli user add dr_editor email=dr_editor@localhost password=password
+ckan_cli user add dr_member email=dr_member@localhost password=password
+
+echo "Creating ${DR_ORG_TITLE} Organisation:"
+
+DR_ORG=$( \
+    curl -LsH "Authorization: ${API_KEY}" \
+    --data "name=${DR_ORG_NAME}&title=${DR_ORG_TITLE}" \
+    ${CKAN_ACTION_URL}/organization_create
+)
+
+DR_ORG_ID=$(echo $DR_ORG | sed -r 's/^(.*)"id": "(.*)",(.*)/\2/')
+
+echo "Assigning test users to ${DR_ORG_TITLE} Organisation:"
+
+curl -LsH "Authorization: ${API_KEY}" \
+    --data "id=${DR_ORG_ID}&object=dr_admin&object_type=user&capacity=admin" \
+    ${CKAN_ACTION_URL}/member_create
+
+curl -LsH "Authorization: ${API_KEY}" \
+    --data "id=${DR_ORG_ID}&object=dr_editor&object_type=user&capacity=editor" \
+    ${CKAN_ACTION_URL}/member_create
+
+curl -LsH "Authorization: ${API_KEY}" \
+    --data "id=${DR_ORG_ID}&object=dr_member&object_type=user&capacity=member" \
+    ${CKAN_ACTION_URL}/member_create
+
+
+echo "Creating test Data Request:"
+
+curl -LsH "Authorization: ${API_KEY}" \
+    --data "title=Test Request&description=This is an example&organization_id=${DR_ORG_ID}" \
+    ${CKAN_ACTION_URL}/create_datarequest
+
+if [ "$VENV_DIR" != "" ]; then
+  deactivate
+fi
diff --git a/.docker/scripts/init-ext.sh b/.docker/scripts/init-ext.sh
index 9e6df825..c6b00ec9 100755
--- a/.docker/scripts/init-ext.sh
+++ b/.docker/scripts/init-ext.sh
@@ -4,12 +4,17 @@
 #
 set -e
 
-PIP="${APP_DIR}/bin/pip"
-cd $WORKDIR
-$PIP install -r "requirements.txt"
-$PIP install -r "requirements-dev.txt"
-$APP_DIR/bin/python setup.py develop
+if [ "$VENV_DIR" != "" ]; then
+  . ${VENV_DIR}/bin/activate
+fi
+pip install -r "requirements-dev.txt"
+pip install -r "requirements.txt"
+python setup.py develop
+installed_name=$(grep '^\s*name=' setup.py |sed "s|[^']*'\([-a-zA-Z0-9]*\)'.*|\1|")
 
 # Validate that the extension was installed correctly.
-if ! $PIP list | grep ckanext-data-qld-theme > /dev/null; then echo "Unable to find the extension in the list"; exit 1; fi
+if ! pip list | grep "$installed_name" > /dev/null; then echo "Unable to find the extension in the list"; exit 1; fi
 
+if [ "$VENV_DIR" != "" ]; then
+  deactivate
+fi
diff --git a/.docker/scripts/init.sh b/.docker/scripts/init.sh
index da423733..eca9485b 100755
--- a/.docker/scripts/init.sh
+++ b/.docker/scripts/init.sh
@@ -9,22 +9,16 @@ CKAN_DISPLAY_NAME="${CKAN_DISPLAY_NAME:-Administrator}"
 CKAN_USER_PASSWORD="${CKAN_USER_PASSWORD:-Password123!}"
 CKAN_USER_EMAIL="${CKAN_USER_EMAIL:-admin@localhost}"
 
-. ${APP_DIR}/bin/activate
-
-ckan_cli () {
-    if (which ckan > /dev/null); then
-        ckan -c ${CKAN_INI} "$@"
-    else
-        paster --plugin=ckan "$@" -c ${CKAN_INI}
-    fi
-}
-ckan_cli db clean || exit 1
-ckan_cli db init || exit 1
+if [ "$VENV_DIR" != "" ]; then
+  . ${VENV_DIR}/bin/activate
+fi
+ckan_cli db clean
+ckan_cli db init
 ckan_cli user add "${CKAN_USER_NAME}"\
  fullname="${CKAN_DISPLAY_NAME}"\
  email="${CKAN_USER_EMAIL}"\
- password="${CKAN_USER_PASSWORD}" || exit 1
-ckan_cli sysadmin add "${CKAN_USER_NAME}" || exit 1
+ password="${CKAN_USER_PASSWORD}"
+ckan_cli sysadmin add "${CKAN_USER_NAME}"
 
 # Create some base test data
-. $WORKDIR/scripts/create-test-data.sh
+. $APP_DIR/scripts/create-test-data.sh
diff --git a/.docker/scripts/serve.sh b/.docker/scripts/serve.sh
index d95eeb70..10f3dead 100755
--- a/.docker/scripts/serve.sh
+++ b/.docker/scripts/serve.sh
@@ -3,10 +3,13 @@ set -e
 
 dockerize -wait tcp://postgres:5432 -timeout 1m
 dockerize -wait tcp://solr:8983 -timeout 1m
+dockerize -wait tcp://redis:6379 -timeout 1m
 
 sed -i "s@SITE_URL@${SITE_URL}@g" $CKAN_INI
 
-. ${APP_DIR}/bin/activate
+if [ "$VENV_DIR" != "" ]; then
+  . ${VENV_DIR}/bin/activate
+fi
 if (which ckan > /dev/null); then
     ckan -c ${CKAN_INI} run
 else
diff --git a/.docker/test.ini b/.docker/test.ini
index 196fc17c..4fa690fa 100644
--- a/.docker/test.ini
+++ b/.docker/test.ini
@@ -92,7 +92,7 @@ ckan.redis.url = redis://redis:6379
 
 # Note: Add ``datastore`` to enable the CKAN DataStore
 #       Add ``datapusher`` to enable DataPusher
-#       Add ``resource_proxy`` to enable resorce proxying and get around the
+#       Add ``resource_proxy`` to enable resource proxying and get around the
 #       same origin policy
 # @todo:setup Cleanup the list to use only required plugins.
 ckan.plugins = stats text_view image_view recline_view datastore datarequests data_qld_resources data_qld_theme qgovext
diff --git a/.env b/.env
index 589b48cd..9afe27bb 100644
--- a/.env
+++ b/.env
@@ -13,7 +13,7 @@
 PROJECT="ckanext-data-qld-theme"
 
 # Docker Compose project name. All containers will have this name.
-COMPOSE_PROJECT_NAME="ckanext-data-qld-theme"
+COMPOSE_PROJECT_NAME="$PROJECT"
 
 # Flag to allow code linting failures.
 ALLOW_LINT_FAIL=0
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 2b6955f0..89fa60d7 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -10,7 +10,7 @@ jobs:
     strategy:
       fail-fast: false
 
-    name: OpenData theme build
+    name: Continuous Integration build
     runs-on: ubuntu-latest
     container: integratedexperts/ci-builder
 
@@ -25,3 +25,16 @@ jobs:
       - name: Test
         run: .circleci/test.sh
         timeout-minutes: 15
+
+      - name: Retrieve screenshots
+        if: failure()
+        run: .circleci/process-artifacts.sh
+        timeout-minutes: 1
+
+      - name: Upload screenshots
+        if: failure()
+        uses: actions/upload-artifact@v2
+        with:
+          name: screenshots
+          path: /tmp/artifacts/behave/screenshots
+        timeout-minutes: 1
diff --git a/docker-compose.yml b/docker-compose.yml
index 555ac3d8..15417d94 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -1,7 +1,7 @@
 version: '2.3'
 
 x-project:
-  &project ckanext-data-qld-theme
+  &project "${PROJECT}"
 
 x-volumes:
   &default-volumes
@@ -29,7 +29,7 @@ services:
       context: .
       dockerfile: .docker/Dockerfile.ckan
       args:
-        SITE_URL: http://ckanext-data-qld-theme.docker.amazee.io
+        SITE_URL: "http://${PROJECT}.docker.amazee.io"
     depends_on:
       - postgres
       - solr
@@ -43,8 +43,8 @@ services:
     environment:
       <<: *default-environment
       AMAZEEIO_HTTP_PORT: 3000
-      LAGOON_LOCALDEV_URL: http://ckanext-data-qld-theme.docker.amazee.io
-      AMAZEEIO_URL: ckanext-data-qld-theme.docker.amazee.io
+      LAGOON_LOCALDEV_URL: "http://${PROJECT}.docker.amazee.io"
+      AMAZEEIO_URL: "${PROJECT}.docker.amazee.io"
 
   postgres:
     image: amazeeio/postgres-ckan
@@ -69,7 +69,7 @@ services:
       <<: *default-environment
 
   redis:
-    image: redis:3
+    image: redis:6-alpine
     <<: *default-user
     environment:
       <<: *default-environment
diff --git a/requirements-dev.txt b/requirements-dev.txt
index c2fe22c3..3d2c551a 100644
--- a/requirements-dev.txt
+++ b/requirements-dev.txt
@@ -4,9 +4,9 @@ Appium-Python-Client<=0.52
 flake8==3.8.3
 nose==1.3.7
 splinter>=0.13.0
--e git+https://github.com/qld-gov-au/ckanext-datarequests@2.1.1-qgov#egg=ckanext-datarequests
--e git+https://github.com/qld-gov-au/ckanext-data-qld@3.0.1#egg=ckanext-data_qld
--e git+https://github.com/qld-gov-au/ckan-ex-qgov@3.1.0#egg=ckan-ex-qgov
+-e git+https://github.com/qld-gov-au/ckanext-datarequests@develop#egg=ckanext-datarequests
+-e git+https://github.com/qld-gov-au/ckanext-data-qld@develop#egg=ckanext-data_qld
+-e git+https://github.com/qld-gov-au/ckan-ex-qgov@develop#egg=ckan-ex-qgov
 -e git+https://github.com/ckan/ckantoolkit@release-0.0.4#egg=ckantoolkit
 -e git+https://github.com/ckan/ckanapi@ckanapi-4.3#egg=ckanapi
 -e git+https://github.com/ckan/ckanext-scheming@release-1.2.0#egg=ckanext-scheming
diff --git a/setup.py b/setup.py
index 4f98d6bc..c670b328 100644
--- a/setup.py
+++ b/setup.py
@@ -5,22 +5,22 @@
 here = path.abspath(path.dirname(__file__))
 
 setup(
-    name='''ckanext-data-qld-theme''',
+    name='ckanext-data-qld-theme',
 
     # Versions should comply with PEP440.  For a discussion on single-sourcing
     # the version across setup.py and the project code, see
     # http://packaging.python.org/en/latest/tutorial.html#version
     version='0.0.1',
 
-    description='''Custom extension for Data QLD''',
-    long_description='''''',
+    description='Custom extension for Data QLD',
+    long_description='',
 
     # The project's main homepage.
     url='https://github.com/qld-gov-au/ckanext-data-qld-theme',
 
     # Author details
-    author='''Salsa Digital''',
-    author_email='''''',
+    author='Salsa Digital',
+    author_email='',
 
     # Choose your license
     license='AGPL',
@@ -31,7 +31,7 @@
         # 3 - Alpha
         # 4 - Beta
         # 5 - Production/Stable
-        'Development Status :: 4 - Beta',
+        'Development Status :: 5 - Production/Stable',
 
         # Pick your license as you wish (should match "license" above)
         'License :: OSI Approved :: GNU Affero General Public License v3 or later (AGPLv3+)',
@@ -41,9 +41,8 @@
         'Programming Language :: Python :: 2.7',
     ],
 
-
     # What does your project relate to?
-    keywords='''CKAN''',
+    keywords='CKAN',
 
     # You can just specify the packages manually here if your project is
     # simple. Or you can use find_packages().
@@ -61,8 +60,7 @@
     # installed, specify them here.  If using Python 2.6 or less, then these
     # have to be included in MANIFEST.in as well.
     include_package_data=True,
-    package_data={
-    },
+    package_data={},
 
     # Although 'package_data' is the preferred approach, in some case you may
     # need to place data files outside of your packages.
diff --git a/test/features/environment.py b/test/features/environment.py
index 9a3d0ee2..fc44f3d7 100644
--- a/test/features/environment.py
+++ b/test/features/environment.py
@@ -14,11 +14,52 @@
 
 # @see .docker/scripts/init.sh for credentials.
 PERSONAS = {
-    'Admin': dict(
-        name=u'admin',
-        email=u'admin@localhost',
-        password=u'Password123!'
-    )
+    'Admin': {
+        'name': u'admin',
+        'email': u'admin@localhost',
+        'password': u'Password123!'
+    },
+    'Unauthenticated': {
+        'name': u'',
+        'email': u'',
+        'password': u''
+    },
+    # This user will not be assigned to any organisations
+    'CKANUser': {
+        'name': u'ckan_user',
+        'email': u'ckan_user@localhost',
+        'password': u'password'
+    },
+    'TestOrgAdmin': {
+        'name': u'test_org_admin',
+        'email': u'test_org_admin@localhost',
+        'password': u'password'
+    },
+    'TestOrgEditor': {
+        'name': u'test_org_editor',
+        'email': u'test_org_editor@localhost',
+        'password': u'password'
+    },
+    'TestOrgMember': {
+        'name': u'test_org_member',
+        'email': u'test_org_member@localhost',
+        'password': u'password'
+    },
+    'DataRequestOrgAdmin': {
+        'name': u'dr_admin',
+        'email': u'dr_admin@localhost',
+        'password': u'password'
+    },
+    'DataRequestOrgEditor': {
+        'name': u'dr_editor',
+        'email': u'dr_editor@localhost',
+        'password': u'password'
+    },
+    'DataRequestOrgMember': {
+        'name': u'dr_member',
+        'email': u'dr_member@localhost',
+        'password': u'password'
+    }
 }
 
 
@@ -27,6 +68,8 @@ def before_all(context):
     context.screenshots_dir = os.path.join(ROOT_PATH, 'test/screenshots')
     # The path where file attachments can be found.
     context.attachment_dir = os.path.join(ROOT_PATH, 'test/fixtures')
+    # The path where emails can be found.
+    context.mail_path = os.path.join(ROOT_PATH, 'test/emails')
 
     # Set base url for all relative links.
     context.base_url = BASE_URL

From 899e95041e6fddaac0012caf9a7c4cb9dbbadaa2 Mon Sep 17 00:00:00 2001
From: ThrawnCA <shell_layer-github@yahoo.com.au>
Date: Fri, 25 Jun 2021 12:08:01 +1000
Subject: [PATCH 3/7] [QOL-8104] move presigned indexing block inside the prod
 block

- non-prod environments disallow everything anyway
---
 ckanext/data_qld_theme/templates/robots.txt | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/ckanext/data_qld_theme/templates/robots.txt b/ckanext/data_qld_theme/templates/robots.txt
index b70c933f..05cc2239 100644
--- a/ckanext/data_qld_theme/templates/robots.txt
+++ b/ckanext/data_qld_theme/templates/robots.txt
@@ -51,6 +51,8 @@ Disallow: /tr/
 Disallow: /uk_UA/
 Disallow: /zh_CN/
 Disallow: /zh_TW/
+# presigned S3 URLs that are only temporary
+Disallow: /ckan-*-attachments-*/
 Crawl-Delay: 10
 {% endraw %}
 {% else %}
@@ -72,6 +74,4 @@ Allow: /api/
 {% endraw %}
 {% endif %}
 
-# presigned S3 URLs that are only temporary
-Disallow: /ckan-*-attachments-*/
 {% endblock %}

From ce4e677dd63f9708a679153800e28a908532cd1e Mon Sep 17 00:00:00 2001
From: ThrawnCA <shell_layer-github@yahoo.com.au>
Date: Fri, 25 Jun 2021 12:17:15 +1000
Subject: [PATCH 4/7] [QOL-8104] oops fix password to match policy

---
 .docker/scripts/create-test-data.sh | 22 +++++++++++++++-------
 test/features/environment.py        | 16 ++++++++--------
 test/features/login.feature         |  6 +++---
 3 files changed, 26 insertions(+), 18 deletions(-)

diff --git a/.docker/scripts/create-test-data.sh b/.docker/scripts/create-test-data.sh
index fd5df67b..af7d35cf 100644
--- a/.docker/scripts/create-test-data.sh
+++ b/.docker/scripts/create-test-data.sh
@@ -10,6 +10,14 @@ if [ "$VENV_DIR" != "" ]; then
   . ${VENV_DIR}/bin/activate
 fi
 
+add_user_if_needed () {
+    echo "Adding user '$2' ($1) with email address [$3]"
+    ckan_cli user "$1" | grep "$1" || ckan_cli user add "$1"\
+        fullname="$2"\
+        email="$3"\
+        password="${4:-Password123!}"
+}
+
 # We know the "admin" sysadmin account exists, so we'll use her API KEY to create further data
 API_KEY=$(ckan_cli user admin | tr -d '\n' | sed -r 's/^(.*)apikey=(\S*)(.*)/\2/')
 
@@ -21,10 +29,10 @@ TEST_ORG_TITLE="Test Organisation"
 
 echo "Creating test users for ${TEST_ORG_TITLE} Organisation:"
 
-ckan_cli user add ckan_user email=ckan_user@localhost password=password
-ckan_cli user add test_org_admin email=test_org_admin@localhost password=password
-ckan_cli user add test_org_editor email=test_org_editor@localhost password=password
-ckan_cli user add test_org_member email=test_org_member@localhost password=password
+add_user_if_needed ckan_user "CKAN User" ckan_user@localhost
+add_user_if_needed test_org_admin "Test Admin" test_org_admin@localhost
+add_user_if_needed test_org_editor "Test Editor" test_org_editor@localhost
+add_user_if_needed test_org_member "Test Member" test_org_member@localhost
 
 echo "Creating ${TEST_ORG_TITLE} Organisation:"
 
@@ -88,9 +96,9 @@ DR_ORG_TITLE="Open Data Administration (data requests)"
 
 echo "Creating test users for ${DR_ORG_TITLE} Organisation:"
 
-ckan_cli user add dr_admin email=dr_admin@localhost password=password
-ckan_cli user add dr_editor email=dr_editor@localhost password=password
-ckan_cli user add dr_member email=dr_member@localhost password=password
+add_user_if_needed dr_admin "Data Request Admin" dr_admin@localhost
+add_user_if_needed dr_editor "Data Request Editor" dr_editor@localhost
+add_user_if_needed dr_member "Data Request Member" dr_member@localhost
 
 echo "Creating ${DR_ORG_TITLE} Organisation:"
 
diff --git a/test/features/environment.py b/test/features/environment.py
index fc44f3d7..8b48e60b 100644
--- a/test/features/environment.py
+++ b/test/features/environment.py
@@ -14,7 +14,7 @@
 
 # @see .docker/scripts/init.sh for credentials.
 PERSONAS = {
-    'Admin': {
+    'SysAdmin': {
         'name': u'admin',
         'email': u'admin@localhost',
         'password': u'Password123!'
@@ -28,37 +28,37 @@
     'CKANUser': {
         'name': u'ckan_user',
         'email': u'ckan_user@localhost',
-        'password': u'password'
+        'password': u'Password123!'
     },
     'TestOrgAdmin': {
         'name': u'test_org_admin',
         'email': u'test_org_admin@localhost',
-        'password': u'password'
+        'password': u'Password123!'
     },
     'TestOrgEditor': {
         'name': u'test_org_editor',
         'email': u'test_org_editor@localhost',
-        'password': u'password'
+        'password': u'Password123!'
     },
     'TestOrgMember': {
         'name': u'test_org_member',
         'email': u'test_org_member@localhost',
-        'password': u'password'
+        'password': u'Password123!'
     },
     'DataRequestOrgAdmin': {
         'name': u'dr_admin',
         'email': u'dr_admin@localhost',
-        'password': u'password'
+        'password': u'Password123!'
     },
     'DataRequestOrgEditor': {
         'name': u'dr_editor',
         'email': u'dr_editor@localhost',
-        'password': u'password'
+        'password': u'Password123!'
     },
     'DataRequestOrgMember': {
         'name': u'dr_member',
         'email': u'dr_member@localhost',
-        'password': u'password'
+        'password': u'Password123!'
     }
 }
 
diff --git a/test/features/login.feature b/test/features/login.feature
index 4e450114..4d7ef8fd 100644
--- a/test/features/login.feature
+++ b/test/features/login.feature
@@ -2,7 +2,7 @@
 Feature: Login
 
     Scenario: Smoke test to ensure Login process works
-        Given "Admin" as the persona
+        Given "SysAdmin" as the persona
         When I go to homepage
         And I click the link with text that contains "Log in"
         And I fill in "login" with "$name"
@@ -13,6 +13,6 @@ Feature: Login
         Then I should see an element with xpath "//a[contains(string(), 'Log out')]"
 
     Scenario: Smoke test to ensure Login step works
-        Given "Admin" as the persona
+        Given "SysAdmin" as the persona
         When I log in
-        Then I take a screenshot
\ No newline at end of file
+        Then I take a screenshot

From 4ef4e10d6aac54d781ea2e949ddebd82c1d6bfc0 Mon Sep 17 00:00:00 2001
From: ThrawnCA <shell_layer-github@yahoo.com.au>
Date: Fri, 25 Jun 2021 12:29:54 +1000
Subject: [PATCH 5/7] [QOL-8104] oops fix username

---
 test/features/navigation.feature | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/features/navigation.feature b/test/features/navigation.feature
index 83ba4abd..261e9518 100644
--- a/test/features/navigation.feature
+++ b/test/features/navigation.feature
@@ -6,7 +6,7 @@ Feature: Navigation
         Then I should see an element with xpath "//a[contains(translate(., 'RD', 'rd'), "request data")]"
 
     Scenario: Check for the presence of the 'Request data' link in header when visiting as a logged in user
-        Given "Admin" as the persona
+        Given "SysAdmin" as the persona
         When I log in
         Then I go to homepage
         # Make the comparison case-insensitive

From 7ce9eb58a1e54e0b50ff547bcb043921bcab650f Mon Sep 17 00:00:00 2001
From: ThrawnCA <shell_layer-github@yahoo.com.au>
Date: Fri, 25 Jun 2021 12:43:16 +1000
Subject: [PATCH 6/7] [QOL-8104] oops restore original org for 'anna-karenina'
 since tests need it

---
 .docker/scripts/create-test-data.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.docker/scripts/create-test-data.sh b/.docker/scripts/create-test-data.sh
index af7d35cf..0850ea82 100644
--- a/.docker/scripts/create-test-data.sh
+++ b/.docker/scripts/create-test-data.sh
@@ -71,10 +71,10 @@ ckan_cli create-test-data
 
 echo "Assigning test Datasets to Organisation..."
 
-echo "Updating annakarenina to use ${TEST_ORG_TITLE} organisation:"
+echo "Updating annakarenina to use 'Department of Health' organisation:"
 package_owner_org_update=$( \
     curl -LsH "Authorization: ${API_KEY}" \
-    --data "id=annakarenina&organization_id=${TEST_ORG_NAME}" \
+    --data "id=annakarenina&organization_id=department-of-health" \
     ${CKAN_ACTION_URL}/package_owner_org_update
 )
 echo ${package_owner_org_update}

From 4114cf4904fab990aafd2ecf84e715c5a5f62114 Mon Sep 17 00:00:00 2001
From: ThrawnCA <shell_layer-github@yahoo.com.au>
Date: Fri, 25 Jun 2021 14:51:21 +1000
Subject: [PATCH 7/7] [QOL-8104] oops retain fixed versions for QGOV extensions

---
 requirements-dev.txt | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/requirements-dev.txt b/requirements-dev.txt
index 3d2c551a..b792923c 100644
--- a/requirements-dev.txt
+++ b/requirements-dev.txt
@@ -4,9 +4,9 @@ Appium-Python-Client<=0.52
 flake8==3.8.3
 nose==1.3.7
 splinter>=0.13.0
--e git+https://github.com/qld-gov-au/ckanext-datarequests@develop#egg=ckanext-datarequests
--e git+https://github.com/qld-gov-au/ckanext-data-qld@develop#egg=ckanext-data_qld
--e git+https://github.com/qld-gov-au/ckan-ex-qgov@develop#egg=ckan-ex-qgov
+-e git+https://github.com/qld-gov-au/ckanext-datarequests@2.1.2-qgov#egg=ckanext-datarequests
+-e git+https://github.com/qld-gov-au/ckanext-data-qld@3.0.3#egg=ckanext-data_qld
+-e git+https://github.com/qld-gov-au/ckan-ex-qgov@3.3.0#egg=ckan-ex-qgov
 -e git+https://github.com/ckan/ckantoolkit@release-0.0.4#egg=ckantoolkit
 -e git+https://github.com/ckan/ckanapi@ckanapi-4.3#egg=ckanapi
 -e git+https://github.com/ckan/ckanext-scheming@release-1.2.0#egg=ckanext-scheming