diff --git a/.ahoy.yml b/.ahoy.yml index 4deb93e8..a649c4c2 100644 --- a/.ahoy.yml +++ b/.ahoy.yml @@ -10,14 +10,19 @@ commands: ahoy title "Building project" ahoy pre-flight ahoy clean - (docker network prune -f > /dev/null && docker network inspect amazeeio-network > /dev/null || docker network create amazeeio-network) + ahoy build-network ahoy up -- --build --force-recreate - ahoy install-dev ahoy install-site ahoy title "Build complete" ahoy doctor ahoy info 1 + build-network: + usage: Ensure that the amazeeio network exists. + cmd: | + docker network prune -f > /dev/null + docker network inspect amazeeio-network > /dev/null || docker network create amazeeio-network + info: usage: Print information about this project. cmd: | @@ -33,7 +38,7 @@ commands: docker-compose up -d "$@" sleep 10 docker-compose logs - ahoy cli "dockerize -wait tcp://ckan:3000 -timeout 1m || (docker-compose logs; exit 1)" + ahoy cli "dockerize -wait tcp://ckan:3000 -timeout 1m" if docker-compose logs | grep -q "\[Error\]"; then docker-compose logs; exit 1; fi if docker-compose logs | grep -q "Exception"; then docker-compose logs; exit 1; fi docker ps -a --filter name=^/${COMPOSE_PROJECT_NAME}_ @@ -41,7 +46,7 @@ commands: down: usage: Stop Docker containers and remove container, images, volumes and networks. - cmd: "if [ -f \"docker-compose.yml\" ]; then docker-compose down --volumes; fi" + cmd: 'if [ -f "docker-compose.yml" ]; then docker-compose down --volumes; fi' start: usage: Start existing Docker containers. @@ -53,7 +58,7 @@ commands: restart: usage: Restart all stopped and running Docker containers. - cmd: docker-compose restart + cmd: docker-compose restart "$@" logs: usage: Show Docker logs. @@ -61,23 +66,21 @@ commands: pull: usage: Pull latest docker images. - cmd: if [ ! -z "$(docker image ls -q)" ]; then docker image ls --format \"{{.Repository}}:{{.Tag}}\" | grep 'amazeeio/|/ckan.*-dev' | grep -v none | xargs -n1 docker pull | cat; fi + cmd: if [ ! -z "$(docker image ls -q)" ]; then docker image ls --format \"{{.Repository}}:{{.Tag}}\" | grep amazeeio/ | grep -v none | xargs -n1 docker pull | cat; fi cli: usage: Start a shell inside CLI container or run a command. - cmd: if \[ "${#}" -ne 0 \]; then docker exec $(docker-compose ps -q ckan) sh -c ". /app/ckan/default/bin/activate; $*"; else docker exec $(docker-compose ps -q ckan) sh -c ". /app/ckan/default/bin/activate && sh"; fi + cmd: if \[ "${#}" -ne 0 \]; then docker exec $(docker-compose ps -q ckan) sh -c '. ${VENV_DIR}/bin/activate; cd $APP_DIR;'" $*"; else docker exec $(docker-compose ps -q ckan) sh -c '. ${VENV_DIR}/bin/activate && cd $APP_DIR && sh'; fi doctor: usage: Find problems with current project setup. cmd: .docker/scripts/doctor.sh "$@" - install-site: usage: Install a site. cmd: | ahoy title "Installing a fresh site" - docker cp -L .docker/test.ini $(docker-compose ps -q ckan):/app/ckan/default/production.ini - ahoy cli "/app/scripts/init.sh" + ahoy cli "./scripts/init.sh" clean: usage: Remove containers and all build files. @@ -96,15 +99,6 @@ commands: git ls-files --others -i --exclude-from=.git/info/exclude | xargs rm -Rf find . -type d -not -path "./.git/*" -empty -delete - install-dev: - usage: Install dependencies. - cmd: | - docker cp -L requirements-dev.txt $(docker-compose ps -q ckan):/app/. - docker cp -L .flake8 $(docker-compose ps -q ckan):/app/. - docker cp -L test $(docker-compose ps -q ckan):/app/. - ahoy cli "pip install -r /app/requirements-dev.txt" - hide: true - flush-redis: usage: Flush Redis cache. cmd: docker exec -i $(docker-compose ps -q redis) redis-cli flushall > /dev/null @@ -112,20 +106,50 @@ commands: lint: usage: Lint code. cmd: | - ahoy cli "flake8 ${@:-/app/ckanext}" || \ + ahoy cli "flake8 ${@:-ckanext}" || \ [ "${ALLOW_LINT_FAIL:-0}" -eq 1 ] test-unit: usage: Run unit tests. cmd: | - ahoy cli "nosetests" || \ + ahoy cli 'nosetests --with-pylons=${CKAN_INI}' || \ [ "${ALLOW_UNIT_FAIL:-0}" -eq 1 ] test-bdd: usage: Run BDD tests. cmd: | - ahoy cli "behave ${*:-/app/test/features}" || \ + ahoy start-ckan-job-worker & + ahoy start-mailmock & + sleep 5 && + ahoy cli "behave ${*:-test/features}" || \ [ "${ALLOW_BDD_FAIL:-0}" -eq 1 ] + ahoy stop-mailmock + ahoy stop-ckan-job-worker + + start-mailmock: + usage: Starts email mock server used for email BDD tests + cmd: | + ahoy title 'Starting mailmock' + ahoy cli 'mailmock -p 8025 -o ${APP_DIR}/test/emails --no-stdout' # for debugging mailmock email output remove --no-stdout + + stop-mailmock: + usage: Stops email mock server used for email BDD tests + cmd: | + ahoy title 'Stopping mailmock' + ahoy cli "killall -2 mailmock" + + start-ckan-job-worker: + usage: Starts CKAN background job worker + cmd: | + ahoy title 'Starting CKAN background job worker' + ahoy cli "ckan_cli jobs clear && \ + ckan_cli jobs worker" + + stop-ckan-job-worker: + usage: Stops CKAN background job worker + cmd: | + ahoy title 'Stopping CKAN background job worker' + ahoy cli "pkill -f 'jobs worker'" # Utilities. title: @@ -161,5 +185,5 @@ entrypoint: export LAGOON_LOCALDEV_URL=http://ckanext-data-qld-theme.docker.amazee.io [ -f .env ] && [ -s .env ] && export $(grep -v '^#' .env | xargs) && if [ -f .env.local ] && [ -s .env.local ]; then export $(grep -v '^#' .env.local | xargs); fi bash -e -c "$0" "$@" - - '{{cmd}}' - - '{{name}}' + - "{{cmd}}" + - "{{name}}" diff --git a/.circleci/process-artifacts.sh b/.circleci/process-artifacts.sh index 8fbc3d20..55bdbef9 100755 --- a/.circleci/process-artifacts.sh +++ b/.circleci/process-artifacts.sh @@ -6,8 +6,8 @@ set -e # Create screenshots directory in case it was not created before. This is to # avoid this script to fail when copying artifacts. -ahoy cli "mkdir -p /app/test/screenshots" +ahoy cli "mkdir -p test/screenshots" # Copy from the app container to the build host for storage. -mkdir -p /tmp/artifacts/behave/screenshots -docker cp "$(docker-compose ps -q ckan)":/app/test/screenshots /tmp/artifacts/behave +mkdir -p /tmp/artifacts/behave +docker cp "$(docker-compose ps -q ckan)":/app/test/screenshots /tmp/artifacts/behave/ diff --git a/.circleci/test.sh b/.circleci/test.sh index 864b4b34..9c584d9b 100755 --- a/.circleci/test.sh +++ b/.circleci/test.sh @@ -5,10 +5,10 @@ set -e echo "==> Lint code" -ahoy lint || exit 1 +ahoy lint echo "==> Run Unit tests" -ahoy test-unit || exit 1 +ahoy test-unit echo "==> Run BDD tests" -ahoy test-bdd || exit 1 +ahoy test-bdd || (ahoy logs; exit 1) diff --git a/.docker/Dockerfile.ckan b/.docker/Dockerfile.ckan index bf8b7dfc..789bf9e1 100644 --- a/.docker/Dockerfile.ckan +++ b/.docker/Dockerfile.ckan @@ -1,42 +1,46 @@ FROM amazeeio/python:2.7-ckan-21.6.0 -ENV WORKDIR=/app - ARG SITE_URL ENV SITE_URL="${SITE_URL}" -ENV APP_DIR=/app/ckan/default +ENV VENV_DIR=/app/ckan/default +ENV APP_DIR=/app ENV CKAN_INI=/app/ckan/default/production.ini +WORKDIR "${APP_DIR}" + ENV DOCKERIZE_VERSION v0.6.1 -RUN apk add --no-cache curl && curl -s -L -O https://github.com/jwilder/dockerize/releases/download/${DOCKERIZE_VERSION}/dockerize-alpine-linux-amd64-${DOCKERIZE_VERSION}.tar.gz \ +RUN apk add --no-cache curl build-base \ + && curl -s -L -O https://github.com/jwilder/dockerize/releases/download/${DOCKERIZE_VERSION}/dockerize-alpine-linux-amd64-${DOCKERIZE_VERSION}.tar.gz \ && tar -C /usr/local/bin -xzvf dockerize-alpine-linux-amd64-${DOCKERIZE_VERSION}.tar.gz \ && rm dockerize-alpine-linux-amd64-${DOCKERIZE_VERSION}.tar.gz # Install CKAN. ENV CKAN_VERSION 2.8.8 -RUN . ${APP_DIR}/bin/activate \ - && cd ${APP_DIR} \ + +RUN . ${VENV_DIR}/bin/activate \ && pip install setuptools==36.1 \ && pip install -e "git+https://github.com/ckan/ckan.git@ckan-${CKAN_VERSION}#egg=ckan" \ - && sed -i "s/psycopg2==2.4.5/psycopg2==2.7.7/g" "${APP_DIR}/src/ckan/requirements.txt" \ - && pip install -r "${APP_DIR}/src/ckan/requirements.txt" \ - && ln -s "${APP_DIR}/src/ckan/who.ini" "${APP_DIR}/who.ini" \ + && sed -i "s/psycopg2==2.4.5/psycopg2==2.7.7/g" "${VENV_DIR}/src/ckan/requirements.txt" \ + && pip install -r "${VENV_DIR}/src/ckan/requirements.txt" \ + && ln -s "${VENV_DIR}/src/ckan/who.ini" "${VENV_DIR}/who.ini" \ && deactivate \ - && ln -s /app/ckan /usr/lib/ckan + && ln -s ${APP_DIR}/ckan /usr/lib/ckan \ + && fix-permissions ${APP_DIR}/ckan -COPY .docker/test.ini /app/ckan/default/production.ini +COPY .docker/test.ini $CKAN_INI -COPY .docker/scripts /app/scripts +# Add current extension and files. +COPY . ${APP_DIR}/ -RUN fix-permissions /app/ckan \ - && chmod +x /app/scripts/*.sh +COPY .docker/scripts ${APP_DIR}/scripts -# Add current extension and files. -COPY ckanext /app/ckanext -COPY requirements.txt requirements-dev.txt setup.cfg setup.py .flake8 /app/ +COPY .docker/scripts/ckan_cli ${VENV_DIR}/bin/ + +RUN chmod +x ${APP_DIR}/scripts/*.sh\ + && chmod +x ${VENV_DIR}/bin/ckan_cli # Init current extension. -RUN /app/scripts/init-ext.sh +RUN ${APP_DIR}/scripts/init-ext.sh ENTRYPOINT ["/sbin/tini", "--", "/lagoon/entrypoints.sh"] CMD ["/app/scripts/serve.sh"] diff --git a/.docker/scripts/ckan_cli b/.docker/scripts/ckan_cli new file mode 100644 index 00000000..fcc045fd --- /dev/null +++ b/.docker/scripts/ckan_cli @@ -0,0 +1,68 @@ +#!/bin/sh + +# Call either 'ckan' (from CKAN >= 2.9) or 'paster' (from CKAN <= 2.8) +# with appropriate syntax, depending on what is present on the system. +# This is intended to smooth the upgrade process from 2.8 to 2.9. +# Eg: +# ckan_cli jobs list +# could become either: +# paster --plugin=ckan jobs list -c /etc/ckan/default/production.ini +# or: +# ckan -c /etc/ckan/default/production.ini jobs list + +# This script is aware of the VIRTUAL_ENV environment variable, and will +# attempt to respect it with similar behaviour to commands like 'pip'. +# Eg placing this script in a virtualenv 'bin' directory will cause it +# to call the 'ckan' or 'paster' command in that directory, while +# placing this script elsewhere will cause it to rely on the VIRTUAL_ENV +# variable, or if that is not set, the system PATH. + +# Since the positioning of the CKAN configuration file is central to the +# differences between 'paster' and 'ckan', this script needs to be aware +# of the config file location. It will use the CKAN_INI environment +# variable if it exists, or default to /etc/ckan/default/production.ini. + +# If 'paster' is being used, the default plugin is 'ckan'. A different +# plugin can be specified by setting the PASTER_PLUGIN environment +# variable. This variable is irrelevant if using the 'ckan' command. + +CKAN_INI="${CKAN_INI:-/etc/ckan/default/production.ini}" +PASTER_PLUGIN="${PASTER_PLUGIN:-ckan}" +# First, look for a command alongside this file +ENV_DIR=$(dirname "$0") +if [ -f "$ENV_DIR/ckan" ]; then + COMMAND=ckan +elif [ -f "$ENV_DIR/paster" ]; then + COMMAND=paster +elif [ "$VIRTUAL_ENV" != "" ]; then + # If command not found alongside this file, check the virtualenv + ENV_DIR="$VIRTUAL_ENV/bin" + if [ -f "$ENV_DIR/ckan" ]; then + COMMAND=ckan + elif [ -f "$ENV_DIR/paster" ]; then + COMMAND=paster + fi +else + # if no virtualenv is active, try the system path + if (which ckan > /dev/null 2>&1); then + ENV_DIR=$(dirname $(which ckan)) + COMMAND=ckan + elif (which paster > /dev/null 2>&1); then + ENV_DIR=$(dirname $(which paster)) + COMMAND=paster + else + echo "Unable to locate 'ckan' or 'paster' command" >&2 + exit 1 + fi +fi + +if [ "$COMMAND" = "ckan" ]; then + echo "Using 'ckan' command from $ENV_DIR with config ${CKAN_INI}..." >&2 + exec $ENV_DIR/ckan -c ${CKAN_INI} "$@" +elif [ "$COMMAND" = "paster" ]; then + echo "Using 'paster' command from $ENV_DIR with config ${CKAN_INI}..." >&2 + exec $ENV_DIR/paster --plugin=$PASTER_PLUGIN "$@" -c ${CKAN_INI} +else + echo "Unable to locate 'ckan' or 'paster' command in $ENV_DIR" >&2 + exit 1 +fi diff --git a/.docker/scripts/create-test-data.sh b/.docker/scripts/create-test-data.sh index 6dc2e1c8..0850ea82 100644 --- a/.docker/scripts/create-test-data.sh +++ b/.docker/scripts/create-test-data.sh @@ -1,4 +1,3 @@ - #!/usr/bin/env sh ## # Create some example content for extension BDD tests. @@ -7,31 +6,131 @@ set -e CKAN_ACTION_URL=http://ckan:3000/api/action -. ${APP_DIR}/bin/activate +if [ "$VENV_DIR" != "" ]; then + . ${VENV_DIR}/bin/activate +fi -ckan_cli () { - if (which ckan > /dev/null); then - ckan -c ${CKAN_INI} "$@" - else - paster --plugin=ckan "$@" -c ${CKAN_INI} - fi +add_user_if_needed () { + echo "Adding user '$2' ($1) with email address [$3]" + ckan_cli user "$1" | grep "$1" || ckan_cli user add "$1"\ + fullname="$2"\ + email="$3"\ + password="${4:-Password123!}" } # We know the "admin" sysadmin account exists, so we'll use her API KEY to create further data API_KEY=$(ckan_cli user admin | tr -d '\n' | sed -r 's/^(.*)apikey=(\S*)(.*)/\2/') +## +# BEGIN: Create a test organisation with test users for admin, editor and member +# +TEST_ORG_NAME=test-organisation +TEST_ORG_TITLE="Test Organisation" + +echo "Creating test users for ${TEST_ORG_TITLE} Organisation:" + +add_user_if_needed ckan_user "CKAN User" ckan_user@localhost +add_user_if_needed test_org_admin "Test Admin" test_org_admin@localhost +add_user_if_needed test_org_editor "Test Editor" test_org_editor@localhost +add_user_if_needed test_org_member "Test Member" test_org_member@localhost + +echo "Creating ${TEST_ORG_TITLE} Organisation:" + +TEST_ORG=$( \ + curl -LsH "Authorization: ${API_KEY}" \ + --data "name=${TEST_ORG_NAME}&title=${TEST_ORG_TITLE}" \ + ${CKAN_ACTION_URL}/organization_create +) + +TEST_ORG_ID=$(echo $TEST_ORG | sed -r 's/^(.*)"id": "(.*)",(.*)/\2/') + +echo "Assigning test users to ${TEST_ORG_TITLE} Organisation:" + +curl -LsH "Authorization: ${API_KEY}" \ + --data "id=${TEST_ORG_ID}&object=test_org_admin&object_type=user&capacity=admin" \ + ${CKAN_ACTION_URL}/member_create + +curl -LsH "Authorization: ${API_KEY}" \ + --data "id=${TEST_ORG_ID}&object=test_org_editor&object_type=user&capacity=editor" \ + ${CKAN_ACTION_URL}/member_create + +curl -LsH "Authorization: ${API_KEY}" \ + --data "id=${TEST_ORG_ID}&object=test_org_member&object_type=user&capacity=member" \ + ${CKAN_ACTION_URL}/member_create +## +# END. +# + # Creating test data hierarchy which creates organisations assigned to datasets ckan_cli create-test-data hierarchy # Creating basic test data which has datasets with resources ckan_cli create-test-data -echo "Updating annakarenina to use department-of-health Organisation:" +# Datasets need to be assigned to an organisation + +echo "Assigning test Datasets to Organisation..." + +echo "Updating annakarenina to use 'Department of Health' organisation:" package_owner_org_update=$( \ - curl -L -s --header "Authorization: ${API_KEY}" \ + curl -LsH "Authorization: ${API_KEY}" \ --data "id=annakarenina&organization_id=department-of-health" \ ${CKAN_ACTION_URL}/package_owner_org_update ) echo ${package_owner_org_update} -deactivate +echo "Updating warandpeace to use ${TEST_ORG_TITLE} organisation:" +package_owner_org_update=$( \ + curl -LsH "Authorization: ${API_KEY}" \ + --data "id=warandpeace&organization_id=${TEST_ORG_NAME}" \ + ${CKAN_ACTION_URL}/package_owner_org_update +) +echo ${package_owner_org_update} + +## +# BEGIN: Create a Data Request organisation with test users for admin, editor and member and default data requests +# +# Data Requests requires a specific organisation to exist in order to create DRs for Data.Qld +DR_ORG_NAME=open-data-administration-data-requests +DR_ORG_TITLE="Open Data Administration (data requests)" + +echo "Creating test users for ${DR_ORG_TITLE} Organisation:" + +add_user_if_needed dr_admin "Data Request Admin" dr_admin@localhost +add_user_if_needed dr_editor "Data Request Editor" dr_editor@localhost +add_user_if_needed dr_member "Data Request Member" dr_member@localhost + +echo "Creating ${DR_ORG_TITLE} Organisation:" + +DR_ORG=$( \ + curl -LsH "Authorization: ${API_KEY}" \ + --data "name=${DR_ORG_NAME}&title=${DR_ORG_TITLE}" \ + ${CKAN_ACTION_URL}/organization_create +) + +DR_ORG_ID=$(echo $DR_ORG | sed -r 's/^(.*)"id": "(.*)",(.*)/\2/') + +echo "Assigning test users to ${DR_ORG_TITLE} Organisation:" + +curl -LsH "Authorization: ${API_KEY}" \ + --data "id=${DR_ORG_ID}&object=dr_admin&object_type=user&capacity=admin" \ + ${CKAN_ACTION_URL}/member_create + +curl -LsH "Authorization: ${API_KEY}" \ + --data "id=${DR_ORG_ID}&object=dr_editor&object_type=user&capacity=editor" \ + ${CKAN_ACTION_URL}/member_create + +curl -LsH "Authorization: ${API_KEY}" \ + --data "id=${DR_ORG_ID}&object=dr_member&object_type=user&capacity=member" \ + ${CKAN_ACTION_URL}/member_create + + +echo "Creating test Data Request:" + +curl -LsH "Authorization: ${API_KEY}" \ + --data "title=Test Request&description=This is an example&organization_id=${DR_ORG_ID}" \ + ${CKAN_ACTION_URL}/create_datarequest + +if [ "$VENV_DIR" != "" ]; then + deactivate +fi diff --git a/.docker/scripts/init-ext.sh b/.docker/scripts/init-ext.sh index 9e6df825..c6b00ec9 100755 --- a/.docker/scripts/init-ext.sh +++ b/.docker/scripts/init-ext.sh @@ -4,12 +4,17 @@ # set -e -PIP="${APP_DIR}/bin/pip" -cd $WORKDIR -$PIP install -r "requirements.txt" -$PIP install -r "requirements-dev.txt" -$APP_DIR/bin/python setup.py develop +if [ "$VENV_DIR" != "" ]; then + . ${VENV_DIR}/bin/activate +fi +pip install -r "requirements-dev.txt" +pip install -r "requirements.txt" +python setup.py develop +installed_name=$(grep '^\s*name=' setup.py |sed "s|[^']*'\([-a-zA-Z0-9]*\)'.*|\1|") # Validate that the extension was installed correctly. -if ! $PIP list | grep ckanext-data-qld-theme > /dev/null; then echo "Unable to find the extension in the list"; exit 1; fi +if ! pip list | grep "$installed_name" > /dev/null; then echo "Unable to find the extension in the list"; exit 1; fi +if [ "$VENV_DIR" != "" ]; then + deactivate +fi diff --git a/.docker/scripts/init.sh b/.docker/scripts/init.sh index da423733..eca9485b 100755 --- a/.docker/scripts/init.sh +++ b/.docker/scripts/init.sh @@ -9,22 +9,16 @@ CKAN_DISPLAY_NAME="${CKAN_DISPLAY_NAME:-Administrator}" CKAN_USER_PASSWORD="${CKAN_USER_PASSWORD:-Password123!}" CKAN_USER_EMAIL="${CKAN_USER_EMAIL:-admin@localhost}" -. ${APP_DIR}/bin/activate - -ckan_cli () { - if (which ckan > /dev/null); then - ckan -c ${CKAN_INI} "$@" - else - paster --plugin=ckan "$@" -c ${CKAN_INI} - fi -} -ckan_cli db clean || exit 1 -ckan_cli db init || exit 1 +if [ "$VENV_DIR" != "" ]; then + . ${VENV_DIR}/bin/activate +fi +ckan_cli db clean +ckan_cli db init ckan_cli user add "${CKAN_USER_NAME}"\ fullname="${CKAN_DISPLAY_NAME}"\ email="${CKAN_USER_EMAIL}"\ - password="${CKAN_USER_PASSWORD}" || exit 1 -ckan_cli sysadmin add "${CKAN_USER_NAME}" || exit 1 + password="${CKAN_USER_PASSWORD}" +ckan_cli sysadmin add "${CKAN_USER_NAME}" # Create some base test data -. $WORKDIR/scripts/create-test-data.sh +. $APP_DIR/scripts/create-test-data.sh diff --git a/.docker/scripts/serve.sh b/.docker/scripts/serve.sh index d95eeb70..10f3dead 100755 --- a/.docker/scripts/serve.sh +++ b/.docker/scripts/serve.sh @@ -3,10 +3,13 @@ set -e dockerize -wait tcp://postgres:5432 -timeout 1m dockerize -wait tcp://solr:8983 -timeout 1m +dockerize -wait tcp://redis:6379 -timeout 1m sed -i "s@SITE_URL@${SITE_URL}@g" $CKAN_INI -. ${APP_DIR}/bin/activate +if [ "$VENV_DIR" != "" ]; then + . ${VENV_DIR}/bin/activate +fi if (which ckan > /dev/null); then ckan -c ${CKAN_INI} run else diff --git a/.docker/test.ini b/.docker/test.ini index 196fc17c..4fa690fa 100644 --- a/.docker/test.ini +++ b/.docker/test.ini @@ -92,7 +92,7 @@ ckan.redis.url = redis://redis:6379 # Note: Add ``datastore`` to enable the CKAN DataStore # Add ``datapusher`` to enable DataPusher -# Add ``resource_proxy`` to enable resorce proxying and get around the +# Add ``resource_proxy`` to enable resource proxying and get around the # same origin policy # @todo:setup Cleanup the list to use only required plugins. ckan.plugins = stats text_view image_view recline_view datastore datarequests data_qld_resources data_qld_theme qgovext diff --git a/.env b/.env index 589b48cd..9afe27bb 100644 --- a/.env +++ b/.env @@ -13,7 +13,7 @@ PROJECT="ckanext-data-qld-theme" # Docker Compose project name. All containers will have this name. -COMPOSE_PROJECT_NAME="ckanext-data-qld-theme" +COMPOSE_PROJECT_NAME="$PROJECT" # Flag to allow code linting failures. ALLOW_LINT_FAIL=0 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2b6955f0..89fa60d7 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -10,7 +10,7 @@ jobs: strategy: fail-fast: false - name: OpenData theme build + name: Continuous Integration build runs-on: ubuntu-latest container: integratedexperts/ci-builder @@ -25,3 +25,16 @@ jobs: - name: Test run: .circleci/test.sh timeout-minutes: 15 + + - name: Retrieve screenshots + if: failure() + run: .circleci/process-artifacts.sh + timeout-minutes: 1 + + - name: Upload screenshots + if: failure() + uses: actions/upload-artifact@v2 + with: + name: screenshots + path: /tmp/artifacts/behave/screenshots + timeout-minutes: 1 diff --git a/ckanext/data_qld_theme/templates/robots.txt b/ckanext/data_qld_theme/templates/robots.txt index 718fb4b8..05cc2239 100644 --- a/ckanext/data_qld_theme/templates/robots.txt +++ b/ckanext/data_qld_theme/templates/robots.txt @@ -51,6 +51,8 @@ Disallow: /tr/ Disallow: /uk_UA/ Disallow: /zh_CN/ Disallow: /zh_TW/ +# presigned S3 URLs that are only temporary +Disallow: /ckan-*-attachments-*/ Crawl-Delay: 10 {% endraw %} {% else %} @@ -71,4 +73,5 @@ Allow: /api/ {% endraw %} {% endif %} -{% endblock %} \ No newline at end of file + +{% endblock %} diff --git a/docker-compose.yml b/docker-compose.yml index 555ac3d8..15417d94 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,7 +1,7 @@ version: '2.3' x-project: - &project ckanext-data-qld-theme + &project "${PROJECT}" x-volumes: &default-volumes @@ -29,7 +29,7 @@ services: context: . dockerfile: .docker/Dockerfile.ckan args: - SITE_URL: http://ckanext-data-qld-theme.docker.amazee.io + SITE_URL: "http://${PROJECT}.docker.amazee.io" depends_on: - postgres - solr @@ -43,8 +43,8 @@ services: environment: <<: *default-environment AMAZEEIO_HTTP_PORT: 3000 - LAGOON_LOCALDEV_URL: http://ckanext-data-qld-theme.docker.amazee.io - AMAZEEIO_URL: ckanext-data-qld-theme.docker.amazee.io + LAGOON_LOCALDEV_URL: "http://${PROJECT}.docker.amazee.io" + AMAZEEIO_URL: "${PROJECT}.docker.amazee.io" postgres: image: amazeeio/postgres-ckan @@ -69,7 +69,7 @@ services: <<: *default-environment redis: - image: redis:3 + image: redis:6-alpine <<: *default-user environment: <<: *default-environment diff --git a/requirements-dev.txt b/requirements-dev.txt index c2fe22c3..b792923c 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -4,9 +4,9 @@ Appium-Python-Client<=0.52 flake8==3.8.3 nose==1.3.7 splinter>=0.13.0 --e git+https://github.com/qld-gov-au/ckanext-datarequests@2.1.1-qgov#egg=ckanext-datarequests --e git+https://github.com/qld-gov-au/ckanext-data-qld@3.0.1#egg=ckanext-data_qld --e git+https://github.com/qld-gov-au/ckan-ex-qgov@3.1.0#egg=ckan-ex-qgov +-e git+https://github.com/qld-gov-au/ckanext-datarequests@2.1.2-qgov#egg=ckanext-datarequests +-e git+https://github.com/qld-gov-au/ckanext-data-qld@3.0.3#egg=ckanext-data_qld +-e git+https://github.com/qld-gov-au/ckan-ex-qgov@3.3.0#egg=ckan-ex-qgov -e git+https://github.com/ckan/ckantoolkit@release-0.0.4#egg=ckantoolkit -e git+https://github.com/ckan/ckanapi@ckanapi-4.3#egg=ckanapi -e git+https://github.com/ckan/ckanext-scheming@release-1.2.0#egg=ckanext-scheming diff --git a/setup.py b/setup.py index 4f98d6bc..c670b328 100644 --- a/setup.py +++ b/setup.py @@ -5,22 +5,22 @@ here = path.abspath(path.dirname(__file__)) setup( - name='''ckanext-data-qld-theme''', + name='ckanext-data-qld-theme', # Versions should comply with PEP440. For a discussion on single-sourcing # the version across setup.py and the project code, see # http://packaging.python.org/en/latest/tutorial.html#version version='0.0.1', - description='''Custom extension for Data QLD''', - long_description='''''', + description='Custom extension for Data QLD', + long_description='', # The project's main homepage. url='https://github.com/qld-gov-au/ckanext-data-qld-theme', # Author details - author='''Salsa Digital''', - author_email='''''', + author='Salsa Digital', + author_email='', # Choose your license license='AGPL', @@ -31,7 +31,7 @@ # 3 - Alpha # 4 - Beta # 5 - Production/Stable - 'Development Status :: 4 - Beta', + 'Development Status :: 5 - Production/Stable', # Pick your license as you wish (should match "license" above) 'License :: OSI Approved :: GNU Affero General Public License v3 or later (AGPLv3+)', @@ -41,9 +41,8 @@ 'Programming Language :: Python :: 2.7', ], - # What does your project relate to? - keywords='''CKAN''', + keywords='CKAN', # You can just specify the packages manually here if your project is # simple. Or you can use find_packages(). @@ -61,8 +60,7 @@ # installed, specify them here. If using Python 2.6 or less, then these # have to be included in MANIFEST.in as well. include_package_data=True, - package_data={ - }, + package_data={}, # Although 'package_data' is the preferred approach, in some case you may # need to place data files outside of your packages. diff --git a/test/features/environment.py b/test/features/environment.py index 9a3d0ee2..8b48e60b 100644 --- a/test/features/environment.py +++ b/test/features/environment.py @@ -14,11 +14,52 @@ # @see .docker/scripts/init.sh for credentials. PERSONAS = { - 'Admin': dict( - name=u'admin', - email=u'admin@localhost', - password=u'Password123!' - ) + 'SysAdmin': { + 'name': u'admin', + 'email': u'admin@localhost', + 'password': u'Password123!' + }, + 'Unauthenticated': { + 'name': u'', + 'email': u'', + 'password': u'' + }, + # This user will not be assigned to any organisations + 'CKANUser': { + 'name': u'ckan_user', + 'email': u'ckan_user@localhost', + 'password': u'Password123!' + }, + 'TestOrgAdmin': { + 'name': u'test_org_admin', + 'email': u'test_org_admin@localhost', + 'password': u'Password123!' + }, + 'TestOrgEditor': { + 'name': u'test_org_editor', + 'email': u'test_org_editor@localhost', + 'password': u'Password123!' + }, + 'TestOrgMember': { + 'name': u'test_org_member', + 'email': u'test_org_member@localhost', + 'password': u'Password123!' + }, + 'DataRequestOrgAdmin': { + 'name': u'dr_admin', + 'email': u'dr_admin@localhost', + 'password': u'Password123!' + }, + 'DataRequestOrgEditor': { + 'name': u'dr_editor', + 'email': u'dr_editor@localhost', + 'password': u'Password123!' + }, + 'DataRequestOrgMember': { + 'name': u'dr_member', + 'email': u'dr_member@localhost', + 'password': u'Password123!' + } } @@ -27,6 +68,8 @@ def before_all(context): context.screenshots_dir = os.path.join(ROOT_PATH, 'test/screenshots') # The path where file attachments can be found. context.attachment_dir = os.path.join(ROOT_PATH, 'test/fixtures') + # The path where emails can be found. + context.mail_path = os.path.join(ROOT_PATH, 'test/emails') # Set base url for all relative links. context.base_url = BASE_URL diff --git a/test/features/login.feature b/test/features/login.feature index 4e450114..4d7ef8fd 100644 --- a/test/features/login.feature +++ b/test/features/login.feature @@ -2,7 +2,7 @@ Feature: Login Scenario: Smoke test to ensure Login process works - Given "Admin" as the persona + Given "SysAdmin" as the persona When I go to homepage And I click the link with text that contains "Log in" And I fill in "login" with "$name" @@ -13,6 +13,6 @@ Feature: Login Then I should see an element with xpath "//a[contains(string(), 'Log out')]" Scenario: Smoke test to ensure Login step works - Given "Admin" as the persona + Given "SysAdmin" as the persona When I log in - Then I take a screenshot \ No newline at end of file + Then I take a screenshot diff --git a/test/features/navigation.feature b/test/features/navigation.feature index 83ba4abd..261e9518 100644 --- a/test/features/navigation.feature +++ b/test/features/navigation.feature @@ -6,7 +6,7 @@ Feature: Navigation Then I should see an element with xpath "//a[contains(translate(., 'RD', 'rd'), "request data")]" Scenario: Check for the presence of the 'Request data' link in header when visiting as a logged in user - Given "Admin" as the persona + Given "SysAdmin" as the persona When I log in Then I go to homepage # Make the comparison case-insensitive