Merge pull request #202 from qld-gov-au/develop

Develop to master

.ahoy.yml

@@ -118,13 +118,14 @@ commands:
   test-bdd:
     usage: Run BDD tests.
     cmd: |
-      ahoy start-ckan-job-worker &
+      ahoy start-ckan-job-workers
       ahoy start-mailmock &
       sleep 5 &&
+      ahoy cli "behave ${*:-test/features}" --tags @smoke && \
       ahoy cli "behave ${*:-test/features}" || \
       [ "${ALLOW_BDD_FAIL:-0}" -eq 1 ]
       ahoy stop-mailmock
-      ahoy stop-ckan-job-worker
+      ahoy stop-ckan-job-workers
 
   start-mailmock:
     usage: Starts email mock server used for email BDD tests
@@ -138,14 +139,36 @@ commands:
       ahoy title 'Stopping mailmock'
       ahoy cli "killall -2 mailmock"
 
-  start-ckan-job-worker:
-    usage: Starts CKAN background job worker
+  start-ckan-job-workers:
+    usage: Starts CKAN background job workers
+    cmd: |
+      ahoy title 'Starting CKAN background job workers'
+      ahoy start-ckan-job-worker-default &
+      ahoy start-ckan-job-worker-priority &
+      ahoy start-ckan-job-worker-bulk &
+
+  start-ckan-job-worker-default:
+    usage: Starts default CKAN background job worker
     cmd: |
       ahoy title 'Starting default CKAN background job worker'
       ahoy cli "ckan_cli jobs clear && \
                 ckan_cli jobs worker"
 
-  stop-ckan-job-worker:
+  start-ckan-job-worker-priority:
+    usage: Starts CKAN background job worker for priority tasks
+    cmd: |
+      ahoy title 'Starting CKAN background job worker priority'
+      ahoy cli "ckan_cli jobs clear priority && \
+                ckan_cli jobs worker priority"
+
+  start-ckan-job-worker-bulk:
+    usage: Starts CKAN background job worker for low-priority tasks
+    cmd: |
+      ahoy title 'Starting CKAN background job worker bulk'
+      ahoy cli "ckan_cli jobs clear bulk && \
+                ckan_cli jobs worker bulk"
+
+  stop-ckan-job-workers:
     usage: Stops CKAN background job worker
     cmd: |
       ahoy title 'Stopping CKAN background job worker'

.circleci/build.sh

@@ -21,4 +21,4 @@ export DOCTOR_CHECK_SSH=0
 export DOCTOR_CHECK_WEBSERVER=0
 export DOCTOR_CHECK_BOOTSTRAP=0
 
-ahoy build
+ahoy build || (ahoy logs; exit 1)

.docker/Dockerfile.ckan

@@ -1,6 +1,8 @@
-FROM amazeeio/python:2.7-ckan-21.7.0
+FROM amazeeio/python:2.7-ckan-21.8.0
 
-ARG SITE_URL
+ARG SITE_URL=http://ckan:3000/
+ARG CKAN_REPO=qld-gov-au/ckan
+ARG CKAN_VERSION=2.8.8-qgov.5
 ENV SITE_URL="${SITE_URL}"
 ENV VENV_DIR=/app/ckan/default
 ENV APP_DIR=/app
@@ -15,13 +17,14 @@ RUN apk add --no-cache curl build-base \
     && rm dockerize-alpine-linux-amd64-${DOCKERIZE_VERSION}.tar.gz
 
 # Install CKAN.
-ENV CKAN_VERSION 2.8.8-qgov.5
 
 RUN . ${VENV_DIR}/bin/activate \
     && pip install setuptools==36.1 \
-    && pip install -e "git+https://github.com/qld-gov-au/ckan.git@ckan-${CKAN_VERSION}#egg=ckan" \
+    && pip install -e "git+https://github.com/${CKAN_REPO}.git@ckan-${CKAN_VERSION}#egg=ckan" \
     && sed -i "s/psycopg2==2.4.5/psycopg2==2.7.7/g" "${VENV_DIR}/src/ckan/requirements.txt" \
-    && pip install -r "${VENV_DIR}/src/ckan/requirements.txt" \
+    && ((test -f "${VENV_DIR}/src/ckan/requirements-py2.txt" && \
+        pip install -r "${VENV_DIR}/src/ckan/requirements-py2.txt") || \
+        pip install -r "${VENV_DIR}/src/ckan/requirements.txt") \
     && ln -s "${VENV_DIR}/src/ckan/who.ini" "${VENV_DIR}/who.ini" \
     && deactivate \
     && ln -s ${APP_DIR}/ckan /usr/lib/ckan \

.docker/scripts/ckan_cli

@@ -58,10 +58,17 @@ fi
 
 if [ "$COMMAND" = "ckan" ]; then
     echo "Using 'ckan' command from $ENV_DIR with config ${CKAN_INI}..." >&2
-    exec $ENV_DIR/ckan -c ${CKAN_INI} "$@"
+    # adjust args to match ckan expectations
+    COMMAND=$(echo "$1" | sed -e 's/create-test-data/seed/')
+    shift
+    exec $ENV_DIR/ckan -c ${CKAN_INI} $COMMAND "$@" $CLICK_ARGS
 elif [ "$COMMAND" = "paster" ]; then
     echo "Using 'paster' command from $ENV_DIR with config ${CKAN_INI}..." >&2
-    exec $ENV_DIR/paster --plugin=$PASTER_PLUGIN "$@" -c ${CKAN_INI}
+    # adjust args to match paster expectations
+    COMMAND=$1
+    shift
+    if [ "$1" = "show" ]; then shift; fi
+    exec $ENV_DIR/paster --plugin=$PASTER_PLUGIN $COMMAND "$@" -c ${CKAN_INI}
 else
     echo "Unable to locate 'ckan' or 'paster' command in $ENV_DIR" >&2
     exit 1

.docker/scripts/create-test-data.sh

@@ -10,16 +10,27 @@ if [ "$VENV_DIR" != "" ]; then
   . ${VENV_DIR}/bin/activate
 fi
 
+CKAN_USER_NAME="${CKAN_USER_NAME:-admin}"
+CKAN_DISPLAY_NAME="${CKAN_DISPLAY_NAME:-Administrator}"
+CKAN_USER_EMAIL="${CKAN_USER_EMAIL:-admin@localhost}"
+
 add_user_if_needed () {
     echo "Adding user '$2' ($1) with email address [$3]"
-    ckan_cli user "$1" | grep "$1" || ckan_cli user add "$1"\
+    ckan_cli user show "$1" | grep "$1" || ckan_cli user add "$1"\
         fullname="$2"\
         email="$3"\
         password="${4:-Password123!}"
 }
 
+add_user_if_needed "$CKAN_USER_NAME" "$CKAN_DISPLAY_NAME" "$CKAN_USER_EMAIL"
+ckan_cli sysadmin add "${CKAN_USER_NAME}"
+
 # We know the "admin" sysadmin account exists, so we'll use her API KEY to create further data
-API_KEY=$(ckan_cli user admin | tr -d '\n' | sed -r 's/^(.*)apikey=(\S*)(.*)/\2/')
+API_KEY=$(ckan_cli user show "${CKAN_USER_NAME}" | tr -d '\n' | sed -r 's/^(.*)apikey=(\S*)(.*)/\2/')
+if [ "$API_KEY" = "None" ]; then
+    echo "No API Key found on ${CKAN_USER_NAME}, generating API Token..."
+    API_KEY=$(ckan_cli user token add "${CKAN_USER_NAME}" test_setup |grep -v '^API Token created' | tr -d '[:space:]')
+fi
 
 ##
 # BEGIN: Create a test organisation with test users for admin, editor and member
@@ -65,16 +76,16 @@ curl -LsH "Authorization: ${API_KEY}" \
 ckan_cli create-test-data hierarchy
 
 # Creating basic test data which has datasets with resources
-ckan_cli create-test-data
+ckan_cli create-test-data basic
 
 # Datasets need to be assigned to an organisation
 
 echo "Assigning test Datasets to Organisation..."
 
-echo "Updating annakarenina to use 'Department of Health' organisation:"
+echo "Updating annakarenina to use ${TEST_ORG_TITLE} organisation:"
 package_owner_org_update=$( \
     curl -LsH "Authorization: ${API_KEY}" \
-    --data "id=annakarenina&organization_id=department-of-health" \
+    --data "id=annakarenina&organization_id=${TEST_ORG_NAME}" \
     ${CKAN_ACTION_URL}/package_owner_org_update
 )
 echo ${package_owner_org_update}
@@ -131,6 +142,14 @@ curl -LsH "Authorization: ${API_KEY}" \
     --data "title=Test Request&description=This is an example&organization_id=${TEST_ORG_ID}" \
     ${CKAN_ACTION_URL}/create_datarequest
 
+##
+# END.
+#
+
+##
+# BEGIN: Create a Reporting organisation with test users
+#
+
 REPORT_ORG_NAME=reporting
 REPORT_ORG_TITLE=Reporting
 
@@ -173,6 +192,15 @@ curl -LsH "Authorization: ${API_KEY}" \
     --data "title=Reporting Request&description=Data Request for reporting&organization_id=${REPORT_ORG_ID}" \
     ${CKAN_ACTION_URL}/create_datarequest
 
+##
+# END.
+#
+
+echo "Creating config value for resource formats:"
+
+curl -LsH "Authorization: ${API_KEY}" \
+    --data '{"ckanext.data_qld.resource_formats": "CSV\r\nHTML\r\nJSON\r\nRDF\r\nTXT\r\nXLS"}' \
+    ${CKAN_ACTION_URL}/config_option_update
 
 if [ "$VENV_DIR" != "" ]; then
   deactivate

.docker/scripts/init.sh

@@ -4,16 +4,12 @@
 #
 set -e
 
-CKAN_USER_NAME="${CKAN_USER_NAME:-admin}"
-CKAN_DISPLAY_NAME="${CKAN_DISPLAY_NAME:-Administrator}"
-CKAN_USER_PASSWORD="${CKAN_USER_PASSWORD:-Password123!}"
-CKAN_USER_EMAIL="${CKAN_USER_EMAIL:-admin@localhost}"
-
 if [ "$VENV_DIR" != "" ]; then
   . ${VENV_DIR}/bin/activate
 fi
-ckan_cli db clean
+CLICK_ARGS="--yes" ckan_cli db clean
 ckan_cli db init
+ckan_cli db upgrade
 
 # Initialise validation tables
 PASTER_PLUGIN=ckanext-validation ckan_cli validation init-db
@@ -32,11 +28,5 @@ PASTER_PLUGIN=ckanext-report ckan_cli report initdb
 # Initialise the QA database tables
 PASTER_PLUGIN=ckanext-qa ckan_cli qa init
 
-ckan_cli user add "${CKAN_USER_NAME}"\
- fullname="${CKAN_DISPLAY_NAME}"\
- email="${CKAN_USER_EMAIL}"\
- password="${CKAN_USER_PASSWORD}"
-ckan_cli sysadmin add "${CKAN_USER_NAME}"
-
 # Create some base test data
 . $APP_DIR/scripts/create-test-data.sh

.docker/scripts/serve.sh

@@ -11,7 +11,7 @@ if [ "$VENV_DIR" != "" ]; then
   . ${VENV_DIR}/bin/activate
 fi
 if (which ckan > /dev/null); then
-    ckan -c ${CKAN_INI} run
+    ckan -c ${CKAN_INI} run -r
 else
-    paster serve --reload ${CKAN_INI}
+    paster serve ${CKAN_INI}
 fi

.docker/test.ini

@@ -22,6 +22,9 @@ host = 0.0.0.0
 port = 3000
 
 [app:main]
+ckan.devserver.host = 0.0.0.0
+ckan.devserver.port = 3000
+
 use = egg:ckan
 full_stack = true
 cache_dir = /tmp/%(ckan.site_id)s/
@@ -94,7 +97,6 @@ ckan.redis.url = redis://redis:6379
 #       Add ``datapusher`` to enable DataPusher
 #       Add ``resource_proxy`` to enable resource proxying and get around the
 #       same origin policy
-# @todo:setup Cleanup the list to use only required plugins.
 ckan.plugins = stats text_view image_view recline_view datastore data_qld_theme datarequests data_qld_resources data_qld_integration scheming_datasets validation dcat qa archiver report qgovext data_qld_reporting ytp_comments
 
 # Define which views should be created by default
@@ -227,7 +229,7 @@ ckanext-archiver.archive_dir = /app/filestore/archive
 ckanext-archiver.cache_url_root = http://dataqld-ckan.docker.amazee.io/resources/
 
 # QA
-qa.resource_format_openness_scores_json = /app/ckanext/data_qld/resource_format_openness_scores.json
+qa.resource_format_openness_scores_json = /app/ckan/default/src/ckanext-data-qld/ckanext/data_qld/resource_format_openness_scores.json
 
 ## Logging configuration
 [loggers]

.flake8

@@ -12,7 +12,7 @@ format = pylint
 show_source = True
 
 max-complexity = 10
-max-line-length = 128
+max-line-length = 127
 
 # List ignore rules one per line.
 ignore =

.github/workflows/test.yml

@@ -1,40 +1,46 @@
 ---
-#based on https://raw.githubusercontent.com/ckan/ckanext-scheming/master/.github/workflows/test.yml
-# alternative https://github.com/ckan/ckan/blob/master/contrib/cookiecutter/ckan_extension/%7B%7Bcookiecutter.project%7D%7D/.github/workflows/test.yml
 name: Tests
 on:
   push:
+  pull_request:
+    branches:
+      - master
 
 jobs:
   test:
     strategy:
       fail-fast: false
+      matrix:
+        ckan-version: [2.8.8-qgov.5]
 
-    name: Continuous Integration build
+    name: Continuous Integration build on CKAN ${{ matrix.ckan-version }}
     runs-on: ubuntu-latest
     container: integratedexperts/ci-builder
+    env:
+      CKAN_REPO: qld-gov-au/ckan
+      CKAN_VERSION: ${{ matrix.ckan-version }}
 
     steps:
       - uses: actions/checkout@v2
-        timeout-minutes: 5
+        timeout-minutes: 2
 
       - name: Build
         run: .circleci/build.sh
         timeout-minutes: 15
 
       - name: Test
         run: .circleci/test.sh
-        timeout-minutes: 20
+        timeout-minutes: 25
 
       - name: Retrieve screenshots
         if: failure()
         run: .circleci/process-artifacts.sh
-        timeout-minutes: 3
+        timeout-minutes: 1
 
       - name: Upload screenshots
         if: failure()
         uses: actions/upload-artifact@v2
         with:
-          name: screenshots
+          name: CKAN ${{ matrix.ckan-version }} screenshots
           path: /tmp/artifacts/behave/screenshots
-        timeout-minutes: 3
+        timeout-minutes: 1

.gitignore

@@ -15,7 +15,13 @@ env/
 build/
 develop-eggs/
 dist/
+downloads/
+eggs/
+lib/
+lib64/
+parts/
 sdist/
+var/
 *.egg-info/
 .installed.cfg
 *.egg
@@ -51,4 +57,9 @@ docs/_build/
 .env.local
 screenshots
 !/test/screenshots/.gitkeep
+
+# PyBuilder
+target/
+
+#Intellij
 .idea

README.md

@@ -1,5 +1,4 @@
-ckanext-data-qld-theme
-======================
+# ckanext-data-qld-theme
 A custom CKAN extension for Data.Qld
 
 [![CircleCI](https://circleci.com/gh/qld-gov-au/ckanext-data-qld-theme/tree/develop.svg?style=shield)](https://circleci.com/gh/qld-gov-au/ckanext-data-qld-theme/tree/develop)

ckanext/data_qld_theme/templates/base.html

@@ -1,14 +1,5 @@
 {% ckan_extends %}
 
-  {% block meta %}
-    {{ super() }}
-    {% if c.controller in ['dataset', 'package', 'ckanext.data_qld.controller:DataQldDataset'] %}
-      {% if c.action in ['read', 'resource_read'] and c.pkg_dict['organization'] and c.pkg_dict['organization']['title'] %}
-          <meta name="DCTERMS.creator" content="c=AU; o=The State of Queensland; ou={{ c.pkg_dict['organization']['title'] }}" scheme="AGLSTERMS.GOLD" />
-      {% endif %}
-    {% endif %}
-  {% endblock %}
-
   {% block styles %}
     {{ super() }}
     <link href="https://fonts.googleapis.com/css?family=Lato:100,300,400,700,900,100italic,300italic,400italic,700italic,900italic" rel="stylesheet" />

ckanext/data_qld_theme/templates/datarequests/new.html

@@ -3,7 +3,7 @@
 {% block subtitle %}{{ _('Create data request') }}{% endblock %}
 
 {% block breadcrumb_content %}
-  <li>{% link_for _('Data requests'), controller='ckanext.datarequests.controllers.ui_controller:DataRequestsUI', action='index' %}</li>
+  <li>{% link_for _('Data requests'), named_route='datarequest.index' %}</li>
   <li class="active"><a href="">{{ _('Create data request') }}</a></li>
 {% endblock %}
 

ckanext/data_qld_theme/templates/datarequests/show.html

@@ -18,15 +18,15 @@
   {% endif %}
 
   {% if h.check_access('open_datarequest', {'id':datarequest_id }) and c.datarequest.closed %}
-      {% link_for _('Re-open'), controller='ckanext.data_qld.controller:DataQldUI', action='open_datarequest', id=datarequest_id, class_='btn btn-success', icon='unlock' %}
+    {% link_for _('Re-open'), named_route='open_datarequest', id=datarequest_id, class_='btn btn-success', icon='unlock' %}
   {% endif %}
 
 {% endblock %}
 
 {% block content_primary_nav %}
-  {{ h.build_nav_icon('show_datarequest', _('Data request'), id=datarequest_id) }}
+  {{ h.build_nav_icon('datarequest.show', _('Data request'), id=datarequest_id) }}
 
   {% if h.show_comments_tab() and h.ytp_comments_enabled() %}
-    {{ h.build_nav_icon('comment_datarequest', _('Comments') + ' ' + h.get_content_type_comments_badge(datarequest_id, 'datarequest'), id=datarequest_id) }}
+    {{ h.build_nav_icon('datarequest.comment', _('Comments') + ' ' + h.get_content_type_comments_badge(datarequest_id, 'datarequest'), id=datarequest_id) }}
   {% endif %}
 {% endblock %}