diff --git a/.ahoy.yml b/.ahoy.yml index a5889462..169c0d6d 100644 --- a/.ahoy.yml +++ b/.ahoy.yml @@ -118,13 +118,14 @@ commands: test-bdd: usage: Run BDD tests. cmd: | - ahoy start-ckan-job-worker & + ahoy start-ckan-job-workers ahoy start-mailmock & sleep 5 && + ahoy cli "behave ${*:-test/features}" --tags @smoke && \ ahoy cli "behave ${*:-test/features}" || \ [ "${ALLOW_BDD_FAIL:-0}" -eq 1 ] ahoy stop-mailmock - ahoy stop-ckan-job-worker + ahoy stop-ckan-job-workers start-mailmock: usage: Starts email mock server used for email BDD tests @@ -138,14 +139,36 @@ commands: ahoy title 'Stopping mailmock' ahoy cli "killall -2 mailmock" - start-ckan-job-worker: - usage: Starts CKAN background job worker + start-ckan-job-workers: + usage: Starts CKAN background job workers + cmd: | + ahoy title 'Starting CKAN background job workers' + ahoy start-ckan-job-worker-default & + ahoy start-ckan-job-worker-priority & + ahoy start-ckan-job-worker-bulk & + + start-ckan-job-worker-default: + usage: Starts default CKAN background job worker cmd: | ahoy title 'Starting default CKAN background job worker' ahoy cli "ckan_cli jobs clear && \ ckan_cli jobs worker" - stop-ckan-job-worker: + start-ckan-job-worker-priority: + usage: Starts CKAN background job worker for priority tasks + cmd: | + ahoy title 'Starting CKAN background job worker priority' + ahoy cli "ckan_cli jobs clear priority && \ + ckan_cli jobs worker priority" + + start-ckan-job-worker-bulk: + usage: Starts CKAN background job worker for low-priority tasks + cmd: | + ahoy title 'Starting CKAN background job worker bulk' + ahoy cli "ckan_cli jobs clear bulk && \ + ckan_cli jobs worker bulk" + + stop-ckan-job-workers: usage: Stops CKAN background job worker cmd: | ahoy title 'Stopping CKAN background job worker' diff --git a/.circleci/build.sh b/.circleci/build.sh index 611dca97..407dd0ec 100755 --- a/.circleci/build.sh +++ b/.circleci/build.sh @@ -21,4 +21,4 @@ export DOCTOR_CHECK_SSH=0 export DOCTOR_CHECK_WEBSERVER=0 export DOCTOR_CHECK_BOOTSTRAP=0 -ahoy build +ahoy build || (ahoy logs; exit 1) diff --git a/.docker/Dockerfile.ckan b/.docker/Dockerfile.ckan index 50461819..3a5995e0 100644 --- a/.docker/Dockerfile.ckan +++ b/.docker/Dockerfile.ckan @@ -1,6 +1,8 @@ -FROM amazeeio/python:2.7-ckan-21.7.0 +FROM amazeeio/python:2.7-ckan-21.8.0 -ARG SITE_URL +ARG SITE_URL=http://ckan:3000/ +ARG CKAN_REPO=qld-gov-au/ckan +ARG CKAN_VERSION=2.8.8-qgov.5 ENV SITE_URL="${SITE_URL}" ENV VENV_DIR=/app/ckan/default ENV APP_DIR=/app @@ -15,13 +17,14 @@ RUN apk add --no-cache curl build-base \ && rm dockerize-alpine-linux-amd64-${DOCKERIZE_VERSION}.tar.gz # Install CKAN. -ENV CKAN_VERSION 2.8.8-qgov.5 RUN . ${VENV_DIR}/bin/activate \ && pip install setuptools==36.1 \ - && pip install -e "git+https://github.com/qld-gov-au/ckan.git@ckan-${CKAN_VERSION}#egg=ckan" \ + && pip install -e "git+https://github.com/${CKAN_REPO}.git@ckan-${CKAN_VERSION}#egg=ckan" \ && sed -i "s/psycopg2==2.4.5/psycopg2==2.7.7/g" "${VENV_DIR}/src/ckan/requirements.txt" \ - && pip install -r "${VENV_DIR}/src/ckan/requirements.txt" \ + && ((test -f "${VENV_DIR}/src/ckan/requirements-py2.txt" && \ + pip install -r "${VENV_DIR}/src/ckan/requirements-py2.txt") || \ + pip install -r "${VENV_DIR}/src/ckan/requirements.txt") \ && ln -s "${VENV_DIR}/src/ckan/who.ini" "${VENV_DIR}/who.ini" \ && deactivate \ && ln -s ${APP_DIR}/ckan /usr/lib/ckan \ diff --git a/.docker/scripts/ckan_cli b/.docker/scripts/ckan_cli index fcc045fd..3cf0b4cc 100644 --- a/.docker/scripts/ckan_cli +++ b/.docker/scripts/ckan_cli @@ -58,10 +58,17 @@ fi if [ "$COMMAND" = "ckan" ]; then echo "Using 'ckan' command from $ENV_DIR with config ${CKAN_INI}..." >&2 - exec $ENV_DIR/ckan -c ${CKAN_INI} "$@" + # adjust args to match ckan expectations + COMMAND=$(echo "$1" | sed -e 's/create-test-data/seed/') + shift + exec $ENV_DIR/ckan -c ${CKAN_INI} $COMMAND "$@" $CLICK_ARGS elif [ "$COMMAND" = "paster" ]; then echo "Using 'paster' command from $ENV_DIR with config ${CKAN_INI}..." >&2 - exec $ENV_DIR/paster --plugin=$PASTER_PLUGIN "$@" -c ${CKAN_INI} + # adjust args to match paster expectations + COMMAND=$1 + shift + if [ "$1" = "show" ]; then shift; fi + exec $ENV_DIR/paster --plugin=$PASTER_PLUGIN $COMMAND "$@" -c ${CKAN_INI} else echo "Unable to locate 'ckan' or 'paster' command in $ENV_DIR" >&2 exit 1 diff --git a/.docker/scripts/create-test-data.sh b/.docker/scripts/create-test-data.sh index de935ac0..60cb2228 100644 --- a/.docker/scripts/create-test-data.sh +++ b/.docker/scripts/create-test-data.sh @@ -10,16 +10,27 @@ if [ "$VENV_DIR" != "" ]; then . ${VENV_DIR}/bin/activate fi +CKAN_USER_NAME="${CKAN_USER_NAME:-admin}" +CKAN_DISPLAY_NAME="${CKAN_DISPLAY_NAME:-Administrator}" +CKAN_USER_EMAIL="${CKAN_USER_EMAIL:-admin@localhost}" + add_user_if_needed () { echo "Adding user '$2' ($1) with email address [$3]" - ckan_cli user "$1" | grep "$1" || ckan_cli user add "$1"\ + ckan_cli user show "$1" | grep "$1" || ckan_cli user add "$1"\ fullname="$2"\ email="$3"\ password="${4:-Password123!}" } +add_user_if_needed "$CKAN_USER_NAME" "$CKAN_DISPLAY_NAME" "$CKAN_USER_EMAIL" +ckan_cli sysadmin add "${CKAN_USER_NAME}" + # We know the "admin" sysadmin account exists, so we'll use her API KEY to create further data -API_KEY=$(ckan_cli user admin | tr -d '\n' | sed -r 's/^(.*)apikey=(\S*)(.*)/\2/') +API_KEY=$(ckan_cli user show "${CKAN_USER_NAME}" | tr -d '\n' | sed -r 's/^(.*)apikey=(\S*)(.*)/\2/') +if [ "$API_KEY" = "None" ]; then + echo "No API Key found on ${CKAN_USER_NAME}, generating API Token..." + API_KEY=$(ckan_cli user token add "${CKAN_USER_NAME}" test_setup |grep -v '^API Token created' | tr -d '[:space:]') +fi ## # BEGIN: Create a test organisation with test users for admin, editor and member @@ -65,16 +76,16 @@ curl -LsH "Authorization: ${API_KEY}" \ ckan_cli create-test-data hierarchy # Creating basic test data which has datasets with resources -ckan_cli create-test-data +ckan_cli create-test-data basic # Datasets need to be assigned to an organisation echo "Assigning test Datasets to Organisation..." -echo "Updating annakarenina to use 'Department of Health' organisation:" +echo "Updating annakarenina to use ${TEST_ORG_TITLE} organisation:" package_owner_org_update=$( \ curl -LsH "Authorization: ${API_KEY}" \ - --data "id=annakarenina&organization_id=department-of-health" \ + --data "id=annakarenina&organization_id=${TEST_ORG_NAME}" \ ${CKAN_ACTION_URL}/package_owner_org_update ) echo ${package_owner_org_update} @@ -131,6 +142,14 @@ curl -LsH "Authorization: ${API_KEY}" \ --data "title=Test Request&description=This is an example&organization_id=${TEST_ORG_ID}" \ ${CKAN_ACTION_URL}/create_datarequest +## +# END. +# + +## +# BEGIN: Create a Reporting organisation with test users +# + REPORT_ORG_NAME=reporting REPORT_ORG_TITLE=Reporting @@ -173,6 +192,15 @@ curl -LsH "Authorization: ${API_KEY}" \ --data "title=Reporting Request&description=Data Request for reporting&organization_id=${REPORT_ORG_ID}" \ ${CKAN_ACTION_URL}/create_datarequest +## +# END. +# + +echo "Creating config value for resource formats:" + +curl -LsH "Authorization: ${API_KEY}" \ + --data '{"ckanext.data_qld.resource_formats": "CSV\r\nHTML\r\nJSON\r\nRDF\r\nTXT\r\nXLS"}' \ + ${CKAN_ACTION_URL}/config_option_update if [ "$VENV_DIR" != "" ]; then deactivate diff --git a/.docker/scripts/init.sh b/.docker/scripts/init.sh index eb7839fa..a380599f 100755 --- a/.docker/scripts/init.sh +++ b/.docker/scripts/init.sh @@ -4,16 +4,12 @@ # set -e -CKAN_USER_NAME="${CKAN_USER_NAME:-admin}" -CKAN_DISPLAY_NAME="${CKAN_DISPLAY_NAME:-Administrator}" -CKAN_USER_PASSWORD="${CKAN_USER_PASSWORD:-Password123!}" -CKAN_USER_EMAIL="${CKAN_USER_EMAIL:-admin@localhost}" - if [ "$VENV_DIR" != "" ]; then . ${VENV_DIR}/bin/activate fi -ckan_cli db clean +CLICK_ARGS="--yes" ckan_cli db clean ckan_cli db init +ckan_cli db upgrade # Initialise validation tables PASTER_PLUGIN=ckanext-validation ckan_cli validation init-db @@ -32,11 +28,5 @@ PASTER_PLUGIN=ckanext-report ckan_cli report initdb # Initialise the QA database tables PASTER_PLUGIN=ckanext-qa ckan_cli qa init -ckan_cli user add "${CKAN_USER_NAME}"\ - fullname="${CKAN_DISPLAY_NAME}"\ - email="${CKAN_USER_EMAIL}"\ - password="${CKAN_USER_PASSWORD}" -ckan_cli sysadmin add "${CKAN_USER_NAME}" - # Create some base test data . $APP_DIR/scripts/create-test-data.sh diff --git a/.docker/scripts/serve.sh b/.docker/scripts/serve.sh index c1aae061..904c45c7 100755 --- a/.docker/scripts/serve.sh +++ b/.docker/scripts/serve.sh @@ -11,7 +11,7 @@ if [ "$VENV_DIR" != "" ]; then . ${VENV_DIR}/bin/activate fi if (which ckan > /dev/null); then - ckan -c ${CKAN_INI} run + ckan -c ${CKAN_INI} run -r else - paster serve --reload ${CKAN_INI} + paster serve ${CKAN_INI} fi diff --git a/.docker/test.ini b/.docker/test.ini index 61c0fff9..7dfe8bb5 100644 --- a/.docker/test.ini +++ b/.docker/test.ini @@ -22,6 +22,9 @@ host = 0.0.0.0 port = 3000 [app:main] +ckan.devserver.host = 0.0.0.0 +ckan.devserver.port = 3000 + use = egg:ckan full_stack = true cache_dir = /tmp/%(ckan.site_id)s/ @@ -94,7 +97,6 @@ ckan.redis.url = redis://redis:6379 # Add ``datapusher`` to enable DataPusher # Add ``resource_proxy`` to enable resource proxying and get around the # same origin policy -# @todo:setup Cleanup the list to use only required plugins. ckan.plugins = stats text_view image_view recline_view datastore data_qld_theme datarequests data_qld_resources data_qld_integration scheming_datasets validation dcat qa archiver report qgovext data_qld_reporting ytp_comments # Define which views should be created by default @@ -227,7 +229,7 @@ ckanext-archiver.archive_dir = /app/filestore/archive ckanext-archiver.cache_url_root = http://dataqld-ckan.docker.amazee.io/resources/ # QA -qa.resource_format_openness_scores_json = /app/ckanext/data_qld/resource_format_openness_scores.json +qa.resource_format_openness_scores_json = /app/ckan/default/src/ckanext-data-qld/ckanext/data_qld/resource_format_openness_scores.json ## Logging configuration [loggers] diff --git a/.flake8 b/.flake8 index 022bf847..d6b0f6b2 100644 --- a/.flake8 +++ b/.flake8 @@ -12,7 +12,7 @@ format = pylint show_source = True max-complexity = 10 -max-line-length = 128 +max-line-length = 127 # List ignore rules one per line. ignore = diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 946f290c..6b657991 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,22 +1,28 @@ --- -#based on https://raw.githubusercontent.com/ckan/ckanext-scheming/master/.github/workflows/test.yml -# alternative https://github.com/ckan/ckan/blob/master/contrib/cookiecutter/ckan_extension/%7B%7Bcookiecutter.project%7D%7D/.github/workflows/test.yml name: Tests on: push: + pull_request: + branches: + - master jobs: test: strategy: fail-fast: false + matrix: + ckan-version: [2.8.8-qgov.5] - name: Continuous Integration build + name: Continuous Integration build on CKAN ${{ matrix.ckan-version }} runs-on: ubuntu-latest container: integratedexperts/ci-builder + env: + CKAN_REPO: qld-gov-au/ckan + CKAN_VERSION: ${{ matrix.ckan-version }} steps: - uses: actions/checkout@v2 - timeout-minutes: 5 + timeout-minutes: 2 - name: Build run: .circleci/build.sh @@ -24,17 +30,17 @@ jobs: - name: Test run: .circleci/test.sh - timeout-minutes: 20 + timeout-minutes: 25 - name: Retrieve screenshots if: failure() run: .circleci/process-artifacts.sh - timeout-minutes: 3 + timeout-minutes: 1 - name: Upload screenshots if: failure() uses: actions/upload-artifact@v2 with: - name: screenshots + name: CKAN ${{ matrix.ckan-version }} screenshots path: /tmp/artifacts/behave/screenshots - timeout-minutes: 3 + timeout-minutes: 1 diff --git a/.gitignore b/.gitignore index 4578f100..7e0716f7 100644 --- a/.gitignore +++ b/.gitignore @@ -15,7 +15,13 @@ env/ build/ develop-eggs/ dist/ +downloads/ +eggs/ +lib/ +lib64/ +parts/ sdist/ +var/ *.egg-info/ .installed.cfg *.egg @@ -51,4 +57,9 @@ docs/_build/ .env.local screenshots !/test/screenshots/.gitkeep + +# PyBuilder +target/ + +#Intellij .idea diff --git a/README.md b/README.md index 0eb3ae7e..caf4c04f 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,4 @@ -ckanext-data-qld-theme -====================== +# ckanext-data-qld-theme A custom CKAN extension for Data.Qld [![CircleCI](https://circleci.com/gh/qld-gov-au/ckanext-data-qld-theme/tree/develop.svg?style=shield)](https://circleci.com/gh/qld-gov-au/ckanext-data-qld-theme/tree/develop) diff --git a/ckanext/data_qld_theme/templates/base.html b/ckanext/data_qld_theme/templates/base.html index b35615aa..4f9f915e 100644 --- a/ckanext/data_qld_theme/templates/base.html +++ b/ckanext/data_qld_theme/templates/base.html @@ -1,14 +1,5 @@ {% ckan_extends %} - {% block meta %} - {{ super() }} - {% if c.controller in ['dataset', 'package', 'ckanext.data_qld.controller:DataQldDataset'] %} - {% if c.action in ['read', 'resource_read'] and c.pkg_dict['organization'] and c.pkg_dict['organization']['title'] %} - - {% endif %} - {% endif %} - {% endblock %} - {% block styles %} {{ super() }} diff --git a/ckanext/data_qld_theme/templates/datarequests/new.html b/ckanext/data_qld_theme/templates/datarequests/new.html index 99ee22aa..cdb7e28c 100644 --- a/ckanext/data_qld_theme/templates/datarequests/new.html +++ b/ckanext/data_qld_theme/templates/datarequests/new.html @@ -3,7 +3,7 @@ {% block subtitle %}{{ _('Create data request') }}{% endblock %} {% block breadcrumb_content %} -
  • {% link_for _('Data requests'), controller='ckanext.datarequests.controllers.ui_controller:DataRequestsUI', action='index' %}
    • +
  • {% link_for _('Data requests'), named_route='datarequest.index' %}
  • {{ _('Create data request') }}
    • {% endblock %} diff --git a/ckanext/data_qld_theme/templates/datarequests/show.html b/ckanext/data_qld_theme/templates/datarequests/show.html index 854d1cea..cadeb052 100644 --- a/ckanext/data_qld_theme/templates/datarequests/show.html +++ b/ckanext/data_qld_theme/templates/datarequests/show.html @@ -18,15 +18,15 @@ {% endif %} {% if h.check_access('open_datarequest', {'id':datarequest_id }) and c.datarequest.closed %} - {% link_for _('Re-open'), controller='ckanext.data_qld.controller:DataQldUI', action='open_datarequest', id=datarequest_id, class_='btn btn-success', icon='unlock' %} + {% link_for _('Re-open'), named_route='open_datarequest', id=datarequest_id, class_='btn btn-success', icon='unlock' %} {% endif %} {% endblock %} {% block content_primary_nav %} - {{ h.build_nav_icon('show_datarequest', _('Data request'), id=datarequest_id) }} + {{ h.build_nav_icon('datarequest.show', _('Data request'), id=datarequest_id) }} {% if h.show_comments_tab() and h.ytp_comments_enabled() %} - {{ h.build_nav_icon('comment_datarequest', _('Comments') + ' ' + h.get_content_type_comments_badge(datarequest_id, 'datarequest'), id=datarequest_id) }} + {{ h.build_nav_icon('datarequest.comment', _('Comments') + ' ' + h.get_content_type_comments_badge(datarequest_id, 'datarequest'), id=datarequest_id) }} {% endif %} {% endblock %} diff --git a/ckanext/data_qld_theme/templates/header.html b/ckanext/data_qld_theme/templates/header.html index 680cdfd6..11b43926 100644 --- a/ckanext/data_qld_theme/templates/header.html +++ b/ckanext/data_qld_theme/templates/header.html @@ -37,7 +37,7 @@ {{ super() }} {% endblock %} {% endblock %} {% if h.is_datarequests_enabled() %} -
  • Request data
    • +
  • Request data
    • {% endif %} @@ -50,7 +50,7 @@
  • {% link_for _('Register'), named_route='user.register', class_='sub' %}
    • {% endif %} {% endblock %} {% if h.is_datarequests_enabled() %} -
  • Request data
    • +
  • Request data
    • {% endif %} diff --git a/ckanext/data_qld_theme/templates/package/group_list.html b/ckanext/data_qld_theme/templates/package/group_list.html index d364709b..b4db86ef 100644 --- a/ckanext/data_qld_theme/templates/package/group_list.html +++ b/ckanext/data_qld_theme/templates/package/group_list.html @@ -5,13 +5,13 @@

    {{ _('Groups') }}

    {% if h.check_access('package_update', {'id':c.pkg_dict.id }) %} - {% set groups = h.get_all_groups() %} + {% set groups = h.groups_available() %} {% if groups %}
    diff --git a/ckanext/data_qld_theme/templates/package/read.html b/ckanext/data_qld_theme/templates/package/read.html new file mode 100644 index 00000000..79bc475a --- /dev/null +++ b/ckanext/data_qld_theme/templates/package/read.html @@ -0,0 +1,8 @@ +{% ckan_extends %} + +{% block meta %} +{{ super() }} +{% if pkg['organization'] and pkg['organization']['title'] %} + +{% endif %} +{% endblock %} diff --git a/ckanext/data_qld_theme/templates/package/resource_read.html b/ckanext/data_qld_theme/templates/package/resource_read.html index c5f91174..613171f3 100644 --- a/ckanext/data_qld_theme/templates/package/resource_read.html +++ b/ckanext/data_qld_theme/templates/package/resource_read.html @@ -1,25 +1,18 @@ {% ckan_extends %} -{% block subtitle %}{{ h.resource_display_name(res) }} - {{ h.dataset_display_name(c.package) }}{% endblock %} - -{% block breadcrumb_content %} +{% set pkg = c.pkg_dict or pkg_dict %} -{% if pkg %} - {% set dataset = h.dataset_display_name(pkg) %} - {% if pkg.organization %} - {% set organization = h.get_translated(pkg.organization, 'title') or pkg.organization.name %} - {% set group_type = pkg.organization.type %} -
  • {% link_for _('Organizations'), named_route='organization.index', named_route=group_type + '_index' %}
    • -
  • {% link_for organization|truncate(30), named_route='organization_read', id=pkg.organization.name, named_route=group_type + '_read' %}
    • - {% else %} -
  • {% link_for _('Datasets'), named_route='package.search' %}
    • - {% endif %} - {% link_for dataset|truncate(30), named_route='dataset_read', id=pkg.name %} - {% else %} -
  • {% link_for _('Datasets'), named_route='package.search' %}
    • -
  • {{ _('Create Dataset') }}
    • +{% block meta %} +{{ super() }} +{% if pkg['organization'] and pkg['organization']['title'] %} + {% endif %} +{% endblock %} +{% block subtitle %}{{ h.resource_display_name(res) }} - {{ h.dataset_display_name(c.package) }}{% endblock %} + +{% block breadcrumb_content %} +{{ super() }}
  • {{ h.resource_display_name(res) }}
    • {% endblock %} @@ -37,7 +30,13 @@