From bb2f59f8df76dc908f1118033efe3e6a6bf1ea30 Mon Sep 17 00:00:00 2001 From: ThrawnCA Date: Fri, 21 Jan 2022 14:37:36 +1000 Subject: [PATCH 1/5] [QOL-7811] sync test config with Publications --- .circleci/build.sh | 2 +- .docker/Dockerfile.ckan | 11 ++++-- .docker/scripts/ckan_cli | 11 +++++- .docker/scripts/create-test-data.sh | 26 ++++++++++++- .docker/scripts/init.sh | 14 +------ .docker/test.ini | 3 ++ .github/workflows/test.yml | 16 +++++--- README.md | 3 +- docker-compose.yml | 2 + test/features/config.feature | 16 +++++--- test/features/dataset_deletion.feature | 7 ++-- test/features/steps/steps.py | 52 ++++++++++++++++---------- 12 files changed, 107 insertions(+), 56 deletions(-) diff --git a/.circleci/build.sh b/.circleci/build.sh index 611dca97..407dd0ec 100755 --- a/.circleci/build.sh +++ b/.circleci/build.sh @@ -21,4 +21,4 @@ export DOCTOR_CHECK_SSH=0 export DOCTOR_CHECK_WEBSERVER=0 export DOCTOR_CHECK_BOOTSTRAP=0 -ahoy build +ahoy build || (ahoy logs; exit 1) diff --git a/.docker/Dockerfile.ckan b/.docker/Dockerfile.ckan index 9d3f89f1..3a5995e0 100644 --- a/.docker/Dockerfile.ckan +++ b/.docker/Dockerfile.ckan @@ -1,6 +1,8 @@ FROM amazeeio/python:2.7-ckan-21.8.0 -ARG SITE_URL +ARG SITE_URL=http://ckan:3000/ +ARG CKAN_REPO=qld-gov-au/ckan +ARG CKAN_VERSION=2.8.8-qgov.5 ENV SITE_URL="${SITE_URL}" ENV VENV_DIR=/app/ckan/default ENV APP_DIR=/app @@ -15,13 +17,14 @@ RUN apk add --no-cache curl build-base \ && rm dockerize-alpine-linux-amd64-${DOCKERIZE_VERSION}.tar.gz # Install CKAN. -ENV CKAN_VERSION 2.8.8-qgov.5 RUN . ${VENV_DIR}/bin/activate \ && pip install setuptools==36.1 \ - && pip install -e "git+https://github.com/qld-gov-au/ckan.git@ckan-${CKAN_VERSION}#egg=ckan" \ + && pip install -e "git+https://github.com/${CKAN_REPO}.git@ckan-${CKAN_VERSION}#egg=ckan" \ && sed -i "s/psycopg2==2.4.5/psycopg2==2.7.7/g" "${VENV_DIR}/src/ckan/requirements.txt" \ - && pip install -r "${VENV_DIR}/src/ckan/requirements.txt" \ + && ((test -f "${VENV_DIR}/src/ckan/requirements-py2.txt" && \ + pip install -r "${VENV_DIR}/src/ckan/requirements-py2.txt") || \ + pip install -r "${VENV_DIR}/src/ckan/requirements.txt") \ && ln -s "${VENV_DIR}/src/ckan/who.ini" "${VENV_DIR}/who.ini" \ && deactivate \ && ln -s ${APP_DIR}/ckan /usr/lib/ckan \ diff --git a/.docker/scripts/ckan_cli b/.docker/scripts/ckan_cli index fcc045fd..3cf0b4cc 100644 --- a/.docker/scripts/ckan_cli +++ b/.docker/scripts/ckan_cli @@ -58,10 +58,17 @@ fi if [ "$COMMAND" = "ckan" ]; then echo "Using 'ckan' command from $ENV_DIR with config ${CKAN_INI}..." >&2 - exec $ENV_DIR/ckan -c ${CKAN_INI} "$@" + # adjust args to match ckan expectations + COMMAND=$(echo "$1" | sed -e 's/create-test-data/seed/') + shift + exec $ENV_DIR/ckan -c ${CKAN_INI} $COMMAND "$@" $CLICK_ARGS elif [ "$COMMAND" = "paster" ]; then echo "Using 'paster' command from $ENV_DIR with config ${CKAN_INI}..." >&2 - exec $ENV_DIR/paster --plugin=$PASTER_PLUGIN "$@" -c ${CKAN_INI} + # adjust args to match paster expectations + COMMAND=$1 + shift + if [ "$1" = "show" ]; then shift; fi + exec $ENV_DIR/paster --plugin=$PASTER_PLUGIN $COMMAND "$@" -c ${CKAN_INI} else echo "Unable to locate 'ckan' or 'paster' command in $ENV_DIR" >&2 exit 1 diff --git a/.docker/scripts/create-test-data.sh b/.docker/scripts/create-test-data.sh index d0197858..02c8215e 100644 --- a/.docker/scripts/create-test-data.sh +++ b/.docker/scripts/create-test-data.sh @@ -10,16 +10,27 @@ if [ "$VENV_DIR" != "" ]; then . ${VENV_DIR}/bin/activate fi +CKAN_USER_NAME="${CKAN_USER_NAME:-admin}" +CKAN_DISPLAY_NAME="${CKAN_DISPLAY_NAME:-Administrator}" +CKAN_USER_EMAIL="${CKAN_USER_EMAIL:-admin@localhost}" + add_user_if_needed () { echo "Adding user '$2' ($1) with email address [$3]" - ckan_cli user "$1" | grep "$1" || ckan_cli user add "$1"\ + ckan_cli user show "$1" | grep "$1" || ckan_cli user add "$1"\ fullname="$2"\ email="$3"\ password="${4:-Password123!}" } +add_user_if_needed "$CKAN_USER_NAME" "$CKAN_DISPLAY_NAME" "$CKAN_USER_EMAIL" +ckan_cli sysadmin add "${CKAN_USER_NAME}" + # We know the "admin" sysadmin account exists, so we'll use her API KEY to create further data -API_KEY=$(ckan_cli user admin | tr -d '\n' | sed -r 's/^(.*)apikey=(\S*)(.*)/\2/') +API_KEY=$(ckan_cli user show "${CKAN_USER_NAME}" | tr -d '\n' | sed -r 's/^(.*)apikey=(\S*)(.*)/\2/') +if [ "$API_KEY" = "None" ]; then + echo "No API Key found on ${CKAN_USER_NAME}, generating API Token..." + API_KEY=$(ckan_cli user token add "${CKAN_USER_NAME}" test_setup |grep -v '^API Token created' | tr -d '[:space:]') +fi ## # BEGIN: Create a test organisation with test users for admin, editor and member @@ -131,6 +142,14 @@ curl -LsH "Authorization: ${API_KEY}" \ --data "title=Test Request&description=This is an example&organization_id=${TEST_ORG_ID}" \ ${CKAN_ACTION_URL}/create_datarequest +## +# END. +# + +## +# BEGIN: Create a Reporting organisation with test users +# + REPORT_ORG_NAME=reporting REPORT_ORG_TITLE=Reporting @@ -173,6 +192,9 @@ curl -LsH "Authorization: ${API_KEY}" \ --data "title=Reporting Request&description=Data Request for reporting&organization_id=${REPORT_ORG_ID}" \ ${CKAN_ACTION_URL}/create_datarequest +## +# END. +# echo "Creating config value for resource formats:" diff --git a/.docker/scripts/init.sh b/.docker/scripts/init.sh index eb7839fa..a380599f 100755 --- a/.docker/scripts/init.sh +++ b/.docker/scripts/init.sh @@ -4,16 +4,12 @@ # set -e -CKAN_USER_NAME="${CKAN_USER_NAME:-admin}" -CKAN_DISPLAY_NAME="${CKAN_DISPLAY_NAME:-Administrator}" -CKAN_USER_PASSWORD="${CKAN_USER_PASSWORD:-Password123!}" -CKAN_USER_EMAIL="${CKAN_USER_EMAIL:-admin@localhost}" - if [ "$VENV_DIR" != "" ]; then . ${VENV_DIR}/bin/activate fi -ckan_cli db clean +CLICK_ARGS="--yes" ckan_cli db clean ckan_cli db init +ckan_cli db upgrade # Initialise validation tables PASTER_PLUGIN=ckanext-validation ckan_cli validation init-db @@ -32,11 +28,5 @@ PASTER_PLUGIN=ckanext-report ckan_cli report initdb # Initialise the QA database tables PASTER_PLUGIN=ckanext-qa ckan_cli qa init -ckan_cli user add "${CKAN_USER_NAME}"\ - fullname="${CKAN_DISPLAY_NAME}"\ - email="${CKAN_USER_EMAIL}"\ - password="${CKAN_USER_PASSWORD}" -ckan_cli sysadmin add "${CKAN_USER_NAME}" - # Create some base test data . $APP_DIR/scripts/create-test-data.sh diff --git a/.docker/test.ini b/.docker/test.ini index 3797e545..a6c38181 100644 --- a/.docker/test.ini +++ b/.docker/test.ini @@ -22,6 +22,9 @@ host = 0.0.0.0 port = 3000 [app:main] +ckan.devserver.host = 0.0.0.0 +ckan.devserver.port = 3000 + use = egg:ckan full_stack = true cache_dir = /tmp/%(ckan.site_id)s/ diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index faaff3b3..7a927b87 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,18 +1,24 @@ --- -#based on https://raw.githubusercontent.com/ckan/ckanext-scheming/master/.github/workflows/test.yml -# alternative https://github.com/ckan/ckan/blob/master/contrib/cookiecutter/ckan_extension/%7B%7Bcookiecutter.project%7D%7D/.github/workflows/test.yml name: Tests on: push: + pull_request: + branches: + - master jobs: test: strategy: fail-fast: false + matrix: + ckan-version: [2.8.8-qgov.5] - name: Continuous Integration build + name: Continuous Integration build on CKAN ${{ matrix.ckan-version }} runs-on: ubuntu-latest container: integratedexperts/ci-builder + env: + CKAN_REPO: qld-gov-au/ckan + CKAN_VERSION: ${{ matrix.ckan-version }} steps: - uses: actions/checkout@v2 @@ -35,6 +41,6 @@ jobs: if: failure() uses: actions/upload-artifact@v2 with: - name: screenshots + name: CKAN ${{ matrix.ckan-version }} screenshots path: /tmp/artifacts/behave/screenshots - timeout-minutes: 3 + timeout-minutes: 1 diff --git a/README.md b/README.md index 0eb3ae7e..caf4c04f 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,4 @@ -ckanext-data-qld-theme -====================== +# ckanext-data-qld-theme A custom CKAN extension for Data.Qld [![CircleCI](https://circleci.com/gh/qld-gov-au/ckanext-data-qld-theme/tree/develop.svg?style=shield)](https://circleci.com/gh/qld-gov-au/ckanext-data-qld-theme/tree/develop) diff --git a/docker-compose.yml b/docker-compose.yml index eef07d0e..4871988a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -24,6 +24,8 @@ services: dockerfile: .docker/Dockerfile.ckan args: SITE_URL: "http://${PROJECT}.docker.amazee.io" + CKAN_REPO: + CKAN_VERSION: depends_on: - postgres - solr diff --git a/test/features/config.feature b/test/features/config.feature index b98d361a..53711996 100644 --- a/test/features/config.feature +++ b/test/features/config.feature @@ -1,9 +1,15 @@ @config Feature: Config - Scenario: Assert that configuration values are available + Scenario: Assert that CSS configuration values are removed Given "SysAdmin" as the persona - When I log in - And I visit "ckan-admin/config" - And I should see "Suggested Description" - Then I should see an element with id "field-ckanext.data_qld.datarequest_suggested_description" + When I log in and go to admin config page + Then I should see "Intro Text" + And I should not see an element with id "field-ckan-main-css" + And I should not see an element with id "field-ckan-site-custom-css" + + Scenario: Assert that Data Request configuration values are available + Given "SysAdmin" as the persona + When I log in and go to admin config page + Then I should see "Suggested Description" + And I should see an element with id "field-ckanext.data_qld.datarequest_suggested_description" diff --git a/test/features/dataset_deletion.feature b/test/features/dataset_deletion.feature index 35c44dc3..71cf587b 100644 --- a/test/features/dataset_deletion.feature +++ b/test/features/dataset_deletion.feature @@ -31,12 +31,11 @@ Feature: Dataset deletion Then I press the element with xpath "//a[@data-module='confirm-action']" And I wait for 5 seconds Then I should see "Briefly describe the reason for deleting this dataset" - Then I should see an element with xpath "//div[@class='modal-footer']//button[@class='btn btn-primary' and @disabled='disabled']" - When I type "it should be longer than 10 character" to "deletion_reason" + And I should see an element with xpath "//div[@class='modal-footer']//button[@class='btn btn-primary' and @disabled='disabled']" + When I type "it should be longer than 10 characters" to "deletion_reason" Then I should not see an element with xpath "//div[@class='modal-footer']//button[@class='btn btn-primary' and @disabled='disabled']" - Then I take a screenshot Then I press the element with xpath "//div[@class='modal-footer']//button[@class='btn btn-primary']" - And I wait for 10 seconds + And I wait for 5 seconds Then I should see "Dataset has been deleted" And I should not see "Dataset deletion" When I go to "/ckan-admin/trash" diff --git a/test/features/steps/steps.py b/test/features/steps/steps.py index 262e92d9..64297f30 100644 --- a/test/features/steps/steps.py +++ b/test/features/steps/steps.py @@ -17,6 +17,11 @@ def go_to_home(context): when_i_visit_url(context, '/') +@step('I go to register page') +def go_to_register_page(context): + when_i_visit_url(context, '/user/register') + + @step('I log in') def log_in(context): assert context.persona @@ -53,14 +58,24 @@ def title_random_text(context): """.format(random.randrange(1000))) +@step('I go to dataset page') +def go_to_dataset_page(context): + when_i_visit_url(context, '/dataset') + + +@step(u'I go to dataset "{name}"') +def go_to_dataset(context, name): + when_i_visit_url(context, '/dataset/' + name) + + @step('I go to organisation page') def go_to_organisation_page(context): when_i_visit_url(context, '/organization') -@step('I go to register page') -def go_to_register_page(context): - when_i_visit_url(context, '/user/register') +@step(u'I set persona var "{key}" to "{value}"') +def set_persona_var(context, key, value): + context.persona[key] = value @step('I log in and go to the data requests page') @@ -105,16 +120,6 @@ def go_to_reporting_page(context): when_i_visit_url(context, '/dashboard/reporting') -@step('I go to dataset page') -def go_to_dataset_page(context): - when_i_visit_url(context, '/dataset') - - -@step(u'I go to dataset "{name}"') -def go_to_dataset(context, name): - when_i_visit_url(context, '/dataset/' + name) - - @step(u'I go to dataset "{name}" comments') def go_to_dataset_comments(context, name): context.execute_steps(u""" @@ -140,11 +145,6 @@ def go_to_data_request_comments(context, subject): """ % (subject)) -@step(u'I set persona var "{key}" to "{value}"') -def set_persona_var(context, key, value): - context.persona[key] = value - - @step(u'I submit a comment with subject "{subject}" and comment "{comment}"') def submit_comment_with_subject_and_comment(context, subject, comment): """ @@ -204,7 +204,7 @@ def create_dataset(context, license, file_format, file): # The default behaving step does not convert base64 emails -# Modifed the default step to decode the payload from base64 +# Modified the default step to decode the payload from base64 @step(u'I should receive a base64 email at "{address}" containing "{text}"') def should_receive_base64_email_containing_text(context, address, text): def filter_contents(mail): @@ -219,3 +219,17 @@ def filter_contents(mail): return text in decoded_payload assert context.mail.user_messages(address, filter_contents) + + +@step('I log in and go to admin config page') +def log_in_go_to_admin_config(context): + assert context.persona + context.execute_steps(u""" + When I log in + And I go to admin config page + """) + + +@step('I go to admin config page') +def go_to_admin_config(context): + when_i_visit_url(context, '/ckan-admin/config') From 7b1bf39fc87a51a1816f8375ee27e4913aaf3ffa Mon Sep 17 00:00:00 2001 From: ThrawnCA Date: Wed, 9 Feb 2022 16:36:41 +1000 Subject: [PATCH 2/5] [QOL-7811] drop controller reference and put metadata in specific templates instead --- ckanext/data_qld_theme/templates/base.html | 9 --------- ckanext/data_qld_theme/templates/package/read.html | 8 ++++++++ .../data_qld_theme/templates/package/resource_read.html | 7 +++++++ 3 files changed, 15 insertions(+), 9 deletions(-) create mode 100644 ckanext/data_qld_theme/templates/package/read.html diff --git a/ckanext/data_qld_theme/templates/base.html b/ckanext/data_qld_theme/templates/base.html index b35615aa..4f9f915e 100644 --- a/ckanext/data_qld_theme/templates/base.html +++ b/ckanext/data_qld_theme/templates/base.html @@ -1,14 +1,5 @@ {% ckan_extends %} - {% block meta %} - {{ super() }} - {% if c.controller in ['dataset', 'package', 'ckanext.data_qld.controller:DataQldDataset'] %} - {% if c.action in ['read', 'resource_read'] and c.pkg_dict['organization'] and c.pkg_dict['organization']['title'] %} - - {% endif %} - {% endif %} - {% endblock %} - {% block styles %} {{ super() }} diff --git a/ckanext/data_qld_theme/templates/package/read.html b/ckanext/data_qld_theme/templates/package/read.html new file mode 100644 index 00000000..2c81c11d --- /dev/null +++ b/ckanext/data_qld_theme/templates/package/read.html @@ -0,0 +1,8 @@ +{% ckan_extends %} + +{% block meta %} +{{ super() }} +{% if c.pkg_dict['organization'] and c.pkg_dict['organization']['title'] %} + +{% endif %} +{% endblock %} diff --git a/ckanext/data_qld_theme/templates/package/resource_read.html b/ckanext/data_qld_theme/templates/package/resource_read.html index b2f94aa2..7a861926 100644 --- a/ckanext/data_qld_theme/templates/package/resource_read.html +++ b/ckanext/data_qld_theme/templates/package/resource_read.html @@ -1,5 +1,12 @@ {% ckan_extends %} +{% block meta %} +{{ super() }} +{% if c.pkg_dict['organization'] and c.pkg_dict['organization']['title'] %} + +{% endif %} +{% endblock %} + {% block subtitle %}{{ h.resource_display_name(res) }} - {{ h.dataset_display_name(c.package) }}{% endblock %} {% block breadcrumb_content %} From dc5e8196dc76e33c94665aa90fdc78a4437829d0 Mon Sep 17 00:00:00 2001 From: ThrawnCA Date: Wed, 2 Mar 2022 14:46:30 +1000 Subject: [PATCH 3/5] [QOL-7811] update route name for data requests - use Flask-compatible name --- ckanext/data_qld_theme/templates/datarequests/show.html | 2 +- test/features/steps/steps.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/ckanext/data_qld_theme/templates/datarequests/show.html b/ckanext/data_qld_theme/templates/datarequests/show.html index 4928edf6..9db87c34 100644 --- a/ckanext/data_qld_theme/templates/datarequests/show.html +++ b/ckanext/data_qld_theme/templates/datarequests/show.html @@ -24,7 +24,7 @@ {% endblock %} {% block content_primary_nav %} - {{ h.build_nav_icon('show_datarequest', _('Data request'), id=datarequest_id) }} + {{ h.build_nav_icon('datarequest.show', _('Data request'), id=datarequest_id) }} {% if h.show_comments_tab() and h.ytp_comments_enabled() %} {{ h.build_nav_icon('comment_datarequest', _('Comments') + ' ' + h.get_content_type_comments_badge(datarequest_id, 'datarequest'), id=datarequest_id) }} diff --git a/test/features/steps/steps.py b/test/features/steps/steps.py index 4e2e53e2..ea9e45e8 100644 --- a/test/features/steps/steps.py +++ b/test/features/steps/steps.py @@ -82,6 +82,7 @@ def go_to_dataset(context, name): def edit_dataset(context, name): when_i_visit_url(context, '/dataset/edit/{}'.format(name)) + @step(u'I go to organisation page') def go_to_organisation_page(context): when_i_visit_url(context, '/organization') From a5fbbb3d5d16b00e4baa2a52d62c505b548dddbc Mon Sep 17 00:00:00 2001 From: ThrawnCA Date: Wed, 2 Mar 2022 15:05:21 +1000 Subject: [PATCH 4/5] [QOL-7811] update route name for data request comments - use Flask-compatible name --- ckanext/data_qld_theme/templates/datarequests/show.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckanext/data_qld_theme/templates/datarequests/show.html b/ckanext/data_qld_theme/templates/datarequests/show.html index 9db87c34..5e779716 100644 --- a/ckanext/data_qld_theme/templates/datarequests/show.html +++ b/ckanext/data_qld_theme/templates/datarequests/show.html @@ -27,6 +27,6 @@ {{ h.build_nav_icon('datarequest.show', _('Data request'), id=datarequest_id) }} {% if h.show_comments_tab() and h.ytp_comments_enabled() %} - {{ h.build_nav_icon('comment_datarequest', _('Comments') + ' ' + h.get_content_type_comments_badge(datarequest_id, 'datarequest'), id=datarequest_id) }} + {{ h.build_nav_icon('datarequest.comment', _('Comments') + ' ' + h.get_content_type_comments_badge(datarequest_id, 'datarequest'), id=datarequest_id) }} {% endif %} {% endblock %} From 8ea9ca083c659a117ee2d94d5b339d79074d3d44 Mon Sep 17 00:00:00 2001 From: ThrawnCA Date: Wed, 2 Mar 2022 15:26:26 +1000 Subject: [PATCH 5/5] [QOL-7811] update route name for organisation link to handle Flask --- .../data_qld_theme/templates/scheming/package/read.html | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/ckanext/data_qld_theme/templates/scheming/package/read.html b/ckanext/data_qld_theme/templates/scheming/package/read.html index da666078..6d3a9618 100644 --- a/ckanext/data_qld_theme/templates/scheming/package/read.html +++ b/ckanext/data_qld_theme/templates/scheming/package/read.html @@ -1,8 +1,13 @@ {% ckan_extends %} {% block breadcrumb_content %} + {% set route_name = 'organization_read' %} + {% if h.ckan_version() > '2.9' %} + {% set route_name = h.map_pylons_to_flask_route_name(route_name) %} + {% endif %}
  • {% link_for _('Organizations'), named_route='organization.index' %}
    • -
  • {% link_for pkg.organization.title, named_route='organization_read', id=pkg.organization.name %}
    • +
  • {% link_for pkg.organization.title, named_route=route_name, id=pkg.organization.name %}
    • +
  • {{ pkg.title }}
    • {% endblock %}