From 2765ae687723b767cc0182789391c1cba71dcbc1 Mon Sep 17 00:00:00 2001 From: antuarc Date: Fri, 12 Nov 2021 15:23:38 +1000 Subject: [PATCH 01/31] [QOL-7811] use named route for data request index page --- ckanext/data_qld_theme/templates/datarequests/new.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckanext/data_qld_theme/templates/datarequests/new.html b/ckanext/data_qld_theme/templates/datarequests/new.html index 99ee22aa..cdb7e28c 100644 --- a/ckanext/data_qld_theme/templates/datarequests/new.html +++ b/ckanext/data_qld_theme/templates/datarequests/new.html @@ -3,7 +3,7 @@ {% block subtitle %}{{ _('Create data request') }}{% endblock %} {% block breadcrumb_content %} -
  • {% link_for _('Data requests'), controller='ckanext.datarequests.controllers.ui_controller:DataRequestsUI', action='index' %}
    • +
  • {% link_for _('Data requests'), named_route='datarequest.index' %}
  • {{ _('Create data request') }}
    • {% endblock %} From 3a96a0d8f32efbb3ac9c42e1d3e8185c166f98a7 Mon Sep 17 00:00:00 2001 From: ThrawnCA Date: Thu, 4 Nov 2021 15:50:25 +1000 Subject: [PATCH 02/31] [QOL-7811][QOL-7906] update route name to report index page --- ckanext/data_qld_theme/templates/report/index.html | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ckanext/data_qld_theme/templates/report/index.html b/ckanext/data_qld_theme/templates/report/index.html index e03b69fd..35b9bd26 100644 --- a/ckanext/data_qld_theme/templates/report/index.html +++ b/ckanext/data_qld_theme/templates/report/index.html @@ -3,7 +3,7 @@ {% block title %}{{ _('Reports') }} - {{ super() }}{% endblock %} {% block breadcrumb_content %} - {{ h.build_nav('reports', _('Reports')) }} + {{ h.build_nav('report.index', _('Reports')) }} {% endblock %} {% block primary %} @@ -15,7 +15,7 @@ {% if loop.index0%3==0 %}
    {% endif %} {% if loop.index0%2==0 %}
    {% endif %}
    - {% set report_url=h.url_for('report', report_name=report.name, **h.report__explicit_default_options(report.name)) %} + {% set report_url=h.url_for('report.view', report_name=report.name, **h.report__explicit_default_options(report.name)) %}
    {{ report.title }}
    @@ -33,7 +33,7 @@
    {% block primary_content_inner %} - + {% endblock%} {% endblock%} {% endblock%} From 3f6646b8b6b5b45978d33869da6b7a2def9c09f5 Mon Sep 17 00:00:00 2001 From: ThrawnCA Date: Tue, 9 Nov 2021 14:06:38 +1000 Subject: [PATCH 03/31] [QOL-7811][QOL-7970] use named route for data request index page --- ckanext/data_qld_theme/templates/header.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ckanext/data_qld_theme/templates/header.html b/ckanext/data_qld_theme/templates/header.html index 680cdfd6..11b43926 100644 --- a/ckanext/data_qld_theme/templates/header.html +++ b/ckanext/data_qld_theme/templates/header.html @@ -37,7 +37,7 @@ {{ super() }} {% endblock %} {% endblock %} {% if h.is_datarequests_enabled() %} -
  • Request data
    • +
  • Request data
    • {% endif %} @@ -50,7 +50,7 @@
  • {% link_for _('Register'), named_route='user.register', class_='sub' %}
    • {% endif %} {% endblock %} {% if h.is_datarequests_enabled() %} -
  • Request data
    • +
  • Request data
    • {% endif %} From 42614c1b369c1946ac4783e77b94b43b01cec299 Mon Sep 17 00:00:00 2001 From: ThrawnCA Date: Fri, 14 Jan 2022 09:58:56 +1000 Subject: [PATCH 04/31] [QOL-7811] update to named routes for compatibility with Flask - this is needed in order to properly test other extensions on CKAN 2.9 --- .../data_qld_theme/templates/datarequests/show.html | 2 +- .../templates/package/resource_read.html | 10 ++++++---- .../templates/package/snippets/package_form.html | 2 +- .../templates/package/snippets/resource_form.html | 2 +- .../templates/package/snippets/resources.html | 11 ----------- 5 files changed, 9 insertions(+), 18 deletions(-) delete mode 100644 ckanext/data_qld_theme/templates/package/snippets/resources.html diff --git a/ckanext/data_qld_theme/templates/datarequests/show.html b/ckanext/data_qld_theme/templates/datarequests/show.html index 854d1cea..4928edf6 100644 --- a/ckanext/data_qld_theme/templates/datarequests/show.html +++ b/ckanext/data_qld_theme/templates/datarequests/show.html @@ -18,7 +18,7 @@ {% endif %} {% if h.check_access('open_datarequest', {'id':datarequest_id }) and c.datarequest.closed %} - {% link_for _('Re-open'), controller='ckanext.data_qld.controller:DataQldUI', action='open_datarequest', id=datarequest_id, class_='btn btn-success', icon='unlock' %} + {% link_for _('Re-open'), named_route='datarequest.open', id=datarequest_id, class_='btn btn-success', icon='unlock' %} {% endif %} {% endblock %} diff --git a/ckanext/data_qld_theme/templates/package/resource_read.html b/ckanext/data_qld_theme/templates/package/resource_read.html index c5f91174..9ebf4088 100644 --- a/ckanext/data_qld_theme/templates/package/resource_read.html +++ b/ckanext/data_qld_theme/templates/package/resource_read.html @@ -37,7 +37,7 @@
      {% block resource_actions_inner %} {% if h.check_access('package_update', {'id':pkg.id }) %} -
    • {% link_for _('Manage'), controller='package', action='resource_edit', id=pkg.name, resource_id=res.id, class_='btn btn-default', icon='wrench' %}
      • +
    • {% link_for _('Manage'), named_route='package.resource_edit', id=pkg.name, resource_id=res.id, class_='btn btn-default', icon='wrench' %}
      • {% endif %} {% if res.url and h.is_url(res.url) %}
    • @@ -64,11 +64,13 @@ diff --git a/ckanext/data_qld_theme/templates/package/snippets/package_form.html b/ckanext/data_qld_theme/templates/package/snippets/package_form.html index 3f5bc45f..e3a83a66 100644 --- a/ckanext/data_qld_theme/templates/package/snippets/package_form.html +++ b/ckanext/data_qld_theme/templates/package/snippets/package_form.html @@ -4,7 +4,7 @@ {% if h.check_access('package_delete', {'id': data.id}) and not data.state == 'deleted' and not h.data_qld_data_driven_application(data.data_driven_application) %} diff --git a/ckanext/data_qld_theme/templates/package/snippets/resource_form.html b/ckanext/data_qld_theme/templates/package/snippets/resource_form.html index 6facb61c..9312934f 100644 --- a/ckanext/data_qld_theme/templates/package/snippets/resource_form.html +++ b/ckanext/data_qld_theme/templates/package/snippets/resource_form.html @@ -3,7 +3,7 @@ {% block delete_button %} {% if data.id %} {% if h.check_access('resource_delete', {'id': data.id}) and not h.data_qld_dataset_data_driven_application(data.package_id) %} - {% block delete_button_text %}{{ _('Delete') }}{% endblock %} + {% block delete_button_text %}{{ _('Delete') }}{% endblock %} {% endif %} {% endif %} {% endblock %} diff --git a/ckanext/data_qld_theme/templates/package/snippets/resources.html b/ckanext/data_qld_theme/templates/package/snippets/resources.html deleted file mode 100644 index 4ecbfecc..00000000 --- a/ckanext/data_qld_theme/templates/package/snippets/resources.html +++ /dev/null @@ -1,11 +0,0 @@ -{% ckan_extends %} - -{% block resources_list %} -
        - {% for resource in resources %} -
      • - {% link_for h.resource_display_name(resource), controller='package', action=action or 'resource_read', truncate=30, id=pkg.name, resource_id=resource.id, inner_span=true %} -
        • - {% endfor %} -
      -{% endblock %} From bc3f0d2e2ef6731fea42862aa1dbafa411ff8173 Mon Sep 17 00:00:00 2001 From: ThrawnCA Date: Fri, 14 Jan 2022 10:57:01 +1000 Subject: [PATCH 05/31] [QOL-7811] handle both Pylons and Flask routing for resource edits - there doesn't seem to be an option that is compatible with both, so include both implementations with a version check --- ckanext/data_qld_theme/templates/package/resource_read.html | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/ckanext/data_qld_theme/templates/package/resource_read.html b/ckanext/data_qld_theme/templates/package/resource_read.html index 9ebf4088..480ba0d3 100644 --- a/ckanext/data_qld_theme/templates/package/resource_read.html +++ b/ckanext/data_qld_theme/templates/package/resource_read.html @@ -37,7 +37,11 @@
        {% block resource_actions_inner %} {% if h.check_access('package_update', {'id':pkg.id }) %} -
      • {% link_for _('Manage'), named_route='package.resource_edit', id=pkg.name, resource_id=res.id, class_='btn btn-default', icon='wrench' %}
        • + {% if h.ckan_version() > '2.9' %} +
      • {% link_for _('Manage'), named_route=pkg.type ~ '_resource.edit', id=pkg.name, resource_id=res.id, class_='btn btn-default', icon='wrench' %}
        • + {% else %} +
      • {% link_for _('Manage'), controller='package', action='resource_edit', id=pkg.name, resource_id=res.id, class_='btn btn-default', icon='wrench' %}
        • + {% endif %} {% endif %} {% if res.url and h.is_url(res.url) %}
      • From 6c7f64c1ef0b296da2a4c47960e34fd4280c89b8 Mon Sep 17 00:00:00 2001 From: ThrawnCA Date: Fri, 14 Jan 2022 11:10:22 +1000 Subject: [PATCH 06/31] [QOL-7811] handle both Pylons and Flask routing for resource deletion - there doesn't seem to be an option that is compatible with both, so include both implementations with a version check --- .../templates/package/snippets/resource_form.html | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/ckanext/data_qld_theme/templates/package/snippets/resource_form.html b/ckanext/data_qld_theme/templates/package/snippets/resource_form.html index 9312934f..29315491 100644 --- a/ckanext/data_qld_theme/templates/package/snippets/resource_form.html +++ b/ckanext/data_qld_theme/templates/package/snippets/resource_form.html @@ -3,7 +3,11 @@ {% block delete_button %} {% if data.id %} {% if h.check_access('resource_delete', {'id': data.id}) and not h.data_qld_dataset_data_driven_application(data.package_id) %} - {% block delete_button_text %}{{ _('Delete') }}{% endblock %} + {% if h.ckan_version() > '2.9' %} + {% block delete_button_text %}{{ _('Delete') }}{% endblock %} + {% else %} + {% block delete_button_text %}{{ _('Delete') }}{% endblock %} + {% endif %} {% endif %} {% endif %} {% endblock %} From b3682e30a9eb3405f10e92ff7c2d1c69c8facd61 Mon Sep 17 00:00:00 2001 From: ThrawnCA Date: Fri, 14 Jan 2022 12:33:43 +1000 Subject: [PATCH 07/31] [QOL-7811] pull common HTML elements outside conditional block --- .../templates/package/snippets/resource_form.html | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/ckanext/data_qld_theme/templates/package/snippets/resource_form.html b/ckanext/data_qld_theme/templates/package/snippets/resource_form.html index 29315491..94ed89b7 100644 --- a/ckanext/data_qld_theme/templates/package/snippets/resource_form.html +++ b/ckanext/data_qld_theme/templates/package/snippets/resource_form.html @@ -3,11 +3,13 @@ {% block delete_button %} {% if data.id %} {% if h.check_access('resource_delete', {'id': data.id}) and not h.data_qld_dataset_data_driven_application(data.package_id) %} + '2.9' %} - {% block delete_button_text %}{{ _('Delete') }}{% endblock %} + href="{% url_for dataset_type ~ '_resource.delete', resource_id=data.id, id=pkg_name %}" data-module="confirm-action" data-module-content="{{ _('Are you sure you want to delete this resource?') }}" {% else %} - {% block delete_button_text %}{{ _('Delete') }}{% endblock %} + href="{{ h.url_for(controller='package', action='resource_delete', resource_id=data.id, id=pkg_name) }}" data-module="confirm-action" data-module-content="{{ _('Are you sure you want to delete this resource?') }}" {% endif %} + >{% block delete_button_text %}{{ _('Delete') }}{% endblock %} {% endif %} {% endif %} {% endblock %} From 1724c2432904c73805b0bb62878e09a21c63579e Mon Sep 17 00:00:00 2001 From: ThrawnCA Date: Fri, 14 Jan 2022 13:02:10 +1000 Subject: [PATCH 08/31] [QOL-7811] pull common HTML elements outside conditional block --- ckanext/data_qld_theme/templates/package/resource_read.html | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/ckanext/data_qld_theme/templates/package/resource_read.html b/ckanext/data_qld_theme/templates/package/resource_read.html index 480ba0d3..b93c599e 100644 --- a/ckanext/data_qld_theme/templates/package/resource_read.html +++ b/ckanext/data_qld_theme/templates/package/resource_read.html @@ -37,11 +37,13 @@
          {% block resource_actions_inner %} {% if h.check_access('package_update', {'id':pkg.id }) %} +
        • {% if h.ckan_version() > '2.9' %} -
        • {% link_for _('Manage'), named_route=pkg.type ~ '_resource.edit', id=pkg.name, resource_id=res.id, class_='btn btn-default', icon='wrench' %}
          • + {% link_for _('Manage'), named_route=pkg.type ~ '_resource.edit', id=pkg.name, resource_id=res.id, class_='btn btn-default', icon='wrench' %} {% else %} -
        • {% link_for _('Manage'), controller='package', action='resource_edit', id=pkg.name, resource_id=res.id, class_='btn btn-default', icon='wrench' %}
          • + {% link_for _('Manage'), controller='package', action='resource_edit', id=pkg.name, resource_id=res.id, class_='btn btn-default', icon='wrench' %} {% endif %} + {% endif %} {% if res.url and h.is_url(res.url) %}
        • From 4b3c8cca42f6157474630a9509bace6e018b32e6 Mon Sep 17 00:00:00 2001 From: ThrawnCA Date: Thu, 20 Jan 2022 15:44:59 +1000 Subject: [PATCH 09/31] [QOL-7811] handle both Pylons and Flask routing for dataset deletion --- .../templates/package/snippets/package_form.html | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/ckanext/data_qld_theme/templates/package/snippets/package_form.html b/ckanext/data_qld_theme/templates/package/snippets/package_form.html index e3a83a66..09ed3b5b 100644 --- a/ckanext/data_qld_theme/templates/package/snippets/package_form.html +++ b/ckanext/data_qld_theme/templates/package/snippets/package_form.html @@ -1,13 +1,18 @@ {% ckan_extends %} + {% block delete_button %} {% if h.check_access('package_delete', {'id': data.id}) and not data.state == 'deleted' and not h.data_qld_data_driven_application(data.data_driven_application) %} + class="btn btn-danger pull-left" + {% if h.ckan_version() > '2.9' %} + href="{{ h.url_for('dataset.delete', id=data.id) }}" + {% else %} + href="{% url_for controller='package', action='delete', id=data.id %}" + {% endif %} + data-module="confirm-action" + data-module-content="{{ _('Deleting this dataset will prevent future use of the dataset’s name. It is highly recommended that dataset issues are rectified rather than deleting and creating a new dataset with the same information.') }}" + data-module-template="{{ h.get_deletion_reason_template() }}"> {% block delete_button_text %}{{ _('Delete') }}{% endblock %} {% resource 'data_qld_theme/dataset-deletion.js' %} From 3903a02461b6a4d9abc22f11942eabd722843a78 Mon Sep 17 00:00:00 2001 From: ThrawnCA Date: Thu, 20 Jan 2022 15:45:22 +1000 Subject: [PATCH 10/31] [QOL-7811] remove unused import --- test/features/steps/steps.py | 1 - 1 file changed, 1 deletion(-) diff --git a/test/features/steps/steps.py b/test/features/steps/steps.py index 169a3650..262e92d9 100644 --- a/test/features/steps/steps.py +++ b/test/features/steps/steps.py @@ -2,7 +2,6 @@ from behaving.personas.steps import * # noqa: F401, F403 from behaving.web.steps import * # noqa: F401, F403 from behaving.web.steps.url import when_i_visit_url -from behaving.mail.steps import * import email import quopri import random From 7ee210ce3de68abfdb17730ca0f8a3885bc66eda Mon Sep 17 00:00:00 2001 From: ThrawnCA Date: Thu, 20 Jan 2022 16:26:32 +1000 Subject: [PATCH 11/31] [QOL-7811] restrict Splinter to Python 2 compatible versions --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 1cb8c797..58e14d7f 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -3,7 +3,7 @@ behaving==2.0.0 Appium-Python-Client<=0.52 flake8==3.8.3 nose==1.3.7 -splinter>=0.13.0 +splinter>=0.13.0,<0.17 git+https://github.com/ckan/ckanapi@ckanapi-4.3#egg=ckanapi git+https://github.com/ckan/ckantoolkit@release-0.0.4#egg=ckantoolkit -e git+https://github.com/ckan/ckanext-dcat.git#egg=ckanext-dcat From b28a41f70039a972c8726fefa0936ecc7cd61c78 Mon Sep 17 00:00:00 2001 From: ThrawnCA Date: Thu, 20 Jan 2022 16:27:42 +1000 Subject: [PATCH 12/31] [QOL-7811] sync changes from ckanext-publications-qld-theme --- .../data_qld_theme/templates/package/group_list.html | 4 ++-- .../templates/package/resource_read.html | 10 ++++------ .../templates/package/snippets/resource_form.html | 5 +++-- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/ckanext/data_qld_theme/templates/package/group_list.html b/ckanext/data_qld_theme/templates/package/group_list.html index d364709b..b4db86ef 100644 --- a/ckanext/data_qld_theme/templates/package/group_list.html +++ b/ckanext/data_qld_theme/templates/package/group_list.html @@ -5,13 +5,13 @@

          {{ _('Groups') }}

          {% if h.check_access('package_update', {'id':c.pkg_dict.id }) %} - {% set groups = h.get_all_groups() %} + {% set groups = h.groups_available() %} {% if groups %}
          diff --git a/ckanext/data_qld_theme/templates/package/resource_read.html b/ckanext/data_qld_theme/templates/package/resource_read.html index b93c599e..b2f94aa2 100644 --- a/ckanext/data_qld_theme/templates/package/resource_read.html +++ b/ckanext/data_qld_theme/templates/package/resource_read.html @@ -56,11 +56,9 @@ {% elif (not res.has_views or not res.can_be_previewed) and not res.url_type == 'upload' %} {{ _('Go to resource') }} {% else %} - {% set res = c.resource %} - {% set size = res.Size %} - {% set format = res.format %} - {% set download_text = 'Download ({0})({1})'.format(size,format) %} - {{ _(download_text) }} + {{ _('Download') }} + {% if res.actual_size or res.size %}({{ res.actual_size or h.format_resource_filesize(res.size) }}){% endif %} + {% if res.format %}({{ res.format }}){% endif %} {% endif %} {% block download_resource_button %} @@ -110,7 +108,7 @@

          {{ _('Data Dictionary') }}

          - {% block resouce_data_dictionary_headers %} + {% block resource_data_dictionary_headers %} diff --git a/ckanext/data_qld_theme/templates/package/snippets/resource_form.html b/ckanext/data_qld_theme/templates/package/snippets/resource_form.html index 94ed89b7..3b3dabba 100644 --- a/ckanext/data_qld_theme/templates/package/snippets/resource_form.html +++ b/ckanext/data_qld_theme/templates/package/snippets/resource_form.html @@ -5,10 +5,11 @@ {% if h.check_access('resource_delete', {'id': data.id}) and not h.data_qld_dataset_data_driven_application(data.package_id) %} '2.9' %} - href="{% url_for dataset_type ~ '_resource.delete', resource_id=data.id, id=pkg_name %}" data-module="confirm-action" data-module-content="{{ _('Are you sure you want to delete this resource?') }}" + href="{% url_for dataset_type ~ '_resource.delete', resource_id=data.id, id=pkg_name %}" {% else %} - href="{{ h.url_for(controller='package', action='resource_delete', resource_id=data.id, id=pkg_name) }}" data-module="confirm-action" data-module-content="{{ _('Are you sure you want to delete this resource?') }}" + href="{{ h.url_for(controller='package', action='resource_delete', resource_id=data.id, id=pkg_name) }}" {% endif %} + data-module="confirm-action" data-module-content="{{ _('Are you sure you want to delete this resource?') }}" >{% block delete_button_text %}{{ _('Delete') }}{% endblock %} {% endif %} {% endif %} From 07c099213ccc7f280333045a99b53847ad272a05 Mon Sep 17 00:00:00 2001 From: ThrawnCA Date: Fri, 21 Jan 2022 09:07:14 +1000 Subject: [PATCH 13/31] [QOL-7811] copy BDD tests from infrastructure project - The theme interacts with almost every other extension, and changes to it may break any of them, so they should all be tested in this repo. --- test/features/admin_reporting.feature | 41 ++++ test/features/comments.feature | 1 - test/features/config.feature | 9 + test/features/data_request.feature | 12 ++ test/features/data_usability_rating.feature | 51 +++++ test/features/data_validation.feature | 38 ++++ test/features/dataset_deletion.feature | 75 +++++++ test/features/de_identified_data.feature | 62 ++++++ test/features/engagement_reporting.feature | 86 ++++++++ test/features/environment.py | 10 + test/features/login_redirect.feature | 62 ++++++ test/features/resource_availability.feature | 220 ++++++++++++++++++++ test/features/resource_freshness.feature | 43 ++++ test/features/schema_metadata.feature | 95 +++++++++ test/features/user_creation.feature | 35 ++++ test/fixtures/csv_resource.csv | 0 test/fixtures/html_resource.html | 1 + test/fixtures/rdf_resource.rdf | 0 test/fixtures/test-resource_schemea.json | 37 ++++ test/fixtures/txt_resource.txt | 0 test/fixtures/xls_resource.xls | 0 21 files changed, 877 insertions(+), 1 deletion(-) create mode 100644 test/features/admin_reporting.feature create mode 100644 test/features/config.feature create mode 100644 test/features/data_request.feature create mode 100644 test/features/data_usability_rating.feature create mode 100644 test/features/data_validation.feature create mode 100644 test/features/dataset_deletion.feature create mode 100644 test/features/de_identified_data.feature create mode 100644 test/features/engagement_reporting.feature create mode 100644 test/features/login_redirect.feature create mode 100644 test/features/resource_availability.feature create mode 100644 test/features/resource_freshness.feature create mode 100644 test/features/schema_metadata.feature create mode 100644 test/features/user_creation.feature create mode 100644 test/fixtures/csv_resource.csv create mode 100644 test/fixtures/html_resource.html create mode 100644 test/fixtures/rdf_resource.rdf create mode 100644 test/fixtures/test-resource_schemea.json create mode 100644 test/fixtures/txt_resource.txt create mode 100644 test/fixtures/xls_resource.xls diff --git a/test/features/admin_reporting.feature b/test/features/admin_reporting.feature new file mode 100644 index 00000000..35e0f0b6 --- /dev/null +++ b/test/features/admin_reporting.feature @@ -0,0 +1,41 @@ +@reporting +Feature: AdminReporting + + Scenario: As an admin user of my organisation, I can view 'My Reports' tab in the dashboard and show the 'Admin Report' with filters and table columns + Given "TestOrgAdmin" as the persona + When I log in + And I visit "dashboard" + And I click the link with text that contains "My Reports" + And I click the link with text that contains "Admin Report" + Then I should see an element with id "organisation" + When I press the element with xpath "//button[contains(string(), 'Show')]" + Then I should see "Organisation: Test Organisation" within 1 seconds + Then I should see an element with xpath "//tr/th[string()='Criteria' and position()=1]" + Then I should see an element with xpath "//tr/th[string()='Figure' and position()=2]" + + + Scenario: As an editor user of my organisation, I can view 'My Reports' tab in the dashboard but I cannot view the 'Admin Report' link + Given "TestOrgEditor" as the persona + When I log in + And I visit "dashboard" + And I click the link with text that contains "My Reports" + Then I should not see an element with xpath "//a[contains(string(), 'Admin Report')]" + + + Scenario: As an admin user of my organisation, when I view my admin report, I can verify the de-identified datasets row exists + Given "TestOrgAdmin" as the persona + When I log in + And I go to my reports page + And I click the link with text that contains "Admin Report" + And I press the element with xpath "//button[contains(string(), 'Show')]" + Then I should see an element with xpath "//tr[@id='de-identified-datasets']/td[contains(@class, 'metric-title') and contains(string(), 'De-identified Datasets') and position()=1]" + Then I should see an element with xpath "//tr[@id='de-identified-datasets']/td[contains(@class, 'metric-data') and position()=2]" + + Scenario: As an admin user of my organisation, when I view my admin report, I can verify the overdue datasets row exists correct + Given "TestOrgAdmin" as the persona + When I log in + And I go to my reports page + And I click the link with text that contains "Admin Report" + And I press the element with xpath "//button[contains(string(), 'Show')]" + Then I should see an element with xpath "//tr[@id='overdue-datasets']/td[contains(@class, 'metric-title') and contains(string(), 'Overdue Datasets') and position()=1]" + Then I should see an element with xpath "//tr[@id='overdue-datasets']/td[contains(@class, 'metric-data') and position()=2]" diff --git a/test/features/comments.feature b/test/features/comments.feature index 4bd1b0e9..8804e83a 100644 --- a/test/features/comments.feature +++ b/test/features/comments.feature @@ -97,7 +97,6 @@ Feature: Comments And I press the element with xpath "//a[@title='Delete comment']" Then I should see "Are you sure you want to delete this comment?" within 1 seconds Then I press the element with xpath "//button[contains(string(), 'Confirm')]" - Then I take a screenshot Then I should not see "This comment was deleted." within 2 seconds And I should see "Comment deleted by Test Admin." within 2 seconds diff --git a/test/features/config.feature b/test/features/config.feature new file mode 100644 index 00000000..b98d361a --- /dev/null +++ b/test/features/config.feature @@ -0,0 +1,9 @@ +@config +Feature: Config + + Scenario: Assert that configuration values are available + Given "SysAdmin" as the persona + When I log in + And I visit "ckan-admin/config" + And I should see "Suggested Description" + Then I should see an element with id "field-ckanext.data_qld.datarequest_suggested_description" diff --git a/test/features/data_request.feature b/test/features/data_request.feature new file mode 100644 index 00000000..0fdc5824 --- /dev/null +++ b/test/features/data_request.feature @@ -0,0 +1,12 @@ +@config +Feature: Data Request + + Scenario Outline: As a logged in user I can request new data + Given "User" as the persona + When I log in + And I visit "datarequest/new" + And I fill in title with random text + And I fill in "description" with "Data Request description" + And I press "Create data request" + Then I should see an element with xpath "//h1[contains(@class, 'page-heading') and contains(text(), 'Test Title ')]" + And I should see an element with xpath "//div[contains(@class, 'notes') and contains(text(), 'Data Request description')]" diff --git a/test/features/data_usability_rating.feature b/test/features/data_usability_rating.feature new file mode 100644 index 00000000..14e5f59a --- /dev/null +++ b/test/features/data_usability_rating.feature @@ -0,0 +1,51 @@ +@data_usability_rating +Feature: Data usability rating + + Scenario: As an admin user of my organisation, when I create a dataset with a HTML resource, I can verify the score is 0 + Given "TestOrgAdmin" as the persona + When I log in + And I create a dataset with license "other-open" and "HTML" resource file "html_resource.html" + Then I wait for 10 seconds + When I reload + Then I should see "Data usability rating" + And I should see an element with xpath "//div[contains(@class, 'qa openness-0')]" + + + Scenario: As an admin user of my organisation, when I create a dataset with an open license and TXT resource, I can verify the score is 1 + Given "TestOrgAdmin" as the persona + When I log in + And I create a dataset with license "other-open" and "TXT" resource file "txt_resource.txt" + Then I wait for 10 seconds + When I reload + Then I should see "Data usability rating" + And I should see an element with xpath "//div[contains(@class, 'qa openness-1')]" + + + Scenario: As an admin user of my organisation, when I create a dataset with an open license and XLS resource, I can verify the score is 2 + Given "TestOrgAdmin" as the persona + When I log in + And I create a dataset with license "other-open" and "XLS" resource file "xls_resource.xls" + Then I wait for 10 seconds + When I reload + Then I should see "Data usability rating" + And I should see an element with xpath "//div[contains(@class, 'qa openness-2')]" + + + Scenario: As an admin user of my organisation, when I create a dataset with an open license and a CSV resource, I can verify the score is 3 + Given "TestOrgAdmin" as the persona + When I log in + And I create a dataset with license "other-open" and "CSV" resource file "csv_resource.csv" + Then I wait for 10 seconds + When I reload + Then I should see "Data usability rating" + And I should see an element with xpath "//div[contains(@class, 'qa openness-3')]" + + + Scenario: As an admin user of my organisation, when I create a dataset with an open license and a RDF resource, I can verify the score is 4 + Given "TestOrgAdmin" as the persona + When I log in + And I create a dataset with license "other-open" and "RDF" resource file "rdf_resource.rdf" + Then I wait for 10 seconds + When I reload + Then I should see "Data usability rating" + And I should see an element with xpath "//div[contains(@class, 'qa openness-4')]" diff --git a/test/features/data_validation.feature b/test/features/data_validation.feature new file mode 100644 index 00000000..0939f070 --- /dev/null +++ b/test/features/data_validation.feature @@ -0,0 +1,38 @@ +@config +Feature: Data Validation + + Scenario Outline: As a sysadmin, admin and editor user of the dataset organisation I cannot see the ' JSON' button + Given "" as the persona + When I log in + And I visit "dataset/new_resource/annakarenina" + And I should not see an element with xpath "//a[contains(string(), ' JSON')]" + + Examples: Users + | User | + | SysAdmin | + | TestOrgAdmin | + | TestOrgEditor | + + + Scenario: As any user, I can view the 'Data Schema' link in the 'Additional Info' table of the resource read-view page + Given "SysAdmin" as the persona + When I log in + And I visit "dataset/new" + When I fill in title with random text + When I fill in "notes" with "Description" + When I fill in "version" with "1.0" + When I fill in "author_email" with "test@me.com" + Then I select "NO" from "de_identified_data" + When I press "Add Data" + And I execute the script "document.getElementById('field-image-url').value='http://example.com/external-data.csv'" + And I fill in "name" with "Test Resource" + And I execute the script "document.getElementById('field-format').value='JSON'" + And I fill in "description" with "Test Resource Description" + And I fill in "size" with "1mb" + And I execute the script "document.getElementById('field-schema-upload').parentNode.parentNode.setAttribute('style', '')" + And I attach the file "test-resource_schemea.json" to "schema_upload" + And I press "Finish" + When I wait for 1 seconds + And I click the link with text that contains "Test Resource" + And I click the link with text that contains "View Schema File" + Then the browser's URL should contain "/schema/show/" diff --git a/test/features/dataset_deletion.feature b/test/features/dataset_deletion.feature new file mode 100644 index 00000000..edf8b9d8 --- /dev/null +++ b/test/features/dataset_deletion.feature @@ -0,0 +1,75 @@ +@dataset_deletion +Feature: Dataset deletion + + Scenario: Sysadmin creates a dataset + Given "SysAdmin" as the persona + When I log in + And I go to "/dataset/new" + Then I fill in "title" with "Dataset deletion" + Then I fill in "notes" with "notes" + Then I execute the script "document.getElementById('field-organizations').value=jQuery('#field-organizations option').filter(function () { return $(this).html() == 'Test Organisation'; }).attr('value')" + Then I select "False" from "private" + Then I fill in "version" with "1" + Then I fill in "author_email" with "test@test.com" + Then I select "NO" from "de_identified_data" + Then I press "save" + And I wait for 10 seconds + Then I execute the script "document.getElementById('field-image-url').value='https://example.com'" + Then I fill in "name" with "res1" + Then I fill in "description" with "description" + Then I fill in "size" with "1024" + Then I select "Resource NOT visible/Pending acknowledgement" from "resource_visibility" + Then I press the element with xpath "//button[@value='go-metadata']" + And I wait for 10 seconds + Then I should see "Data and Resources" + + Scenario: Sysadmin creates a dataset + Given "SysAdmin" as the persona + When I log in + And I go to "/dataset/new" + Then I fill in "title" with "Dataset deletion" + Then I fill in "notes" with "notes" + Then I execute the script "document.getElementById('field-organizations').value=jQuery('#field-organizations option').filter(function () { return $(this).html() == 'Test Organisation'; }).attr('value')" + Then I select "False" from "private" + Then I fill in "version" with "1" + Then I fill in "author_email" with "test@test.com" + Then I press "save" + And I wait for 10 seconds + Then I execute the script "document.getElementById('field-image-url').value='https://example.com'" + Then I fill in "name" with "res1" + Then I fill in "description" with "description" + Then I press the element with xpath "//button[@value='go-metadata']" + And I wait for 10 seconds + Then I should see "Data and Resources" + + Scenario: Sysadmin deletes a dataset + Given "SysAdmin" as the persona + When I log in + And I go to "/dataset/edit/dataset-deletion" + Then I should see an element with xpath "//a[@data-module='confirm-action']" + Then I press the element with xpath "//a[@data-module='confirm-action']" + And I wait for 5 seconds + Then I should see "Briefly describe the reason for deleting this dataset" + Then I should see an element with xpath "//div[@class='modal-footer']//button[@class='btn btn-primary' and @disabled='disabled']" + When I type "it should be longer than 10 character" to "deletion_reason" + Then I should not see an element with xpath "//div[@class='modal-footer']//button[@class='btn btn-primary' and @disabled='disabled']" + Then I take a screenshot + Then I press the element with xpath "//div[@class='modal-footer']//button[@class='btn btn-primary']" + And I wait for 10 seconds + Then I should see "Dataset has been deleted" + And I should not see "Dataset deletion" + When I go to "/ckan-admin/trash" + Then I should see "Dataset deletion" + Then I press the element with xpath "//button[@name='purge-packages']" + + Scenario: Sysadmin deletes a dataset + Given "SysAdmin" as the persona + When I log in + And I go to "/dataset/edit/dataset-deletion" + Then I should see an element with xpath "//a[@data-module='confirm-action']" + Then I press the element with xpath "//a[@data-module='confirm-action']" + And I wait for 5 seconds + Then I should not see "Dataset deletion" + And I go to "/ckan-admin/trash" + Then I should see "Purge deleted datasets forever and irreversibly" + Then I press the element with xpath "//button[@name='purge-packages']" diff --git a/test/features/de_identified_data.feature b/test/features/de_identified_data.feature new file mode 100644 index 00000000..f456f9df --- /dev/null +++ b/test/features/de_identified_data.feature @@ -0,0 +1,62 @@ +@schema_metadata +Feature: De-identified data + + Scenario Outline: An editor, admin or sysadmin user, when I go to the dataset new page, the field field-de_identified_data should be visible with the correct values + Given "" as the persona + When I log in + And I go to "/dataset/new" + Then I should see an element with id "field-de_identified_data" + Then I should see an element with xpath "//select[@id='field-de_identified_data']/option[@value='YES']" + Then I should see an element with xpath "//select[@id='field-de_identified_data']/option[@value='NO']" + Then I should see an element with xpath "//select[@id='field-de_identified_data']/option[@value='']" + Then I should not see an element with xpath "//select[@id='field-de_identified_data']/option[@selected='' and @value='YES']" + Then I should not see an element with xpath "//select[@id='field-de_identified_data']/option[@selected='' and @value='NO']" + Then I should not see an element with xpath "//select[@id='field-de_identified_data']/option[@selected='' and @value='']" + + Examples: Users + | User | + | SysAdmin | + | TestOrgAdmin | + | TestOrgEditor | + + + Scenario Outline: An editor, admin or sysadmin user, when I go to the edit dataset page, the field field-de_identified_data should be visible with the correct values + Given "" as the persona + When I log in + And I go to "/dataset/edit/warandpeace" + Then I should see an element with id "field-de_identified_data" + Then I should see an element with xpath "//select[@id='field-de_identified_data']/option[@value='YES']" + Then I should see an element with xpath "//select[@id='field-de_identified_data']/option[@selected='' and @value='NO']" + + Examples: Users + | User | + | SysAdmin | + | TestOrgAdmin | + | TestOrgEditor | + + Scenario Outline: An editor, admin or sysadmin user can view the de-identified data + Given "" as the persona + When I log in + + And I go to "/dataset/warandpeace" + Then I should see "Contains de-identified data" + Then I should see an element with xpath "//th[contains(text(), 'Contains de-identified data')]/following-sibling::td[contains(text(), 'NO')]" + + When I go to "/api/3/action/package_show?id=warandpeace" + Then I should see an element with xpath "//body/*[contains(text(), '"de_identified_data":')]" + + Examples: Users + | User | + | SysAdmin | + | TestOrgAdmin | + | TestOrgEditor | + + Scenario: Unauthenticated user cannot view the de-identified data + Given "Unauthenticated" as the persona + When I go to "/dataset/warandpeace" + Then I should not see "Contains de-identified data" + Then I should not see an element with xpath "//th[contains(text(), 'Contains de-identified data')]/following-sibling::td[contains(text(), 'NO')]" + + And I go to "/api/3/action/package_show?id=warandpeace" + Then I should not see an element with xpath "//body/*[contains(text(), '"de_identified_data":')]" + diff --git a/test/features/engagement_reporting.feature b/test/features/engagement_reporting.feature new file mode 100644 index 00000000..3bef795f --- /dev/null +++ b/test/features/engagement_reporting.feature @@ -0,0 +1,86 @@ +@reporting +Feature: Engagement Reporting + + Scenario Outline: As a user with admin or editor role capacity of an organisation, I can view 'My Reports' tab in the dashboard and show the engagement report with filters + Given "" as the persona + When I log in + And I visit "dashboard" + And I click the link with text that contains "My Reports" + And I click the link with text that contains "Engagement Report" + Then I should see an element with id "organisation" + And I should see an element with id "start_date" + And I fill in "start_date" with "01-01-2019" + And I should see an element with id "end_date" + And I fill in "end_date" with "01-01-2020" + When I press the element with xpath "//button[contains(string(), 'Show')]" + Then I should see "Organisation: Test Organisation" within 1 seconds + And I should see "01/01/2019 - 01/01/2020" within 1 seconds + + Examples: Users + | User | + | TestOrgAdmin | + | TestOrgEditor | + + + Scenario: As a data request organisation admin, when I view my engagement report, I can verify the number of data requests is correct and increments + Given "DataRequestOrgAdmin" as the persona + When I log in + And I go to my reports page + And I click the link with text that contains "Engagement Report" + And I press the element with xpath "//button[contains(string(), 'Show')]" + Then I should see an element with xpath "//tr[@id='datarequests-total']/td[contains(@class, 'metric-title') and string()='Data requests' and position()=1]" + Then I should see an element with xpath "//tr[@id='datarequests-total']/td[contains(@class, 'metric-data') and string()='0' and position()=2]" + When I create a datarequest + And I go to my reports page + And I click the link with text that contains "Engagement Report" + And I press the element with xpath "//button[contains(string(), 'Show')]" + Then I should see an element with xpath "//tr[@id='datarequests-total']/td[contains(@class, 'metric-title') and string()='Data requests' and position()=1]" + Then I should see an element with xpath "//tr[@id='datarequests-total']/td[contains(@class, 'metric-data') and string()='1' and position()=2]" + + + Scenario: As an admin user of my organisation, when I view my engagement report, I can verify the number of dataset followers is correct and increments + Given "ReportingOrgAdmin" as the persona + When I log in + And I go to my reports page + And I click the link with text that contains "Engagement Report" + And I press the element with xpath "//button[contains(string(), 'Show')]" + Then I should see an element with xpath "//tr[@id='dataset-followers']/td[contains(@class, 'metric-title') and string()='Dataset followers' and position()=1]" + Then I should see an element with xpath "//tr[@id='dataset-followers']/td[contains(@class, 'metric-data') and string()='0' and position()=2]" + When I go to dataset "reporting" + And I press the element with xpath "//a[@class='btn btn-success' and contains(string(), 'Follow')]" + And I go to my reports page + And I click the link with text that contains "Engagement Report" + Then I should see an element with xpath "//tr[@id='dataset-followers']/td[contains(@class, 'metric-title') and string()='Dataset followers' and position()=1]" + Then I should see an element with xpath "//tr[@id='dataset-followers']/td[contains(@class, 'metric-data') and string()='1' and position()=2]" + + + Scenario: As an admin user of my organisation, when I view my engagement report, I can verify the number of dataset comments is correct and increments + Given "ReportingOrgAdmin" as the persona + When I log in + And I go to my reports page + And I click the link with text that contains "Engagement Report" + And I press the element with xpath "//button[contains(string(), 'Show')]" + Then I should see an element with xpath "//tr[@id='dataset-comments']/td[contains(@class, 'metric-title') and string()='Dataset comments' and position()=1]" + Then I should see an element with xpath "//tr[@id='dataset-comments']/td[contains(@class, 'metric-data') and string()='0' and position()=2]" + When I go to dataset "reporting" comments + And I submit a comment with subject "Test subject" and comment "This is a test comment" + And I go to my reports page + And I click the link with text that contains "Engagement Report" + Then I should see an element with xpath "//tr[@id='dataset-comments']/td[contains(@class, 'metric-title') and string()='Dataset comments' and position()=1]" + Then I should see an element with xpath "//tr[@id='dataset-comments']/td[contains(@class, 'metric-data') and string()='1' and position()=2]" + + + Scenario: As an admin user of my organisation, when I view my engagement report, I can verify the number of data request comments is correct and increments + Given "ReportingOrgAdmin" as the persona + When I log in + And I go to my reports page + And I click the link with text that contains "Engagement Report" + And I press the element with xpath "//button[contains(string(), 'Show')]" + Then I should see an element with xpath "//tr[@id='datarequest-comments']/td[contains(@class, 'metric-title') and string()='Data request comments' and position()=1]" + Then I should see an element with xpath "//tr[@id='datarequest-comments']/td[contains(@class, 'metric-data') and string()='0' and position()=2]" + When I go to data request "Reporting Request" comments + And I submit a comment with subject "Test subject" and comment "This is a test comment" + And I go to my reports page + And I click the link with text that contains "Engagement Report" + Then I should see an element with xpath "//tr[@id='datarequest-comments']/td[contains(@class, 'metric-title') and string()='Data request comments' and position()=1]" + Then I should see an element with xpath "//tr[@id='datarequest-comments']/td[contains(@class, 'metric-data') and string()='1' and position()=2]" diff --git a/test/features/environment.py b/test/features/environment.py index f94a3ccc..2dbac9b9 100644 --- a/test/features/environment.py +++ b/test/features/environment.py @@ -61,6 +61,16 @@ 'name': u'dr_member', 'email': u'dr_member@localhost', 'password': u'Password123!' + }, + 'ReportingOrgAdmin': { + 'name': u'report_admin', + 'email': u'report_admin@localhost', + 'password': u'Password123!' + }, + 'ReportingOrgEditor': { + 'name': u'report_editor', + 'email': u'report_editor@localhost', + 'password': u'Password123!' } } diff --git a/test/features/login_redirect.feature b/test/features/login_redirect.feature new file mode 100644 index 00000000..531761f3 --- /dev/null +++ b/test/features/login_redirect.feature @@ -0,0 +1,62 @@ +@login_redirect +Feature: Login Redirection + + @dashboard_login + Scenario: As an unauthenticated user, when I visit the /dashboard URL I see the login page + Given "Unauthenticated" as the persona + When I visit "/dashboard" + Then I should see an element with xpath "//h1[contains(string(), 'Login')]" + + @dashboard_login + Scenario: As an unauthenticated user, when I visit the /dashboard/ URL I see the login page + Given "Unauthenticated" as the persona + When I visit "/dashboard/" + Then I should see an element with xpath "//h1[contains(string(), 'Login')]" + + @user_edit + Scenario: As an unauthenticated organisation member, when I visit the /user/edit URL I see the login page. Upon logging in I am taken to the /user/edit page + Given "TestOrgMember" as the persona + When I visit "/user/edit" + Then I should see an element with xpath "//h1[contains(string(), 'Login')]" + When I log in directly + Then I should see "Change details" + + @dataset_setup + Scenario: As a Sysadmin I set the visibility of a public record to private for the following scenarios + Given "SysAdmin" as the persona + When I log in + Then I visit "/dataset/edit/annakarenina" + When I select "True" from "private" + And I fill in "author_email" with "test@example.com" + And I press "Update Dataset" + Then I should see an element with xpath "//span[contains(string(), 'Private')]" + + @private_dataset + Scenario: As an unauthenticated user, when I visit the URL of a private dataset I see the login page + Given "Unauthenticated" as the persona + When I visit "/dataset/annakarenina" + Then I should see an element with xpath "//h1[contains(string(), 'Login')]" + + @public_dataset + Scenario: As an unauthenticated user, when I visit the URL of a public dataset I see the dataset without needing to login + Given "Unauthenticated" as the persona + When I visit "/dataset/warandpeace" + Then I should see an element with xpath "//h1[contains(string(), 'A Wonderful Story')]" + And I should not see an element with xpath "//h1[contains(string(), 'Login')]" + + @private_dataset + Scenario: As an unauthenticated organisation member, when I visit the URL of a private dataset I see the login page. Upon logging in I am taken to the private dataset + Given "TestOrgMember" as the persona + When I visit "/dataset/annakarenina" + Then I should see an element with xpath "//h1[contains(string(), 'Login')]" + When I log in directly + Then I should see an element with xpath "//h1[contains(string(), 'A Novel By Tolstoy')]" + And I should see an element with xpath "//span[contains(string(), 'Private')]" + + @private_dataset + Scenario: As an authenticated organisation member, when I visit the URL of a dataset private to my organisation I am taken to the private dataset + Given "TestOrgMember" as the persona + When I log in + Then I visit "/dataset/annakarenina" + Then I should see an element with xpath "//h1[contains(string(), 'A Novel By Tolstoy')]" + And I should see an element with xpath "//span[contains(string(), 'Private')]" diff --git a/test/features/resource_availability.feature b/test/features/resource_availability.feature new file mode 100644 index 00000000..8978f349 --- /dev/null +++ b/test/features/resource_availability.feature @@ -0,0 +1,220 @@ +@resource_visibility +Feature: Re-identification risk governance acknowledgement or Resource visibility + + + Scenario: Sysadmin creates dataset with Contains de-identified data is YES + Given "SysAdmin" as the persona + When I log in + And I go to "/dataset/new" + Then I fill in "title" with "Contains de-identified data - YES" + Then I fill in "notes" with "notes" + Then I execute the script "document.getElementById('field-organizations').value=jQuery('#field-organizations option').filter(function () { return $(this).html() == 'Test Organisation'; }).attr('value')" + Then I select "False" from "private" + Then I fill in "version" with "1" + Then I fill in "author_email" with "test@test.com" + Then I select "YES" from "de_identified_data" + Then I press "save" + And I wait for 10 seconds + Then I execute the script "document.getElementById('field-image-url').value='https://example.com'" + Then I fill in "name" with "res1" + Then I fill in "description" with "description" + Then I fill in "size" with "1024" + Then I select "Resource NOT visible/Pending acknowledgement" from "resource_visibility" + Then I press the element with xpath "//button[@value='go-metadata']" + And I wait for 10 seconds + Then I should see "Data and Resources" + + + Scenario: Sysadmin creates dataset with Contains de-identified data is NO + Given "SysAdmin" as the persona + When I log in + And I go to "/dataset/new" + Then I fill in "title" with "Contains de-identified data - NO" + Then I fill in "notes" with "notes" + Then I execute the script "document.getElementById('field-organizations').value=jQuery('#field-organizations option').filter(function () { return $(this).html() == 'Test Organisation'; }).attr('value')" + Then I select "False" from "private" + Then I fill in "version" with "1" + Then I fill in "author_email" with "test@test.com" + Then I select "NO" from "de_identified_data" + Then I press "save" + And I wait for 10 seconds + Then I execute the script "document.getElementById('field-image-url').value='https://example.com'" + Then I fill in "name" with "res1" + Then I fill in "description" with "description" + Then I fill in "size" with "1024" + Then I select "Resource NOT visible/Pending acknowledgement" from "resource_visibility" + Then I press the element with xpath "//button[@value='go-metadata']" + And I wait for 10 seconds + Then I should see "Data and Resources" + + + Scenario Outline: User creates resource for a dataset with Contains de-identified data is YES + Given "" as the persona + When I log in + + ### + # Create resource that available for non-logged in user. + ### + And I go to "/dataset/new_resource/contains-de-identified-data-yes" + + # Check field visibility, xpath element start with 1. + Then I should see an element with id "field-resource_visibility" + Then I should see an element with xpath "//select[@id='field-resource_visibility']/option[2][@disabled]" + + # Create the resource, with error. + Then I execute the script "document.getElementById('field-image-url').value='https://example.com'" + Then I fill in "name" with "resource created by and is available" + Then I fill in "description" with "description" + Then I fill in "size" with "1024" + Then I press "save" + And I wait for 10 seconds + Then I should see "This dataset has been recorded as containing de-identified data." + + # Create the resource, with success. + Then I select "Appropriate steps have been taken to minimise personal information re-identification risk prior to publishing" from "resource_visibility" + Then I press "save" + And I wait for 10 seconds + Then I should see "resource created by and is available" + + ### + # Create resource that NOT available for non-logged in user. + ### + And I go to "/dataset/new_resource/contains-de-identified-data-yes" + + Then I execute the script "document.getElementById('field-image-url').value='https://example.com'" + Then I fill in "name" with "resource created by and is NOT available" + Then I fill in "description" with "description" + Then I fill in "size" with "1024" + Then I select "Resource NOT visible/Pending acknowledgement" from "resource_visibility" + Then I press "save" + And I wait for 10 seconds + Then I should see "resource created by and is NOT available" + + # Verify the result as non-logged in user. + Given "Unauthenticated" as the persona + And I go to "/dataset/contains-de-identified-data-yes" + Then I should see "resource created by and is available" + Then I should not see "resource created by and is NOT available" + + Examples: Users + | User | + | SysAdmin | + | TestOrgAdmin | + | TestOrgEditor | + + + Scenario Outline: User creates resource for a dataset with Contains de-identified data is NO + Given "" as the persona + When I log in + + ### + # Create resource that available for non-logged in user. + ### + And I go to "/dataset/new_resource/contains-de-identified-data-no" + + # Check field visibility, xpath element start with 1. + Then I should see an element with id "field-resource_visibility" + Then I should see an element with xpath "//select[@id='field-resource_visibility']/option[3][@disabled]" + + # Create the resource, with success. + Then I execute the script "document.getElementById('field-image-url').value='https://example.com'" + Then I fill in "name" with "resource created by and is available" + Then I fill in "description" with "description" + Then I fill in "size" with "1024" + Then I select "Resource visible and re-identification risk governance acknowledgement not required" from "resource_visibility" + Then I press "save" + And I wait for 10 seconds + Then I should see "resource created by and is available" + + ### + # Create resource that available for non-logged in user with resource_visibility blank. + ### + And I go to "/dataset/new_resource/contains-de-identified-data-no" + + # Create the resource, with success. + Then I execute the script "document.getElementById('field-image-url').value='https://example.com'" + Then I fill in "name" with "resource created by and is available with blank resource_visibility" + Then I fill in "description" with "description" + Then I fill in "size" with "1024" + Then I press "save" + And I wait for 10 seconds + Then I should see "resource created by and is available with blank resource_visibility" + + ### + # Create resource that NOT available for non-logged in user. + ### + And I go to "/dataset/new_resource/contains-de-identified-data-no" + + Then I execute the script "document.getElementById('field-image-url').value='https://example.com'" + Then I fill in "name" with "resource created by and is NOT available" + Then I fill in "description" with "description" + Then I fill in "size" with "1024" + Then I select "Resource NOT visible/Pending acknowledgement" from "resource_visibility" + Then I press "save" + And I wait for 10 seconds + Then I should see "resource created by and is NOT available" + + # Verify the result as non-logged in user. + Given "Unauthenticated" as the persona + And I go to "/dataset/contains-de-identified-data-no" + Then I should see "resource created by and is available" + Then I should see "resource created by and is available with blank resource_visibility" + Then I should not see "resource created by and is NOT available" + + Examples: Users + | User | + | SysAdmin | + | TestOrgAdmin | + | TestOrgEditor | + + + Scenario Outline: An editor, admin or sysadmin user views a dataset resource additional info + Given "" as the persona + When I log in + + And I go to "/dataset/contains-de-identified-data-yes" + Then I press the element with xpath "//a[@title='resource created by SysAdmin and is available']" + Then I should see "Re-identification risk governance acknowledgement/Resource visibility" + + Examples: Users + | User | + | SysAdmin | + | TestOrgAdmin | + | TestOrgEditor | + + + Scenario: A Member views a dataset resource additional info + Given "TestOrgMember" as the persona + When I log in + + And I go to "/dataset/contains-de-identified-data-yes" + Then I press the element with xpath "//a[@title='resource created by SysAdmin and is available']" + Then I should not see "Re-identification risk governance acknowledgement/Resource visibility" + + Scenario: A general user views a dataset resource additional info + Given "Unauthenticated" as the persona + + And I go to "/dataset/contains-de-identified-data-yes" + Then I press the element with xpath "//a[@title='resource created by SysAdmin and is available']" + Then I should not see "Re-identification risk governance acknowledgement/Resource visibility" + + + Scenario: Clean up + Given "SysAdmin" as the persona + When I log in + And I go to "/dataset/edit/contains-de-identified-data-yes" + Then I press the element with xpath "//a[@data-module='confirm-action']" + And I wait for 5 seconds + When I type "it should be longer than 10 character" to "deletion_reason" + Then I press the element with xpath "//div[@class='modal-footer']//button[@class='btn btn-primary']" + And I wait for 10 seconds + + And I go to "/dataset/edit/contains-de-identified-data-no" + Then I press the element with xpath "//a[@data-module='confirm-action']" + And I wait for 5 seconds + When I type "it should be longer than 10 character" to "deletion_reason" + Then I press the element with xpath "//div[@class='modal-footer']//button[@class='btn btn-primary']" + And I wait for 10 seconds + + When I go to "/ckan-admin/trash" + Then I press the element with xpath "//button[@name='purge-packages']" diff --git a/test/features/resource_freshness.feature b/test/features/resource_freshness.feature new file mode 100644 index 00000000..706d4b4e --- /dev/null +++ b/test/features/resource_freshness.feature @@ -0,0 +1,43 @@ +@resource_freshness +Feature: Resource freshness + + Scenario Outline: An editor, admin or sysadmin user, when I go to the dataset new page, the text 'Next update due' should not be visible + Given "" as the persona + When I log in + And I go to "/dataset/new" + Then I should not see "Next update due" + + Examples: Users + | User | + | SysAdmin | + | TestOrgAdmin | + | TestOrgEditor | + + + Scenario Outline: An editor, admin or sysadmin user, when I go to the dataset new page and select 'monthly' update frequency, then the text 'Next update due' should be visible + Given "" as the persona + When I log in + And I go to "/dataset/new" + And I select "monthly" from "update_frequency" + Then I should see "Next update due" + + Examples: Users + | User | + | SysAdmin | + | TestOrgAdmin | + | TestOrgEditor | + + + + Scenario Outline: An editor, admin or sysadmin user, when I go to the edit dataset page, the text 'Next update due' should be visible + Given "" as the persona + When I log in + And I go to "/dataset/edit/warandpeace" + And I select "monthly" from "update_frequency" + Then I should see "Next update due" + + Examples: Users + | User | + | SysAdmin | + | TestOrgAdmin | + | TestOrgEditor | diff --git a/test/features/schema_metadata.feature b/test/features/schema_metadata.feature new file mode 100644 index 00000000..97c7f7bc --- /dev/null +++ b/test/features/schema_metadata.feature @@ -0,0 +1,95 @@ +@schema_metadata +Feature: SchemaMetadata + + + Scenario: When a go to the dataset new page, the field field-author_email should not be visible + Given "SysAdmin" as the persona + When I log in + And I go to "/dataset/new" + Then I should see an element with id "field-author_email" + + Scenario: When a go to the dataset new page, the field field-maintainer_email should not be visible + Given "SysAdmin" as the persona + When I log in + And I go to "/dataset/new" + Then I should not see an element with id "field-maintainer_email" + + Scenario: When I create resource without a description, I should see the following error Description: Missing value + Given "SysAdmin" as the persona + When I log in + And I go to "/dataset/new_resource/warandpeace" + And I press the element with xpath "//button[contains(string(), 'Add')]" + Then I should see "Description: Missing value" + + Scenario: When I create resource without a name, I should see the following error Name: Missing value + Given "SysAdmin" as the persona + When I log in + When I visit "/dataset/new_resource/warandpeace" + And I press the element with xpath "//button[contains(string(), 'Add')]" + Then I should see "Name: Missing value" + + Scenario: When viewing the HTML source code of a dataset page, the structured data script is visible + Given "SysAdmin" as the persona + When I log in + When I go to "/dataset/warandpeace" + Then I should see an element with xpath "//link[@type='application/ld+json']" + + Scenario Outline: Check value of de_identified_data dropdown field + Given "" as the persona + When I log in + And I go to "/dataset/new" + Then I should see an element with id "field-de_identified_data" + Then I should see an element with xpath "//select[@id='field-de_identified_data']/option[@value='YES']" + Then I should see an element with xpath "//select[@id='field-de_identified_data']/option[@value='NO']" + Then I should see an element with xpath "//select[@id='field-de_identified_data']/option[@value='']" + Then I should not see an element with xpath "//select[@id='field-de_identified_data']/option[@selected='' and @value='YES']" + Then I should not see an element with xpath "//select[@id='field-de_identified_data']/option[@selected='' and @value='NO']" + Then I should not see an element with xpath "//select[@id='field-de_identified_data']/option[@selected='' and @value='']" + + Examples: Users + | User | + | SysAdmin | + | TestOrgAdmin | + | TestOrgEditor | + + + Scenario Outline: Edit existing dataset, field de_identified_data value should be NO + Given "" as the persona + When I log in + And I go to "/dataset/edit/warandpeace" + Then I should see an element with id "field-de_identified_data" + Then I should see an element with xpath "//select[@id='field-de_identified_data']/option[@value='YES']" + Then I should see an element with xpath "//select[@id='field-de_identified_data']/option[@selected='' and @value='NO']" + + Examples: Users + | User | + | SysAdmin | + | TestOrgAdmin | + | TestOrgEditor | + + Scenario Outline: When viewing existing dataset, field de_identified_data should be NO + Given "" as the persona + When I log in + + And I go to "/dataset/warandpeace" + Then I should see "Contains de-identified data" + Then I should see an element with xpath "//th[contains(text(), 'Contains de-identified data')]/following-sibling::td[contains(text(), 'NO')]" + + When I go to "/api/3/action/package_show?id=warandpeace" + Then I should see an element with xpath "//body/*[contains(text(), '"de_identified_data":')]" + + Examples: Users + | User | + | SysAdmin | + | TestOrgAdmin | + | TestOrgEditor | + + Scenario: Non logged-in user should not see de_identified_data value. + Given "Unauthenticated" as the persona + When I go to "/dataset/warandpeace" + Then I should not see "Contains de-identified data" + Then I should not see an element with xpath "//th[contains(text(), 'Contains de-identified data')]/following-sibling::td[contains(text(), 'NO')]" + + And I go to "/api/3/action/package_show?id=warandpeace" + Then I should not see an element with xpath "//body/*[contains(text(), '"de_identified_data":')]" + diff --git a/test/features/user_creation.feature b/test/features/user_creation.feature new file mode 100644 index 00000000..de822307 --- /dev/null +++ b/test/features/user_creation.feature @@ -0,0 +1,35 @@ +@user_creation +Feature: User creation + + Scenario: SysAdmin can create 'Excluded display name words' in ckan admin config + Given "SysAdmin" as the persona + When I log in + Then I go to "/ckan-admin/config" + Then I should see "Excluded display name words" + Then I fill in "ckanext.data_qld.excluded_display_name_words" with "gov" + Then I press "save" + + + Scenario: SysAdmin create a new user to the site. + Given "SysAdmin" as the persona + When I log in + When I go to "/user/register" + Then I should see "Displayed name" + Then I fill in "name" with "publisher_user" + Then I fill in "fullname" with "gov user" + Then I press "save" + And I wait for 10 seconds + Then I should not see "The username cannot contain the word 'publisher'. Please enter another username." + Then I should not see "The displayed name cannot contain certain words such as 'publisher', 'QLD Government' or similar. Please enter another display name." + + + Scenario: Non logged-in user register to the site. + Given "Unauthenticated" as the persona + When I go to "/user/register" + Then I should see "Displayed name" + Then I fill in "name" with "publisher_user" + Then I fill in "fullname" with "gov user" + Then I press "save" + And I wait for 10 seconds + Then I should see "The username cannot contain the word 'publisher'. Please enter another username." + Then I should see "The displayed name cannot contain certain words such as 'publisher', 'QLD Government' or similar. Please enter another display name." diff --git a/test/fixtures/csv_resource.csv b/test/fixtures/csv_resource.csv new file mode 100644 index 00000000..e69de29b diff --git a/test/fixtures/html_resource.html b/test/fixtures/html_resource.html new file mode 100644 index 00000000..79e991d0 --- /dev/null +++ b/test/fixtures/html_resource.html @@ -0,0 +1 @@ +Foo diff --git a/test/fixtures/rdf_resource.rdf b/test/fixtures/rdf_resource.rdf new file mode 100644 index 00000000..e69de29b diff --git a/test/fixtures/test-resource_schemea.json b/test/fixtures/test-resource_schemea.json new file mode 100644 index 00000000..bb870653 --- /dev/null +++ b/test/fixtures/test-resource_schemea.json @@ -0,0 +1,37 @@ +{ + "primaryKey": "id", + "fields": [ + { + "name": "id", + "title": "Measurement identifier", + "type": "integer" + }, + { + "name": "location", + "title": "Measurement location code", + "type": "string", + "constraints": { + "enum": ["A", "B", "C", "D"] + } + }, + { + "name": "date", + "title": "Measurement date", + "type": "date", + "format": "%d/%m/%Y" + }, + { + "name": "measurement", + "title": "Measure of the oblique fractal impedance at noon", + "type": "number", + "constraints": { + "required": true + } + }, + { + "name": "observations", + "title": "Extra observations", + "type": "string" + } + ] +} \ No newline at end of file diff --git a/test/fixtures/txt_resource.txt b/test/fixtures/txt_resource.txt new file mode 100644 index 00000000..e69de29b diff --git a/test/fixtures/xls_resource.xls b/test/fixtures/xls_resource.xls new file mode 100644 index 00000000..e69de29b From 2c0213e4de7efd67e573e1d042aef5860c3daaa0 Mon Sep 17 00:00:00 2001 From: ThrawnCA Date: Fri, 21 Jan 2022 10:09:02 +1000 Subject: [PATCH 14/31] [QOL-7811] sync more test config with infrastructure - add 'priority' and 'bulk' batch workers - update extensions to latest tag - add test config for resource formats --- .ahoy.yml | 32 ++++++++++++++++++++++++----- .docker/Dockerfile.ckan | 2 +- .docker/scripts/create-test-data.sh | 6 ++++++ .docker/scripts/serve.sh | 2 +- requirements-dev.txt | 13 ++++++------ 5 files changed, 42 insertions(+), 13 deletions(-) diff --git a/.ahoy.yml b/.ahoy.yml index a5889462..f8e3aaa7 100644 --- a/.ahoy.yml +++ b/.ahoy.yml @@ -118,13 +118,13 @@ commands: test-bdd: usage: Run BDD tests. cmd: | - ahoy start-ckan-job-worker & + ahoy start-ckan-job-workers ahoy start-mailmock & sleep 5 && ahoy cli "behave ${*:-test/features}" || \ [ "${ALLOW_BDD_FAIL:-0}" -eq 1 ] ahoy stop-mailmock - ahoy stop-ckan-job-worker + ahoy stop-ckan-job-workers start-mailmock: usage: Starts email mock server used for email BDD tests @@ -138,14 +138,36 @@ commands: ahoy title 'Stopping mailmock' ahoy cli "killall -2 mailmock" - start-ckan-job-worker: - usage: Starts CKAN background job worker + start-ckan-job-workers: + usage: Starts CKAN background job workers + cmd: | + ahoy title 'Starting CKAN background job workers' + ahoy start-ckan-job-worker-default & + ahoy start-ckan-job-worker-priority & + ahoy start-ckan-job-worker-bulk & + + start-ckan-job-worker-default: + usage: Starts default CKAN background job worker cmd: | ahoy title 'Starting default CKAN background job worker' ahoy cli "ckan_cli jobs clear && \ ckan_cli jobs worker" - stop-ckan-job-worker: + start-ckan-job-worker-priority: + usage: Starts CKAN background job worker for priority tasks + cmd: | + ahoy title 'Starting CKAN background job worker priority' + ahoy cli "ckan_cli jobs clear priority && \ + ckan_cli jobs worker priority" + + start-ckan-job-worker-bulk: + usage: Starts CKAN background job worker for low-priority tasks + cmd: | + ahoy title 'Starting CKAN background job worker bulk' + ahoy cli "ckan_cli jobs clear bulk && \ + ckan_cli jobs worker bulk" + + stop-ckan-job-workers: usage: Stops CKAN background job worker cmd: | ahoy title 'Stopping CKAN background job worker' diff --git a/.docker/Dockerfile.ckan b/.docker/Dockerfile.ckan index 50461819..9d3f89f1 100644 --- a/.docker/Dockerfile.ckan +++ b/.docker/Dockerfile.ckan @@ -1,4 +1,4 @@ -FROM amazeeio/python:2.7-ckan-21.7.0 +FROM amazeeio/python:2.7-ckan-21.8.0 ARG SITE_URL ENV SITE_URL="${SITE_URL}" diff --git a/.docker/scripts/create-test-data.sh b/.docker/scripts/create-test-data.sh index de935ac0..a258fc98 100644 --- a/.docker/scripts/create-test-data.sh +++ b/.docker/scripts/create-test-data.sh @@ -174,6 +174,12 @@ curl -LsH "Authorization: ${API_KEY}" \ ${CKAN_ACTION_URL}/create_datarequest +echo "Creating config value for resource formats:" + +curl -LsH "Authorization: ${API_KEY}" \ + --data '{"ckanext.data_qld.resource_formats": "CSV\r\nHTML\r\nJSON\r\nRDF\r\nTXT\r\nXLS"}' \ + ${CKAN_ACTION_URL}/config_option_update + if [ "$VENV_DIR" != "" ]; then deactivate fi diff --git a/.docker/scripts/serve.sh b/.docker/scripts/serve.sh index c1aae061..10f3dead 100755 --- a/.docker/scripts/serve.sh +++ b/.docker/scripts/serve.sh @@ -13,5 +13,5 @@ fi if (which ckan > /dev/null); then ckan -c ${CKAN_INI} run else - paster serve --reload ${CKAN_INI} + paster serve ${CKAN_INI} fi diff --git a/requirements-dev.txt b/requirements-dev.txt index 58e14d7f..f7e69d35 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -4,16 +4,17 @@ Appium-Python-Client<=0.52 flake8==3.8.3 nose==1.3.7 splinter>=0.13.0,<0.17 + git+https://github.com/ckan/ckanapi@ckanapi-4.3#egg=ckanapi git+https://github.com/ckan/ckantoolkit@release-0.0.4#egg=ckantoolkit -e git+https://github.com/ckan/ckanext-dcat.git#egg=ckanext-dcat -e git+https://github.com/ckan/ckanext-scheming.git@release-1.2.0#egg=ckanext-scheming -e git+https://github.com/frictionlessdata/ckanext-validation.git#egg=ckanext-validation --e git+https://github.com/qld-gov-au/ckan-ex-qgov.git@3.3.0#egg=ckanext-qgov --e git+https://github.com/qld-gov-au/ckanext-archiver.git@2.1.1-qgov.6#egg=ckanext-archiver --e git+https://github.com/qld-gov-au/ckanext-data-qld.git@4.0.0#egg=ckanext-data-qld --e git+https://github.com/qld-gov-au/ckanext-datarequests.git@2.1.3-qgov#egg=ckanext-datarequests --e git+https://github.com/qld-gov-au/ckanext-qa.git@2.0.3-qgov.3#egg=ckanext-qa +-e git+https://github.com/qld-gov-au/ckan-ex-qgov.git@3.3.1#egg=ckanext-qgov +-e git+https://github.com/qld-gov-au/ckanext-archiver.git@2.1.1-qgov.7#egg=ckanext-archiver +-e git+https://github.com/qld-gov-au/ckanext-data-qld.git@4.0.1#egg=ckanext-data-qld +-e git+https://github.com/qld-gov-au/ckanext-datarequests.git@2.2.0-qgov#egg=ckanext-datarequests +-e git+https://github.com/qld-gov-au/ckanext-qa.git@2.0.3-qgov.4#egg=ckanext-qa -e git+https://github.com/qld-gov-au/ckanext-report.git@0.2#egg=ckanext-report --e git+https://github.com/qld-gov-au/ckanext-ytp-comments.git@2.4.0-qgov#egg=ckanext-ytp-comments +-e git+https://github.com/qld-gov-au/ckanext-ytp-comments.git@2.5.0-qgov#egg=ckanext-ytp-comments From a2d59cb2f777675e6a138775f85bed511a43e949 Mon Sep 17 00:00:00 2001 From: ThrawnCA Date: Fri, 21 Jan 2022 10:29:33 +1000 Subject: [PATCH 15/31] [QOL-7811] fix path to resource openness score config --- .docker/test.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.docker/test.ini b/.docker/test.ini index 61c0fff9..3797e545 100644 --- a/.docker/test.ini +++ b/.docker/test.ini @@ -227,7 +227,7 @@ ckanext-archiver.archive_dir = /app/filestore/archive ckanext-archiver.cache_url_root = http://dataqld-ckan.docker.amazee.io/resources/ # QA -qa.resource_format_openness_scores_json = /app/ckanext/data_qld/resource_format_openness_scores.json +qa.resource_format_openness_scores_json = /app/ckan/default/src/ckanext-data-qld/ckanext/data_qld/resource_format_openness_scores.json ## Logging configuration [loggers] From 3161a1a9dc7690d4e86c15ad5f42be384f9c4cb5 Mon Sep 17 00:00:00 2001 From: ThrawnCA Date: Fri, 21 Jan 2022 11:02:00 +1000 Subject: [PATCH 16/31] [QOL-7811] oops drop tests that were specific to Publications --- test/features/dataset_deletion.feature | 31 -------------------------- 1 file changed, 31 deletions(-) diff --git a/test/features/dataset_deletion.feature b/test/features/dataset_deletion.feature index edf8b9d8..35c44dc3 100644 --- a/test/features/dataset_deletion.feature +++ b/test/features/dataset_deletion.feature @@ -23,25 +23,6 @@ Feature: Dataset deletion And I wait for 10 seconds Then I should see "Data and Resources" - Scenario: Sysadmin creates a dataset - Given "SysAdmin" as the persona - When I log in - And I go to "/dataset/new" - Then I fill in "title" with "Dataset deletion" - Then I fill in "notes" with "notes" - Then I execute the script "document.getElementById('field-organizations').value=jQuery('#field-organizations option').filter(function () { return $(this).html() == 'Test Organisation'; }).attr('value')" - Then I select "False" from "private" - Then I fill in "version" with "1" - Then I fill in "author_email" with "test@test.com" - Then I press "save" - And I wait for 10 seconds - Then I execute the script "document.getElementById('field-image-url').value='https://example.com'" - Then I fill in "name" with "res1" - Then I fill in "description" with "description" - Then I press the element with xpath "//button[@value='go-metadata']" - And I wait for 10 seconds - Then I should see "Data and Resources" - Scenario: Sysadmin deletes a dataset Given "SysAdmin" as the persona When I log in @@ -61,15 +42,3 @@ Feature: Dataset deletion When I go to "/ckan-admin/trash" Then I should see "Dataset deletion" Then I press the element with xpath "//button[@name='purge-packages']" - - Scenario: Sysadmin deletes a dataset - Given "SysAdmin" as the persona - When I log in - And I go to "/dataset/edit/dataset-deletion" - Then I should see an element with xpath "//a[@data-module='confirm-action']" - Then I press the element with xpath "//a[@data-module='confirm-action']" - And I wait for 5 seconds - Then I should not see "Dataset deletion" - And I go to "/ckan-admin/trash" - Then I should see "Purge deleted datasets forever and irreversibly" - Then I press the element with xpath "//button[@name='purge-packages']" From 3cdde9ce474c47f6c79dd276aad2c312c378f72c Mon Sep 17 00:00:00 2001 From: ThrawnCA Date: Fri, 21 Jan 2022 11:05:44 +1000 Subject: [PATCH 17/31] [QOL-7811] move test dataset into test org to match test expectations --- .docker/scripts/create-test-data.sh | 4 ++-- test/features/google_analytics.feature | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.docker/scripts/create-test-data.sh b/.docker/scripts/create-test-data.sh index a258fc98..d0197858 100644 --- a/.docker/scripts/create-test-data.sh +++ b/.docker/scripts/create-test-data.sh @@ -71,10 +71,10 @@ ckan_cli create-test-data echo "Assigning test Datasets to Organisation..." -echo "Updating annakarenina to use 'Department of Health' organisation:" +echo "Updating annakarenina to use ${TEST_ORG_TITLE} organisation:" package_owner_org_update=$( \ curl -LsH "Authorization: ${API_KEY}" \ - --data "id=annakarenina&organization_id=department-of-health" \ + --data "id=annakarenina&organization_id=${TEST_ORG_NAME}" \ ${CKAN_ACTION_URL}/package_owner_org_update ) echo ${package_owner_org_update} diff --git a/test/features/google_analytics.feature b/test/features/google_analytics.feature index 12351611..303ddf37 100644 --- a/test/features/google_analytics.feature +++ b/test/features/google_analytics.feature @@ -14,4 +14,4 @@ Feature: GoogleAnalytics When I go to Dataset page And I click the link with text that contains "A Novel By Tolstoy" And I click the link with text that contains "Full text" - Then I should see an element with xpath "//meta[@name='DCTERMS.creator' and @content='c=AU; o=The State of Queensland; ou=Department of Health' and @scheme='AGLSTERMS.GOLD']" \ No newline at end of file + Then I should see an element with xpath "//meta[@name='DCTERMS.creator' and @content='c=AU; o=The State of Queensland; ou=Test Organisation' and @scheme='AGLSTERMS.GOLD']" From 5c2c31ba482ba27f4cc254bb9bb87209316b9c70 Mon Sep 17 00:00:00 2001 From: ThrawnCA Date: Fri, 21 Jan 2022 11:50:44 +1000 Subject: [PATCH 18/31] [QOL-7811] extend test timeout to match infrastructure --- .github/workflows/test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 946f290c..faaff3b3 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -16,7 +16,7 @@ jobs: steps: - uses: actions/checkout@v2 - timeout-minutes: 5 + timeout-minutes: 2 - name: Build run: .circleci/build.sh @@ -24,7 +24,7 @@ jobs: - name: Test run: .circleci/test.sh - timeout-minutes: 20 + timeout-minutes: 25 - name: Retrieve screenshots if: failure() From bb2f59f8df76dc908f1118033efe3e6a6bf1ea30 Mon Sep 17 00:00:00 2001 From: ThrawnCA Date: Fri, 21 Jan 2022 14:37:36 +1000 Subject: [PATCH 19/31] [QOL-7811] sync test config with Publications --- .circleci/build.sh | 2 +- .docker/Dockerfile.ckan | 11 ++++-- .docker/scripts/ckan_cli | 11 +++++- .docker/scripts/create-test-data.sh | 26 ++++++++++++- .docker/scripts/init.sh | 14 +------ .docker/test.ini | 3 ++ .github/workflows/test.yml | 16 +++++--- README.md | 3 +- docker-compose.yml | 2 + test/features/config.feature | 16 +++++--- test/features/dataset_deletion.feature | 7 ++-- test/features/steps/steps.py | 52 ++++++++++++++++---------- 12 files changed, 107 insertions(+), 56 deletions(-) diff --git a/.circleci/build.sh b/.circleci/build.sh index 611dca97..407dd0ec 100755 --- a/.circleci/build.sh +++ b/.circleci/build.sh @@ -21,4 +21,4 @@ export DOCTOR_CHECK_SSH=0 export DOCTOR_CHECK_WEBSERVER=0 export DOCTOR_CHECK_BOOTSTRAP=0 -ahoy build +ahoy build || (ahoy logs; exit 1) diff --git a/.docker/Dockerfile.ckan b/.docker/Dockerfile.ckan index 9d3f89f1..3a5995e0 100644 --- a/.docker/Dockerfile.ckan +++ b/.docker/Dockerfile.ckan @@ -1,6 +1,8 @@ FROM amazeeio/python:2.7-ckan-21.8.0 -ARG SITE_URL +ARG SITE_URL=http://ckan:3000/ +ARG CKAN_REPO=qld-gov-au/ckan +ARG CKAN_VERSION=2.8.8-qgov.5 ENV SITE_URL="${SITE_URL}" ENV VENV_DIR=/app/ckan/default ENV APP_DIR=/app @@ -15,13 +17,14 @@ RUN apk add --no-cache curl build-base \ && rm dockerize-alpine-linux-amd64-${DOCKERIZE_VERSION}.tar.gz # Install CKAN. -ENV CKAN_VERSION 2.8.8-qgov.5 RUN . ${VENV_DIR}/bin/activate \ && pip install setuptools==36.1 \ - && pip install -e "git+https://github.com/qld-gov-au/ckan.git@ckan-${CKAN_VERSION}#egg=ckan" \ + && pip install -e "git+https://github.com/${CKAN_REPO}.git@ckan-${CKAN_VERSION}#egg=ckan" \ && sed -i "s/psycopg2==2.4.5/psycopg2==2.7.7/g" "${VENV_DIR}/src/ckan/requirements.txt" \ - && pip install -r "${VENV_DIR}/src/ckan/requirements.txt" \ + && ((test -f "${VENV_DIR}/src/ckan/requirements-py2.txt" && \ + pip install -r "${VENV_DIR}/src/ckan/requirements-py2.txt") || \ + pip install -r "${VENV_DIR}/src/ckan/requirements.txt") \ && ln -s "${VENV_DIR}/src/ckan/who.ini" "${VENV_DIR}/who.ini" \ && deactivate \ && ln -s ${APP_DIR}/ckan /usr/lib/ckan \ diff --git a/.docker/scripts/ckan_cli b/.docker/scripts/ckan_cli index fcc045fd..3cf0b4cc 100644 --- a/.docker/scripts/ckan_cli +++ b/.docker/scripts/ckan_cli @@ -58,10 +58,17 @@ fi if [ "$COMMAND" = "ckan" ]; then echo "Using 'ckan' command from $ENV_DIR with config ${CKAN_INI}..." >&2 - exec $ENV_DIR/ckan -c ${CKAN_INI} "$@" + # adjust args to match ckan expectations + COMMAND=$(echo "$1" | sed -e 's/create-test-data/seed/') + shift + exec $ENV_DIR/ckan -c ${CKAN_INI} $COMMAND "$@" $CLICK_ARGS elif [ "$COMMAND" = "paster" ]; then echo "Using 'paster' command from $ENV_DIR with config ${CKAN_INI}..." >&2 - exec $ENV_DIR/paster --plugin=$PASTER_PLUGIN "$@" -c ${CKAN_INI} + # adjust args to match paster expectations + COMMAND=$1 + shift + if [ "$1" = "show" ]; then shift; fi + exec $ENV_DIR/paster --plugin=$PASTER_PLUGIN $COMMAND "$@" -c ${CKAN_INI} else echo "Unable to locate 'ckan' or 'paster' command in $ENV_DIR" >&2 exit 1 diff --git a/.docker/scripts/create-test-data.sh b/.docker/scripts/create-test-data.sh index d0197858..02c8215e 100644 --- a/.docker/scripts/create-test-data.sh +++ b/.docker/scripts/create-test-data.sh @@ -10,16 +10,27 @@ if [ "$VENV_DIR" != "" ]; then . ${VENV_DIR}/bin/activate fi +CKAN_USER_NAME="${CKAN_USER_NAME:-admin}" +CKAN_DISPLAY_NAME="${CKAN_DISPLAY_NAME:-Administrator}" +CKAN_USER_EMAIL="${CKAN_USER_EMAIL:-admin@localhost}" + add_user_if_needed () { echo "Adding user '$2' ($1) with email address [$3]" - ckan_cli user "$1" | grep "$1" || ckan_cli user add "$1"\ + ckan_cli user show "$1" | grep "$1" || ckan_cli user add "$1"\ fullname="$2"\ email="$3"\ password="${4:-Password123!}" } +add_user_if_needed "$CKAN_USER_NAME" "$CKAN_DISPLAY_NAME" "$CKAN_USER_EMAIL" +ckan_cli sysadmin add "${CKAN_USER_NAME}" + # We know the "admin" sysadmin account exists, so we'll use her API KEY to create further data -API_KEY=$(ckan_cli user admin | tr -d '\n' | sed -r 's/^(.*)apikey=(\S*)(.*)/\2/') +API_KEY=$(ckan_cli user show "${CKAN_USER_NAME}" | tr -d '\n' | sed -r 's/^(.*)apikey=(\S*)(.*)/\2/') +if [ "$API_KEY" = "None" ]; then + echo "No API Key found on ${CKAN_USER_NAME}, generating API Token..." + API_KEY=$(ckan_cli user token add "${CKAN_USER_NAME}" test_setup |grep -v '^API Token created' | tr -d '[:space:]') +fi ## # BEGIN: Create a test organisation with test users for admin, editor and member @@ -131,6 +142,14 @@ curl -LsH "Authorization: ${API_KEY}" \ --data "title=Test Request&description=This is an example&organization_id=${TEST_ORG_ID}" \ ${CKAN_ACTION_URL}/create_datarequest +## +# END. +# + +## +# BEGIN: Create a Reporting organisation with test users +# + REPORT_ORG_NAME=reporting REPORT_ORG_TITLE=Reporting @@ -173,6 +192,9 @@ curl -LsH "Authorization: ${API_KEY}" \ --data "title=Reporting Request&description=Data Request for reporting&organization_id=${REPORT_ORG_ID}" \ ${CKAN_ACTION_URL}/create_datarequest +## +# END. +# echo "Creating config value for resource formats:" diff --git a/.docker/scripts/init.sh b/.docker/scripts/init.sh index eb7839fa..a380599f 100755 --- a/.docker/scripts/init.sh +++ b/.docker/scripts/init.sh @@ -4,16 +4,12 @@ # set -e -CKAN_USER_NAME="${CKAN_USER_NAME:-admin}" -CKAN_DISPLAY_NAME="${CKAN_DISPLAY_NAME:-Administrator}" -CKAN_USER_PASSWORD="${CKAN_USER_PASSWORD:-Password123!}" -CKAN_USER_EMAIL="${CKAN_USER_EMAIL:-admin@localhost}" - if [ "$VENV_DIR" != "" ]; then . ${VENV_DIR}/bin/activate fi -ckan_cli db clean +CLICK_ARGS="--yes" ckan_cli db clean ckan_cli db init +ckan_cli db upgrade # Initialise validation tables PASTER_PLUGIN=ckanext-validation ckan_cli validation init-db @@ -32,11 +28,5 @@ PASTER_PLUGIN=ckanext-report ckan_cli report initdb # Initialise the QA database tables PASTER_PLUGIN=ckanext-qa ckan_cli qa init -ckan_cli user add "${CKAN_USER_NAME}"\ - fullname="${CKAN_DISPLAY_NAME}"\ - email="${CKAN_USER_EMAIL}"\ - password="${CKAN_USER_PASSWORD}" -ckan_cli sysadmin add "${CKAN_USER_NAME}" - # Create some base test data . $APP_DIR/scripts/create-test-data.sh diff --git a/.docker/test.ini b/.docker/test.ini index 3797e545..a6c38181 100644 --- a/.docker/test.ini +++ b/.docker/test.ini @@ -22,6 +22,9 @@ host = 0.0.0.0 port = 3000 [app:main] +ckan.devserver.host = 0.0.0.0 +ckan.devserver.port = 3000 + use = egg:ckan full_stack = true cache_dir = /tmp/%(ckan.site_id)s/ diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index faaff3b3..7a927b87 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,18 +1,24 @@ --- -#based on https://raw.githubusercontent.com/ckan/ckanext-scheming/master/.github/workflows/test.yml -# alternative https://github.com/ckan/ckan/blob/master/contrib/cookiecutter/ckan_extension/%7B%7Bcookiecutter.project%7D%7D/.github/workflows/test.yml name: Tests on: push: + pull_request: + branches: + - master jobs: test: strategy: fail-fast: false + matrix: + ckan-version: [2.8.8-qgov.5] - name: Continuous Integration build + name: Continuous Integration build on CKAN ${{ matrix.ckan-version }} runs-on: ubuntu-latest container: integratedexperts/ci-builder + env: + CKAN_REPO: qld-gov-au/ckan + CKAN_VERSION: ${{ matrix.ckan-version }} steps: - uses: actions/checkout@v2 @@ -35,6 +41,6 @@ jobs: if: failure() uses: actions/upload-artifact@v2 with: - name: screenshots + name: CKAN ${{ matrix.ckan-version }} screenshots path: /tmp/artifacts/behave/screenshots - timeout-minutes: 3 + timeout-minutes: 1 diff --git a/README.md b/README.md index 0eb3ae7e..caf4c04f 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,4 @@ -ckanext-data-qld-theme -====================== +# ckanext-data-qld-theme A custom CKAN extension for Data.Qld [![CircleCI](https://circleci.com/gh/qld-gov-au/ckanext-data-qld-theme/tree/develop.svg?style=shield)](https://circleci.com/gh/qld-gov-au/ckanext-data-qld-theme/tree/develop) diff --git a/docker-compose.yml b/docker-compose.yml index eef07d0e..4871988a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -24,6 +24,8 @@ services: dockerfile: .docker/Dockerfile.ckan args: SITE_URL: "http://${PROJECT}.docker.amazee.io" + CKAN_REPO: + CKAN_VERSION: depends_on: - postgres - solr diff --git a/test/features/config.feature b/test/features/config.feature index b98d361a..53711996 100644 --- a/test/features/config.feature +++ b/test/features/config.feature @@ -1,9 +1,15 @@ @config Feature: Config - Scenario: Assert that configuration values are available + Scenario: Assert that CSS configuration values are removed Given "SysAdmin" as the persona - When I log in - And I visit "ckan-admin/config" - And I should see "Suggested Description" - Then I should see an element with id "field-ckanext.data_qld.datarequest_suggested_description" + When I log in and go to admin config page + Then I should see "Intro Text" + And I should not see an element with id "field-ckan-main-css" + And I should not see an element with id "field-ckan-site-custom-css" + + Scenario: Assert that Data Request configuration values are available + Given "SysAdmin" as the persona + When I log in and go to admin config page + Then I should see "Suggested Description" + And I should see an element with id "field-ckanext.data_qld.datarequest_suggested_description" diff --git a/test/features/dataset_deletion.feature b/test/features/dataset_deletion.feature index 35c44dc3..71cf587b 100644 --- a/test/features/dataset_deletion.feature +++ b/test/features/dataset_deletion.feature @@ -31,12 +31,11 @@ Feature: Dataset deletion Then I press the element with xpath "//a[@data-module='confirm-action']" And I wait for 5 seconds Then I should see "Briefly describe the reason for deleting this dataset" - Then I should see an element with xpath "//div[@class='modal-footer']//button[@class='btn btn-primary' and @disabled='disabled']" - When I type "it should be longer than 10 character" to "deletion_reason" + And I should see an element with xpath "//div[@class='modal-footer']//button[@class='btn btn-primary' and @disabled='disabled']" + When I type "it should be longer than 10 characters" to "deletion_reason" Then I should not see an element with xpath "//div[@class='modal-footer']//button[@class='btn btn-primary' and @disabled='disabled']" - Then I take a screenshot Then I press the element with xpath "//div[@class='modal-footer']//button[@class='btn btn-primary']" - And I wait for 10 seconds + And I wait for 5 seconds Then I should see "Dataset has been deleted" And I should not see "Dataset deletion" When I go to "/ckan-admin/trash" diff --git a/test/features/steps/steps.py b/test/features/steps/steps.py index 262e92d9..64297f30 100644 --- a/test/features/steps/steps.py +++ b/test/features/steps/steps.py @@ -17,6 +17,11 @@ def go_to_home(context): when_i_visit_url(context, '/') +@step('I go to register page') +def go_to_register_page(context): + when_i_visit_url(context, '/user/register') + + @step('I log in') def log_in(context): assert context.persona @@ -53,14 +58,24 @@ def title_random_text(context): """.format(random.randrange(1000))) +@step('I go to dataset page') +def go_to_dataset_page(context): + when_i_visit_url(context, '/dataset') + + +@step(u'I go to dataset "{name}"') +def go_to_dataset(context, name): + when_i_visit_url(context, '/dataset/' + name) + + @step('I go to organisation page') def go_to_organisation_page(context): when_i_visit_url(context, '/organization') -@step('I go to register page') -def go_to_register_page(context): - when_i_visit_url(context, '/user/register') +@step(u'I set persona var "{key}" to "{value}"') +def set_persona_var(context, key, value): + context.persona[key] = value @step('I log in and go to the data requests page') @@ -105,16 +120,6 @@ def go_to_reporting_page(context): when_i_visit_url(context, '/dashboard/reporting') -@step('I go to dataset page') -def go_to_dataset_page(context): - when_i_visit_url(context, '/dataset') - - -@step(u'I go to dataset "{name}"') -def go_to_dataset(context, name): - when_i_visit_url(context, '/dataset/' + name) - - @step(u'I go to dataset "{name}" comments') def go_to_dataset_comments(context, name): context.execute_steps(u""" @@ -140,11 +145,6 @@ def go_to_data_request_comments(context, subject): """ % (subject)) -@step(u'I set persona var "{key}" to "{value}"') -def set_persona_var(context, key, value): - context.persona[key] = value - - @step(u'I submit a comment with subject "{subject}" and comment "{comment}"') def submit_comment_with_subject_and_comment(context, subject, comment): """ @@ -204,7 +204,7 @@ def create_dataset(context, license, file_format, file): # The default behaving step does not convert base64 emails -# Modifed the default step to decode the payload from base64 +# Modified the default step to decode the payload from base64 @step(u'I should receive a base64 email at "{address}" containing "{text}"') def should_receive_base64_email_containing_text(context, address, text): def filter_contents(mail): @@ -219,3 +219,17 @@ def filter_contents(mail): return text in decoded_payload assert context.mail.user_messages(address, filter_contents) + + +@step('I log in and go to admin config page') +def log_in_go_to_admin_config(context): + assert context.persona + context.execute_steps(u""" + When I log in + And I go to admin config page + """) + + +@step('I go to admin config page') +def go_to_admin_config(context): + when_i_visit_url(context, '/ckan-admin/config') From 7b1bf39fc87a51a1816f8375ee27e4913aaf3ffa Mon Sep 17 00:00:00 2001 From: ThrawnCA Date: Wed, 9 Feb 2022 16:36:41 +1000 Subject: [PATCH 20/31] [QOL-7811] drop controller reference and put metadata in specific templates instead --- ckanext/data_qld_theme/templates/base.html | 9 --------- ckanext/data_qld_theme/templates/package/read.html | 8 ++++++++ .../data_qld_theme/templates/package/resource_read.html | 7 +++++++ 3 files changed, 15 insertions(+), 9 deletions(-) create mode 100644 ckanext/data_qld_theme/templates/package/read.html diff --git a/ckanext/data_qld_theme/templates/base.html b/ckanext/data_qld_theme/templates/base.html index b35615aa..4f9f915e 100644 --- a/ckanext/data_qld_theme/templates/base.html +++ b/ckanext/data_qld_theme/templates/base.html @@ -1,14 +1,5 @@ {% ckan_extends %} - {% block meta %} - {{ super() }} - {% if c.controller in ['dataset', 'package', 'ckanext.data_qld.controller:DataQldDataset'] %} - {% if c.action in ['read', 'resource_read'] and c.pkg_dict['organization'] and c.pkg_dict['organization']['title'] %} - - {% endif %} - {% endif %} - {% endblock %} - {% block styles %} {{ super() }} diff --git a/ckanext/data_qld_theme/templates/package/read.html b/ckanext/data_qld_theme/templates/package/read.html new file mode 100644 index 00000000..2c81c11d --- /dev/null +++ b/ckanext/data_qld_theme/templates/package/read.html @@ -0,0 +1,8 @@ +{% ckan_extends %} + +{% block meta %} +{{ super() }} +{% if c.pkg_dict['organization'] and c.pkg_dict['organization']['title'] %} + +{% endif %} +{% endblock %} diff --git a/ckanext/data_qld_theme/templates/package/resource_read.html b/ckanext/data_qld_theme/templates/package/resource_read.html index b2f94aa2..7a861926 100644 --- a/ckanext/data_qld_theme/templates/package/resource_read.html +++ b/ckanext/data_qld_theme/templates/package/resource_read.html @@ -1,5 +1,12 @@ {% ckan_extends %} +{% block meta %} +{{ super() }} +{% if c.pkg_dict['organization'] and c.pkg_dict['organization']['title'] %} + +{% endif %} +{% endblock %} + {% block subtitle %}{{ h.resource_display_name(res) }} - {{ h.dataset_display_name(c.package) }}{% endblock %} {% block breadcrumb_content %} From cf87e37555affdc01ced9624e70e7638ea5bc288 Mon Sep 17 00:00:00 2001 From: ThrawnCA Date: Wed, 9 Feb 2022 16:36:41 +1000 Subject: [PATCH 21/31] [QOL-7811] drop controller reference and put metadata in specific templates instead --- ckanext/data_qld_theme/templates/base.html | 9 --------- ckanext/data_qld_theme/templates/package/read.html | 8 ++++++++ .../data_qld_theme/templates/package/resource_read.html | 7 +++++++ 3 files changed, 15 insertions(+), 9 deletions(-) create mode 100644 ckanext/data_qld_theme/templates/package/read.html diff --git a/ckanext/data_qld_theme/templates/base.html b/ckanext/data_qld_theme/templates/base.html index b35615aa..4f9f915e 100644 --- a/ckanext/data_qld_theme/templates/base.html +++ b/ckanext/data_qld_theme/templates/base.html @@ -1,14 +1,5 @@ {% ckan_extends %} - {% block meta %} - {{ super() }} - {% if c.controller in ['dataset', 'package', 'ckanext.data_qld.controller:DataQldDataset'] %} - {% if c.action in ['read', 'resource_read'] and c.pkg_dict['organization'] and c.pkg_dict['organization']['title'] %} - - {% endif %} - {% endif %} - {% endblock %} - {% block styles %} {{ super() }} diff --git a/ckanext/data_qld_theme/templates/package/read.html b/ckanext/data_qld_theme/templates/package/read.html new file mode 100644 index 00000000..2c81c11d --- /dev/null +++ b/ckanext/data_qld_theme/templates/package/read.html @@ -0,0 +1,8 @@ +{% ckan_extends %} + +{% block meta %} +{{ super() }} +{% if c.pkg_dict['organization'] and c.pkg_dict['organization']['title'] %} + +{% endif %} +{% endblock %} diff --git a/ckanext/data_qld_theme/templates/package/resource_read.html b/ckanext/data_qld_theme/templates/package/resource_read.html index b2f94aa2..7a861926 100644 --- a/ckanext/data_qld_theme/templates/package/resource_read.html +++ b/ckanext/data_qld_theme/templates/package/resource_read.html @@ -1,5 +1,12 @@ {% ckan_extends %} +{% block meta %} +{{ super() }} +{% if c.pkg_dict['organization'] and c.pkg_dict['organization']['title'] %} + +{% endif %} +{% endblock %} + {% block subtitle %}{{ h.resource_display_name(res) }} - {{ h.dataset_display_name(c.package) }}{% endblock %} {% block breadcrumb_content %} From efafb31bde73e12fba8f6787eac75596527956aa Mon Sep 17 00:00:00 2001 From: ThrawnCA Date: Wed, 9 Feb 2022 16:51:56 +1000 Subject: [PATCH 22/31] [QOL-8595] ensure 'six' is new enough for our dependencies - see https://github.com/jazzband/pathlib2/issues/81 --- requirements-dev.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements-dev.txt b/requirements-dev.txt index f7e69d35..52a1808a 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -4,6 +4,7 @@ Appium-Python-Client<=0.52 flake8==3.8.3 nose==1.3.7 splinter>=0.13.0,<0.17 +six>=1.13.0 git+https://github.com/ckan/ckanapi@ckanapi-4.3#egg=ckanapi git+https://github.com/ckan/ckantoolkit@release-0.0.4#egg=ckantoolkit From 545e4fc2c011d901e533d4af55232d341c19ea25 Mon Sep 17 00:00:00 2001 From: ThrawnCA Date: Thu, 10 Feb 2022 08:43:56 +1000 Subject: [PATCH 23/31] [QOL-8595] make template better match core so we can move to CKAN 2.9 more easily --- .../templates/package/read.html | 4 ++-- .../templates/package/resource_read.html | 24 ++++--------------- 2 files changed, 7 insertions(+), 21 deletions(-) diff --git a/ckanext/data_qld_theme/templates/package/read.html b/ckanext/data_qld_theme/templates/package/read.html index 2c81c11d..79bc475a 100644 --- a/ckanext/data_qld_theme/templates/package/read.html +++ b/ckanext/data_qld_theme/templates/package/read.html @@ -2,7 +2,7 @@ {% block meta %} {{ super() }} -{% if c.pkg_dict['organization'] and c.pkg_dict['organization']['title'] %} - +{% if pkg['organization'] and pkg['organization']['title'] %} + {% endif %} {% endblock %} diff --git a/ckanext/data_qld_theme/templates/package/resource_read.html b/ckanext/data_qld_theme/templates/package/resource_read.html index 7a861926..613171f3 100644 --- a/ckanext/data_qld_theme/templates/package/resource_read.html +++ b/ckanext/data_qld_theme/templates/package/resource_read.html @@ -1,32 +1,18 @@ {% ckan_extends %} +{% set pkg = c.pkg_dict or pkg_dict %} + {% block meta %} {{ super() }} -{% if c.pkg_dict['organization'] and c.pkg_dict['organization']['title'] %} - +{% if pkg['organization'] and pkg['organization']['title'] %} + {% endif %} {% endblock %} {% block subtitle %}{{ h.resource_display_name(res) }} - {{ h.dataset_display_name(c.package) }}{% endblock %} {% block breadcrumb_content %} - -{% if pkg %} - {% set dataset = h.dataset_display_name(pkg) %} - {% if pkg.organization %} - {% set organization = h.get_translated(pkg.organization, 'title') or pkg.organization.name %} - {% set group_type = pkg.organization.type %} -
        • {% link_for _('Organizations'), named_route='organization.index', named_route=group_type + '_index' %}
          • -
        • {% link_for organization|truncate(30), named_route='organization_read', id=pkg.organization.name, named_route=group_type + '_read' %}
          • - {% else %} -
        • {% link_for _('Datasets'), named_route='package.search' %}
          • - {% endif %} - {% link_for dataset|truncate(30), named_route='dataset_read', id=pkg.name %} - {% else %} -
        • {% link_for _('Datasets'), named_route='package.search' %}
          • -
        • {{ _('Create Dataset') }}
          • -{% endif %} - +{{ super() }}
        • {{ h.resource_display_name(res) }}
          • {% endblock %} From 475ff11523dc96b98ab115e331a228bfaf6f3ae3 Mon Sep 17 00:00:00 2001 From: ThrawnCA Date: Thu, 10 Feb 2022 10:41:02 +1000 Subject: [PATCH 24/31] [QOL-8595] sync tests with ckanext-data-qld - also update ckanext-qgov in tests, to ensure backwards compatibility --- .ahoy.yml | 1 + .docker/scripts/create-test-data.sh | 2 +- .docker/test.ini | 1 - .flake8 | 2 +- .github/workflows/test.yml | 2 +- docker-compose.yml | 2 + requirements-dev.txt | 4 +- test/features/config.feature | 2 + test/features/dataset_deletion.feature | 1 - test/features/login_redirect.feature | 27 ++++++------ test/features/steps/steps.py | 60 +++++++++++++++----------- test/features/user_creation.feature | 2 +- 12 files changed, 61 insertions(+), 45 deletions(-) diff --git a/.ahoy.yml b/.ahoy.yml index f8e3aaa7..169c0d6d 100644 --- a/.ahoy.yml +++ b/.ahoy.yml @@ -121,6 +121,7 @@ commands: ahoy start-ckan-job-workers ahoy start-mailmock & sleep 5 && + ahoy cli "behave ${*:-test/features}" --tags @smoke && \ ahoy cli "behave ${*:-test/features}" || \ [ "${ALLOW_BDD_FAIL:-0}" -eq 1 ] ahoy stop-mailmock diff --git a/.docker/scripts/create-test-data.sh b/.docker/scripts/create-test-data.sh index d0197858..1821213b 100644 --- a/.docker/scripts/create-test-data.sh +++ b/.docker/scripts/create-test-data.sh @@ -65,7 +65,7 @@ curl -LsH "Authorization: ${API_KEY}" \ ckan_cli create-test-data hierarchy # Creating basic test data which has datasets with resources -ckan_cli create-test-data +ckan_cli create-test-data basic # Datasets need to be assigned to an organisation diff --git a/.docker/test.ini b/.docker/test.ini index 3797e545..f33da6f4 100644 --- a/.docker/test.ini +++ b/.docker/test.ini @@ -94,7 +94,6 @@ ckan.redis.url = redis://redis:6379 # Add ``datapusher`` to enable DataPusher # Add ``resource_proxy`` to enable resource proxying and get around the # same origin policy -# @todo:setup Cleanup the list to use only required plugins. ckan.plugins = stats text_view image_view recline_view datastore data_qld_theme datarequests data_qld_resources data_qld_integration scheming_datasets validation dcat qa archiver report qgovext data_qld_reporting ytp_comments # Define which views should be created by default diff --git a/.flake8 b/.flake8 index 022bf847..d6b0f6b2 100644 --- a/.flake8 +++ b/.flake8 @@ -12,7 +12,7 @@ format = pylint show_source = True max-complexity = 10 -max-line-length = 128 +max-line-length = 127 # List ignore rules one per line. ignore = diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index faaff3b3..1b4e7950 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -29,7 +29,7 @@ jobs: - name: Retrieve screenshots if: failure() run: .circleci/process-artifacts.sh - timeout-minutes: 3 + timeout-minutes: 1 - name: Upload screenshots if: failure() diff --git a/docker-compose.yml b/docker-compose.yml index eef07d0e..f43b8030 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -39,6 +39,8 @@ services: AMAZEEIO_HTTP_PORT: 3000 LAGOON_LOCALDEV_URL: "http://${PROJECT}.docker.amazee.io" AMAZEEIO_URL: "${PROJECT}.docker.amazee.io" + stdin_open: true + tty: true postgres: image: amazeeio/postgres-ckan diff --git a/requirements-dev.txt b/requirements-dev.txt index 52a1808a..c73cd9ab 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -10,12 +10,12 @@ git+https://github.com/ckan/ckanapi@ckanapi-4.3#egg=ckanapi git+https://github.com/ckan/ckantoolkit@release-0.0.4#egg=ckantoolkit -e git+https://github.com/ckan/ckanext-dcat.git#egg=ckanext-dcat -e git+https://github.com/ckan/ckanext-scheming.git@release-1.2.0#egg=ckanext-scheming --e git+https://github.com/frictionlessdata/ckanext-validation.git#egg=ckanext-validation --e git+https://github.com/qld-gov-au/ckan-ex-qgov.git@3.3.1#egg=ckanext-qgov +-e git+https://github.com/qld-gov-au/ckan-ex-qgov.git@4.0.0#egg=ckanext-qgov -e git+https://github.com/qld-gov-au/ckanext-archiver.git@2.1.1-qgov.7#egg=ckanext-archiver -e git+https://github.com/qld-gov-au/ckanext-data-qld.git@4.0.1#egg=ckanext-data-qld -e git+https://github.com/qld-gov-au/ckanext-datarequests.git@2.2.0-qgov#egg=ckanext-datarequests -e git+https://github.com/qld-gov-au/ckanext-qa.git@2.0.3-qgov.4#egg=ckanext-qa -e git+https://github.com/qld-gov-au/ckanext-report.git@0.2#egg=ckanext-report +-e git+https://github.com/qld-gov-au/ckanext-validation.git#egg=ckanext-validation -e git+https://github.com/qld-gov-au/ckanext-ytp-comments.git@2.5.0-qgov#egg=ckanext-ytp-comments diff --git a/test/features/config.feature b/test/features/config.feature index b98d361a..cb619842 100644 --- a/test/features/config.feature +++ b/test/features/config.feature @@ -7,3 +7,5 @@ Feature: Config And I visit "ckan-admin/config" And I should see "Suggested Description" Then I should see an element with id "field-ckanext.data_qld.datarequest_suggested_description" + And I should not see an element with id "field-ckan-main-css" + And I should not see an element with id "field-ckan-site-custom-css" diff --git a/test/features/dataset_deletion.feature b/test/features/dataset_deletion.feature index 35c44dc3..f91ea371 100644 --- a/test/features/dataset_deletion.feature +++ b/test/features/dataset_deletion.feature @@ -34,7 +34,6 @@ Feature: Dataset deletion Then I should see an element with xpath "//div[@class='modal-footer']//button[@class='btn btn-primary' and @disabled='disabled']" When I type "it should be longer than 10 character" to "deletion_reason" Then I should not see an element with xpath "//div[@class='modal-footer']//button[@class='btn btn-primary' and @disabled='disabled']" - Then I take a screenshot Then I press the element with xpath "//div[@class='modal-footer']//button[@class='btn btn-primary']" And I wait for 10 seconds Then I should see "Dataset has been deleted" diff --git a/test/features/login_redirect.feature b/test/features/login_redirect.feature index 531761f3..030db00c 100644 --- a/test/features/login_redirect.feature +++ b/test/features/login_redirect.feature @@ -2,22 +2,23 @@ Feature: Login Redirection @dashboard_login - Scenario: As an unauthenticated user, when I visit the /dashboard URL I see the login page - Given "Unauthenticated" as the persona - When I visit "/dashboard" - Then I should see an element with xpath "//h1[contains(string(), 'Login')]" + Scenario Outline: As an unauthenticated user, when I visit the dashboard URL I see the login page + Given "TestOrgMember" as the persona + When I visit "" + Then I should see a login link + When I log in directly + Then I should see "News feed" - @dashboard_login - Scenario: As an unauthenticated user, when I visit the /dashboard/ URL I see the login page - Given "Unauthenticated" as the persona - When I visit "/dashboard/" - Then I should see an element with xpath "//h1[contains(string(), 'Login')]" + Examples: Dashboard URLs + | URL | + | /dashboard | + | /dashboard/ | @user_edit - Scenario: As an unauthenticated organisation member, when I visit the /user/edit URL I see the login page. Upon logging in I am taken to the /user/edit page + Scenario: As an unauthenticated organisation member, when I visit the user edit URL I see the login page. Upon logging in I am taken to the user edit page Given "TestOrgMember" as the persona When I visit "/user/edit" - Then I should see an element with xpath "//h1[contains(string(), 'Login')]" + Then I should see a login link When I log in directly Then I should see "Change details" @@ -35,7 +36,7 @@ Feature: Login Redirection Scenario: As an unauthenticated user, when I visit the URL of a private dataset I see the login page Given "Unauthenticated" as the persona When I visit "/dataset/annakarenina" - Then I should see an element with xpath "//h1[contains(string(), 'Login')]" + Then I should see a login link @public_dataset Scenario: As an unauthenticated user, when I visit the URL of a public dataset I see the dataset without needing to login @@ -48,7 +49,7 @@ Feature: Login Redirection Scenario: As an unauthenticated organisation member, when I visit the URL of a private dataset I see the login page. Upon logging in I am taken to the private dataset Given "TestOrgMember" as the persona When I visit "/dataset/annakarenina" - Then I should see an element with xpath "//h1[contains(string(), 'Login')]" + Then I should see a login link When I log in directly Then I should see an element with xpath "//h1[contains(string(), 'A Novel By Tolstoy')]" And I should see an element with xpath "//span[contains(string(), 'Private')]" diff --git a/test/features/steps/steps.py b/test/features/steps/steps.py index 262e92d9..3bbf561c 100644 --- a/test/features/steps/steps.py +++ b/test/features/steps/steps.py @@ -12,12 +12,17 @@ def get_current_url(context): context.browser.evaluate_script("document.documentElement.clientWidth") -@step('I go to homepage') +@step(u'I go to homepage') def go_to_home(context): when_i_visit_url(context, '/') -@step('I log in') +@step(u'I go to register page') +def go_to_register_page(context): + when_i_visit_url(context, '/user/register') + + +@step(u'I log in') def log_in(context): assert context.persona context.execute_steps(u""" @@ -27,7 +32,7 @@ def log_in(context): """) -@step('I log in directly') +@step(u'I log in directly') def log_in_directly(context): """ This differs to the `log_in` function above by logging in directly to a page where the user login form is presented @@ -44,6 +49,13 @@ def log_in_directly(context): """) +@step(u'I should see a login link') +def login_link_visible(context): + context.execute_steps(u""" + Then I should see an element with xpath "//h1[contains(string(), 'Login')]" + """) + + @step('I fill in title with random text') def title_random_text(context): @@ -53,17 +65,27 @@ def title_random_text(context): """.format(random.randrange(1000))) -@step('I go to organisation page') -def go_to_organisation_page(context): - when_i_visit_url(context, '/organization') +@step(u'I go to dataset page') +def go_to_dataset_page(context): + when_i_visit_url(context, '/dataset') -@step('I go to register page') -def go_to_register_page(context): - when_i_visit_url(context, '/user/register') +@step(u'I go to dataset "{name}"') +def go_to_dataset(context, name): + when_i_visit_url(context, '/dataset/' + name) + + +@step(u'I edit the "{name}" dataset') +def edit_dataset(context, name): + when_i_visit_url(context, '/dataset/edit/{}'.format(name)) + +@step(u'I go to organisation page') +def go_to_organisation_page(context): + when_i_visit_url(context, '/organization') -@step('I log in and go to the data requests page') + +@step(u'I log in and go to the data requests page') def log_in_go_to_datarequest_page(context): assert context.persona context.execute_steps(u""" @@ -72,12 +94,12 @@ def log_in_go_to_datarequest_page(context): """) -@step('I go to the data requests page') +@step(u'I go to the data requests page') def go_to_datarequest_page(context): when_i_visit_url(context, '/datarequest') -@step('I log in and create a datarequest') +@step(u'I log in and create a datarequest') def log_in_create_a_datarequest(context): assert context.persona @@ -87,7 +109,7 @@ def log_in_create_a_datarequest(context): """) -@step('I create a datarequest') +@step(u'I create a datarequest') def create_datarequest(context): assert context.persona @@ -100,21 +122,11 @@ def create_datarequest(context): """) -@step('I go to my reports page') +@step(u'I go to my reports page') def go_to_reporting_page(context): when_i_visit_url(context, '/dashboard/reporting') -@step('I go to dataset page') -def go_to_dataset_page(context): - when_i_visit_url(context, '/dataset') - - -@step(u'I go to dataset "{name}"') -def go_to_dataset(context, name): - when_i_visit_url(context, '/dataset/' + name) - - @step(u'I go to dataset "{name}" comments') def go_to_dataset_comments(context, name): context.execute_steps(u""" diff --git a/test/features/user_creation.feature b/test/features/user_creation.feature index de822307..0e9c289d 100644 --- a/test/features/user_creation.feature +++ b/test/features/user_creation.feature @@ -25,7 +25,7 @@ Feature: User creation Scenario: Non logged-in user register to the site. Given "Unauthenticated" as the persona - When I go to "/user/register" + When I go to register page Then I should see "Displayed name" Then I fill in "name" with "publisher_user" Then I fill in "fullname" with "gov user" From ac2a0f2bdcf968ec21e87b48bb3e70680225c3cf Mon Sep 17 00:00:00 2001 From: ThrawnCA Date: Thu, 10 Feb 2022 12:03:07 +1000 Subject: [PATCH 25/31] [QOL-8595] use 'develop' branch of ckanext-data-qld for testing - we should probably combine these two extensions at some point --- .docker/Dockerfile.ckan | 11 +++++++---- .docker/scripts/ckan_cli | 11 +++++++++-- .docker/scripts/create-test-data.sh | 15 +++++++++++++-- .docker/scripts/init.sh | 13 +------------ .docker/scripts/serve.sh | 2 +- .docker/test.ini | 3 +++ .github/workflows/test.yml | 16 +++++++++++----- .gitignore | 11 +++++++++++ docker-compose.yml | 2 ++ requirements-dev.txt | 4 ++-- test/features/steps/steps.py | 5 ++++- 11 files changed, 64 insertions(+), 29 deletions(-) diff --git a/.docker/Dockerfile.ckan b/.docker/Dockerfile.ckan index 9d3f89f1..3a5995e0 100644 --- a/.docker/Dockerfile.ckan +++ b/.docker/Dockerfile.ckan @@ -1,6 +1,8 @@ FROM amazeeio/python:2.7-ckan-21.8.0 -ARG SITE_URL +ARG SITE_URL=http://ckan:3000/ +ARG CKAN_REPO=qld-gov-au/ckan +ARG CKAN_VERSION=2.8.8-qgov.5 ENV SITE_URL="${SITE_URL}" ENV VENV_DIR=/app/ckan/default ENV APP_DIR=/app @@ -15,13 +17,14 @@ RUN apk add --no-cache curl build-base \ && rm dockerize-alpine-linux-amd64-${DOCKERIZE_VERSION}.tar.gz # Install CKAN. -ENV CKAN_VERSION 2.8.8-qgov.5 RUN . ${VENV_DIR}/bin/activate \ && pip install setuptools==36.1 \ - && pip install -e "git+https://github.com/qld-gov-au/ckan.git@ckan-${CKAN_VERSION}#egg=ckan" \ + && pip install -e "git+https://github.com/${CKAN_REPO}.git@ckan-${CKAN_VERSION}#egg=ckan" \ && sed -i "s/psycopg2==2.4.5/psycopg2==2.7.7/g" "${VENV_DIR}/src/ckan/requirements.txt" \ - && pip install -r "${VENV_DIR}/src/ckan/requirements.txt" \ + && ((test -f "${VENV_DIR}/src/ckan/requirements-py2.txt" && \ + pip install -r "${VENV_DIR}/src/ckan/requirements-py2.txt") || \ + pip install -r "${VENV_DIR}/src/ckan/requirements.txt") \ && ln -s "${VENV_DIR}/src/ckan/who.ini" "${VENV_DIR}/who.ini" \ && deactivate \ && ln -s ${APP_DIR}/ckan /usr/lib/ckan \ diff --git a/.docker/scripts/ckan_cli b/.docker/scripts/ckan_cli index fcc045fd..3cf0b4cc 100644 --- a/.docker/scripts/ckan_cli +++ b/.docker/scripts/ckan_cli @@ -58,10 +58,17 @@ fi if [ "$COMMAND" = "ckan" ]; then echo "Using 'ckan' command from $ENV_DIR with config ${CKAN_INI}..." >&2 - exec $ENV_DIR/ckan -c ${CKAN_INI} "$@" + # adjust args to match ckan expectations + COMMAND=$(echo "$1" | sed -e 's/create-test-data/seed/') + shift + exec $ENV_DIR/ckan -c ${CKAN_INI} $COMMAND "$@" $CLICK_ARGS elif [ "$COMMAND" = "paster" ]; then echo "Using 'paster' command from $ENV_DIR with config ${CKAN_INI}..." >&2 - exec $ENV_DIR/paster --plugin=$PASTER_PLUGIN "$@" -c ${CKAN_INI} + # adjust args to match paster expectations + COMMAND=$1 + shift + if [ "$1" = "show" ]; then shift; fi + exec $ENV_DIR/paster --plugin=$PASTER_PLUGIN $COMMAND "$@" -c ${CKAN_INI} else echo "Unable to locate 'ckan' or 'paster' command in $ENV_DIR" >&2 exit 1 diff --git a/.docker/scripts/create-test-data.sh b/.docker/scripts/create-test-data.sh index 1821213b..38b48c72 100644 --- a/.docker/scripts/create-test-data.sh +++ b/.docker/scripts/create-test-data.sh @@ -10,16 +10,27 @@ if [ "$VENV_DIR" != "" ]; then . ${VENV_DIR}/bin/activate fi +CKAN_USER_NAME="${CKAN_USER_NAME:-admin}" +CKAN_DISPLAY_NAME="${CKAN_DISPLAY_NAME:-Administrator}" +CKAN_USER_EMAIL="${CKAN_USER_EMAIL:-admin@localhost}" + add_user_if_needed () { echo "Adding user '$2' ($1) with email address [$3]" - ckan_cli user "$1" | grep "$1" || ckan_cli user add "$1"\ + ckan_cli user show "$1" | grep "$1" || ckan_cli user add "$1"\ fullname="$2"\ email="$3"\ password="${4:-Password123!}" } +add_user_if_needed "$CKAN_USER_NAME" "$CKAN_DISPLAY_NAME" "$CKAN_USER_EMAIL" +ckan_cli sysadmin add "${CKAN_USER_NAME}" + # We know the "admin" sysadmin account exists, so we'll use her API KEY to create further data -API_KEY=$(ckan_cli user admin | tr -d '\n' | sed -r 's/^(.*)apikey=(\S*)(.*)/\2/') +API_KEY=$(ckan_cli user show "${CKAN_USER_NAME}" | tr -d '\n' | sed -r 's/^(.*)apikey=(\S*)(.*)/\2/') +if [ "$API_KEY" = "None" ]; then + echo "No API Key found on ${CKAN_USER_NAME}, generating API Token..." + API_KEY=$(ckan_cli user token add "${CKAN_USER_NAME}" test_setup |grep -v '^API Token created' | tr -d '[:space:]') +fi ## # BEGIN: Create a test organisation with test users for admin, editor and member diff --git a/.docker/scripts/init.sh b/.docker/scripts/init.sh index eb7839fa..46b0a307 100755 --- a/.docker/scripts/init.sh +++ b/.docker/scripts/init.sh @@ -4,15 +4,10 @@ # set -e -CKAN_USER_NAME="${CKAN_USER_NAME:-admin}" -CKAN_DISPLAY_NAME="${CKAN_DISPLAY_NAME:-Administrator}" -CKAN_USER_PASSWORD="${CKAN_USER_PASSWORD:-Password123!}" -CKAN_USER_EMAIL="${CKAN_USER_EMAIL:-admin@localhost}" - if [ "$VENV_DIR" != "" ]; then . ${VENV_DIR}/bin/activate fi -ckan_cli db clean +CLICK_ARGS="--yes" ckan_cli db clean ckan_cli db init # Initialise validation tables @@ -32,11 +27,5 @@ PASTER_PLUGIN=ckanext-report ckan_cli report initdb # Initialise the QA database tables PASTER_PLUGIN=ckanext-qa ckan_cli qa init -ckan_cli user add "${CKAN_USER_NAME}"\ - fullname="${CKAN_DISPLAY_NAME}"\ - email="${CKAN_USER_EMAIL}"\ - password="${CKAN_USER_PASSWORD}" -ckan_cli sysadmin add "${CKAN_USER_NAME}" - # Create some base test data . $APP_DIR/scripts/create-test-data.sh diff --git a/.docker/scripts/serve.sh b/.docker/scripts/serve.sh index 10f3dead..904c45c7 100755 --- a/.docker/scripts/serve.sh +++ b/.docker/scripts/serve.sh @@ -11,7 +11,7 @@ if [ "$VENV_DIR" != "" ]; then . ${VENV_DIR}/bin/activate fi if (which ckan > /dev/null); then - ckan -c ${CKAN_INI} run + ckan -c ${CKAN_INI} run -r else paster serve ${CKAN_INI} fi diff --git a/.docker/test.ini b/.docker/test.ini index f33da6f4..7dfe8bb5 100644 --- a/.docker/test.ini +++ b/.docker/test.ini @@ -22,6 +22,9 @@ host = 0.0.0.0 port = 3000 [app:main] +ckan.devserver.host = 0.0.0.0 +ckan.devserver.port = 3000 + use = egg:ckan full_stack = true cache_dir = /tmp/%(ckan.site_id)s/ diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 1b4e7950..6b657991 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,18 +1,24 @@ --- -#based on https://raw.githubusercontent.com/ckan/ckanext-scheming/master/.github/workflows/test.yml -# alternative https://github.com/ckan/ckan/blob/master/contrib/cookiecutter/ckan_extension/%7B%7Bcookiecutter.project%7D%7D/.github/workflows/test.yml name: Tests on: push: + pull_request: + branches: + - master jobs: test: strategy: fail-fast: false + matrix: + ckan-version: [2.8.8-qgov.5] - name: Continuous Integration build + name: Continuous Integration build on CKAN ${{ matrix.ckan-version }} runs-on: ubuntu-latest container: integratedexperts/ci-builder + env: + CKAN_REPO: qld-gov-au/ckan + CKAN_VERSION: ${{ matrix.ckan-version }} steps: - uses: actions/checkout@v2 @@ -35,6 +41,6 @@ jobs: if: failure() uses: actions/upload-artifact@v2 with: - name: screenshots + name: CKAN ${{ matrix.ckan-version }} screenshots path: /tmp/artifacts/behave/screenshots - timeout-minutes: 3 + timeout-minutes: 1 diff --git a/.gitignore b/.gitignore index 4578f100..7e0716f7 100644 --- a/.gitignore +++ b/.gitignore @@ -15,7 +15,13 @@ env/ build/ develop-eggs/ dist/ +downloads/ +eggs/ +lib/ +lib64/ +parts/ sdist/ +var/ *.egg-info/ .installed.cfg *.egg @@ -51,4 +57,9 @@ docs/_build/ .env.local screenshots !/test/screenshots/.gitkeep + +# PyBuilder +target/ + +#Intellij .idea diff --git a/docker-compose.yml b/docker-compose.yml index f43b8030..f0c99c99 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -24,6 +24,8 @@ services: dockerfile: .docker/Dockerfile.ckan args: SITE_URL: "http://${PROJECT}.docker.amazee.io" + CKAN_REPO: + CKAN_VERSION: depends_on: - postgres - solr diff --git a/requirements-dev.txt b/requirements-dev.txt index c73cd9ab..b7d23825 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -3,8 +3,8 @@ behaving==2.0.0 Appium-Python-Client<=0.52 flake8==3.8.3 nose==1.3.7 -splinter>=0.13.0,<0.17 six>=1.13.0 +splinter>=0.13.0,<0.17 git+https://github.com/ckan/ckanapi@ckanapi-4.3#egg=ckanapi git+https://github.com/ckan/ckantoolkit@release-0.0.4#egg=ckantoolkit @@ -12,7 +12,7 @@ git+https://github.com/ckan/ckantoolkit@release-0.0.4#egg=ckantoolkit -e git+https://github.com/ckan/ckanext-scheming.git@release-1.2.0#egg=ckanext-scheming -e git+https://github.com/qld-gov-au/ckan-ex-qgov.git@4.0.0#egg=ckanext-qgov -e git+https://github.com/qld-gov-au/ckanext-archiver.git@2.1.1-qgov.7#egg=ckanext-archiver --e git+https://github.com/qld-gov-au/ckanext-data-qld.git@4.0.1#egg=ckanext-data-qld +-e git+https://github.com/qld-gov-au/ckanext-data-qld.git@develop#egg=ckanext-data-qld -e git+https://github.com/qld-gov-au/ckanext-datarequests.git@2.2.0-qgov#egg=ckanext-datarequests -e git+https://github.com/qld-gov-au/ckanext-qa.git@2.0.3-qgov.4#egg=ckanext-qa -e git+https://github.com/qld-gov-au/ckanext-report.git@0.2#egg=ckanext-report diff --git a/test/features/steps/steps.py b/test/features/steps/steps.py index 3bbf561c..d9c269c1 100644 --- a/test/features/steps/steps.py +++ b/test/features/steps/steps.py @@ -19,7 +19,10 @@ def go_to_home(context): @step(u'I go to register page') def go_to_register_page(context): - when_i_visit_url(context, '/user/register') + context.execute_steps(u""" + When I go to homepage + And I click the link with text that contains "Register" + """) @step(u'I log in') From 314918e095497774b726673d784a30c7a9426d64 Mon Sep 17 00:00:00 2001 From: ThrawnCA Date: Thu, 10 Feb 2022 17:15:10 +1000 Subject: [PATCH 26/31] [QOL-8595] drop cleanup scenario that may be causing a race condition --- .circleci/build.sh | 2 +- setup.py | 5 +---- test/features/resource_availability.feature | 23 +-------------------- 3 files changed, 3 insertions(+), 27 deletions(-) diff --git a/.circleci/build.sh b/.circleci/build.sh index 611dca97..407dd0ec 100755 --- a/.circleci/build.sh +++ b/.circleci/build.sh @@ -21,4 +21,4 @@ export DOCTOR_CHECK_SSH=0 export DOCTOR_CHECK_WEBSERVER=0 export DOCTOR_CHECK_BOOTSTRAP=0 -ahoy build +ahoy build || (ahoy logs; exit 1) diff --git a/setup.py b/setup.py index c670b328..fa200af9 100644 --- a/setup.py +++ b/setup.py @@ -47,7 +47,7 @@ # You can just specify the packages manually here if your project is # simple. Or you can use find_packages(). packages=find_packages(exclude=['contrib', 'docs', 'tests*']), - # namespace_packages=['ckanext'], + namespace_packages=['ckanext'], install_requires=[ # CKAN extensions should not list dependencies here, but in a separate @@ -75,9 +75,6 @@ [ckan.plugins] data_qld_theme=ckanext.data_qld_theme.plugin:DataQldThemePlugin - [paste.paster_command] - migrate_extras = ckanext.data_qld.commands:MigrateExtras - [babel.extractors] ckan = ckan.lib.extract:extract_ckan ''', diff --git a/test/features/resource_availability.feature b/test/features/resource_availability.feature index 8978f349..d02a8701 100644 --- a/test/features/resource_availability.feature +++ b/test/features/resource_availability.feature @@ -132,7 +132,7 @@ Feature: Re-identification risk governance acknowledgement or Resource visibilit And I go to "/dataset/new_resource/contains-de-identified-data-no" # Create the resource, with success. - Then I execute the script "document.getElementById('field-image-url').value='https://example.com'" + Then I execute the script "document.getElementById('field-image-url').value='http://ckanext-data-qld.docker.amazee.io/'" Then I fill in "name" with "resource created by and is available with blank resource_visibility" Then I fill in "description" with "description" Then I fill in "size" with "1024" @@ -197,24 +197,3 @@ Feature: Re-identification risk governance acknowledgement or Resource visibilit And I go to "/dataset/contains-de-identified-data-yes" Then I press the element with xpath "//a[@title='resource created by SysAdmin and is available']" Then I should not see "Re-identification risk governance acknowledgement/Resource visibility" - - - Scenario: Clean up - Given "SysAdmin" as the persona - When I log in - And I go to "/dataset/edit/contains-de-identified-data-yes" - Then I press the element with xpath "//a[@data-module='confirm-action']" - And I wait for 5 seconds - When I type "it should be longer than 10 character" to "deletion_reason" - Then I press the element with xpath "//div[@class='modal-footer']//button[@class='btn btn-primary']" - And I wait for 10 seconds - - And I go to "/dataset/edit/contains-de-identified-data-no" - Then I press the element with xpath "//a[@data-module='confirm-action']" - And I wait for 5 seconds - When I type "it should be longer than 10 character" to "deletion_reason" - Then I press the element with xpath "//div[@class='modal-footer']//button[@class='btn btn-primary']" - And I wait for 10 seconds - - When I go to "/ckan-admin/trash" - Then I press the element with xpath "//button[@name='purge-packages']" From 587c543b6786f730111ea51e6cae30dfa8473826 Mon Sep 17 00:00:00 2001 From: ThrawnCA Date: Fri, 11 Feb 2022 13:48:35 +1000 Subject: [PATCH 27/31] [QOL-8595] oops restore 'example.com' reference in testing --- test/features/resource_availability.feature | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/features/resource_availability.feature b/test/features/resource_availability.feature index d02a8701..80d81608 100644 --- a/test/features/resource_availability.feature +++ b/test/features/resource_availability.feature @@ -132,7 +132,7 @@ Feature: Re-identification risk governance acknowledgement or Resource visibilit And I go to "/dataset/new_resource/contains-de-identified-data-no" # Create the resource, with success. - Then I execute the script "document.getElementById('field-image-url').value='http://ckanext-data-qld.docker.amazee.io/'" + Then I execute the script "document.getElementById('field-image-url').value='https://example.com'" Then I fill in "name" with "resource created by and is available with blank resource_visibility" Then I fill in "description" with "description" Then I fill in "size" with "1024" From dc5e8196dc76e33c94665aa90fdc78a4437829d0 Mon Sep 17 00:00:00 2001 From: ThrawnCA Date: Wed, 2 Mar 2022 14:46:30 +1000 Subject: [PATCH 28/31] [QOL-7811] update route name for data requests - use Flask-compatible name --- ckanext/data_qld_theme/templates/datarequests/show.html | 2 +- test/features/steps/steps.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/ckanext/data_qld_theme/templates/datarequests/show.html b/ckanext/data_qld_theme/templates/datarequests/show.html index 4928edf6..9db87c34 100644 --- a/ckanext/data_qld_theme/templates/datarequests/show.html +++ b/ckanext/data_qld_theme/templates/datarequests/show.html @@ -24,7 +24,7 @@ {% endblock %} {% block content_primary_nav %} - {{ h.build_nav_icon('show_datarequest', _('Data request'), id=datarequest_id) }} + {{ h.build_nav_icon('datarequest.show', _('Data request'), id=datarequest_id) }} {% if h.show_comments_tab() and h.ytp_comments_enabled() %} {{ h.build_nav_icon('comment_datarequest', _('Comments') + ' ' + h.get_content_type_comments_badge(datarequest_id, 'datarequest'), id=datarequest_id) }} diff --git a/test/features/steps/steps.py b/test/features/steps/steps.py index 4e2e53e2..ea9e45e8 100644 --- a/test/features/steps/steps.py +++ b/test/features/steps/steps.py @@ -82,6 +82,7 @@ def go_to_dataset(context, name): def edit_dataset(context, name): when_i_visit_url(context, '/dataset/edit/{}'.format(name)) + @step(u'I go to organisation page') def go_to_organisation_page(context): when_i_visit_url(context, '/organization') From a5fbbb3d5d16b00e4baa2a52d62c505b548dddbc Mon Sep 17 00:00:00 2001 From: ThrawnCA Date: Wed, 2 Mar 2022 15:05:21 +1000 Subject: [PATCH 29/31] [QOL-7811] update route name for data request comments - use Flask-compatible name --- ckanext/data_qld_theme/templates/datarequests/show.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckanext/data_qld_theme/templates/datarequests/show.html b/ckanext/data_qld_theme/templates/datarequests/show.html index 9db87c34..5e779716 100644 --- a/ckanext/data_qld_theme/templates/datarequests/show.html +++ b/ckanext/data_qld_theme/templates/datarequests/show.html @@ -27,6 +27,6 @@ {{ h.build_nav_icon('datarequest.show', _('Data request'), id=datarequest_id) }} {% if h.show_comments_tab() and h.ytp_comments_enabled() %} - {{ h.build_nav_icon('comment_datarequest', _('Comments') + ' ' + h.get_content_type_comments_badge(datarequest_id, 'datarequest'), id=datarequest_id) }} + {{ h.build_nav_icon('datarequest.comment', _('Comments') + ' ' + h.get_content_type_comments_badge(datarequest_id, 'datarequest'), id=datarequest_id) }} {% endif %} {% endblock %} From 8ea9ca083c659a117ee2d94d5b339d79074d3d44 Mon Sep 17 00:00:00 2001 From: ThrawnCA Date: Wed, 2 Mar 2022 15:26:26 +1000 Subject: [PATCH 30/31] [QOL-7811] update route name for organisation link to handle Flask --- .../data_qld_theme/templates/scheming/package/read.html | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/ckanext/data_qld_theme/templates/scheming/package/read.html b/ckanext/data_qld_theme/templates/scheming/package/read.html index da666078..6d3a9618 100644 --- a/ckanext/data_qld_theme/templates/scheming/package/read.html +++ b/ckanext/data_qld_theme/templates/scheming/package/read.html @@ -1,8 +1,13 @@ {% ckan_extends %} {% block breadcrumb_content %} + {% set route_name = 'organization_read' %} + {% if h.ckan_version() > '2.9' %} + {% set route_name = h.map_pylons_to_flask_route_name(route_name) %} + {% endif %}
        • {% link_for _('Organizations'), named_route='organization.index' %}
          • -
        • {% link_for pkg.organization.title, named_route='organization_read', id=pkg.organization.name %}
          • +
        • {% link_for pkg.organization.title, named_route=route_name, id=pkg.organization.name %}
          • +
        • {{ pkg.title }}
          • {% endblock %} From 935f42aa2002065e0e90559ff71d0ed0fb208741 Mon Sep 17 00:00:00 2001 From: ThrawnCA Date: Tue, 15 Mar 2022 10:00:33 +1000 Subject: [PATCH 31/31] [QOL-8595] fix route name for re-opening data requests --- ckanext/data_qld_theme/templates/datarequests/show.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckanext/data_qld_theme/templates/datarequests/show.html b/ckanext/data_qld_theme/templates/datarequests/show.html index 4928edf6..0c72aac9 100644 --- a/ckanext/data_qld_theme/templates/datarequests/show.html +++ b/ckanext/data_qld_theme/templates/datarequests/show.html @@ -18,7 +18,7 @@ {% endif %} {% if h.check_access('open_datarequest', {'id':datarequest_id }) and c.datarequest.closed %} - {% link_for _('Re-open'), named_route='datarequest.open', id=datarequest_id, class_='btn btn-success', icon='unlock' %} + {% link_for _('Re-open'), named_route='open_datarequest', id=datarequest_id, class_='btn btn-success', icon='unlock' %} {% endif %} {% endblock %}
          {{ _('Column') }} {{ _('Type') }}