diff --git a/ckanext/datavicmain/cli/maintain.py b/ckanext/datavicmain/cli/maintain.py index 1fe0f094..8973d90d 100644 --- a/ckanext/datavicmain/cli/maintain.py +++ b/ckanext/datavicmain/cli/maintain.py @@ -3,23 +3,21 @@ import copy import datetime import logging -import csv -import openpyxl import mimetypes from os import path, stat -from typing import Any -from sqlalchemy.orm import Query from itertools import groupby +from os import path +from typing import Any from urllib.parse import urlparse -import click -import tqdm - import ckan.logic.validators as validators import ckan.model as model import ckan.plugins.toolkit as tk +import click +import openpyxl +import tqdm from ckan.lib.munge import munge_title_to_name from ckan.lib.search import rebuild from ckan.lib.uploader import get_resource_uploader @@ -29,6 +27,8 @@ from ckanext.harvest.model import HarvestObject, HarvestSource from ckanext.datastore.backend import get_all_resources_ids_in_datastore +from ckanext.harvest.model import HarvestObject, HarvestSource +from sqlalchemy.orm import Query log = logging.getLogger(__name__) @@ -576,7 +576,7 @@ def _search_in_batch(datasets: list[dict[str, Any]]) -> dict[str, dict[str, str] def _get_default_values_for_missing_fields( dataset: dict[str, Any], field: str -) -> "str": +) -> str: """Get values for missing fields""" if field == "date_created_data_asset": return _get_date_created(dataset) @@ -590,6 +590,7 @@ def _get_default_values_for_missing_fields( return "official" elif field == "category": return _get_category(dataset) + return "" def _get_date_created(pkg: dict[str, Any]) -> str: @@ -615,7 +616,7 @@ def _get_category(pkg: dict[str, Any]) -> str: """ if groups := pkg.get("groups"): return groups[0].get("id") - return + return "" @maintain.command(u"update-broken-urls", @@ -624,7 +625,8 @@ def update_broken_urls(): """Change resources urls' protocols from http to https listed in XLSX file""" file = path.join( - path.dirname(__file__), "data/DTF Content list bulk URL change 20231017.xlsx" + path.dirname(__file__), + "data/DTF Content list bulk URL change 20231017.xlsx", ) wb = openpyxl.load_workbook(file) ws = wb.active @@ -641,15 +643,14 @@ def update_broken_urls(): if not resource: click.secho( - f"Resource <{title}> with URL <{url}> does not exist", - fg="red" + f"Resource <{title}> with URL <{url}> does not exist", fg="red" ) continue resource.url = row[XLSX_IDX_NEW_URL].value click.secho( f"URL of resource <{title}> has been updated to <{resource.url}>", - fg="green" + fg="green", ) model.Session.commit() @@ -825,3 +826,18 @@ def get_resources_by_size(empty: bool, limit: bool, restricted: bool): f"Found {resources.count()} resources...", fg="green", ) + + +@maintain.command("make-datatables-view-prioritized") +def make_datatables_view_prioritized(): + """Check if there are resources that have recline_view and datatables_view and + reorder them so that datatables_view is first.""" + resources = model.Session.query(Resource).all() + number_reordered = 0 + for resource in tqdm.tqdm(resources): + result = tk.get_action("datavic_datatables_view_prioritize")( + {"ignore_auth": True}, {"resource_id": resource.id} + ) + if result.get("updated"): + number_reordered += 1 + click.secho(f"Reordered {number_reordered} resources", fg="green") diff --git a/ckanext/datavicmain/config.py b/ckanext/datavicmain/config.py index 5a3ca737..412b7530 100644 --- a/ckanext/datavicmain/config.py +++ b/ckanext/datavicmain/config.py @@ -4,6 +4,7 @@ CONFIG_PAGES_BASE_URL = "ckan.pages.base_url" CONFIG_DTV_URL = "ckanext.datavicmain.dtv.url" +CONFIG_DTV_MAX_SIZE_LIMIT = "ckanext.datavicmain.dtv.max_size_limit" CONFIG_DTV_EXTERNAL_LINK = "ckanext.datavicmain.dtv.external_link" @@ -15,5 +16,9 @@ def get_dtv_url() -> str: return tk.config.get(CONFIG_DTV_URL, "") +def get_dtv_max_size_limit() -> int: + return tk.config.get(CONFIG_DTV_MAX_SIZE_LIMIT, "157286400") + + def get_dtv_external_link() -> str: - return tk.config.get(CONFIG_DTV_EXTERNAL_LINK, "") \ No newline at end of file + return tk.config.get(CONFIG_DTV_EXTERNAL_LINK, "") diff --git a/ckanext/datavicmain/helpers.py b/ckanext/datavicmain/helpers.py index 99b979c4..1f6d4ba6 100644 --- a/ckanext/datavicmain/helpers.py +++ b/ckanext/datavicmain/helpers.py @@ -1,5 +1,4 @@ from __future__ import annotations -from json import tool import math import os @@ -8,26 +7,23 @@ import logging import json import base64 -from typing import Any +from typing import Any, Optional from urllib.parse import urlsplit, urljoin from flask import Blueprint -import ckan.plugins as plugins import ckan.model as model import ckan.authz as authz import ckan.plugins.toolkit as toolkit from ckanext.harvest.model import HarvestObject from ckanext.activity.model.activity import Activity -from ckanext.mailcraft.utils import get_mailer -from ckanext.mailcraft.exception import MailerException -from . import utils, const +from . import utils, const, config as conf from ckanext.datavicmain.config import get_dtv_url, get_dtv_external_link -mailer = get_mailer() + log = logging.getLogger(__name__) WORKFLOW_STATUS_OPTIONS = [ "draft", @@ -368,6 +364,43 @@ def datavic_org_uploads_allowed(org_id: str) -> bool: return flake["data"].get(org.id, False) +def get_group(group: Optional[str] = None, + include_datasets: bool = False) -> dict[str, Any]: + if group is None: + return {} + try: + return toolkit.get_action("group_show")( + {}, + {"id": group, "include_datasets": include_datasets} + ) + except (toolkit.NotFound, toolkit.ValidationError, toolkit.NotAuthorized): + return {} + + +def dtv_exceeds_max_size_limit(resource_id: str) -> bool: + """Check if DTV resource exceeds the maximum file size limit + + Args: + resource_id (str): DTV resource id + + Returns: + bool: return True if dtv resource exceeds maximum file size limit set + in ckan config "ckanext.datavicmain.dtv.max_size_limit", + otherwise - False + """ + try: + resource = toolkit.get_action("resource_show")({}, {"id": resource_id}) + except (toolkit.ObjectNotFound, toolkit.NotAuthorized): + return True + + limit = conf.get_dtv_max_size_limit() + filesize = resource.get("filesize") + if filesize and int(filesize) >= int(limit): + return True + + return False + + def datavic_get_org_roles() -> list[str]: return ["admin", "editor", "member"] @@ -560,7 +593,7 @@ def _group_tree_parents(id_, type_="organization"): def add_current_organisation( - avalable_organisations: list[dict[str, Any]], current_org: dict[str, Any] + available_organisations: list[dict[str, Any]], current_org: dict[str, Any] ): """When user doesn't have an access to an organisation, it won't be included for a list of available organisations. Include it there, but check if it's @@ -568,19 +601,19 @@ def add_current_organisation( current_org_included = False - for organization in avalable_organisations: + for organization in available_organisations: if organization["id"] == current_org["id"]: current_org_included = True break if not current_org_included: - avalable_organisations.append(current_org) + available_organisations.append(current_org) - return avalable_organisations + return available_organisations def datavic_max_image_size(): - """Return max size for image configurate for portal""" + """Return max size for image configuration for portal""" return toolkit.config["ckan.max_image_size"] @@ -658,3 +691,33 @@ def datavic_allowable_parent_orgs(org_id: str = None) -> list[dict[str, Any]]: continue orgs.append(org) return orgs + + +def has_user_capacity( + org_id: str, + current_user_id: str, + capacity: Optional[str] = None) -> bool: + """Check if the current user has an appropriate capacity in the certain organization + + Args: + org_id (str): the id or name of the organization + current_user_id (str): the id or name of the user + capacity (str): restrict the members returned to those with a given capacity, + e.g. 'member', 'editor', 'admin', 'public', 'private' + (optional, default: None) + + Returns: + bool: True for success, False otherwise + """ + try: + members = toolkit.get_action("member_list")( + {}, + {"id": org_id, "object_type": "user", "capacity": capacity} + ) + members_id = [member[0] for member in members] + if current_user_id in members_id: + return True + except (toolkit.ObjectNotFound, toolkit.NotAuthorized): + return False + + return False diff --git a/ckanext/datavicmain/iar_ckan_dataset.yaml b/ckanext/datavicmain/iar_ckan_dataset.yaml index d37182c5..4d80c020 100644 --- a/ckanext/datavicmain/iar_ckan_dataset.yaml +++ b/ckanext/datavicmain/iar_ckan_dataset.yaml @@ -22,6 +22,7 @@ dataset_fields: - field_name: alias preset: dataset_alias + display_group: General - field_name: notes label: Description @@ -152,6 +153,7 @@ dataset_fields: label: "Off" validators: default(true) boolean_validator output_validators: boolean_validator + display_group: General # Security fields diff --git a/ckanext/datavicmain/logic/action.py b/ckanext/datavicmain/logic/action.py index bbf8a96f..99a0585a 100644 --- a/ckanext/datavicmain/logic/action.py +++ b/ckanext/datavicmain/logic/action.py @@ -94,7 +94,10 @@ def organization_update(next_, context, data_dict): except ckanapi.NotFound: continue - patch = {f: result[f] for f in tracked_fields if f in result} + patch = { + f: result[f] + for f in tracked_fields if f in result + } if 'image_url' in tracked_fields and result.get('image_display_url'): grp_uloader: uploader.PUploader = uploader.get_uploader('group') @@ -106,19 +109,14 @@ def organization_update(next_, context, data_dict): ckan.call_action('organization_patch', data_dict=patch, files={ "image_upload": (result['image_url'], file_data)}) else: - ckan.action.organization_patch( - id=remote["id"], - **patch, - ) - - return result + ckan.action.organization_patch(id=remote["id"], **patch) def _is_org_changed( - old_org: dict[str, Any], new_org: dict[str, Any], tracked_fields: list[str] + old_org: model.Group, new_org: dict[str, Any], tracked_fields: list[str] ) -> bool: for field_name in tracked_fields: - if old_org.get(field_name) != new_org.get(field_name): + if old_org.__dict__.get(field_name) != new_org.get(field_name): return True return False @@ -151,14 +149,22 @@ def organization_list( """Restrict organisations. Force all_fields and include_extras, because we need visibility field to be here. Throw out extra fields later if it's not all_fields initially""" - all_fields = data_dict.pop("all_fields", False) + all_fields = data_dict.get("all_fields", False) - data_dict.update({"all_fields": True, "include_extras": True}) + if all_fields: + data_dict.update({"include_extras": True}) context["_skip_restriction_check"] = True org_list: types.ActionResult.OrganizationList = next_(context, data_dict) + if not all_fields: + orgs = model.Session.query(model.Group)\ + .filter(model.Group.name.in_(org_list)) + + # Intead of all all_fields, lets get the ID from the Objet as it much faster + org_list = [{'id': org.id , 'name': org.name} for org in orgs] + filtered_orgs = _hide_restricted_orgs(context, org_list) if not all_fields: @@ -221,7 +227,7 @@ def _show_errors_in_sibling_resources( "list[type.ErrorDict]", valid_errors.error_dict['resources'])[-1] except (KeyError, IndexError): error_dict = valid_errors.error_dict - + pkg_dict = toolkit.get_action("package_show")( context, {"id": data_dict["package_id"]} ) @@ -362,3 +368,57 @@ def send_delwp_data_request(context, data_dict): return {"success": False} return {"success": True} + + +@validate(vic_schema.datatables_view_prioritize) +def datavic_datatables_view_prioritize( + context: Context, data_dict: DataDict +) -> types.DataDict: + """Check if the datatables view is prioritized over the recline view. + If not, swap their order. + """ + toolkit.check_access("vic_datatables_view_prioritize", context, data_dict) + + resource_id = data_dict["resource_id"] + res_views = sorted( + model.Session.query(model.ResourceView) + .filter(model.ResourceView.resource_id == resource_id) + .all(), + key=lambda x: x.order, + ) + datatables_views = _filter_views(res_views, "datatables_view") + recline_views = _filter_views(res_views, "recline_view") + + if not ( + datatables_views + and recline_views + and datatables_views[0].order > recline_views[0].order + ): + return {"updated": False} + + datatables_views[0].order, recline_views[0].order = ( + recline_views[0].order, + datatables_views[0].order, + ) + order = [view.id for view in sorted(res_views, key=lambda x: x.order)] + toolkit.get_action("resource_view_reorder")( + {"ignore_auth": True}, {"id": resource_id, "order": order} + ) + return {"updated": True} + + +@toolkit.chained_action +def resource_view_create(next_, context, data_dict): + result = next_(context, data_dict) + if data_dict["view_type"] == "datatables_view": + toolkit.get_action("datavic_datatables_view_prioritize")( + {"ignore_auth": True}, {"resource_id": data_dict["resource_id"]} + ) + return result + + +def _filter_views( + res_views: list[model.ResourceView], view_type: str +) -> list[model.ResourceView]: + """Return a list of views with the given view type.""" + return [view for view in res_views if view.view_type == view_type] diff --git a/ckanext/datavicmain/logic/auth.py b/ckanext/datavicmain/logic/auth.py index 43c575ad..6cadc5b6 100644 --- a/ckanext/datavicmain/logic/auth.py +++ b/ckanext/datavicmain/logic/auth.py @@ -85,3 +85,57 @@ def user_show(context: Context, data_dict: DataDict) -> AuthResult: return {"success": True} return {"success": False} + + +def _has_user_capacity_in_org(org_id: str, roles: list) -> bool: + """ Check if the current user has the necessary capacity in the certain + organization + + Args: + org_id (str): id of the organization + roles (list): list of necessary member roles in the organization + + Returns: + bool: True if the current user has the necessary capacity in the certain + organization, False - otherwise + """ + if authz.users_role_for_group_or_org( + group_id=org_id, + user_name=tk.current_user.name) in roles: + return True + return False + + +@tk.chained_auth_function +def package_activity_list(next_auth, context, data_dict): + pkg_dict = tk.get_action("package_show")( + context, + {"id": data_dict["id"]}, + ) + allowed_roles = ["admin", "editor"] + is_user_collaborator = authz.user_is_collaborator_on_dataset( + tk.current_user.id, pkg_dict["id"], allowed_roles + ) + has_user_capacity = _has_user_capacity_in_org( + pkg_dict["owner_org"], allowed_roles + ) + + if has_user_capacity or is_user_collaborator: + return next_auth(context, data_dict) + return {"success": False} + + +@tk.chained_auth_function +def organization_activity_list(next_auth, context, data_dict): + allowed_roles = ["admin", "editor"] + if _has_user_capacity_in_org(data_dict["id"], allowed_roles): + return next_auth(context, data_dict) + return {"success": False} + + +@tk.chained_auth_function +def organization_activity_list(next_auth, context, data_dict): + allowed_roles = ["admin", "editor"] + if _has_user_capacity_in_org(data_dict["id"], allowed_roles): + return next_auth(context, data_dict) + return {"success": False} diff --git a/ckanext/datavicmain/logic/schema.py b/ckanext/datavicmain/logic/schema.py index 27dd86f3..08d04128 100644 --- a/ckanext/datavicmain/logic/schema.py +++ b/ckanext/datavicmain/logic/schema.py @@ -47,3 +47,12 @@ def delwp_data_request_schema( "message": [not_missing, unicode_safe], "package_id": [not_missing, unicode_safe, package_id_or_name_exists], } + + +@validator_args +def datatables_view_prioritize(not_empty): + return { + "resource_id": [ + not_empty, + ], + } diff --git a/ckanext/datavicmain/organisation_schema.yaml b/ckanext/datavicmain/organisation_schema.yaml index 8ef1ee04..d33769db 100644 --- a/ckanext/datavicmain/organisation_schema.yaml +++ b/ckanext/datavicmain/organisation_schema.yaml @@ -10,6 +10,7 @@ fields: form_attrs: data-module: slug-preview-target form_placeholder: My Organization + required: true - field_name: name label: URL @@ -28,10 +29,6 @@ fields: validators: scheming_required form_placeholder: http://example.com/my-image.jpg - - field_name: parent - label: Parent - form_snippet: org_hierarchy.html - - field_name: visibility label: Visibility form_snippet: vic_org_visibility.html @@ -43,3 +40,9 @@ fields: label: Unrestricted - value: restricted label: Restricted + + - field_name: parent + label: Parent + form_snippet: org_hierarchy.html + validators: ignore_empty + required: true diff --git a/ckanext/datavicmain/plugins.py b/ckanext/datavicmain/plugins.py index 484c3313..ae7df7e7 100644 --- a/ckanext/datavicmain/plugins.py +++ b/ckanext/datavicmain/plugins.py @@ -5,8 +5,6 @@ import calendar import logging from typing import Any -from six import text_type -from typing import Any from datetime import datetime import ckan.authz as authz @@ -242,6 +240,8 @@ def get_helpers(self): 'get_digital_twin_resources': helpers.get_digital_twin_resources, 'url_for_dtv_config': helpers.url_for_dtv_config, "datavic_org_uploads_allowed": helpers.datavic_org_uploads_allowed, + "get_group": helpers.get_group, + "dtv_exceeds_max_size_limit": helpers.dtv_exceeds_max_size_limit, "datavic_user_is_a_member_of_org": helpers.datavic_user_is_a_member_of_org, "datavic_is_pending_request_to_join_org": helpers.datavic_is_pending_request_to_join_org, "datavic_is_org_restricted": helpers.datavic_is_org_restricted, @@ -258,6 +258,7 @@ def get_helpers(self): "datavic_get_org_roles": helpers.datavic_get_org_roles, "datavic_get_user_roles_in_org": helpers.datavic_get_user_roles_in_org, "datavic_allowable_parent_orgs": helpers.datavic_allowable_parent_orgs, + "has_user_capacity": helpers.has_user_capacity, } ## IConfigurer interface ## diff --git a/ckanext/datavicmain/templates/admin/trash.html b/ckanext/datavicmain/templates/admin/trash.html index 3da96d85..eef68359 100644 --- a/ckanext/datavicmain/templates/admin/trash.html +++ b/ckanext/datavicmain/templates/admin/trash.html @@ -4,7 +4,7 @@ {% set truncate = truncate or 180 %} {% set truncate_title = truncate_title or 80 %} -
- {{ h.csrf_input() }} -
{% endblock %} diff --git a/ckanext/datavicmain/templates/organization/read_base.html b/ckanext/datavicmain/templates/organization/read_base.html index 151c6cc9..a53fec4f 100644 --- a/ckanext/datavicmain/templates/organization/read_base.html +++ b/ckanext/datavicmain/templates/organization/read_base.html @@ -14,4 +14,15 @@ {{ super() }} {% endblock %} -{% endblock %} \ No newline at end of file +{% endblock %} + +{% block content_primary_nav %} + {{ h.build_nav_icon(group_type + '.read', h.humanize_entity_type('package', dataset_type, 'content tab') or _('Datasets'), id=group_dict.name, icon='sitemap') }} + {{ h.build_nav_icon(group_type + '.about', _('About'), id=group_dict.name, icon='info-circle') }} + + {% set is_admin = h.has_user_capacity(group_dict.name, g.userobj.id, 'admin') %} + {% set is_editor = h.has_user_capacity(group_dict.name, g.userobj.id, 'editor') %} + {% if g.userobj.sysadmin or is_admin or is_editor %} + {{ h.build_nav_icon('activity.organization_activity', _('Activity Stream'), id=group_dict.name, offset=0, icon='clock') }} + {% endif %} +{% endblock %} diff --git a/ckanext/datavicmain/templates/package/base.html b/ckanext/datavicmain/templates/package/base.html index 89793d72..5477a8ff 100644 --- a/ckanext/datavicmain/templates/package/base.html +++ b/ckanext/datavicmain/templates/package/base.html @@ -1,7 +1,7 @@ {% ckan_extends %} {% block breadcrumb_content %} - {#} check if user has an access to parent org before showing it in breadcrumb {#} + {# check if user has an access to parent org before showing it in breadcrumb #} {% if pkg %} {% set dataset = h.dataset_display_name(pkg) %} diff --git a/ckanext/datavicmain/templates/package/read.html b/ckanext/datavicmain/templates/package/read.html index dec5e328..02996ed9 100644 --- a/ckanext/datavicmain/templates/package/read.html +++ b/ckanext/datavicmain/templates/package/read.html @@ -23,9 +23,6 @@

{% endblock %} {% endblock %} -{% block package_tags %} -{% endblock %} - {% block package_resources %} {% set delwp_dataset = h.vic_iar_is_delwp_dataset(pkg) %} {% set delwp_restricted = h.vic_iar_is_delwp_dataset_restricted(pkg) %} diff --git a/ckanext/datavicmain/templates/package/read_base.html b/ckanext/datavicmain/templates/package/read_base.html index 2574c28d..8ae0a704 100644 --- a/ckanext/datavicmain/templates/package/read_base.html +++ b/ckanext/datavicmain/templates/package/read_base.html @@ -2,8 +2,12 @@ {% block content_primary_nav %} {{ h.build_nav_icon('dataset.read', _('Dataset'), id=pkg.name, icon=None) }} - {{ h.build_nav_icon('dataset.groups', _('Categories'), id=pkg.name, icon=None) }} - {{ h.build_nav_icon('activity.package_activity', _('Activity Stream'), id=pkg.name, icon=None) }} + + {% set is_admin = h.has_user_capacity(pkg.organization.id, g.userobj.id, 'admin') %} + {% set is_editor = h.has_user_capacity(pkg.organization.id, g.userobj.id, 'editor') %} + {% if g.userobj.sysadmin or is_admin or is_editor %} + {{ h.build_nav_icon('activity.package_activity', _('Activity Stream'), id=pkg.name, icon=None) }} + {% endif %} {% if h.group_resources_by_temporal_range(pkg.resources) | length > 1 %} {{ h.build_nav_icon('datavicmain.historical', _('Historical Data and Resources'), package_type=pkg.type, package_id=pkg.name, icon=None) }} diff --git a/ckanext/datavicmain/templates/package/snippets/datavic_dtv.html b/ckanext/datavicmain/templates/package/snippets/datavic_dtv.html index c5f99c5d..68b5888e 100644 --- a/ckanext/datavicmain/templates/package/snippets/datavic_dtv.html +++ b/ckanext/datavicmain/templates/package/snippets/datavic_dtv.html @@ -1,9 +1,10 @@ {% set dtv_resources = h.get_digital_twin_resources(pkg.id)|map(attribute="id")|list %} {% set dtv_preview = pkg.dtv_preview %} +{% set dtv_exceeds_limit = h.dtv_exceeds_max_size_limit(dtv_resources[0]) %} {% set dtv_url = h.datavic_get_dtv_url() %} {% set dtv_external_link = h.datavic_get_dtv_url(ext_link=True) %} -{% if dtv_resources and dtv_url and dtv_preview is not sameas false %} +{% if dtv_resources and dtv_url and not dtv_exceeds_limit and dtv_preview is not sameas false %}

{{ _('Map preview') }}

diff --git a/ckanext/datavicmain/templates/scheming/form_snippets/org_hierarchy.html b/ckanext/datavicmain/templates/scheming/form_snippets/org_hierarchy.html index eccd5cf8..05c90baa 100644 --- a/ckanext/datavicmain/templates/scheming/form_snippets/org_hierarchy.html +++ b/ckanext/datavicmain/templates/scheming/form_snippets/org_hierarchy.html @@ -6,7 +6,11 @@ {% set visibility = data.visibility or "unrestricted" %} {% endif %} -{% call form.input_block("field-parent", label=_("Parent")) %} +{% call form.input_block( + "field-parent", + label=_("Parent"), + is_required=h.scheming_field_required(field)) +%} + + {%- snippet 'scheming/form_snippets/help_text.html', field=field -%} +
+ diff --git a/ckanext/datavicmain_home/templates/scheming/form_snippets/vic_text.html b/ckanext/datavicmain_home/templates/scheming/form_snippets/vic_text.html new file mode 100644 index 00000000..c0eb7779 --- /dev/null +++ b/ckanext/datavicmain_home/templates/scheming/form_snippets/vic_text.html @@ -0,0 +1,19 @@ +{#} alterations: type=field.input_type {#} + +{% import 'macros/form.html' as form %} + +{% call form.input( + field.field_name, + id='field-' + field.field_name, + label=h.scheming_language_text(field.label), + placeholder=h.scheming_language_text(field.form_placeholder), + value=data[field.field_name], + error=errors[field.field_name], + classes=field.classes if 'classes' in field else ['control-medium'], + attrs=dict({"class": "form-control"}, **(field.get('form_attrs', {}))), + is_required=h.scheming_field_required(field), + type=field.input_type + ) +%} + {%- snippet 'scheming/form_snippets/help_text.html', field=field -%} +{% endcall %} diff --git a/ckanext/datavicmain_home/templates/scheming/form_snippets/vic_upload.html b/ckanext/datavicmain_home/templates/scheming/form_snippets/vic_upload.html new file mode 100644 index 00000000..6fe584a7 --- /dev/null +++ b/ckanext/datavicmain_home/templates/scheming/form_snippets/vic_upload.html @@ -0,0 +1,32 @@ +{% import 'macros/form.html' as form %} + +
+ + + {% if data.image_id %} +
+ {% set file_info = h.files_link_details(data.image_id) %} + +
+ Uploaded Image + +
+ + + +
+ {% endif %} + +
+ + + {%- if field.help_text -%} + {% set text = h.scheming_language_text(field.help_text) %} + {{- form.info( + text=text|safe if field.get('help_allow_html', false) else text, + inline=field.get('help_inline', false) + ) -}} + {%- endif -%} + +
+
diff --git a/ckanext/datavicmain_home/templates/vic_home/add_or_edit.html b/ckanext/datavicmain_home/templates/vic_home/add_or_edit.html new file mode 100644 index 00000000..5016e9fa --- /dev/null +++ b/ckanext/datavicmain_home/templates/vic_home/add_or_edit.html @@ -0,0 +1,18 @@ +{% extends "vic_home/base.html" %} + +{% block primary_content_inner %} +
+ {{ h.csrf_input() }} + + {% snippet 'vic_home/render_fields.html', fields=schema.fields, data=data, errors=errors %} + +
+ {{ _('Cancel') }} + +
+
+{% endblock %} + +{% block vic_info_content %} +

{{ _('Create a new entity to display on a home page') }}

+{% endblock %} diff --git a/ckanext/datavicmain_home/templates/vic_home/base.html b/ckanext/datavicmain_home/templates/vic_home/base.html new file mode 100644 index 00000000..0e48e399 --- /dev/null +++ b/ckanext/datavicmain_home/templates/vic_home/base.html @@ -0,0 +1,47 @@ +{% extends "admin/base.html" %} + +{% import 'macros/form.html' as form %} + +{% block breadcrumb_content %} +
  • {{ h.nav_link(_('Manage home items'), named_route='datavic_home.manage') }}
    • +{% endblock %} + +{% block primary %} +
    + {% block primary_content %} + {% block primary_content_inner %} + {% endblock %} + {% endblock %} +
    +{% endblock %} + +{% block secondary %} + +{% endblock %} + +{% block styles %} + {% asset "vic_home/css-global" %} + + {{ super() }} +{% endblock %} + +{% block scripts %} + {% asset "vic_home/js-global" %} + + {{ super() }} +{% endblock %} diff --git a/ckanext/datavicmain_home/templates/vic_home/manage.html b/ckanext/datavicmain_home/templates/vic_home/manage.html new file mode 100644 index 00000000..0f4a0c6b --- /dev/null +++ b/ckanext/datavicmain_home/templates/vic_home/manage.html @@ -0,0 +1,53 @@ +{% extends "vic_home/base.html" %} + +{% block primary_content_inner %} +

    + {{ _("Add new item") }} +

    + + + + + + + + + + + + + + + + + + + + + + {% for home_item in home_items %} + + + + + + + + {% endfor %} + +
    {{ _('Title') }}{{ _('Description') }}{{ _('State') }}{{ _('Type') }}{{ _('Actions') }}
    {{ home_item.title }}{{ home_item.description | truncate (70) }}{{ home_item.state }}{{ home_item.section_type }} +
    + + + + + + + +
    +
    +{% endblock %} + +{% block vic_info_content %} +

    {{ _('You can manage the sections on the home page here.') }}

    +{% endblock %} diff --git a/ckanext/datavicmain_home/templates/vic_home/render_fields.html b/ckanext/datavicmain_home/templates/vic_home/render_fields.html new file mode 100644 index 00000000..75d21c97 --- /dev/null +++ b/ckanext/datavicmain_home/templates/vic_home/render_fields.html @@ -0,0 +1,3 @@ +{% for field in fields if field.form_snippet is not none %} + {% snippet 'scheming/snippets/form_field.html', field=field, data=data, errors=errors %} +{% endfor %} diff --git a/ckanext/datavicmain_home/tests/conftest.py b/ckanext/datavicmain_home/tests/conftest.py new file mode 100644 index 00000000..a2c86ae8 --- /dev/null +++ b/ckanext/datavicmain_home/tests/conftest.py @@ -0,0 +1,41 @@ +from __future__ import annotations + +from io import BytesIO + +import pytest +import factory +from pytest_factoryboy import register + + +from ckan.tests.factories import CKANFactory + +from ckanext.datavicmain_home.model import HomeSectionItem +from ckanext.datavicmain_home.tests.helpers import MockFileStorage, PNG_IMAGE + + +@pytest.fixture() +def clean_db(reset_db, migrate_db_for): + reset_db() + + migrate_db_for("datavicmain_home") + migrate_db_for("files") + + +@register +class HomeSectionItemFactory(CKANFactory): + class Meta: + model = HomeSectionItem + action = "create_section_item" + + title = factory.Faker("sentence") + description = factory.Faker("sentence") + # image_id = factory.Faker("uuid4") + upload = factory.LazyAttribute( + lambda _: MockFileStorage(BytesIO(PNG_IMAGE), "image.png") + ) + + url = factory.Faker("url") + entity_url = factory.Faker("url") + state = HomeSectionItem.State.active + section_type = HomeSectionItem.SectionType.news + weight = 0 diff --git a/ckanext/datavicmain_home/tests/helpers.py b/ckanext/datavicmain_home/tests/helpers.py new file mode 100644 index 00000000..3a161617 --- /dev/null +++ b/ckanext/datavicmain_home/tests/helpers.py @@ -0,0 +1,3 @@ +from werkzeug.datastructures import FileStorage as MockFileStorage # noqa + +PNG_IMAGE = b"\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00\x01\x00\x00\x00\x01\x08\x06\x00\x00\x00\x1f\x15\xc4\x89\x00\x00\x00\nIDATx\xdac\xf8\x0f\x00\x01\x01\x01\x00\xa7(\xa5\x8d\x00\x00\x00\x00IEND\xaeB`\x82" diff --git a/ckanext/datavicmain_home/tests/logic/test_action.py b/ckanext/datavicmain_home/tests/logic/test_action.py new file mode 100644 index 00000000..d8cdf889 --- /dev/null +++ b/ckanext/datavicmain_home/tests/logic/test_action.py @@ -0,0 +1,197 @@ +from __future__ import annotations + +from typing import TypedDict +from io import BytesIO + +import pytest + +import ckan.plugins.toolkit as tk +from ckan.tests.helpers import call_action + +from ckanext.datavicmain_home.model import HomeSectionItem +from ckanext.datavicmain_home.tests.helpers import MockFileStorage, PNG_IMAGE + + +class HomeSectionData(TypedDict): + id: str + title: str + description: str + image_id: str + url: str + entity_url: str + state: str + section_type: str + weight: int + created_at: str + modified_at: str + url_in_new_tab: bool + + +@pytest.mark.usefixtures("with_plugins", "clean_db") +class TestHomeSectionItemCreate: + def test_basic_create(self, home_section_item_factory): + home_section_item: HomeSectionData = home_section_item_factory() + + assert home_section_item["id"] + assert home_section_item["title"] + assert home_section_item["description"] + assert home_section_item["image_id"] + assert home_section_item["url"] + assert home_section_item["entity_url"] + assert home_section_item["state"] == HomeSectionItem.State.active + assert home_section_item["weight"] == 0 + assert home_section_item["created_at"] + assert home_section_item["modified_at"] + assert home_section_item["url_in_new_tab"] == False + assert home_section_item["section_type"] + + def test_create_with_invalid_url(self, home_section_item_factory): + with pytest.raises(tk.ValidationError): + home_section_item_factory(url="invalid-url") + + def test_create_with_invalid_state(self, home_section_item_factory): + with pytest.raises(tk.ValidationError): + home_section_item_factory(state="invalid-state") + + def test_create_without_title(self, home_section_item_factory): + with pytest.raises(tk.ValidationError): + home_section_item_factory(title="") + + def test_create_without_description(self, home_section_item_factory): + home_section_item = home_section_item_factory(description="") + + assert home_section_item["description"] == "" + + +@pytest.mark.usefixtures("with_plugins", "clean_db") +class TestHomeSectionItemDelete: + def test_basic_delete(self, home_section_item_factory): + home_section_item: HomeSectionData = home_section_item_factory() + + call_action("delete_section_item", id=home_section_item["id"]) + + result = call_action( + "get_section_items_by_section_type", + section_type=HomeSectionItem.SectionType.news, + ) + + assert not result + + def test_delete_with_invalid_id(self): + with pytest.raises(tk.ValidationError): + call_action("delete_section_item", id="invalid-id") + + +@pytest.mark.usefixtures("with_plugins", "clean_db") +class TestHomeSectionItemUpdate: + def test_basic_update(self, home_section_item_factory): + home_section_item: HomeSectionData = home_section_item_factory() + + new_title = "New title" + new_description = "New description" + new_url = "https://example.com" + new_state = HomeSectionItem.State.inactive + new_section_type = HomeSectionItem.SectionType.data + new_weight = 11 + + result = call_action( + "update_section_item", + id=home_section_item["id"], + title=new_title, + description=new_description, + upload=MockFileStorage(BytesIO(PNG_IMAGE), "image.png"), + url=new_url, + entity_url=new_url, + state=new_state, + section_type=new_section_type, + weight=new_weight, + url_in_new_tab=True, + ) + + assert result["title"] == new_title + assert result["description"] == new_description + assert result["image_id"] != home_section_item["image_id"] + assert result["url"] == new_url + assert result["entity_url"] == new_url + assert result["state"] == new_state + assert result["section_type"] == new_section_type + assert result["weight"] == new_weight + assert result["url_in_new_tab"] == True + + def test_update_with_invalid_url(self, home_section_item_factory): + home_section_item = home_section_item_factory() + + with pytest.raises(tk.ValidationError): + call_action( + "update_section_item", + id=home_section_item["id"], + url="invalid-url", + ) + + def test_update_with_invalid_state(self, home_section_item_factory): + home_section_item = home_section_item_factory() + + with pytest.raises(tk.ValidationError): + call_action( + "update_section_item", + id=home_section_item["id"], + state="invalid-state", + ) + + def test_update_without_title(self, home_section_item_factory): + home_section_item = home_section_item_factory() + + result = call_action( + "update_section_item", + id=home_section_item["id"], + title="", + ) + + assert result["title"] == "" + + def test_update_without_description(self, home_section_item_factory): + home_section_item = home_section_item_factory() + + result = call_action( + "update_section_item", + id=home_section_item["id"], + description="", + ) + + assert result["description"] == "" + + +@pytest.mark.usefixtures("with_plugins", "clean_db") +class TestGetItemsBySectionType: + def test_get_items_by_section_type(self, home_section_item_factory): + home_section_item: HomeSectionData = home_section_item_factory() + + result = call_action( + "get_section_items_by_section_type", + section_type=HomeSectionItem.SectionType.news, + ) + + assert result[0]["id"] == home_section_item["id"] + + def test_no_items(self): + result = call_action( + "get_section_items_by_section_type", + section_type=HomeSectionItem.SectionType.news, + ) + + assert not result + + +@pytest.mark.usefixtures("with_plugins", "clean_db") +class TestGetAllItems: + def test_get_all_items(self, home_section_item_factory): + home_section_item: HomeSectionData = home_section_item_factory() + + result = call_action("get_all_section_items") + + assert result[0]["id"] == home_section_item["id"] + + def test_no_items(self): + result = call_action("get_all_section_items") + + assert not result diff --git a/ckanext/datavicmain_home/tests/test_helpers.py b/ckanext/datavicmain_home/tests/test_helpers.py new file mode 100644 index 00000000..b84f4a55 --- /dev/null +++ b/ckanext/datavicmain_home/tests/test_helpers.py @@ -0,0 +1,45 @@ +import pytest + +import ckan.plugins.toolkit as tk + + +@pytest.mark.usefixtures("with_plugins", "clean_db") +class TestGetHomeSections: + def test_no_items(self): + assert not tk.h.vic_home_get_sections() + + def test_one_item(self, home_section_item_factory): + home_section_item = home_section_item_factory(section_type="test") + assert tk.h.vic_home_get_sections() == [ + home_section_item["section_type"] + ] + + def test_reoder(self, home_section_item_factory): + home_section_item_factory(section_type="test") + home_section_item_factory(section_type="test 2", weight="-1") + + assert tk.h.vic_home_get_sections() == ["test 2", "test"] + + +@pytest.mark.usefixtures("with_plugins", "clean_db") +class TestGetSectionItemsBySectionType: + def test_no_items(self): + assert not tk.h.get_item_by_section_type("test") + + def test_one_item(self, home_section_item_factory): + home_section_item = home_section_item_factory(section_type="test") + + assert tk.h.get_item_by_section_type("test") == [home_section_item] + + def test_two_items(self, home_section_item_factory): + home_section_item_factory(section_type="test") + home_section_item_factory(section_type="test") + + assert len(tk.h.get_item_by_section_type("test")) == 2 + + def test_two_items_different_section(self, home_section_item_factory): + home_section_item_factory(section_type="test") + home_section_item_factory(section_type="test 2") + + assert len(tk.h.get_item_by_section_type("test 2")) == 1 + assert len(tk.h.get_item_by_section_type("test")) == 1 diff --git a/ckanext/datavicmain_home/tests/test_utils.py b/ckanext/datavicmain_home/tests/test_utils.py new file mode 100644 index 00000000..812c905f --- /dev/null +++ b/ckanext/datavicmain_home/tests/test_utils.py @@ -0,0 +1,9 @@ +from ckanext.datavicmain_home.utils import get_config_schema + + +class TestGetConfigSchema: + def test_get_config_schema(self): + schema = get_config_schema() + + assert schema + assert isinstance(schema, dict) diff --git a/ckanext/datavicmain_home/utils.py b/ckanext/datavicmain_home/utils.py new file mode 100644 index 00000000..b064a3d9 --- /dev/null +++ b/ckanext/datavicmain_home/utils.py @@ -0,0 +1,14 @@ +from __future__ import annotations + +from typing import Any + + +def get_config_schema() -> dict[Any, Any]: + from ckanext.scheming.plugins import _load_schemas, _expand_schemas + + schemas = _load_schemas( + ["ckanext.datavicmain_home:config_schema.yaml"], "schema_id" + ) + expanded_schemas = _expand_schemas(schemas) + + return expanded_schemas["datavicmain_home_item"] diff --git a/ckanext/datavicmain_home/views.py b/ckanext/datavicmain_home/views.py new file mode 100644 index 00000000..7b83b82f --- /dev/null +++ b/ckanext/datavicmain_home/views.py @@ -0,0 +1,200 @@ +from __future__ import annotations + +import logging +from typing import Union, Any, cast + +from flask.views import MethodView +from flask import Blueprint, jsonify + +from ckan import types, model +from ckan.plugins import toolkit as tk +from ckan.types import Response +from ckan.logic import parse_params +from ckan.lib.navl.dictization_functions import convert + +from ckanext.scheming.validation import validators_from_string + +from ckanext.datavicmain_home import utils + +log = logging.getLogger(__name__) + +datavic_home = Blueprint("datavic_home", __name__, url_prefix="/vic-home") + + +class HomeManage(MethodView): + def get(self) -> str: + return tk.render( + "vic_home/manage.html", + extra_vars={ + "home_items": tk.get_action("get_all_section_items")({}, {}), + }, + ) + + def post(self) -> Union[str, Response]: + tk.h.flash_success("Hello world") + + return tk.redirect_to("datavic_home.manage") + + +class HomeItemCreateOrUpdate(MethodView): + def validate_data( + self, + data: dict[str, Any], + schema: dict[str, Any], + errors: types.FlattenErrorDict, + ) -> tuple[dict[str, Any], types.FlattenErrorDict]: + for field_name in data: + field: dict[str, Any] | None = tk.h.scheming_field_by_name( + schema["fields"], field_name + ) + + if not field: + continue + + if "validators" not in field: + continue + + for validator in validators_from_string( + field["validators"], field, schema + ): + try: + convert(validator, field_name, data, errors, context={}) + except tk.StopOnError: + return data, errors + + return data, errors + + +class HomeItemCreate(HomeItemCreateOrUpdate): + def get(self) -> str: + return tk.render( + "vic_home/add_or_edit.html", + extra_vars={ + "errors": {}, + "data": {}, + "schema": utils.get_config_schema(), + }, + ) + + def post(self) -> Union[str, Response]: + data = parse_params(tk.request.form) + schema = utils.get_config_schema() + + if file := tk.request.files.get("upload"): + data["upload"] = file + + errors: types.FlattenErrorDict = dict((key, []) for key in data) + data, errors = self.validate_data(data, schema, errors) + + if any(list(errors.values())): + return tk.render( + "vic_home/add_or_edit.html", + extra_vars={ + "errors": errors, + "data": data, + "schema": schema, + }, + ) + + tk.get_action("create_section_item")({}, data) + + tk.h.flash_success("The item has been created") + + return tk.redirect_to("datavic_home.manage") + + +class HomeItemEdit(HomeItemCreateOrUpdate): + action = "update_section_item" + + def get(self, item_id: str) -> str: + try: + data = tk.get_action("get_section_item")({}, {"id": item_id}) + except tk.ObjectNotFound: + tk.abort(404) + + return tk.render( + "vic_home/add_or_edit.html", + extra_vars={ + "errors": {}, + "data": data, + "schema": utils.get_config_schema(), + }, + ) + + def post(self, item_id: str) -> Union[str, Response]: + try: + tk.get_action("get_section_item")({}, {"id": item_id}) + except tk.ObjectNotFound: + tk.abort(404) + + data = parse_params(tk.request.form) + schema = utils.get_config_schema() + + if file := tk.request.files.get("upload"): + data["upload"] = file + + data["id"] = item_id + + errors: types.FlattenErrorDict = dict((key, []) for key in data) + data, errors = self.validate_data(data, schema, errors) + + if any(list(errors.values())): + return tk.render( + "vic_home/add_or_edit.html", + extra_vars={ + "errors": errors, + "data": data, + "schema": schema, + }, + ) + + tk.get_action("update_section_item")({}, data) + + tk.h.flash_success("The item has been updated") + + return tk.redirect_to("datavic_home.manage") + + +class HomeItemDelete(MethodView): + def post(self, item_id: str) -> Union[str, Response]: + try: + tk.get_action("get_section_item")({}, {"id": item_id}) + except tk.ObjectNotFound: + tk.abort(404) + + tk.get_action("delete_section_item")({}, {"id": item_id}) + + tk.h.flash_success("The item has been deleted") + + return tk.redirect_to("datavic_home.manage") + + +def section_autocomplete() -> Response: + q = tk.request.args.get("incomplete", "") + + if not q: + return jsonify({"ResultSet": {"Result": []}}) + + return jsonify( + { + "ResultSet": { + "Result": [ + {"Name": section} + for section in tk.h.vic_home_get_sections() + ] + } + } + ) + + +datavic_home.add_url_rule("/manage", view_func=HomeManage.as_view("manage")) +datavic_home.add_url_rule("/new", view_func=HomeItemCreate.as_view("new")) +datavic_home.add_url_rule( + "/edit/", view_func=HomeItemEdit.as_view("edit") +) +datavic_home.add_url_rule( + "/delete/", view_func=HomeItemDelete.as_view("delete") +) +datavic_home.add_url_rule( + "/section_autocomplete", view_func=section_autocomplete +) diff --git a/setup.py b/setup.py index 691660c0..4b4e561a 100644 --- a/setup.py +++ b/setup.py @@ -29,6 +29,7 @@ """ [ckan.plugins] datavicmain_dataset = ckanext.datavicmain.plugins:DatasetForm + datavicmain_home = ckanext.datavicmain_home.plugins:DatavicHomePlugin [fanstatic.libraries] """, diff --git a/test.ini b/test.ini index 6a05282c..296d8c87 100644 --- a/test.ini +++ b/test.ini @@ -13,16 +13,19 @@ use = config:../ckan/test-core.ini ckan.plugins = activity hierarchy - datavicmain_dataset + datavicmain_dataset datavicmain_home datavic_iar_theme scheming_datasets scheming_organizations alias + files oidc_pkce flakes pages workflow mailcraft +ckanext.files.storage.default.type = files:redis + scheming.dataset_schemas = ckanext.datavicmain:iar_ckan_dataset.yaml scheming.organization_schemas =