Release 2.9.0 [PROD]

ckanext/datavicmain/cli/maintain.py

@@ -3,23 +3,21 @@
 import copy
 import datetime
 import logging
-import csv
-import openpyxl
 import mimetypes
 
 from os import path, stat
-from typing import Any
-from sqlalchemy.orm import Query
 from itertools import groupby
+from os import path
+from typing import Any
 from urllib.parse import urlparse
 
-import click
-import tqdm
-
 import ckan.logic.validators as validators
 import ckan.model as model
 import ckan.plugins.toolkit as tk
 
+import click
+import openpyxl
+import tqdm
 from ckan.lib.munge import munge_title_to_name
 from ckan.lib.search import rebuild
 from ckan.lib.uploader import get_resource_uploader
@@ -29,6 +27,8 @@
 from ckanext.harvest.model import HarvestObject, HarvestSource
 from ckanext.datastore.backend import get_all_resources_ids_in_datastore
 
+from ckanext.harvest.model import HarvestObject, HarvestSource
+from sqlalchemy.orm import Query
 
 log = logging.getLogger(__name__)
 
@@ -576,7 +576,7 @@ def _search_in_batch(datasets: list[dict[str, Any]]) -> dict[str, dict[str, str]
 
 def _get_default_values_for_missing_fields(
     dataset: dict[str, Any], field: str
-) -> "str":
+) -> str:
     """Get values for missing fields"""
     if field == "date_created_data_asset":
         return _get_date_created(dataset)
@@ -590,6 +590,7 @@ def _get_default_values_for_missing_fields(
         return "official"
     elif field == "category":
         return _get_category(dataset)
+    return ""
 
 
 def _get_date_created(pkg: dict[str, Any]) -> str:
@@ -615,7 +616,7 @@ def _get_category(pkg: dict[str, Any]) -> str:
     """
     if groups := pkg.get("groups"):
         return groups[0].get("id")
-    return
+    return ""
 
 
 @maintain.command(u"update-broken-urls",
@@ -624,7 +625,8 @@ def update_broken_urls():
     """Change resources urls' protocols from http to https listed in XLSX file"""
 
     file = path.join(
-        path.dirname(__file__), "data/DTF Content list bulk URL change 20231017.xlsx"
+        path.dirname(__file__),
+        "data/DTF Content list bulk URL change 20231017.xlsx",
     )
     wb = openpyxl.load_workbook(file)
     ws = wb.active
@@ -641,15 +643,14 @@ def update_broken_urls():
 
         if not resource:
             click.secho(
-                f"Resource <{title}> with URL <{url}> does not exist",
-                fg="red"
+                f"Resource <{title}> with URL <{url}> does not exist", fg="red"
             )
             continue
 
         resource.url = row[XLSX_IDX_NEW_URL].value
         click.secho(
             f"URL of resource <{title}> has been updated to <{resource.url}>",
-            fg="green"
+            fg="green",
         )
 
         model.Session.commit()
@@ -825,3 +826,18 @@ def get_resources_by_size(empty: bool, limit: bool, restricted: bool):
         f"Found {resources.count()} resources...",
         fg="green",
     )
+
+
+@maintain.command("make-datatables-view-prioritized")
+def make_datatables_view_prioritized():
+    """Check if there are resources that have recline_view and datatables_view and
+    reorder them so that datatables_view is first."""
+    resources = model.Session.query(Resource).all()
+    number_reordered = 0
+    for resource in tqdm.tqdm(resources):
+        result = tk.get_action("datavic_datatables_view_prioritize")(
+            {"ignore_auth": True}, {"resource_id": resource.id}
+        )
+        if result.get("updated"):
+            number_reordered += 1
+    click.secho(f"Reordered {number_reordered} resources", fg="green")

ckanext/datavicmain/config.py

@@ -4,6 +4,7 @@
 
 CONFIG_PAGES_BASE_URL = "ckan.pages.base_url"
 CONFIG_DTV_URL = "ckanext.datavicmain.dtv.url"
+CONFIG_DTV_MAX_SIZE_LIMIT = "ckanext.datavicmain.dtv.max_size_limit"
 CONFIG_DTV_EXTERNAL_LINK = "ckanext.datavicmain.dtv.external_link"
 
 
@@ -15,5 +16,9 @@ def get_dtv_url() -> str:
     return tk.config.get(CONFIG_DTV_URL, "")
 
 
+def get_dtv_max_size_limit() -> int:
+    return tk.config.get(CONFIG_DTV_MAX_SIZE_LIMIT, "157286400")
+
+
 def get_dtv_external_link() -> str:
-    return tk.config.get(CONFIG_DTV_EXTERNAL_LINK, "")
+    return tk.config.get(CONFIG_DTV_EXTERNAL_LINK, "")

ckanext/datavicmain/helpers.py

@@ -1,5 +1,4 @@
 from __future__ import annotations
-from json import tool
 
 import math
 import os
@@ -8,26 +7,23 @@
 import logging
 import json
 import base64
-from typing import Any
+from typing import Any, Optional
 
 from urllib.parse import urlsplit, urljoin
 
 from flask import Blueprint
 
-import ckan.plugins as plugins
 import ckan.model as model
 import ckan.authz as authz
 import ckan.plugins.toolkit as toolkit
 
 from ckanext.harvest.model import HarvestObject
 from ckanext.activity.model.activity import Activity
-from ckanext.mailcraft.utils import get_mailer
-from ckanext.mailcraft.exception import MailerException
 
-from . import utils, const
+from . import utils, const, config as conf
 from ckanext.datavicmain.config import get_dtv_url, get_dtv_external_link
 
-mailer = get_mailer()
+
 log = logging.getLogger(__name__)
 WORKFLOW_STATUS_OPTIONS = [
     "draft",
@@ -368,6 +364,43 @@ def datavic_org_uploads_allowed(org_id: str) -> bool:
     return flake["data"].get(org.id, False)
 
 
+def get_group(group: Optional[str] = None,
+              include_datasets: bool = False) -> dict[str, Any]:
+    if group is None:
+        return {}
+    try:
+        return toolkit.get_action("group_show")(
+            {},
+            {"id": group, "include_datasets": include_datasets}
+        )
+    except (toolkit.NotFound, toolkit.ValidationError, toolkit.NotAuthorized):
+        return {}
+
+
+def dtv_exceeds_max_size_limit(resource_id: str) -> bool:
+    """Check if DTV resource exceeds the maximum file size limit
+
+    Args:
+        resource_id (str): DTV resource id
+
+    Returns:
+        bool: return True if dtv resource exceeds maximum file size limit set
+            in ckan config "ckanext.datavicmain.dtv.max_size_limit",
+            otherwise - False  
+    """
+    try:
+        resource = toolkit.get_action("resource_show")({}, {"id": resource_id})
+    except (toolkit.ObjectNotFound, toolkit.NotAuthorized):
+        return True
+
+    limit = conf.get_dtv_max_size_limit()
+    filesize = resource.get("filesize")
+    if filesize and int(filesize) >= int(limit):
+        return True
+
+    return False
+
+
 def datavic_get_org_roles() -> list[str]:
     return ["admin", "editor", "member"]
 
@@ -560,27 +593,27 @@ def _group_tree_parents(id_, type_="organization"):
 
 
 def add_current_organisation(
-    avalable_organisations: list[dict[str, Any]], current_org: dict[str, Any]
+    available_organisations: list[dict[str, Any]], current_org: dict[str, Any]
 ):
     """When user doesn't have an access to an organisation, it won't be included
     for a list of available organisations. Include it there, but check if it's
     not already there"""
 
     current_org_included = False
 
-    for organization in avalable_organisations:
+    for organization in available_organisations:
         if organization["id"] == current_org["id"]:
             current_org_included = True
             break
 
     if not current_org_included:
-        avalable_organisations.append(current_org)
+        available_organisations.append(current_org)
 
-    return avalable_organisations
+    return available_organisations
 
 
 def datavic_max_image_size():
-    """Return max size for image configurate for portal"""
+    """Return max size for image configuration for portal"""
     return toolkit.config["ckan.max_image_size"]
 
 
@@ -658,3 +691,33 @@ def datavic_allowable_parent_orgs(org_id: str = None) -> list[dict[str, Any]]:
             continue
         orgs.append(org)
     return orgs
+
+
+def has_user_capacity(
+    org_id: str,
+    current_user_id: str, 
+    capacity: Optional[str] = None) -> bool:
+    """Check if the current user has an appropriate capacity in the certain organization
+
+    Args:
+        org_id (str): the id or name of the organization
+        current_user_id (str): the id or name of the user
+        capacity (str): restrict the members returned to those with a given capacity,
+                        e.g. 'member', 'editor', 'admin', 'public', 'private'
+                        (optional, default: None)
+
+    Returns:
+        bool: True for success, False otherwise
+    """
+    try:
+        members = toolkit.get_action("member_list")(
+            {},
+            {"id": org_id, "object_type": "user", "capacity": capacity}
+        )
+        members_id = [member[0] for member in members]
+        if current_user_id in members_id:
+            return True
+    except (toolkit.ObjectNotFound, toolkit.NotAuthorized):
+        return False
+
+    return False

ckanext/datavicmain/iar_ckan_dataset.yaml

@@ -22,6 +22,7 @@ dataset_fields:
 
 - field_name: alias
   preset: dataset_alias
+  display_group: General
 
 - field_name: notes
   label: Description
@@ -152,6 +153,7 @@ dataset_fields:
       label: "Off"
   validators: default(true) boolean_validator
   output_validators: boolean_validator
+  display_group: General
 
   # Security fields