From 753c0caf13e1db6edf16d1826c600f98b592c0cc Mon Sep 17 00:00:00 2001 From: Jesse Vickery Date: Fri, 4 Oct 2024 20:00:42 +0000 Subject: [PATCH] feat(dev): harvest schema; - Used ckanext-scheming for harvest plugin. - Added en/fr schema. - Added templating. --- ckanext/canada/cli.py | 29 +-- ckanext/canada/harvesters.py | 16 +- ckanext/canada/helpers.py | 7 + ckanext/canada/plugins.py | 182 ++++++----------- ckanext/canada/schemas/harvest.yaml | 112 +++++++++++ ckanext/canada/templates/admin/base.html | 2 +- ckanext/canada/templates/package/base.html | 3 + .../templates/package/base_form_page.html | 14 +- .../canada/templates/package/edit_base.html | 9 +- ckanext/canada/templates/package/new.html | 9 + ckanext/canada/templates/package/read.html | 9 +- .../canada/templates/package/read_base.html | 34 +++- ckanext/canada/templates/package/search.html | 38 ++-- .../display_snippets/harvest_source_type.html | 5 + .../form_snippets/harvest_sources.html | 18 ++ .../package/snippets/package_form.html | 111 +++++----- .../templates/snippets/dataset_facets.html | 189 ++++++++++-------- .../canada/templates/snippets/facet_list.html | 12 +- .../templates/snippets/package_item.html | 12 +- ckanext/canada/templates/source/admin.html | 22 ++ .../canada/templates/source/admin_base.html | 19 -- ckanext/canada/templates/source/base.html | 6 + ckanext/canada/templates/source/job/list.html | 67 +++++++ .../canada/templates/source/read_base.html | 36 ++++ ckanext/canada/validators.py | 46 ++--- ckanext/canada/view.py | 1 - 26 files changed, 654 insertions(+), 354 deletions(-) create mode 100644 ckanext/canada/schemas/harvest.yaml create mode 100644 ckanext/canada/templates/package/new.html create mode 100644 ckanext/canada/templates/scheming/display_snippets/harvest_source_type.html create mode 100644 ckanext/canada/templates/scheming/form_snippets/harvest_sources.html create mode 100644 ckanext/canada/templates/source/admin.html delete mode 100644 ckanext/canada/templates/source/admin_base.html create mode 100644 ckanext/canada/templates/source/base.html create mode 100644 ckanext/canada/templates/source/job/list.html create mode 100644 ckanext/canada/templates/source/read_base.html diff --git a/ckanext/canada/cli.py b/ckanext/canada/cli.py index a0a2c46cb..4b20d9f2d 100644 --- a/ckanext/canada/cli.py +++ b/ckanext/canada/cli.py @@ -43,7 +43,7 @@ import ckanext.datastore.backend.postgres as datastore from ckanext.canada import triggers -from ckanext.canada.harvesters import PORTAL_SYNC_ID +from ckanext.canada.harvesters import PORTAL_SYNC_ID, HARVESTER_ID BOM = "\N{bom}" @@ -1908,11 +1908,11 @@ def _drop_function(name, verbose=False): pass -@canada.command(short_help="Creates the Portal Sync Harvester if it does not already exist.") +@canada.command(short_help="Creates harvest database tables.") @click.option('-f', '--refresh', is_flag=True, type=click.BOOL, help='Forces the refresh of all the source objects in the database.') @click.option('-q', '--quiet', is_flag=True, type=click.BOOL, help='Suppresses human inspection.') -def init_portal_harvester(refresh=False, quiet=False): - """Creates the Portal Sync Harvester if it does not already exist.""" +def init_harvest_plugin(refresh=False, quiet=False): + """Creates harvest database tables.""" # check to see if the harvet_object table exists. # bad workaround, but subclassing a plugin does not allow @@ -1960,24 +1960,3 @@ def init_portal_harvester(refresh=False, quiet=False): model.Session.query(model.Package).filter(model.Package.id == PORTAL_SYNC_ID).delete() model.Session.commit() _success_message('Successfully purged harvest source objects.') - - try: - package = get_action('package_show')(_get_site_user_context(), {'id': PORTAL_SYNC_ID}) - if package.get('state') != 'active': - raise NotFound - _success_message('Portal Sync Harvester already exists.') - except NotFound: - _error_message('Portal Sync Harvester does not exist, creating it now...') - pkg_dict = { - 'type': 'harvest', - 'id': PORTAL_SYNC_ID, - 'name': PORTAL_SYNC_ID, - 'title': 'Portal Sync', - 'source_type': 'portal_sync', - 'url': toolkit.config.get('ckan.site_url', 'registry'), - 'source': 'registry', - 'target': 'portal', - } - get_action('package_create')(_get_site_user_context(), pkg_dict) - _success_message('Created the Portal Sync Harvester.') - pass diff --git a/ckanext/canada/harvesters.py b/ckanext/canada/harvesters.py index 711ea978e..3ffd10a3b 100644 --- a/ckanext/canada/harvesters.py +++ b/ckanext/canada/harvesters.py @@ -5,6 +5,7 @@ from ckanext.harvest.interfaces import IHarvester PORTAL_SYNC_ID = 'portal_sync_harvester' +HARVESTER_ID = 'portal_sync' class PortalSync(plugins.SingletonPlugin): @@ -14,10 +15,19 @@ class PortalSync(plugins.SingletonPlugin): """ plugins.implements(IHarvester) + def info(self): return { - 'name': 'portal_sync', - 'title': plugins.toolkit._('Portal Sync'), - 'description': plugins.toolkit._('Syncs Datasets, Resources, Views, and DataStores from the Registry to the Portal.'), + 'name': HARVESTER_ID, + 'title': 'Portal Sync', + 'title_translated': { + 'en': 'Portal Sync', + 'fr': 'FR Portal Sync FR', + }, + 'description': 'Syncs Datasets, Resources, Views, and DataStores from the Registry to the Portal.', + 'description_translated': { + 'en': 'Syncs Datasets, Resources, Views, and DataStores from the Registry to the Portal.', + 'fr': 'FR Syncs Datasets, Resources, Views, and DataStores from the Registry to the Portal. FR', + }, 'form_config_interface': 'Text' } diff --git a/ckanext/canada/helpers.py b/ckanext/canada/helpers.py index fe3f77591..e5768f95c 100755 --- a/ckanext/canada/helpers.py +++ b/ckanext/canada/helpers.py @@ -901,3 +901,10 @@ def is_user_locked(user_name): return True return False + + +def get_harvester_info(source_type=None): + harvesters_info = t.get_action('harvesters_info_show')({'user': g.user}, {}) + for harvester_info in harvesters_info: + if harvester_info.get('name') == source_type: + return harvester_info diff --git a/ckanext/canada/plugins.py b/ckanext/canada/plugins.py index cd98d0985..8899d1a86 100755 --- a/ckanext/canada/plugins.py +++ b/ckanext/canada/plugins.py @@ -13,9 +13,6 @@ from ckan.lib.app_globals import set_app_global from ckan.plugins.core import plugin_loaded -from ckan.logic.schema import default_extras_schema -from ckan.logic.converters import convert_to_extras, convert_from_extras - from ckan.plugins.toolkit import ( g, h, @@ -48,6 +45,7 @@ from ckanext.security.plugin import CkanSecurityPlugin from ckanext.harvest.plugin import Harvest from ckanext.harvest.views import harvester +from ckanext.harvest.logic import validators as harvest_validators from ckanext.canada.view import ( canada_views, CanadaDatasetEditView, @@ -201,46 +199,72 @@ class CanadaHarvestPlugin(Harvest): """ p.implements(p.IValidators, inherit=True) - disallow_views = ['harvester.edit', - 'harvester.delete'] + disabled_endpoints = ['harvester.about'] - # IValidators - def get_validators(self): - return {'canada_harvester_id': - validators.canada_harvester_id, - 'canada_harvester_type': - validators.canada_harvester_type, - 'canada_harvester_source_type': - validators.canada_harvester_source_type, - 'canada_harvester_url': - validators.canada_harvester_url, - 'canada_harvester_source': - validators.canada_harvester_source, - 'canada_harvester_target': - validators.canada_harvester_target, - 'canada_harvester_title': - validators.canada_harvester_title,} + # IDatasetForm + def package_types(self): + """Use ckanext-scheming""" + return [] - # IActions - def get_actions(self): - action_functions = super(CanadaHarvestPlugin, self).get_actions() - action_functions['package_show'] = logic.portal_sync_package_show - return action_functions + # ITemplateHelpers + def get_helpers(self): + helper_functions = super(CanadaHarvestPlugin, self).get_helpers() + helper_functions['get_harvester_info'] = helpers.get_harvester_info + return helper_functions + + + # IFacets + def dataset_facets(self, facets_dict, package_type): + if package_type != 'harvest': + return facets_dict + + return {'frequency': _('Frequency'), + 'source_type': _('Source Type'), + 'organization': _('Organization'),} + + + # IValidators + def get_validators(self): + return {'portal_sync_id': + validators.portal_sync_id, + 'portal_sync_limit': + validators.portal_sync_limit, + 'harvest_source_url_validator': + harvest_validators.harvest_source_url_validator, + 'harvest_source_type_exists': + harvest_validators.harvest_source_type_exists, + 'harvest_source_config_validator': + harvest_validators.harvest_source_config_validator, + 'harvest_source_extra_validator': + harvest_validators.harvest_source_extra_validator, + 'harvest_source_frequency_exists': + harvest_validators.harvest_source_frequency_exists, + 'dataset_type_exists': + harvest_validators.dataset_type_exists, + 'harvest_source_convert_from_config': + harvest_validators.harvest_source_convert_from_config, + 'harvest_source_id_exists': + harvest_validators.harvest_source_id_exists, + 'harvest_job_exists': + harvest_validators.harvest_job_exists, + 'harvest_object_extras_validator': + harvest_validators.harvest_object_extras_validator,} # IAuthFunctions def get_auth_functions(self): auth_functions = super(CanadaHarvestPlugin, self).get_auth_functions() auth_functions['harvest_log_list'] = auth.harvest_log_list + #TODO: check other auth functions from ckanext-harvest that need limitations?? return auth_functions #IBlueprint def get_blueprint(self): """Custom blueprints for the Portal Sync single harvest source.""" - harvester.before_request(self._redirect_harvest_dataset_endpoints) + harvester.before_request(self._redirect_harvest_endpoints) return [harvester] @@ -250,94 +274,10 @@ def _not_sysadmin(self, contextual_user=None): return not contextual_user or not is_sysadmin(contextual_user) - def _redirect_harvest_dataset_endpoints(self): - if self._not_sysadmin(): + def _redirect_harvest_endpoints(self): + if self._not_sysadmin() or request.endpoint in self.disabled_endpoints: return abort(404) - #FIXME: redirect loop!! based on self.disallow_views?? - # return h.redirect_to('harvester.admin', id=PORTAL_SYNC_ID) - - - def _redirect_harvest_views(self): - if self._not_sysadmin(): - return abort(404) - #FIXME: redirect loop!! based on self.disallow_views?? - # return h.redirect_to('harvester.admin', id=PORTAL_SYNC_ID) - - - #IDatasetForm - def prepare_dataset_blueprint(self, package_type, blueprint): - # type: (str,Blueprint) -> Blueprint - """Redirect Harvest endpoints accessed from /harvest/""" - if package_type == 'harvest': - blueprint.before_request(self._redirect_harvest_dataset_endpoints) - return blueprint - - #IDatasetForm - def prepare_resource_blueprint(self, package_type, blueprint): - # type: (str,Blueprint) -> Blueprint - """Redirect Harvest endpoints accessed from /harvest/""" - if package_type == 'harvest': - blueprint.before_request(self._redirect_harvest_dataset_endpoints) - return blueprint - - - #IDatasetForm - def validate(self, context, data_dict, schema, action): - """Only sysadmins can create a harvest source.""" - if data_dict.get('type') == 'harvest' and self._not_sysadmin(context.get('user')): - return data_dict, {'type': [ - "Unsupported dataset type: {t}".format(t=data_dict.get('type'))]} - - - #IDatasetForm - def create_package_schema(self): - ignore = get_validator('ignore') - canada_harvester_id = get_validator('canada_harvester_id') - canada_harvester_type = get_validator('canada_harvester_type') - canada_harvester_source_type = get_validator('canada_harvester_source_type') - canada_harvester_url = get_validator('canada_harvester_url') - canada_harvester_source = get_validator('canada_harvester_source') - canada_harvester_target = get_validator('canada_harvester_target') - canada_harvester_title = get_validator('canada_harvester_title') - - return { - 'id': [canada_harvester_id], - 'name': [canada_harvester_id], - 'type': [canada_harvester_type], - 'source_type': [canada_harvester_source_type, convert_to_extras], - 'url': [canada_harvester_url], - 'source': [canada_harvester_source, convert_to_extras], - 'target': [canada_harvester_target, convert_to_extras], - 'title': [canada_harvester_title], - 'extras': default_extras_schema(), - '__extras': [ignore], - } - - - #IDatasetForm - def update_package_schema(self): - return self.create_package_schema() - - - #IDatasetForm - def show_package_schema(self): - not_empty = get_validator('not_empty') - ignore = get_validator('ignore') - package_id_exists = get_validator('package_id_exists') - - return { - 'id': [not_empty, package_id_exists], - 'name': [not_empty], - 'type': [not_empty], - 'source_type': [convert_from_extras, not_empty], - 'url': [not_empty], - 'source': [convert_from_extras, not_empty], - 'target': [convert_from_extras, not_empty], - 'title': [not_empty], - 'extras': default_extras_schema(), - '__extras': [ignore], - } class CanadaDatasetsPlugin(SchemingDatasetsPlugin): @@ -388,7 +328,7 @@ def get_blueprint(self): return blueprints - def _redirect_pd_dataset_endpoints(blueprint): + def _redirect_dataset_endpoints(blueprint): """ Runs before request for /dataset and /dataset//resource @@ -404,7 +344,8 @@ def _redirect_pd_dataset_endpoints(blueprint): return h.redirect_to('canada.type_redirect', resource_name=package_type) if package_type == 'harvest': - return abort(404) + if '_resource.' in request.endpoint or '.resources' in request.endpoint or not is_sysadmin(g.user): + return abort(404) #IDatasetForm @@ -430,7 +371,7 @@ def prepare_dataset_blueprint(self, package_type, blueprint): strict_slashes=False ) # redirect PD endpoints accessed from /dataset/ - blueprint.before_request(self._redirect_pd_dataset_endpoints) + blueprint.before_request(self._redirect_dataset_endpoints) return blueprint @@ -450,7 +391,7 @@ def prepare_resource_blueprint(self, package_type, blueprint): methods=['GET', 'POST'] ) # redirect PD endpoints accessed from /dataset//resource - blueprint.before_request(self._redirect_pd_dataset_endpoints) + blueprint.before_request(self._redirect_dataset_endpoints) return blueprint # IDataValidation @@ -824,7 +765,9 @@ def update_config(self, config): ckanext.canada:schemas/dataset.yaml ckanext.canada:schemas/info.yaml ckanext.canada:schemas/prop.yaml -""" +""" + ( + "ckanext.canada:schemas/harvest.yaml" if plugin_loaded('canada_harvest') else "" +) config['scheming.organization_schemas'] = 'ckanext.canada:schemas/organization.yaml' # Pretty output for Feeds @@ -854,6 +797,9 @@ def update_config(self, config): def dataset_facets(self, facets_dict, package_type): ''' Update the facets_dict and return it. ''' + if package_type == 'harvest': + return facets_dict + facets_dict.update({ 'portal_type': _('Portal Type'), 'organization': _('Organization'), diff --git a/ckanext/canada/schemas/harvest.yaml b/ckanext/canada/schemas/harvest.yaml new file mode 100644 index 000000000..b977b9187 --- /dev/null +++ b/ckanext/canada/schemas/harvest.yaml @@ -0,0 +1,112 @@ +scheming_version: 2 +dataset_type: harvest + + +form_languages: +- en +- fr +# machine translated metadata: +alternate_languages: + en: + - en-t-fr + fr: + - fr-t-en + + +sidebar_show_fields: +- url +- source_type +- frequency + + +# +# DATASET FIELDS +# +dataset_fields: + + +- field_name: id + create_validators: canada_validate_generate_uuid + unicode name_validator package_id_does_not_exist portal_sync_id portal_sync_limit + display_snippet: null + form_snippet: null + + +- field_name: name + form_snippet: null + validators: canada_output_none if_empty_same_as(id) unicode not_empty portal_sync_id + + +- preset: canada_title +- preset: canada_notes + + +- field_name: url + label: + en: Source URL + fr: FR Source URL FR + help_text: + en: The URL of the remote source to harvest from. + fr: FR The URL of the remote source to harvest from. FR + required: true + validators: scheming_required not_empty unicode_safe harvest_source_url_validator + form_attrs: + size: 110 + class: form-control + + +- field_name: source_type + label: + en: Source Type + fr: FR Source Type FR + form_snippet: harvest_sources.html + display_snippet: harvest_source_type.html + required: true + validators: scheming_required not_empty unicode_safe harvest_source_type_exists + + +- field_name: frequency + label: + en: Harvesting Frequency + fr: FR Harvesting Frequency FR + help_text: + en: Frequency in which the source is harvested. + fr: FR Frequency in which the source is harvested. FR + required: true + form_snippet: select.html + display_snippet: select.html + validators: scheming_required scheming_choices unicode_safe harvest_source_frequency_exists + choices: + - label: + en: Manual + fr: FR Manual FR + value: MANUAL + - label: + en: Monthly + fr: FR Monthly FR + value: MONTHLY + - label: + en: Weekly + fr: FR Weekly FR + value: WEEKLY + - label: + en: BiWeekly + fr: FR BiWeekly FR + value: BIWEEKLY + - label: + en: Daily + fr: FR Daily FR + value: DAILY + - label: + en: Always Run + fr: FR Always Run FR + value: ALWAYS + + +- preset: canada_owner_org + + +# +# RESOURCE FIELDS +# +resource_fields: [] diff --git a/ckanext/canada/templates/admin/base.html b/ckanext/canada/templates/admin/base.html index 854c3a250..0aef6bc14 100644 --- a/ckanext/canada/templates/admin/base.html +++ b/ckanext/canada/templates/admin/base.html @@ -16,7 +16,7 @@ {{ h.build_nav_icon('canada.ckanadmin_publish', _('Publish Records'), icon='cloud-upload') }} {{ h.build_nav_icon('canada.ckanadmin_job_queue', _('Job Queue'), icon='tasks') }} {% if h.plugin_loaded('canada_harvest') %} - {{ h.build_nav_icon('harvester.admin', _('Portal Sync'), id='portal_sync_harvester', icon='refresh') }} + {{ h.build_nav_icon('harvest.search', _('Harvesters'), icon='exchange') }} {% endif %} {{ h.build_extra_admin_nav() }} {% else %} diff --git a/ckanext/canada/templates/package/base.html b/ckanext/canada/templates/package/base.html index 7ba59a427..c381cdabe 100644 --- a/ckanext/canada/templates/package/base.html +++ b/ckanext/canada/templates/package/base.html @@ -4,6 +4,9 @@ {% set client_lang = h.lang() %} {% block breadcrumb_content %} + {% if pkg.type == 'harvest' %} +
  • {% link_for _('Harvest Sources'), named_route='harvest.search' %}
    • + {% endif %} {% set pkg_url = h.url_for(pkg.type ~ '.read', id=pkg.id if is_activity_archive else pkg.name, **({'activity_id': request.args['activity_id']} if 'activity_id' in request.args else {})) %} {% if g.action != 'new' and pkg %} {% set dataset = h.get_translated(pkg, 'title') %} diff --git a/ckanext/canada/templates/package/base_form_page.html b/ckanext/canada/templates/package/base_form_page.html index 822787575..d9f419938 100644 --- a/ckanext/canada/templates/package/base_form_page.html +++ b/ckanext/canada/templates/package/base_form_page.html @@ -2,7 +2,19 @@ {% block breadcrumb_content %} {% if g.is_registry %} - {{ h.build_nav('dataset.new', _('Create Dataset')) }} + {% if dataset_type != 'harvest' %} + {{ h.build_nav('dataset.new', _('Create Dataset')) }} + {% else %} + {{ h.build_nav('harvest.new', _('Create Harvest Source')) }} + {% endif %} + {% endif %} +{% endblock %} + +{% block subtitle %} + {% if dataset_type != 'harvest' %} + {{ super() }} + {% else %} + {{ _('Create Harvest Source') }} {% endif %} {% endblock %} diff --git a/ckanext/canada/templates/package/edit_base.html b/ckanext/canada/templates/package/edit_base.html index be6915830..91cf0540e 100644 --- a/ckanext/canada/templates/package/edit_base.html +++ b/ckanext/canada/templates/package/edit_base.html @@ -1,9 +1,16 @@ {% ckan_extends %} {% block content_primary_nav %} - {% if pkg.type != 'prop' %} {# suggested datasets have no resources #} + {% if pkg.type != 'prop' and pkg.type != 'harvest' %} {# suggested datasets have no resources #} {{ super() }} {% endif %} + {% if pkg.type == 'harvest' %} + {{ h.build_nav_icon(pkg.type ~ '.edit', _('Edit metadata'), id=pkg.name, icon='pencil') }} + {% endif %} +{% endblock %} + +{% block content_action %} + {% link_for _('View Harvest Source') if pkg.type == 'harvest' else _('View dataset'), named_route=pkg.type ~ '.read', id=pkg.name, class_='btn btn-default', icon='eye' %} {% endblock %} {% block secondary_content %} diff --git a/ckanext/canada/templates/package/new.html b/ckanext/canada/templates/package/new.html new file mode 100644 index 000000000..7b5f4cf1e --- /dev/null +++ b/ckanext/canada/templates/package/new.html @@ -0,0 +1,9 @@ +{% ckan_extends %} + +{% block subtitle %} + {% if dataset_type != 'harvest' %} + {{ super() }} + {% else %} + {{ _('Create Harvest Source') }} + {% endif %} +{% endblock %} diff --git a/ckanext/canada/templates/package/read.html b/ckanext/canada/templates/package/read.html index 3dc68efc9..b2911609e 100755 --- a/ckanext/canada/templates/package/read.html +++ b/ckanext/canada/templates/package/read.html @@ -159,9 +159,16 @@

    {{ _('Made available by the ') + owner_org_title }}

    {% block package_resources %} - {% if pkg.type != 'prop' %} + {% if pkg.type != 'prop' and pkg.type != 'harvest' %} {{ super() }} {% endif %} + {% if pkg.type == 'harvest' %} + {% set harvest_source = harvest_source or h.get_harvest_source() %} +
    +

    {{ _('Harvested Datasets') }}

    + {{ h.package_list_for_source(harvest_source.id) }} +
    + {% endif %} {% endblock %} {% block package_item_apps %}
    diff --git a/ckanext/canada/templates/package/read_base.html b/ckanext/canada/templates/package/read_base.html index a95f9bb28..9e5590547 100644 --- a/ckanext/canada/templates/package/read_base.html +++ b/ckanext/canada/templates/package/read_base.html @@ -19,15 +19,24 @@ {% if pkg.type in h.recombinant_get_types() %} {% set dataset_label = _(pkg.title) %} {% endif %} + {% if pkg.type == 'harvest' %} + {% set dataset_label = _('Harvest Source') %} + {% endif %} {{ h.build_nav_icon(pkg.type + '.read', dataset_label, id=pkg.name) }} {{ h.build_nav_icon(pkg.type + '.activity', _('Activity Stream'), id=pkg.name) }} {% endif %} {% endblock %} {% block content_action %} - {% if pkg.type not in h.recombinant_get_types() %} + {% if pkg.type not in h.recombinant_get_types() and pkg.type != 'harvest' %} {{ super() }} {% endif %} + {% if h.plugin_loaded('canada_harvest') %} + {% set harvest_source = harvest_source or h.get_harvest_source() %} + {% if pkg.type == 'harvest' and h.check_access('harvest_source_update', {'id': harvest_source.id }) %} + {{ h.nav_link(_('Admin'), named_route='harvester.admin', id=harvest_source.name, class_='btn btn-default', icon='wrench')}} + {% endif %} + {% endif %} {% endblock %} {% block secondary_content %} @@ -59,6 +68,17 @@ {% endif %} {% endif %} {% endfor %} + {% if pkg.type == 'harvest'and h.plugin_loaded('canada_harvest') %} + {% set harvest_source = harvest_source or h.get_harvest_source() %} +
  • +
    +
    +
    {{ _('Harvested Datasets:') }}
    +
    {{ h.package_count_for_source(harvest_source.id) }}
    +
    +
    +
    • + {% endif %} {%- set start_date = h.date_field("time_period_coverage_start", pkg) -%} {%- set end_date = h.date_field("time_period_coverage_end", pkg) -%} {% if start_date or end_date %} @@ -145,11 +165,13 @@ {% endif %} -
  • - -  {{_('Atom Feed')}} - -
    • + {% if pkg.type != 'harvest' %} +
  • + +  {{_('Atom Feed')}} + +
    • + {% endif %} {% if pkg.spatial_representation_type %} {% set srep = h.scheming_field_by_name(schema.dataset_fields, 'spatial_representation_type') %}
  • {{ h.scheming_language_text(srep.label) }}{{ _(':') }} diff --git a/ckanext/canada/templates/package/search.html b/ckanext/canada/templates/package/search.html index d0613e1c8..565dd4072 100644 --- a/ckanext/canada/templates/package/search.html +++ b/ckanext/canada/templates/package/search.html @@ -8,34 +8,42 @@ {%- block api_access_info -%}{%- endblock -%} - {% block page_primary_action %}{% endblock %} + {% block page_primary_action %} + {% if dataset_type == 'harvest' %} +
    + {{ h.snippet('snippets/add_source_button.html', dataset_type=dataset_type) }} +
    + {% endif %} + {% endblock %} - {% block subtitle %}{{ _('Search Datasets') }}{% endblock %} + {% block subtitle %}{{ _('Search Datasets') if dataset_type != 'harvest' else _('Search Harvest Sources') }}{% endblock %} {% block breadcrumb_content %} {% if g.is_registry %} -
  • {{ h.nav_link(_('Datasets'), named_route='dataset.search', highlight_actions = 'new index') }}
    • +
  • {{ h.nav_link(_('Datasets') if dataset_type != 'harvest' else _('Harvest Sources'), named_route=dataset_type ~ '.search', highlight_actions = 'new index') }}
    • {% else %}
  • {% link_for _('Search Open Government'), named_route='dataset_search' %}
    • {% endif %} {% endblock %} {% block secondary_content %} -

    {{ _('Search Filters') }} {{ _('Clear All') }}

    - {% snippet 'snippets/dataset_facets.html', show_org_facet=true %} +

    {{ _('Search Filters') }} {{ _('Clear All') }}

    + {% snippet 'snippets/dataset_facets.html', show_org_facet=true, dataset_type=dataset_type %} {% block extra_facets %} - {% if g.is_registry %} + {% if g.is_registry and dataset_type != 'harvest' %} {{ h.snippet('snippets/publish_facet.html', facet_ranges=facet_ranges) }} {% endif %} {% endblock %} - {% set license = h.get_license('ca-ogl-lgo') %} - {% if license != None %} -
    - {% if h.lang() == 'en' %} - View the {{ license.title }} - {% else %} - Voir la {{ license.title_fra[:1].lower() }}{{ license.title_fra[1:] }} - {% endif %} -
    + {% if dataset_type != 'harvest' %} + {% set license = h.get_license('ca-ogl-lgo') %} + {% if license != None %} +
    + {% if h.lang() == 'en' %} + View the {{ license.title }} + {% else %} + Voir la {{ license.title_fra[:1].lower() }}{{ license.title_fra[1:] }} + {% endif %} +
    + {% endif %} {% endif %} {% endblock %} diff --git a/ckanext/canada/templates/scheming/display_snippets/harvest_source_type.html b/ckanext/canada/templates/scheming/display_snippets/harvest_source_type.html new file mode 100644 index 000000000..36fdb4531 --- /dev/null +++ b/ckanext/canada/templates/scheming/display_snippets/harvest_source_type.html @@ -0,0 +1,5 @@ +{% set harvester = h.get_harvester_info(data[field.field_name]) %} + +{% if harvester %} + {{ h.get_translated(harvester, 'title') }} +{% endif %} diff --git a/ckanext/canada/templates/scheming/form_snippets/harvest_sources.html b/ckanext/canada/templates/scheming/form_snippets/harvest_sources.html new file mode 100644 index 000000000..21284ce9d --- /dev/null +++ b/ckanext/canada/templates/scheming/form_snippets/harvest_sources.html @@ -0,0 +1,18 @@ +
    + {% if field.required %}* {% endif %} + +
    + {% for harvester in h.harvesters_info() %} + {% set checked = False %} + {# select first option if nothing in data #} + {% if data.get(field.field_name) == harvester['name'] or (not data.get(field.field_name) and loop.first) %} + {% set checked = True %} + {% endif %} + + {% endfor %} +
    +
    diff --git a/ckanext/canada/templates/scheming/package/snippets/package_form.html b/ckanext/canada/templates/scheming/package/snippets/package_form.html index d75aa0c07..b3f74b3d0 100644 --- a/ckanext/canada/templates/scheming/package/snippets/package_form.html +++ b/ckanext/canada/templates/scheming/package/snippets/package_form.html @@ -1,67 +1,84 @@ {% ckan_extends %} {% block stages %} - {% if form_style != 'edit' %} + {% if form_style != 'edit' and dataset_type != 'harvest' %} {{ h.snippet('package/snippets/stages.html', stages=stage, dataset_type=dataset_type) }} {% endif %} {% endblock %} {% block basic_fields %} -
      -
    • -
      - - {{_('Catalogue Metadata')}} - - {% set m1 = _('Open Data Metadata Element Set') %} - {% if dataset_type == 'info' %} - {% set m1 = _('Open Information Metadata Element Set') %} - {% endif %} -
      -
      - {{_('This section of the form only displays system-populated data elements used to describe the record type and the placement in the registry')}} + {% if dataset_type != 'harvest' %} +
        +
      • +
        + + {{_('Catalogue Metadata')}} + + {% set m1 = _('Open Data Metadata Element Set') %} + {% if dataset_type == 'info' %} + {% set m1 = _('Open Information Metadata Element Set') %} + {% endif %} +
        +
        + {{_('This section of the form only displays system-populated data elements used to describe the record type and the placement in the registry')}} +
        +
        + + {% set f_value = '' %} + {%- if dataset_type == 'dataset' -%} + {% set f_value = _('Open Data') %} + {%- elif dataset_type == 'info' -%} + {% set f_value = _('Open Information') %} + {%- elif dataset_type == 'prop' -%}\ + {% set f_value = _('Suggested Dataset') %} + {%- endif -%} + + {{_('The portal to which the metadata record belongs (Open Data or Open Information)')}} +
        +
        + + + {{_('The name of the metadata scheme used (including profile name)')}} +
        +
        + + + {{_('The version of the metadata scheme used (version of the profile)')}} +
        +
        + + + {{_('A unique phrase or string which identifies the metadata record within the Open Government Portal')}} +
        -
        - - {% set f_value = '' %} - {%- if dataset_type == 'dataset' -%} - {% set f_value = _('Open Data') %} - {%- elif dataset_type == 'info' -%} - {% set f_value = _('Open Information') %} - {%- elif dataset_type == 'prop' -%}\ - {% set f_value = _('Suggested Dataset') %} - {%- endif -%} - - {{_('The portal to which the metadata record belongs (Open Data or Open Information)')}} -
        -
        - - - {{_('The name of the metadata scheme used (including profile name)')}} -
        -
        - - - {{_('The version of the metadata scheme used (version of the profile)')}} -
        -
        - - - {{_('A unique phrase or string which identifies the metadata record within the Open Government Portal')}} -
        -
      -
      -
      • -
    + + + + {% endif %} {{ super() }} {% endblock %} +{% block disclaimer %} + {% if dataset_type != 'harvest' %} + {{ super() }} + {% endif %} +{% endblock %} + +{% block ckan_phase %} + {% if dataset_type != 'harvest' %} + {{ super() }} + {% endif %} +{% endblock %} + {% block form_actions %} - {# custom hack for suggested datasets #} {% if dataset_type == 'prop' %}
    + {% elif dataset_type == 'harvest' %} +
    + +
    {% else %} {{ super() }} {% endif %} diff --git a/ckanext/canada/templates/snippets/dataset_facets.html b/ckanext/canada/templates/snippets/dataset_facets.html index 0463b31fe..678a448b4 100644 --- a/ckanext/canada/templates/snippets/dataset_facets.html +++ b/ckanext/canada/templates/snippets/dataset_facets.html @@ -1,101 +1,130 @@ -{% if g.is_registry %} - {% set type_choices = [ - {'value': 'dataset', 'label': _('Open Data')}, - {'value': 'dialogue', 'label': _('Open Dialogue')}, - {'value': 'info', 'label': _('Open Information')}, - {'value': 'prop', 'label': _('Suggested Datasets')},] - %} -{% else %} - {% set type_choices = [ - {'value': 'dataset', 'label': _('Open Data')}, - {'value': 'info', 'label': _('Open Information')}, - {'value': 'prop', 'label': _('Suggested Datasets')},] - %} -{% endif %} +{% set dataset_type = dataset_type or 'dataset' %} + +{% if dataset_type != 'harvest' %} -{% snippet 'snippets/facet_list.html', - title=g.facet_titles['portal_type'], - name='portal_type', - scheming_choices=type_choices, - extras=extras %} + {% if g.is_registry %} + {% set type_choices = [ + {'value': 'dataset', 'label': _('Open Data')}, + {'value': 'dialogue', 'label': _('Open Dialogue')}, + {'value': 'info', 'label': _('Open Information')}, + {'value': 'prop', 'label': _('Suggested Datasets')},] + %} + {% else %} + {% set type_choices = [ + {'value': 'dataset', 'label': _('Open Data')}, + {'value': 'info', 'label': _('Open Information')}, + {'value': 'prop', 'label': _('Suggested Datasets')},] + %} + {% endif %} -{% if g.is_registry and 'status' in request.params or ('portal_type') in request.params.items() %} {% snippet 'snippets/facet_list.html', - title=g.facet_titles['status'], - name='status', - scheming_choices=[{'value': 'department_contacted', 'label': _('Request sent to data owner - awaiting response')}], + title=g.facet_titles['portal_type'], + name='portal_type', + scheming_choices=type_choices, extras=extras %} -{% endif %} -{% if g.is_registry %} - {% set collection_choices = h.scheming_get_preset('canada_collection').choices + [{'value': 'pd', 'label': _('Proactive Publication')}] %} - {% for t in h.recombinant_get_types() %} - {% do collection_choices.append({'value': t, 'label': _(h.recombinant_get_chromo(t).title)}) %} - {% endfor %} -{% else %} - {% set collection_choices = h.scheming_get_preset('canada_collection').choices %} -{% endif %} + {% if g.is_registry and 'status' in request.params or ('portal_type') in request.params.items() %} + {% snippet 'snippets/facet_list.html', + title=g.facet_titles['status'], + name='status', + scheming_choices=[{'value': 'department_contacted', 'label': _('Request sent to data owner - awaiting response')}], + extras=extras %} + {% endif %} -{% snippet 'snippets/facet_list.html', - title=g.facet_titles['collection'], - name='collection', - scheming_choices=collection_choices, - extras=extras, - unlimit=is_registry %} + {% if g.is_registry %} + {% set collection_choices = h.scheming_get_preset('canada_collection').choices + [{'value': 'pd', 'label': _('Proactive Publication')}] %} + {% for t in h.recombinant_get_types() %} + {% do collection_choices.append({'value': t, 'label': _(h.recombinant_get_chromo(t).title)}) %} + {% endfor %} + {% else %} + {% set collection_choices = h.scheming_get_preset('canada_collection').choices %} + {% endif %} -{% if not g.is_registry %} {% snippet 'snippets/facet_list.html', - title=g.facet_titles['jurisdiction'], - name='jurisdiction', - scheming_choices=h.scheming_get_preset('canada_jurisdiction').choices, - extras=extras %} -{% endif %} + title=g.facet_titles['collection'], + name='collection', + scheming_choices=collection_choices, + extras=extras, + unlimit=is_registry %} -{% if show_org_facet %} - {% snippet 'snippets/facet_list.html', - title=g.facet_titles['organization'], - name='organization' %} -{% endif %} + {% if not g.is_registry %} + {% snippet 'snippets/facet_list.html', + title=g.facet_titles['jurisdiction'], + name='jurisdiction', + scheming_choices=h.scheming_get_preset('canada_jurisdiction').choices, + extras=extras %} + {% endif %} -{% if h.lang() == 'fr' %} - {% snippet 'snippets/facet_list.html', title=g.facet_titles['keywords_fra'], - name='keywords_fra', extras=extras %} -{% else %} - {% snippet 'snippets/facet_list.html', title=g.facet_titles['keywords'], - name='keywords', extras=extras %} -{% endif %} + {% if show_org_facet %} + {% snippet 'snippets/facet_list.html', + title=g.facet_titles['organization'], + name='organization' %} + {% endif %} -{% if g.is_registry %} - {% if g.facets.collection|length != 1 or 'fgp' not in g.facets.collection %} - {# Hide subject on maps search #} + {% if h.lang() == 'fr' %} + {% snippet 'snippets/facet_list.html', title=g.facet_titles['keywords_fra'], + name='keywords_fra', extras=extras %} + {% else %} + {% snippet 'snippets/facet_list.html', title=g.facet_titles['keywords'], + name='keywords', extras=extras %} + {% endif %} + + {% if g.is_registry %} + {% if g.facets.collection|length != 1 or 'fgp' not in g.facets.collection %} + {# Hide subject on maps search #} + {% snippet 'snippets/facet_list.html', + title=g.facet_titles['subject'], + name='subject', + scheming_choices=h.scheming_get_preset('canada_subject').choices, + extras=extras %} + {% endif %} + {% else %} {% snippet 'snippets/facet_list.html', title=g.facet_titles['subject'], name='subject', scheming_choices=h.scheming_get_preset('canada_subject').choices, extras=extras %} {% endif %} + + {% snippet 'snippets/facet_list.html', + title=g.facet_titles['res_format'], + name='res_format', + scheming_choices=h.scheming_get_preset('canada_resource_format').choices, + extras=extras %} + + {% snippet 'snippets/facet_list.html', + title=g.facet_titles['res_type'], + name='res_type', + scheming_choices=h.scheming_get_preset('canada_resource_type').choices, + extras=extras %} + + {% snippet 'snippets/facet_list.html', + title=g.facet_titles['frequency'], + name='frequency', + scheming_choices=h.scheming_get_preset('canada_frequency').choices, + extras=extras %} + {% else %} + + {% set schema = h.scheming_get_dataset_schema(dataset_type) %} + {% set frequency_field = h.scheming_field_by_name(schema.dataset_fields, 'frequency') %} + {% snippet 'snippets/facet_list.html', - title=g.facet_titles['subject'], - name='subject', - scheming_choices=h.scheming_get_preset('canada_subject').choices, + title=g.facet_titles['frequency'], + scheming_choices=frequency_field.choices, + name='frequency', extras=extras %} -{% endif %} -{% snippet 'snippets/facet_list.html', - title=g.facet_titles['res_format'], - name='res_format', - scheming_choices=h.scheming_get_preset('canada_resource_format').choices, - extras=extras %} - -{% snippet 'snippets/facet_list.html', - title=g.facet_titles['res_type'], - name='res_type', - scheming_choices=h.scheming_get_preset('canada_resource_type').choices, - extras=extras %} - -{% snippet 'snippets/facet_list.html', - title=g.facet_titles['frequency'], - name='frequency', - scheming_choices=h.scheming_get_preset('canada_frequency').choices, - extras=extras %} + {% snippet 'snippets/facet_list.html', + title=g.facet_titles['source_type'], + name='source_type', + harvesters=h.harvesters_info(), + extras=extras %} + + {% if show_org_facet %} + {% snippet 'snippets/facet_list.html', + title=g.facet_titles['organization'], + name='organization' %} + {% endif %} + +{% endif %} diff --git a/ckanext/canada/templates/snippets/facet_list.html b/ckanext/canada/templates/snippets/facet_list.html index 6d2e39705..f16f520a5 100644 --- a/ckanext/canada/templates/snippets/facet_list.html +++ b/ckanext/canada/templates/snippets/facet_list.html @@ -18,8 +18,16 @@
    {% endif %} - {% else %} + {% elif package.type not in h.recombinant_get_types() and package.type != 'harvest' %}
    {{ _('Resource Formats:') }} @@ -134,6 +134,16 @@

    + {% elif package.type == 'harvest' %} +
    +
    + {{ _('Source Type:') }} +
    +
    + {% set harvester = h.get_harvester_info(package.source_type) %} +

    {{ h.get_translated(harvester, 'title') }}

    +
    +
    {% endif %} {% endblock %} diff --git a/ckanext/canada/templates/source/admin.html b/ckanext/canada/templates/source/admin.html new file mode 100644 index 000000000..5d8d107b8 --- /dev/null +++ b/ckanext/canada/templates/source/admin.html @@ -0,0 +1,22 @@ +{% ckan_extends %} + +{% block primary_content_inner %} +
    + +

    {{ _('About') }}

    +

    {{ h.markdown_extract(h.get_translated(harvest_source, 'notes')) }}

    + +

    {{ _('Last Harvest Job') }}

    + {% if harvest_source.status and harvest_source.status.last_job %} + {% snippet "snippets/job_details.html", job=harvest_source.status.last_job %} +
    + + + {{ _('View full job report') }} + +
    + {% else %} +

    {{ _('No jobs yet for this source') }}

    + {% endif %} +
    +{% endblock %} diff --git a/ckanext/canada/templates/source/admin_base.html b/ckanext/canada/templates/source/admin_base.html deleted file mode 100644 index 04b7b443e..000000000 --- a/ckanext/canada/templates/source/admin_base.html +++ /dev/null @@ -1,19 +0,0 @@ -{% ckan_extends %} - -{% block secondary_content %} -
    -
    -

    -  {{ _('Portal Sync') }} -

    -
    -
    -

    {{ _("View the current Portal Sync queue, running jobs, and failed syncs.") }}

    -
    -
    -{% endblock %} - -{% block page_header_tabs %} - {{ h.build_nav_icon('harvester.admin', _('Dashboard'), id=harvest_source.name, icon='dashboard') }} - {{ h.build_nav_icon('harvester.job_list', _('Jobs'), source=harvest_source.name, icon='reorder') }} -{% endblock %} diff --git a/ckanext/canada/templates/source/base.html b/ckanext/canada/templates/source/base.html new file mode 100644 index 000000000..65742c7ad --- /dev/null +++ b/ckanext/canada/templates/source/base.html @@ -0,0 +1,6 @@ +{% ckan_extends %} + +{% block breadcrumb_content %} +
  • {% link_for _('Harvest Sources'), named_route='harvest.search' %}
    • +
  • {% link_for h.get_translated(harvest_source, 'title')|truncate(80), named_route='harvest.read', id=harvest_source.id %}
    • +{% endblock %} diff --git a/ckanext/canada/templates/source/job/list.html b/ckanext/canada/templates/source/job/list.html new file mode 100644 index 000000000..b947ba30a --- /dev/null +++ b/ckanext/canada/templates/source/job/list.html @@ -0,0 +1,67 @@ +{% ckan_extends %} + +{% block primary_content_inner %} +
    + +

    {{ _('About') }}

    +

    {{ h.markdown_extract(h.get_translated(harvest_source, 'notes')) }}

    + +

    {{ _('Harvest Jobs') }}

    + + {% if jobs|length == 0 %} +

    {{ _('No jobs yet for this source') }}

    + {% else %} +
      + {% for job in jobs %} +
    • +
      +

      + + {{ _('Job: ') }} {{ job.id }} + + {% if job.status != 'Finished' %} + {{ job.status }} + {% endif %} +

      +

      + {{ _('Started:') }} + + {{ h.render_datetime(job.gather_started, with_hours=True) or _('Not yet') }} + + — + {{ _('Finished:') }} + + {{ h.render_datetime(job.finished, with_hours=True) or _('Not yet') }} + +

      +
      + {% if job.status == 'Finished' %} +
        + {% if 'errored' in job.stats and job.stats['errored'] > 0 %} +
      • + + {{ job.stats['errored'] }} {{ _('errors') }} + +
        • + {% endif %} + {% for action in ['added', 'updated', 'deleted', 'not modified'] %} +
      • + + {% if action in job.stats and job.stats[action] > 0 %} + {{ job.stats[action] }} + {% else %} + 0 + {% endif %} + {{ _(action) }} + +
        • + {% endfor %} +
      + {% endif %} +
      • + {% endfor %} +
    + {% endif %} + +
    +{% endblock %} diff --git a/ckanext/canada/templates/source/read_base.html b/ckanext/canada/templates/source/read_base.html new file mode 100644 index 000000000..7828af721 --- /dev/null +++ b/ckanext/canada/templates/source/read_base.html @@ -0,0 +1,36 @@ +{% ckan_extends %} + +{% block secondary_content %} +
    +
    +
    + {{ _('Additional Information') }} +
    +
    +
      +
    • + {{ _('Source URL:') }}  + {{- harvest_source.url -}} +
      • + {% set harvester = h.get_harvester_info(harvest_source.source_type) %} +
    • + {{ _('Source Type:') }}  + {{- h.get_translated(harvester, 'title') -}} +
      • + {% set schema = h.scheming_get_dataset_schema('harvest') %} + {% set frequency_field = h.scheming_field_by_name(schema.dataset_fields, 'frequency') %} +
    • + {{ _('Harvesting Frequency:') }}  + {{- h.scheming_choices_label(h.scheming_field_choices(frequency_field), harvest_source.frequency) -}} +
      • +
    • +
      +
      +
      {{ _('Harvested Datasets:') }}
      +
      {{ h.package_count_for_source(harvest_source.id) }}
      +
      +
      +
      • +
    +
    +{% endblock %} diff --git a/ckanext/canada/validators.py b/ckanext/canada/validators.py index 32b964dde..a28d83bbf 100644 --- a/ckanext/canada/validators.py +++ b/ckanext/canada/validators.py @@ -535,36 +535,16 @@ def protect_registry_access(key, data, errors, context): raise StopOnError -def canada_harvester_id(value): - """Forces value for singular harvester for Portal Sync.""" - return PORTAL_SYNC_ID - - -def canada_harvester_type(value): - """Forces value for singular harvester for Portal Sync.""" - return 'harvest' - - -def canada_harvester_source_type(value): - """Forces value for singular harvester for Portal Sync.""" - return 'portal_sync' - - -def canada_harvester_url(value): - """Forces value for singular harvester for Portal Sync.""" - return config.get('ckan.site_url', 'registry') - - -def canada_harvester_source(value): - """Forces value for singular harvester for Portal Sync.""" - return 'registry' - - -def canada_harvester_target(value): - """Forces value for singular harvester for Portal Sync.""" - return 'portal' - - -def canada_harvester_title(value): - """Forces value for singular harvester for Portal Sync.""" - return 'Portal Sync' +def portal_sync_id(key, data, errors, context): + """Forces singular Package ID for portal_sync source type. Only one is allowed.""" + source_type = data.get(key[:-1] + ('source_type',)) + if source_type == 'portal_sync': + data[key] = PORTAL_SYNC_ID + + +def portal_sync_limit(value, context): + if value == PORTAL_SYNC_ID: + existing = model.Package.get(PORTAL_SYNC_ID) + if existing: + raise Invalid(_('There is already a Portal Sync harvester. Only one is allowed.')) + return value diff --git a/ckanext/canada/view.py b/ckanext/canada/view.py index bbe97f03e..18f9c139d 100644 --- a/ckanext/canada/view.py +++ b/ckanext/canada/view.py @@ -68,7 +68,6 @@ from ckanext.canada.urlsafe import url_part_unescape, url_part_escape from ckanext.canada.helpers import canada_date_str_to_datetime -from ckanext.canada.harvesters import PORTAL_SYNC_ID from io import StringIO