diff --git a/ckanext/qdes_schema/blueprint.py b/ckanext/qdes_schema/blueprint.py index c200a96e..9d798cfe 100644 --- a/ckanext/qdes_schema/blueprint.py +++ b/ckanext/qdes_schema/blueprint.py @@ -196,17 +196,32 @@ def datasets_schema_validation(id): extra_vars['publish_activities'] = helpers.get_publish_activities(pkg) # Process unpublish status. - unpublish_log_id = request.params.get('unpublish', None) - if unpublish_log_id == "0": - extra_vars['unpublish'] = 0 - elif unpublish_log_id: - extra_vars['unpublish'] = 1 if helpers.is_unpublish_pending(unpublish_log_id) else '' + unpublish_log_ids = request.params.get('unpublish', None) + if unpublish_log_ids is None: + extra_vars['unpublish'] = None + elif unpublish_log_ids: + unpublished = False + for unpublish_log_id in unpublish_log_ids: + unpublished = helpers.is_unpublish_pending(unpublish_log_id) + if not unpublished: + break + extra_vars['unpublish'] = 1 if unpublished else 0 return render('package/publish_metadata.html', extra_vars=extra_vars) def unpublish_external_dataset_resource(id): # Check the user has permission to clone the dataset + + def _jsonfy(data): + json_data = [] + if type(data) is str: + return [json.loads(data.replace('\'', '"'))] + else: + for d in data: + json_data.append(json.loads(d)) + return json_data + context = { 'model': model, 'user': c.user, @@ -219,39 +234,52 @@ def unpublish_external_dataset_resource(id): data = clean_dict(dict_fns.unflatten(tuplize_dict(parse_params( request.form)))) - pkg = get_action('package_show')({}, {'id': id}) - + unpublish_resources = [] + schemas = [] + schema_resources = _jsonfy(data.get('schema_resources')) + for res_schema in schema_resources: + unpublish_resources.append(res_schema.get('resource_id')) + schemas.append(res_schema.get('destination')) # Create job. - resource_to_unpublish = {} + resource_to_unpublish = [] for resource in pkg.get('resources', []): - if resource.get('id') == data.get('unpublish_resource'): - resource_to_unpublish = resource + if resource.get('id') in unpublish_resources: + resource_to_unpublish.append(resource) # Add to publish log. - unpublish = 0 - try: - publish_log = get_action('create_publish_log')({}, { - 'dataset_id': pkg.get('id'), - 'resource_id': resource_to_unpublish.get('id'), - 'trigger': constants.PUBLISH_TRIGGER_MANUAL, - 'destination': data.get('schema'), - 'status': constants.PUBLISH_STATUS_PENDING, - 'action': constants.PUBLISH_ACTION_DELETE - }) - - # Add to job worker queue. - if publish_log: - # Improvements for job worker visibility when troubleshooting via logs - job_title = f'Unpublish external dataset resource: dataset_id={publish_log.dataset_id}, resource_id={publish_log.resource_id}, destination={publish_log.destination}' - toolkit.enqueue_job(jobs.unpublish_external_distribution, [publish_log.id, c.user], title=job_title) - unpublish = publish_log.id - - except Exception as e: - log.error(str(e)) - - return h.redirect_to('/dataset/{}/publish?unpublish={}'.format(id, unpublish)) - + unpublished = [] + for res, schema in zip(resource_to_unpublish, schemas): + unpublish = _unpublish_resource(res, pkg, schema) + if unpublish != 0: + unpublished.append(unpublish) + + + return h.redirect_to('/dataset/{}/publish?unpublish={}'.format(id, unpublished)) + +def _unpublish_resource(resource, pkg, schema): + unpublish = 0 + try: + publish_log = get_action('create_publish_log')({}, { + 'dataset_id': pkg.get('id'), + 'resource_id': resource.get('id'), + 'trigger': constants.PUBLISH_TRIGGER_MANUAL, + 'destination': schema, + 'status': constants.PUBLISH_STATUS_PENDING, + 'action': constants.PUBLISH_ACTION_DELETE + }) + + # Add to job worker queue. + + if publish_log: + # Improvements for job worker visibility when troubleshooting via logs + job_title = f'Unpublish external dataset resource: dataset_id={publish_log.dataset_id}, resource_id={publish_log.resource_id}, destination={publish_log.destination}' + toolkit.enqueue_job(jobs.unpublish_external_distribution, [publish_log.id, c.user], title=job_title) + unpublish = publish_log.id + + except Exception as e: + log.error(str(e)) + return unpublish def _get_term_obj(field_value, vocab_service_name): if isinstance(field_value, list): diff --git a/ckanext/qdes_schema/dataqld_dataset.json b/ckanext/qdes_schema/dataqld_dataset.json index 5c80ab8f..4c5175b9 100644 --- a/ckanext/qdes_schema/dataqld_dataset.json +++ b/ckanext/qdes_schema/dataqld_dataset.json @@ -468,7 +468,7 @@ "display_snippet": "text.html", "display_property": "dcterms:creator", "preset": "qdes_secure_vocab_service_autocomplete", - "vocabulary_service_name": "creator", + "vocabulary_service_name": "point-of-contact", "display_group": "contacts", "form_attrs": { "data-module": "qdes_autocomplete", @@ -493,7 +493,7 @@ "form_snippet": "text.html", "display_snippet": "qdes_secure_vocabulary_text.html", "display_property": "prov:agent", - "vocabulary_service_name": "the-party", + "vocabulary_service_name": "point-of-contact", "form_attrs": { "data-module": "qdes_autocomplete", "data-module-source": "/ckan-admin/vocabulary-services/secure-autocomplete/{vocabularyServiceName}?incomplete=?", @@ -602,7 +602,7 @@ "label": "License", "display_property": "dcterms:license", "preset": "controlled_vocabulary_single_select", - "vocabulary_service_name": "license", + "vocabulary_service_name": "dataset_license", "display_group": "rights & licensing", "required": true, "form_include_blank_choice": true, @@ -752,7 +752,7 @@ "label": "License", "display_property": "dcterms:license", "preset": "controlled_vocabulary_single_select", - "vocabulary_service_name": "license", + "vocabulary_service_name": "distribution_license", "form_attrs": { "data-module": "qdes_autocomplete", "data-module-title": "title" diff --git a/ckanext/qdes_schema/fanstatic/publish.js b/ckanext/qdes_schema/fanstatic/publish.js index 57d600fa..2e211da8 100644 --- a/ckanext/qdes_schema/fanstatic/publish.js +++ b/ckanext/qdes_schema/fanstatic/publish.js @@ -246,23 +246,25 @@ }); $(document).ready(function () { - var $unPublishEl = $('.unpublish-wrapper'); - var $resourceEl = $unPublishEl.find('#unpublish-resource'); - var $destinationEl = $unPublishEl.find('#schema-unpublish'); + var $unPublishEl = $('#unpublish-wrapper-content'); + var $resourceEl = $unPublishEl.find('schema-resources'); + var $res_inputs = $('#unpublish-wrapper-content').find('input[type=checkbox]'); + var $unPublishBtnEl = $unPublishEl.find('.unpublish'); var toggleBtnOnOff = function () { - if ($resourceEl.val() !== 'none' && $destinationEl.val() !== 'none') { - $unPublishBtnEl.attr('disabled', false); - } else { + var $res_checks = $('#unpublish-wrapper-content').find('input[type=checkbox]:checked'); + if ($res_checks.length == 0) { $unPublishBtnEl.attr('disabled', true); + } else { + $unPublishBtnEl.attr('disabled', false); } } - $resourceEl.on('change', function () { + $res_inputs.on('change', function () { toggleBtnOnOff(); }); - $destinationEl.on('change', function () { + $res_inputs.on('change', function () { toggleBtnOnOff(); }); diff --git a/ckanext/qdes_schema/fanstatic/qdes-tree-selector.js b/ckanext/qdes_schema/fanstatic/qdes-tree-selector.js index d70fe400..532ff6c0 100644 --- a/ckanext/qdes_schema/fanstatic/qdes-tree-selector.js +++ b/ckanext/qdes_schema/fanstatic/qdes-tree-selector.js @@ -132,6 +132,51 @@ if (enableButton) { $wrapperEl.find('.add-selected-node-tree').removeAttr('disabled'); } + + // Support for quantityKind. + // + // If selected node is parent, collect all children quantity_kind. + // If selected node is not parent, collect current quantity_kind. + // If quantity_kind array is null, show all available option. + // If quantity_kind array is not null, show only option that has the same quantity_kind, + // check the js variable for the option, and then re-render the dropdown. + if (isMultiGrp) { + var selectEl = $wrapperEl.find('select') + var selectOptionVar = window[selectEl.attr('data-field-name').replace(/-/g, '')] + var quantityKinds = [] + var getQuantityKinds = function (item, recursive = false) { + if (item.data.quantity_kind !== null && quantityKinds.indexOf(item.data.quantity_kind) === -1) { + quantityKinds.push(item.data.quantity_kind) + } + + if (recursive && item.children !== null) { + item.children.forEach(function (childItem) { + getQuantityKinds(childItem) + }); + } + } + + // Get all quantity kinds recursively. + getQuantityKinds(data.node) + + // Re-render the dropdown. + selectEl.html('') + if (quantityKinds.length > 0) { + // Collect all quantity kind options. + selectEl.append('') + selectOptionVar.forEach(function (item) { + if (quantityKinds.indexOf(item.quantity_kind) > -1) { + selectEl.append('') + } + }) + } + else { + // Render all if no quantity kinds. + selectOptionVar.forEach(function (item) { + selectEl.append('') + }) + } + } }, enhanceTitle: function (e, data) { if (data.node.children !== null) { @@ -213,7 +258,6 @@ var valObj = []; var selectedValue = {}; - // Get existing value if any. if (val.length > 0) { valObj = JSON.parse(val); @@ -247,6 +291,13 @@ // Remove value on the other field if (isMultiGrp) { + // Reset. + var selectOptionVar = window[$otherFieldEl.attr('data-field-name').replace(/-/g, '')] + $otherFieldEl.html(''); + selectOptionVar.forEach(function (item) { + $otherFieldEl.append('') + }); + $otherFieldEl.val(''); $otherFieldEl.change(); } diff --git a/ckanext/qdes_schema/fanstatic/spinner.js b/ckanext/qdes_schema/fanstatic/spinner.js new file mode 100644 index 00000000..801576a8 --- /dev/null +++ b/ckanext/qdes_schema/fanstatic/spinner.js @@ -0,0 +1,10 @@ +function removeLoader() { + var $loaderEl = jQuery('.loader'); + $loaderEl.fadeOut(500, function () { + // fadeOut complete. Remove the loading div + $loaderEl.remove(); //makes page more lightweight + }); +} +jQuery(window).on('load', function () { + removeLoader(); +}); diff --git a/ckanext/qdes_schema/fanstatic/webassets.yml b/ckanext/qdes_schema/fanstatic/webassets.yml index 07898f29..2e395d4c 100644 --- a/ckanext/qdes_schema/fanstatic/webassets.yml +++ b/ckanext/qdes_schema/fanstatic/webassets.yml @@ -85,3 +85,10 @@ accordion: - vendor/jquery contents: - accordion.js + +spinner: + extra: + preload: + - vendor/jquery + contents: + - spinner.js diff --git a/ckanext/qdes_schema/helpers.py b/ckanext/qdes_schema/helpers.py index 244db9b2..06152df6 100644 --- a/ckanext/qdes_schema/helpers.py +++ b/ckanext/qdes_schema/helpers.py @@ -307,6 +307,9 @@ def convert_relationships_to_related_resources(relationships): def get_qld_bounding_box_config(): return config.get('ckanext.qdes_schema.qld_bounding_box', None) +def get_au_bounding_box_config(): + return config.get('ckanext.qdes_schema.au_bounding_box', None) + def get_default_map_zoom(): return config.get('ckanext.qdes_schema.default_map_zoom', None) or 5 @@ -587,6 +590,14 @@ def get_distribution_naming(pkg, resource): return resource.get('name') +def get_portal_naming(destination): + if destination == constants.PUBLISH_EXTERNAL_IDENTIFIER_DATA_QLD_SCHEMA: + return 'Opendata' + elif destination == constants.PUBLISH_EXTERNAL_IDENTIFIER_QSPATIAL_SCHEMA: + return 'QSpatial' + elif destination == constants.PUBLISH_EXTERNAL_IDENTIFIER_SIR_SCHEMA: + return 'SIR' + def get_last_success_publish_log(resource): last_success_log = PublishLog.get_recent_resource_log( @@ -751,13 +762,7 @@ def get_publish_activities(pkg): processed_unpublished_date = resource_publish_log.date_processed # Get portal. - portal = '' - if resource_publish_log.destination == constants.PUBLISH_EXTERNAL_IDENTIFIER_DATA_QLD_SCHEMA: - portal = 'Opendata' - elif resource_publish_log.destination == constants.PUBLISH_EXTERNAL_IDENTIFIER_QSPATIAL_SCHEMA: - portal = 'QSpatial' - elif resource_publish_log.destination == constants.PUBLISH_EXTERNAL_IDENTIFIER_SIR_SCHEMA: - portal = 'SIR' + portal = get_portal_naming(resource_publish_log.destination) # Process the published date. if processed_date: diff --git a/ckanext/qdes_schema/jobs.py b/ckanext/qdes_schema/jobs.py index 221391e3..29e1f242 100644 --- a/ckanext/qdes_schema/jobs.py +++ b/ckanext/qdes_schema/jobs.py @@ -38,7 +38,6 @@ def publish_to_external_catalogue(publish_log_id, user): package_dict = {} external_pkg_dict = {} - try: site_user = get_action(u'get_site_user')({u'ignore_auth': True}, {}) context = {u'user': site_user[u'name']} @@ -385,6 +384,11 @@ def _build_and_clean_up_dataqld(des_package_dict, external_package_dict=None, re # Add the resource to package. qld_pkg_dict['resources'] = [qld_resource_dict] + # Set resource package_id for resources + for resource in qld_pkg_dict.get('resources'): + if not resource.get('package_id'): + resource['package_id'] = qld_pkg_dict.get('id') + # Manual Mapping for dataset field. update_freq = helpers.map_update_schedule(des_package_dict['update_schedule'], constants.PUBLISH_EXTERNAL_IDENTIFIER_DATA_QLD_SCHEMA) diff --git a/ckanext/qdes_schema/plugin.py b/ckanext/qdes_schema/plugin.py index 857e7484..2f2fbdbe 100644 --- a/ckanext/qdes_schema/plugin.py +++ b/ckanext/qdes_schema/plugin.py @@ -236,6 +236,7 @@ def before_search(self, search_params): if temporal_coverage_from and not temporal_coverage_to: search_params['fq'] += ' +temporal_start:[' + temporal_coverage_from + ' TO *]' + search_params['fq'] += ' OR (temporal_start:[* TO ' + temporal_coverage_from + '] AND temporal_end:[' + temporal_coverage_from + ' TO *])' if temporal_coverage_from and temporal_coverage_to: # Need to make sure to use the last day of the selected month, @@ -244,8 +245,8 @@ def before_search(self, search_params): last_day = calendar.monthrange(int(to_date[0]), int(to_date[1]))[1] temporal_coverage_to = temporal_coverage_to + '-' + str(last_day) - search_params['fq'] += ' +temporal_start:[' + temporal_coverage_from + ' TO ' + temporal_coverage_to + ']' - search_params['fq'] += 'OR temporal_end:[' + temporal_coverage_from + ' TO ' + temporal_coverage_to + ']' + search_params['fq'] += ' +(temporal_start:[' + temporal_coverage_from + ' TO ' + temporal_coverage_to + '] OR temporal_end:[' + temporal_coverage_from + ' TO ' + temporal_coverage_to + '])' + search_params['fq'] += ' OR (temporal_start:[* TO ' + temporal_coverage_from + '] AND temporal_end:[' + temporal_coverage_to + ' TO *])' return search_params @@ -324,6 +325,7 @@ def get_helpers(self): 'get_all_relationships': helpers.get_all_relationships, 'convert_relationships_to_related_resources': helpers.convert_relationships_to_related_resources, 'get_qld_bounding_box_config': helpers.get_qld_bounding_box_config, + 'get_au_bounding_box_config': helpers.get_au_bounding_box_config, 'get_default_map_zoom': helpers.get_default_map_zoom, 'get_package_dict': helpers.get_package_dict, 'get_invalid_uris': helpers.get_invalid_uris, @@ -341,6 +343,7 @@ def get_helpers(self): 'resource_has_published_to_external_schema': helpers.resource_has_published_to_external_schema, 'get_publish_activities': helpers.get_publish_activities, 'get_distribution_naming': helpers.get_distribution_naming, + 'get_portal_naming': helpers.get_portal_naming, 'get_published_distributions': helpers.get_published_distributions, 'get_state_list': helpers.get_state_list, 'get_pkg_title': helpers.get_pkg_title, @@ -390,14 +393,14 @@ def dataset_facets(self, facets_dict, package_type): facets_dict.pop('organization') facets_dict['collection_package_id'] = 'Collections' - facets_dict['type'] = plugins.toolkit._('Dataset or Data service') + facets_dict['type'] = plugins.toolkit._('Dataset or Data Service') facets_dict['general_classification'] = plugins.toolkit._('General classification') facets_dict['topic_labels'] = plugins.toolkit._('Topic or theme') facets_dict['publication_status_label'] = plugins.toolkit._('Status') facets_dict['service_status_label'] = plugins.toolkit._('Status') facets_dict['classification_and_access_restrictions_label'] = plugins.toolkit._('Access restrictions') facets_dict['resource_format_labels'] = plugins.toolkit._('Primary format') - facets_dict['standards_label'] = plugins.toolkit._('Data service standards') + facets_dict['standards_label'] = plugins.toolkit._('Data Service standards') facets_dict['temporal_start'] = plugins.toolkit._('Temporal start') facets_dict['temporal_end'] = plugins.toolkit._('Temporal end') facets_dict['temporal_coverage_from'] = plugins.toolkit._('Temporal coverage from') diff --git a/ckanext/qdes_schema/qdes_ckan_dataset.json b/ckanext/qdes_schema/qdes_ckan_dataset.json index 06e28188..c372ce45 100644 --- a/ckanext/qdes_schema/qdes_ckan_dataset.json +++ b/ckanext/qdes_schema/qdes_ckan_dataset.json @@ -622,7 +622,7 @@ "label": "License", "display_property": "dcterms:license", "preset": "controlled_vocabulary_single_select", - "vocabulary_service_name": "license", + "vocabulary_service_name": "dataset_license", "display_group": "rights and licensing", "recommended": true, "form_include_blank_choice": true, @@ -770,10 +770,10 @@ }, { "field_name": "data_services", - "label": "Data service", + "label": "Data Service", "display_property": "dcat:accessService", "preset": "controlled_dataservice_multi_select", - "validators": "ignore_missing qdes_validate_data_service_is_exist", + "validators": "ignore_missing ignore_empty qdes_validate_data_service_is_exist", "form_attrs": { "data-module": "qdes_autocomplete" }, @@ -807,7 +807,7 @@ "label": "License", "display_property": "dcterms:license", "preset": "controlled_vocabulary_single_select", - "vocabulary_service_name": "license", + "vocabulary_service_name": "distribution_license", "form_attrs": { "data-module": "qdes_autocomplete", "data-module-title": "title" diff --git a/ckanext/qdes_schema/qdes_dataservice.json b/ckanext/qdes_schema/qdes_dataservice.json index 7c6d3348..dba2a174 100644 --- a/ckanext/qdes_schema/qdes_dataservice.json +++ b/ckanext/qdes_schema/qdes_dataservice.json @@ -70,7 +70,7 @@ "display_property": "dcterms:language", "display_group": "description", "preset": "controlled_vocabulary_multi_select", - "vocabulary_service_name": "dataset_language", + "vocabulary_service_name": "dataservice_language", "form_attrs": { "data-module": "qdes_autocomplete", "data-module-title": "title" @@ -84,7 +84,7 @@ "display_property": "dcat:theme", "display_group": "description", "preset": "controlled_vocabulary_tree_multiple", - "vocabulary_service_name": "topic", + "vocabulary_service_name": "dataservice_topic", "validators": "scheming_required", "form_include_blank_choice": true, "label_plural": "topic or themes", @@ -235,7 +235,7 @@ "label": "Nature of their responsibility", "form_snippet": "select.html", "display_property": "dcat:hadRole", - "vocabulary_service_name": "nature-of-their-responsibility", + "vocabulary_service_name": "dataservice-nature-of-their-responsibility", "choices_helper": "scheming_vocabulary_service_choices", "form_attrs": { "class": "form-control", @@ -251,7 +251,7 @@ "display_property": "dcterms:accessRights", "display_group": "rights and licensing", "preset": "controlled_vocabulary_multi_select", - "vocabulary_service_name": "classification_and_access_restrictions", + "vocabulary_service_name": "dataservice_classification_and_access_restrictions", "form_attrs": { "data-module": "qdes_autocomplete", "data-module-title": "title" @@ -273,7 +273,7 @@ "label": "License", "display_property": "dcterms:license", "preset": "controlled_vocabulary_single_select", - "vocabulary_service_name": "license", + "vocabulary_service_name": "dataservice_license", "display_group": "rights and licensing", "recommended": true, "form_include_blank_choice": true, diff --git a/ckanext/qdes_schema/templates/package/available_datasets.html b/ckanext/qdes_schema/templates/package/available_datasets.html index 428c3158..2e21a523 100644 --- a/ckanext/qdes_schema/templates/package/available_datasets.html +++ b/ckanext/qdes_schema/templates/package/available_datasets.html @@ -10,6 +10,6 @@ {% endfor %} {% else %} -