Release QDES 1.3.0 (#238)

* Add spinner * Add spiner js * [SUPDESQ-25] - added support for validating empty json array (#219) * [SUPDESQ-37] Unpublish all (#212) * Remove select and add table * Make unpublish separate method * Fix the validation error for multiple resources * Improve table styling Create portal naming helper * Add hidden input for the destination schema name * Iterate through the publish log * Small refactor over schema definition for unpublish * [SUPDESQ-39] - added the spatial map input form and geometry validation (#213) * [SUPDESQ-72] updated data service label (#217) * [SUPDESQ-73] added new filter for temporal coverage (#216) * [SUPDESQ-35] - added support for quantity kind (#215) * [SUPDESQ-34] - updated vocab name for shared vocab * [SUPDESQ-35] - fix issue when grouping quantity kinds (#224) * Rename the vobaulary_names according qdes schema (#226) * Fix the issue with unpublish (#227) * [SUPDESQ-47] moved loader to snippets * [SUPDESQ-47] add new form to have loader * Fix typo in 'Needs republishing' * [SUPDESQ-47] - fix spinner issue (#230) * [SUPDESQ-35] removed recursive and dimensionless (#225) * [SUPDESQ-37] Unpublish issues (#228) * Fix the issue with unpublish * Fix the plural message and logic in unpublishing * Fix bugs * Plural/singulare message for queued unpublishing * Fix typos * Removed unused code Co-authored-by: Mark Calvert <[email protected]> * Fixed issue with publishing returning Internal Server error if new resource did not have `package_id` set Co-authored-by: Nathan Perry <[email protected]> Co-authored-by: Konstantin Sivakov <[email protected]> Co-authored-by: Awang <[email protected]>
salsadigitalauorg · May 25, 2022 · c0eca15 · c0eca15
1 parent c529b36
commit c0eca15
Show file tree

Hide file tree

Showing 19 changed files with 268 additions and 93 deletions.
diff --git a/ckanext/qdes_schema/blueprint.py b/ckanext/qdes_schema/blueprint.py
@@ -196,17 +196,32 @@ def datasets_schema_validation(id):
     extra_vars['publish_activities'] = helpers.get_publish_activities(pkg)
 
     # Process unpublish status.
-    unpublish_log_id = request.params.get('unpublish', None)
-    if unpublish_log_id == "0":
-        extra_vars['unpublish'] = 0
-    elif unpublish_log_id:
-        extra_vars['unpublish'] = 1 if helpers.is_unpublish_pending(unpublish_log_id) else ''
+    unpublish_log_ids = request.params.get('unpublish', None)
+    if unpublish_log_ids is None:
+        extra_vars['unpublish'] = None
+    elif unpublish_log_ids:
+        unpublished = False
+        for unpublish_log_id in unpublish_log_ids:
+            unpublished = helpers.is_unpublish_pending(unpublish_log_id)
+            if not unpublished:
+                break
+        extra_vars['unpublish'] = 1 if unpublished else 0
 
     return render('package/publish_metadata.html', extra_vars=extra_vars)
 
 
 def unpublish_external_dataset_resource(id):
     # Check the user has permission to clone the dataset
+
+    def _jsonfy(data):
+        json_data = []
+        if type(data) is str:
+            return [json.loads(data.replace('\'', '"'))]
+        else:
+            for d in data:
+                json_data.append(json.loads(d))
+            return json_data
+
     context = {
         'model': model,
         'user': c.user,
@@ -219,39 +234,52 @@ def unpublish_external_dataset_resource(id):
 
     data = clean_dict(dict_fns.unflatten(tuplize_dict(parse_params(
         request.form))))
-
     pkg = get_action('package_show')({}, {'id': id})
-
+    unpublish_resources = []
+    schemas = []
+    schema_resources = _jsonfy(data.get('schema_resources'))
+    for res_schema in schema_resources:
+        unpublish_resources.append(res_schema.get('resource_id'))
+        schemas.append(res_schema.get('destination'))
     # Create job.
-    resource_to_unpublish = {}
+    resource_to_unpublish = []
     for resource in pkg.get('resources', []):
-        if resource.get('id') == data.get('unpublish_resource'):
-            resource_to_unpublish = resource
+        if resource.get('id') in unpublish_resources:
+            resource_to_unpublish.append(resource)
 
     # Add to publish log.
-    unpublish = 0
-    try:
-        publish_log = get_action('create_publish_log')({}, {
-            'dataset_id': pkg.get('id'),
-            'resource_id': resource_to_unpublish.get('id'),
-            'trigger': constants.PUBLISH_TRIGGER_MANUAL,
-            'destination': data.get('schema'),
-            'status': constants.PUBLISH_STATUS_PENDING,
-            'action': constants.PUBLISH_ACTION_DELETE
-        })
-
-        # Add to job worker queue.
-        if publish_log:
-            # Improvements for job worker visibility when troubleshooting via logs
-            job_title = f'Unpublish external dataset resource: dataset_id={publish_log.dataset_id}, resource_id={publish_log.resource_id}, destination={publish_log.destination}'
-            toolkit.enqueue_job(jobs.unpublish_external_distribution, [publish_log.id, c.user], title=job_title)
-            unpublish = publish_log.id
-
-    except Exception as e:
-        log.error(str(e))
-
-    return h.redirect_to('/dataset/{}/publish?unpublish={}'.format(id, unpublish))
-
+    unpublished = []
+    for res, schema in zip(resource_to_unpublish, schemas):
+        unpublish = _unpublish_resource(res, pkg, schema)
+        if unpublish != 0:
+            unpublished.append(unpublish)
+
+
+    return h.redirect_to('/dataset/{}/publish?unpublish={}'.format(id, unpublished))
+
+def _unpublish_resource(resource, pkg, schema):
+        unpublish = 0
+        try:
+            publish_log = get_action('create_publish_log')({}, {
+                'dataset_id': pkg.get('id'),
+                'resource_id': resource.get('id'),
+                'trigger': constants.PUBLISH_TRIGGER_MANUAL,
+                'destination': schema,
+                'status': constants.PUBLISH_STATUS_PENDING,
+                'action': constants.PUBLISH_ACTION_DELETE
+            })
+
+            # Add to job worker queue.
+
+            if publish_log:
+                # Improvements for job worker visibility when troubleshooting via logs
+                job_title = f'Unpublish external dataset resource: dataset_id={publish_log.dataset_id}, resource_id={publish_log.resource_id}, destination={publish_log.destination}'
+                toolkit.enqueue_job(jobs.unpublish_external_distribution, [publish_log.id, c.user], title=job_title)
+                unpublish = publish_log.id
+
+        except Exception as e:
+            log.error(str(e))
+        return unpublish
 
 def _get_term_obj(field_value, vocab_service_name):
     if isinstance(field_value, list):

diff --git a/ckanext/qdes_schema/dataqld_dataset.json b/ckanext/qdes_schema/dataqld_dataset.json
@@ -468,7 +468,7 @@
       "display_snippet": "text.html",
       "display_property": "dcterms:creator",
       "preset": "qdes_secure_vocab_service_autocomplete",
-      "vocabulary_service_name": "creator",
+      "vocabulary_service_name": "point-of-contact",
       "display_group": "contacts",
       "form_attrs": {
         "data-module": "qdes_autocomplete",
@@ -493,7 +493,7 @@
           "form_snippet": "text.html",
           "display_snippet": "qdes_secure_vocabulary_text.html",
           "display_property": "prov:agent",
-          "vocabulary_service_name": "the-party",
+          "vocabulary_service_name": "point-of-contact",
           "form_attrs": {
             "data-module": "qdes_autocomplete",
             "data-module-source": "/ckan-admin/vocabulary-services/secure-autocomplete/{vocabularyServiceName}?incomplete=?",
@@ -602,7 +602,7 @@
       "label": "License",
       "display_property": "dcterms:license",
       "preset": "controlled_vocabulary_single_select",
-      "vocabulary_service_name": "license",
+      "vocabulary_service_name": "dataset_license",
       "display_group": "rights & licensing",
       "required": true,
       "form_include_blank_choice": true,
@@ -752,7 +752,7 @@
       "label": "License",
       "display_property": "dcterms:license",
       "preset": "controlled_vocabulary_single_select",
-      "vocabulary_service_name": "license",
+      "vocabulary_service_name": "distribution_license",
       "form_attrs": {
         "data-module": "qdes_autocomplete",
         "data-module-title": "title"

diff --git a/ckanext/qdes_schema/fanstatic/publish.js b/ckanext/qdes_schema/fanstatic/publish.js
@@ -246,23 +246,25 @@
     });
 
     $(document).ready(function () {
-        var $unPublishEl = $('.unpublish-wrapper');
-        var $resourceEl = $unPublishEl.find('#unpublish-resource');
-        var $destinationEl = $unPublishEl.find('#schema-unpublish');
+        var $unPublishEl = $('#unpublish-wrapper-content');
+        var $resourceEl = $unPublishEl.find('schema-resources');
+        var $res_inputs = $('#unpublish-wrapper-content').find('input[type=checkbox]');
+
         var $unPublishBtnEl = $unPublishEl.find('.unpublish');
         var toggleBtnOnOff = function () {
-            if ($resourceEl.val() !== 'none' && $destinationEl.val() !== 'none') {
-                $unPublishBtnEl.attr('disabled', false);
-            } else {
+            var $res_checks = $('#unpublish-wrapper-content').find('input[type=checkbox]:checked');
+            if ($res_checks.length == 0) {
                 $unPublishBtnEl.attr('disabled', true);
+            } else {
+                $unPublishBtnEl.attr('disabled', false);
             }
         }
 
-        $resourceEl.on('change', function () {
+        $res_inputs.on('change', function () {
             toggleBtnOnOff();
         });
 
-        $destinationEl.on('change', function () {
+        $res_inputs.on('change', function () {
             toggleBtnOnOff();
         });
 

diff --git a/ckanext/qdes_schema/fanstatic/qdes-tree-selector.js b/ckanext/qdes_schema/fanstatic/qdes-tree-selector.js
@@ -132,6 +132,51 @@
                         if (enableButton) {
                             $wrapperEl.find('.add-selected-node-tree').removeAttr('disabled');
                         }
+
+                        // Support for quantityKind.
+                        //
+                        // If selected node is parent, collect all children quantity_kind.
+                        // If selected node is not parent, collect current quantity_kind.
+                        // If quantity_kind array is null, show all available option.
+                        // If quantity_kind array is not null, show only option that has the same quantity_kind,
+                        // check the js variable for the option, and then re-render the dropdown.
+                        if (isMultiGrp) {
+                            var selectEl = $wrapperEl.find('select')
+                            var selectOptionVar = window[selectEl.attr('data-field-name').replace(/-/g, '')]
+                            var quantityKinds = []
+                            var getQuantityKinds = function (item, recursive = false) {
+                                if (item.data.quantity_kind !== null && quantityKinds.indexOf(item.data.quantity_kind) === -1) {
+                                    quantityKinds.push(item.data.quantity_kind)
+                                }
+
+                                if (recursive && item.children !== null) {
+                                    item.children.forEach(function (childItem) {
+                                        getQuantityKinds(childItem)
+                                    });
+                                }
+                            }
+
+                            // Get all quantity kinds recursively.
+                            getQuantityKinds(data.node)
+
+                            // Re-render the dropdown.
+                            selectEl.html('')
+                            if (quantityKinds.length > 0) {
+                                // Collect all quantity kind options.
+                                selectEl.append('<option title="" value=""></option>')
+                                selectOptionVar.forEach(function (item) {
+                                    if (quantityKinds.indexOf(item.quantity_kind) > -1) {
+                                        selectEl.append('<option title="' + item.title + '" value="' + item.value + '">' + item.text + '</option>')
+                                    }
+                                })
+                            }
+                            else {
+                                // Render all if no quantity kinds.
+                                selectOptionVar.forEach(function (item) {
+                                    selectEl.append('<option title="' + item.title + '" value="' + item.value + '">' + item.text + '</option>')
+                                })
+                            }
+                        }
                     },
                     enhanceTitle: function (e, data) {
                         if (data.node.children !== null) {
@@ -213,7 +258,6 @@
                 var valObj = [];
                 var selectedValue = {};
 
-
                 // Get existing value if any.
                 if (val.length > 0) {
                     valObj = JSON.parse(val);
@@ -247,6 +291,13 @@
 
                 // Remove value on the other field
                 if (isMultiGrp) {
+                    // Reset.
+                    var selectOptionVar = window[$otherFieldEl.attr('data-field-name').replace(/-/g, '')]
+                    $otherFieldEl.html('');
+                    selectOptionVar.forEach(function (item) {
+                        $otherFieldEl.append('<option title="' + item.title + '" value="' + item.value + '">' + item.text + '</option>')
+                    });
+
                     $otherFieldEl.val('');
                     $otherFieldEl.change();
                 }

diff --git a/ckanext/qdes_schema/fanstatic/spinner.js b/ckanext/qdes_schema/fanstatic/spinner.js
@@ -0,0 +1,10 @@
+function removeLoader() {
+    var $loaderEl = jQuery('.loader');
+    $loaderEl.fadeOut(500, function () {
+        // fadeOut complete. Remove the loading div
+        $loaderEl.remove(); //makes page more lightweight
+    });
+}
+jQuery(window).on('load', function () {
+    removeLoader();
+});
diff --git a/ckanext/qdes_schema/fanstatic/webassets.yml b/ckanext/qdes_schema/fanstatic/webassets.yml
@@ -85,3 +85,10 @@ accordion:
       - vendor/jquery
   contents:
     - accordion.js
+
+spinner:
+  extra:
+    preload:
+      - vendor/jquery
+  contents:
+    - spinner.js
diff --git a/ckanext/qdes_schema/helpers.py b/ckanext/qdes_schema/helpers.py
@@ -307,6 +307,9 @@ def convert_relationships_to_related_resources(relationships):
 def get_qld_bounding_box_config():
     return config.get('ckanext.qdes_schema.qld_bounding_box', None)
 
+def get_au_bounding_box_config():
+    return config.get('ckanext.qdes_schema.au_bounding_box', None)
+
 
 def get_default_map_zoom():
     return config.get('ckanext.qdes_schema.default_map_zoom', None) or 5
@@ -587,6 +590,14 @@ def get_distribution_naming(pkg, resource):
 
     return resource.get('name')
 
+def get_portal_naming(destination):
+    if destination == constants.PUBLISH_EXTERNAL_IDENTIFIER_DATA_QLD_SCHEMA:
+        return 'Opendata'
+    elif destination == constants.PUBLISH_EXTERNAL_IDENTIFIER_QSPATIAL_SCHEMA:
+        return  'QSpatial'
+    elif destination == constants.PUBLISH_EXTERNAL_IDENTIFIER_SIR_SCHEMA:
+        return  'SIR'
+
 
 def get_last_success_publish_log(resource):
     last_success_log = PublishLog.get_recent_resource_log(
@@ -751,13 +762,7 @@ def get_publish_activities(pkg):
                 processed_unpublished_date = resource_publish_log.date_processed
 
             # Get portal.
-            portal = ''
-            if resource_publish_log.destination == constants.PUBLISH_EXTERNAL_IDENTIFIER_DATA_QLD_SCHEMA:
-                portal = 'Opendata'
-            elif resource_publish_log.destination == constants.PUBLISH_EXTERNAL_IDENTIFIER_QSPATIAL_SCHEMA:
-                portal = 'QSpatial'
-            elif resource_publish_log.destination == constants.PUBLISH_EXTERNAL_IDENTIFIER_SIR_SCHEMA:
-                portal = 'SIR'
+            portal = get_portal_naming(resource_publish_log.destination)
 
             # Process the published date.
             if processed_date:

diff --git a/ckanext/qdes_schema/jobs.py b/ckanext/qdes_schema/jobs.py
@@ -38,7 +38,6 @@ def publish_to_external_catalogue(publish_log_id, user):
 
     package_dict = {}
     external_pkg_dict = {}
-
     try:
         site_user = get_action(u'get_site_user')({u'ignore_auth': True}, {})
         context = {u'user': site_user[u'name']}
@@ -385,6 +384,11 @@ def _build_and_clean_up_dataqld(des_package_dict, external_package_dict=None, re
         # Add the resource to package.
         qld_pkg_dict['resources'] = [qld_resource_dict]
 
+    # Set resource package_id for resources
+    for resource in qld_pkg_dict.get('resources'):
+        if not resource.get('package_id'):
+            resource['package_id'] = qld_pkg_dict.get('id')
+
     # Manual Mapping for dataset field.
     update_freq = helpers.map_update_schedule(des_package_dict['update_schedule'],
                                               constants.PUBLISH_EXTERNAL_IDENTIFIER_DATA_QLD_SCHEMA)

diff --git a/ckanext/qdes_schema/plugin.py b/ckanext/qdes_schema/plugin.py
@@ -236,6 +236,7 @@ def before_search(self, search_params):
 
             if temporal_coverage_from and not temporal_coverage_to:
                 search_params['fq'] += ' +temporal_start:[' + temporal_coverage_from + ' TO *]'
+                search_params['fq'] += ' OR (temporal_start:[* TO ' + temporal_coverage_from + '] AND temporal_end:[' + temporal_coverage_from + ' TO *])'
 
             if temporal_coverage_from and temporal_coverage_to:
                 # Need to make sure to use the last day of the selected month,
@@ -244,8 +245,8 @@ def before_search(self, search_params):
                 last_day = calendar.monthrange(int(to_date[0]), int(to_date[1]))[1]
                 temporal_coverage_to = temporal_coverage_to + '-' + str(last_day)
 
-                search_params['fq'] += ' +temporal_start:[' + temporal_coverage_from + ' TO ' + temporal_coverage_to + ']'
-                search_params['fq'] += 'OR temporal_end:[' + temporal_coverage_from + ' TO ' + temporal_coverage_to + ']'
+                search_params['fq'] += ' +(temporal_start:[' + temporal_coverage_from + ' TO ' + temporal_coverage_to + '] OR temporal_end:[' + temporal_coverage_from + ' TO ' + temporal_coverage_to + '])'
+                search_params['fq'] += ' OR (temporal_start:[* TO ' + temporal_coverage_from + '] AND temporal_end:[' + temporal_coverage_to + ' TO *])'
 
         return search_params
 
@@ -324,6 +325,7 @@ def get_helpers(self):
             'get_all_relationships': helpers.get_all_relationships,
             'convert_relationships_to_related_resources': helpers.convert_relationships_to_related_resources,
             'get_qld_bounding_box_config': helpers.get_qld_bounding_box_config,
+            'get_au_bounding_box_config': helpers.get_au_bounding_box_config,
             'get_default_map_zoom': helpers.get_default_map_zoom,
             'get_package_dict': helpers.get_package_dict,
             'get_invalid_uris': helpers.get_invalid_uris,
@@ -341,6 +343,7 @@ def get_helpers(self):
             'resource_has_published_to_external_schema': helpers.resource_has_published_to_external_schema,
             'get_publish_activities': helpers.get_publish_activities,
             'get_distribution_naming': helpers.get_distribution_naming,
+            'get_portal_naming': helpers.get_portal_naming,
             'get_published_distributions': helpers.get_published_distributions,
             'get_state_list': helpers.get_state_list,
             'get_pkg_title': helpers.get_pkg_title,
@@ -390,14 +393,14 @@ def dataset_facets(self, facets_dict, package_type):
             facets_dict.pop('organization')
 
         facets_dict['collection_package_id'] = 'Collections'
-        facets_dict['type'] = plugins.toolkit._('Dataset or Data service')
+        facets_dict['type'] = plugins.toolkit._('Dataset or Data Service')
         facets_dict['general_classification'] = plugins.toolkit._('General classification')
         facets_dict['topic_labels'] = plugins.toolkit._('Topic or theme')
         facets_dict['publication_status_label'] = plugins.toolkit._('Status')
         facets_dict['service_status_label'] = plugins.toolkit._('Status')
         facets_dict['classification_and_access_restrictions_label'] = plugins.toolkit._('Access restrictions')
         facets_dict['resource_format_labels'] = plugins.toolkit._('Primary format')
-        facets_dict['standards_label'] = plugins.toolkit._('Data service standards')
+        facets_dict['standards_label'] = plugins.toolkit._('Data Service standards')
         facets_dict['temporal_start'] = plugins.toolkit._('Temporal start')
         facets_dict['temporal_end'] = plugins.toolkit._('Temporal end')
         facets_dict['temporal_coverage_from'] = plugins.toolkit._('Temporal coverage from')