Merge branch 'canada-v2.10' into staging

# Conflicts:
#	ckanext/canada/i18n/ckanext-canada.pot
#	ckanext/canada/i18n/en/LC_MESSAGES/ckanext-canada.po
#	ckanext/canada/i18n/fr/LC_MESSAGES/ckanext-canada.po
### RESOLVED.

.circleci/config.yml

@@ -17,7 +17,7 @@ orbs:
 jobs:
   test:
     docker:
-      - image: openknowledge/ckan-dev:2.9  # primary executor
+      - image: ckan/ckan-dev:2.10.4  # primary executor
         name: ckan
         environment:
           PGHOST: postgres
@@ -39,7 +39,7 @@ jobs:
           CKAN_REDIS_URL: redis://redis:6379/1
       - image: opendatacanada/solr  # dependency service
         name: solr
-      - image: postgres:9.6  # dependency service
+      - image: postgres:13.14  # dependency service
         name: postgres
         environment:
           POSTGRES_USER: postgres
@@ -54,33 +54,41 @@ jobs:
             - "0c:02:84:7e:c0:46:16:eb:9b:5e:1d:c9:f8:3b:5a:3d"
       - checkout
       - run:
-          name: Set CKAN to open-data/ckan@canada-py3
+          name: Set Python Version to 3.9
+          command: |
+            apk add --no-cache python3=3.9.18-r0 --repository=https://dl-cdn.alpinelinux.org/alpine/v3.15/main
+            ln -sf python3 /usr/local/bin/python
+            python3 -m ensurepip
+            pip3 install --no-cache --upgrade pip setuptools
+            python3 --version
+      - run:
+          name: Set CKAN to open-data/[email protected]
           command: |
             pip uninstall -y ckan
             git --git-dir=\/srv\/app\/src\/ckan\/.git --work-tree=\/srv\/app\/src\/ckan\/ remote add canada https://github.com/open-data/ckan.git
-            git --git-dir=\/srv\/app\/src\/ckan\/.git --work-tree=\/srv\/app\/src\/ckan\/ fetch canada canada-py3
-            git --git-dir=\/srv\/app\/src\/ckan\/.git --work-tree=\/srv\/app\/src\/ckan\/ checkout -b canada-py3 canada/canada-py3
+            git --git-dir=\/srv\/app\/src\/ckan\/.git --work-tree=\/srv\/app\/src\/ckan\/ fetch canada canada-v2.10
+            git --git-dir=\/srv\/app\/src\/ckan\/.git --work-tree=\/srv\/app\/src\/ckan\/ checkout -b canada-v2.10 canada/canada-v2.10
             git --git-dir=\/srv\/app\/src\/ckan\/.git --work-tree=\/srv\/app\/src\/ckan\/ pull
       - run:
           name: Install requirements
           command: |
             apk add jpeg-dev
-            pip install setuptools==44.1.0
+            pip install --upgrade setuptools==70.0.0
             pip install --upgrade pip==23.2.1
             pip install -e \/srv\/app\/src\/ckan\/ -r \/srv\/app\/src\/ckan\/requirements.txt -r \/srv\/app\/src\/ckan\/dev-requirements.txt
             pip install -e 'git+https://github.com/ckan/ckanapi.git#egg=ckanapi' -r 'https://raw.githubusercontent.com/ckan/ckanapi/master/requirements.txt'
             pip install -e . -r .\/requirements.txt -r .\/test-requirements.txt
             pip install -e 'git+https://github.com/ckan/ckanext-fluent.git#egg=ckanext-fluent' -r 'https://raw.githubusercontent.com/ckan/ckanext-fluent/master/requirements.txt'
-            pip install -e 'git+https://github.com/open-data/ckanext-recombinant.git#egg=ckanext-recombinant' -r 'https://raw.githubusercontent.com/open-data/ckanext-recombinant/master/requirements.txt'
+            pip install -e 'git+https://github.com/open-data/ckanext-recombinant.git@canada-v2.10#egg=ckanext-recombinant' -r 'https://raw.githubusercontent.com/open-data/ckanext-recombinant/canada-v2.10/requirements.txt'
             pip install -e 'git+https://github.com/ckan/ckanext-scheming.git#egg=ckanext-scheming'
-            pip install -e 'git+https://github.com/open-data/ckanext-validation.git@canada-py3#egg=ckanext-validation' -r 'https://raw.githubusercontent.com/open-data/ckanext-validation/canada-py3/requirements.txt' -r 'https://raw.githubusercontent.com/open-data/ckanext-validation/canada-py3/dev-requirements.txt'
-            pip install -e 'git+https://github.com/open-data/ckanext-xloader.git@canada-py3#egg=ckanext-xloader' -r 'https://raw.githubusercontent.com/open-data/ckanext-xloader/canada-py3/requirements.txt' -r 'https://raw.githubusercontent.com/open-data/ckanext-xloader/canada-py3/dev-requirements.txt'
+            pip install -e 'git+https://github.com/open-data/ckanext-validation.git@canada-v2.10#egg=ckanext-validation' -r 'https://raw.githubusercontent.com/open-data/ckanext-validation/canada-v2.10/requirements.txt' -r 'https://raw.githubusercontent.com/open-data/ckanext-validation/canada-v2.10/dev-requirements.txt'
+            pip install -e 'git+https://github.com/open-data/ckanext-xloader.git@canada-v2.10#egg=ckanext-xloader' -r 'https://raw.githubusercontent.com/open-data/ckanext-xloader/canada-v2.10/requirements.txt' -r 'https://raw.githubusercontent.com/open-data/ckanext-xloader/canada-v2.10/dev-requirements.txt'
             pip install -e 'git+https://github.com/ckan/ckantoolkit.git#egg=ckantoolkit' -r 'https://raw.githubusercontent.com/ckan/ckantoolkit/master/requirements.txt'
-            pip install -e 'git+https://github.com/open-data/goodtables.git@canada-py3#egg=goodtables' -r 'https://raw.githubusercontent.com/open-data/goodtables/canada-py3/requirements.txt'
-            pip install -e 'git+https://github.com/open-data/ckanext-security.git@canada-py3#egg=ckanext-security' -r 'https://raw.githubusercontent.com/open-data/ckanext-security/canada-py3/requirements.txt'
+            pip install -e 'git+https://github.com/open-data/frictionless-py.git@canada-v2.10#egg=frictionless' -r 'https://raw.githubusercontent.com/open-data/frictionless-py/canada-v2.10/requirements.txt'
+            pip install -e 'git+https://github.com/open-data/ckanext-security.git@canada-v2.10#egg=ckanext-security' -r 'https://raw.githubusercontent.com/open-data/ckanext-security/canada-v2.10/requirements.txt'
             find \/srv\/app\/ -name '*.pyc' -delete
       - run:
-          name: Setup extension (CKAN 2.9 PY3)
+          name: Setup extension (CKAN 2.10 PY3.9)
           command: |
             ln -s \/srv\/app\/src\/ckan\/test-core.ini .\/links\/test-core.ini
             ln -s \/srv\/app\/src\/ckan\/who.ini .\/links\/who.ini
@@ -93,12 +101,14 @@ jobs:
             ckan -c test-core.ini datastore set-permissions | psql -U postgres --set ON_ERROR_STOP=1
             ckan -c test-core.ini canada update-triggers
             ckan -c test-core.ini recombinant create-triggers -a
+            ckan -c test-core.ini db upgrade
+            ckan -c test-core.ini db pending-migrations --apply
             python3 .\/bin\/download_country.py
       - run:
           name: Run all tests
           command: |
             mkdir -p ~/junit/result
-            pytest --ckan-ini=test-core.ini ckanext/canada/tests --junitxml=~/junit/result/junit.xml
+            pytest --ckan-ini=test-core.ini --disable-warnings ckanext/canada/tests --junitxml=~/junit/result/junit.xml
       - store_test_results:
           path: ~/junit
       - slack/notify:

bin/ati_to_datastore_create.py

@@ -28,23 +28,23 @@ def org_info():
             break
         except ckanapi.errors.CKANAPIError:
             count += 1
-            print >> sys.stderr, 'Error read org list from open.canada.ca'
+            print('Error read org list from open.canada.ca', file=sys.stderr)
             time.sleep(2)
     res = {}
     for rec in orgs:
         count = 0
         while count <=50:
             try:
-                print >> sys.stderr, 'read org ' + rec['name']
+                print('read org ' + rec['name'], file=sys.stderr)
                 org = site.action.organization_show(id=rec['id'])
                 break
             except ckanapi.errors.CKANAPIError:
                 count += 1
                 org = None
-                print >> sys.stderr, 'Error read org ' + rec['name']
+                print('Error read org ' + rec['name'], file=sys.stderr)
                 time.sleep(2)
         if not org:
-            print >> sys.stderr, 'Network error'
+            print('Network error', file=sys.stderr)
             sys.exit(-1)
         extras = org['extras']
         ati_email = None
@@ -66,7 +66,7 @@ def org_info():
     rec['org_title_fr'] = title[1]
     rec['ati_email'] = org_dict.get(rec['owner_org'], '')
 
-print json.dumps({
+print(json.dumps({
     'resource_id': '19383ca2-b01a-487d-88f7-e1ffbc7d39c2',
     'fields': [
         {'id':'year', 'type':'int'},
@@ -85,4 +85,4 @@ def org_info():
     'indexes': ['year', 'month'],
     'force': True,
     'records': list(csv),
-})
+}))

bin/filter/filter_nap5.py

@@ -28,7 +28,8 @@
 ]
 
 def main():
-    table = yaml.load(open(TABLE_YAML, 'r'))
+    with open(TABLE_YAML, 'r') as f:
+        table = yaml.load(f, yaml.Loader)
     for field in table['resources'][0]['fields']:
         if field['datastore_id'] == 'indicators':
             break

bin/import_xml2obd.py

@@ -30,11 +30,11 @@
 
 import csv
 
-from azure.storage.blob import BlockBlobService
+from azure.storage.blob import BlobServiceClient
 from azure.common import AzureMissingResourceHttpError
 from azure.storage.blob import ContentSettings
 
-import urllib2
+import urllib
 import ckanapi
 import ckan
 from ckanapi.errors import CKANAPIError
@@ -106,7 +106,7 @@ def base64md5str(val):
 
 def read_presets(filename):
     with open(filename, 'r') as f:
-        _presets = yaml.load(f)
+        _presets = yaml.load(f, yaml.Loader)
     return _presets
 
 
@@ -379,7 +379,7 @@ def upload_resources(remote_site, api_key, jsonfile, resource_directory):
                 target_pkg = site.action.package_show(id=rec['id'])
             except (NotFound, NotAuthorized):
                 target_pkg = None
-            except (CKANAPIError, urllib2.URLError), e:
+            except (CKANAPIError, urllib2.URLError) as e:
                 retries -= 1
                 if retries > 0:
                     time.sleep(1)
@@ -517,7 +517,7 @@ def pull_docs(conf_file, local_dir):
     src = RemoteStorage(src_user, src_key, src_container)
 
     all_docs = src.new_docs()
-    print 'total source files ', len(all_docs)
+    print('total source files ', len(all_docs))
     xmls = [x for x in all_docs if x[-4:]=='.xml' and x[:-4] in all_docs and
             x[:-4] + '.ind' in all_docs]
     docs = [x[:-4] for x in xmls]
@@ -652,7 +652,7 @@ def duplicate_docs(file_dir, site_url):
                     count += 1
                 except (NotFound, NotAuthorized):
                     target_pkg = None
-                except (CKANAPIError, urllib2.URLError), e:
+                except (CKANAPIError, urllib2.URLError) as e:
                     sys.stdout.write(
                         json.dumps([
                             rec['id'],
@@ -696,6 +696,8 @@ def de_dup2(site_url):
         print(vl)
 
 def main():
+    sys.exit("This script has been deprecated. Use ckanapi load instead.")
+
     global audience, canada_resource_type,canada_subject
     global canada_resource_language, organizations
     global canada_resource_format

bin/link_check.py

@@ -210,7 +210,7 @@ def get_resources(self):
                 urls = {}
         if new_url:
             self.test_links(new_url, urls)
-        print ('total record count: ', count)
+        print('total record count: ', count)
 
     def dumpBrokenLink(self, csvfile):
         outf=open(csvfile, 'wb')

bin/migrate/migrate_cansim_export.py

@@ -66,7 +66,7 @@ def main():
                     if rec["subject"] in subject_replacements:
                         rec["subject"] = subject_replacements[rec["subject"]]
                     else:
-                        print >> sys.stderr, 'Invalid subject "{0}" for {1}'.format(rec["subject"], rec["id"])
+                        print('Invalid subject "{0}" for {1}'.format(rec["subject"], rec["id"]), file=sys.stderr)
                         continue
                 rec['author_email'] = '[email protected]'
                 rec['maintainer_email'] = '[email protected]'
@@ -122,7 +122,7 @@ def main():
                 print(simplejson.dumps(rec))
 
         except IOError:
-            print >> sys.stderr, 'Error while reading line.'
+            print('Error while reading line.', file=sys.stderr)
 
     except KeyError:
         if 'warehouse' in sys.argv:

ckanext/canada/assets/datatables/canada_datatables.css

@@ -63,16 +63,16 @@ div.canada-disabled-view-alert{
   min-width: 50%;
   width: 50%;
   margin-bottom: 5px;
-  padding-left: 16px;
+  padding-left: 24px;
   padding-right: 10px;
   position: relative;
 }
 .buttons-collection.buttons-colvis ~ .dt-button-collection.fixed a.buttons-columnVisibility::before{
   content: '\f096';
   font: normal normal normal 14px/1 FontAwesome;
   position: absolute;
-  left: 0;
-  top: 2px;
+  left: 6px;
+  top: 6px;
 }
 .buttons-collection.buttons-colvis ~ .dt-button-collection.fixed a.buttons-columnVisibility.active::before{
   content: '\f046';

ckanext/canada/assets/internal/canada_guess_mimetype.js

@@ -16,6 +16,7 @@ window.addEventListener('load', function(){
                     .children('#s2id_field-format')
                     .addClass('conrtol-medium')
                     .removeClass('form-control')
+                    .removeClass('form-select')
                     .css({'display': 'block'});
 
     }
@@ -33,13 +34,15 @@ window.addEventListener('load', function(){
     }
 
     function _guess_mimetype(url){
+      let tokenFieldName = $('meta[name="csrf_field_name"]').attr('content');
+      let tokenValue = $('meta[name="' + tokenFieldName + '"]').attr('content');
+      payload = {'url': url};
+      payload[tokenFieldName] = tokenValue;
       $.ajax({
         'url': '/api/action/canada_guess_mimetype',
         'type': 'POST',
         'dataType': 'JSON',
-        'data': {
-          'url': url,
-        },
+        'data': payload,
         'complete': function(_data){
           if( _data.responseJSON ){  // we have response JSON
             if( _data.responseJSON.success ){  // successful format guess