Merge branch 'canada-v2.9' into staging-v2.9

.circleci/config.yml

@@ -0,0 +1,115 @@
+version: 2.1
+parameters:
+  GHA_Event:
+    type: string
+    default: ""
+  GHA_Actor:
+    type: string
+    default: ""
+  GHA_Action:
+    type: string
+    default: ""
+  GHA_Meta:
+    type: string
+    default: ""
+orbs:
+  slack: circleci/[email protected]
+jobs:
+  test:
+    docker:
+      - image: openknowledge/ckan-dev:2.8  # primary executor
+        name: ckan
+        environment:
+          PGHOST: postgres
+          PGDATABASE: postgres
+          PGUSER: postgres
+          PGPASSWORD: pass
+          CKAN_POSTGRES_DB: ckan_test
+          CKAN_DATASTORE_POSTGRES_DB: datastore_test
+          CKAN_POSTGRES_USER: ckan_default
+          CKAN_DATASTORE_POSTGRES_READ_USER: datastore_read
+          CKAN_DATASTORE_POSTGRES_WRITE_USER: datastore_write
+          CKAN_POSTGRES_PWD: pass
+          CKAN_DATASTORE_POSTGRES_READ_PWD: pass
+          CKAN_DATASTORE_POSTGRES_WRITE_PWD: pass
+          CKAN_SQLALCHEMY_URL: postgresql://ckan_default:pass@postgres/ckan_test
+          CKAN_DATASTORE_WRITE_URL: postgresql://datastore_write:pass@postgres/datastore_test
+          CKAN_DATASTORE_READ_URL: postgresql://datastore_read:pass@postgres/datastore_test
+          CKAN_SOLR_URL: http://solr:8983/solr/ckan_registry
+          CKAN_REDIS_URL: redis://redis:6379/1
+      - image: opendatacanada/solr  # dependency service
+        name: solr
+      - image: postgres:9.6  # dependency service
+        name: postgres
+        environment:
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: pass
+          POSTGRES_DB: postgres
+      - image: redis:3  # dependency service
+        name: redis
+    steps:
+      - add_ssh_keys:
+          fingerprints:
+            - "94:db:2e:03:93:db:67:64:e3:8c:32:b0:d1:c6:7a:b7"
+            - "0c:02:84:7e:c0:46:16:eb:9b:5e:1d:c9:f8:3b:5a:3d"
+      - checkout
+      - run:
+          name: Set CKAN to open-data/[email protected]
+          command: |
+            pip uninstall -y ckan
+            git --git-dir=\/srv\/app\/src\/ckan\/.git --work-tree=\/srv\/app\/src\/ckan\/ remote add canada https://github.com/open-data/ckan.git
+            git --git-dir=\/srv\/app\/src\/ckan\/.git --work-tree=\/srv\/app\/src\/ckan\/ fetch canada canada-v2.9
+            git --git-dir=\/srv\/app\/src\/ckan\/.git --work-tree=\/srv\/app\/src\/ckan\/ checkout -b canada-v2.9 canada/canada-v2.9
+            git --git-dir=\/srv\/app\/src\/ckan\/.git --work-tree=\/srv\/app\/src\/ckan\/ pull
+      - run:
+          name: Install requirements
+          command: |
+            apk add jpeg-dev
+            pip install setuptools==36.1
+            pip install --upgrade pip==19.2.1
+            pip install -e \/srv\/app\/src\/ckan\/ -r \/srv\/app\/src\/ckan\/requirements-py2.txt -r \/srv\/app\/src\/ckan\/dev-requirements.txt
+            pip install -e 'git+https://github.com/ckan/ckanapi.git#egg=ckanapi' -r 'https://raw.githubusercontent.com/ckan/ckanapi/master/requirements.txt'
+            pip install -e . -r .\/requirements.txt -r .\/test-requirements.txt
+            pip install -e 'git+https://github.com/ckan/ckanext-fluent.git#egg=ckanext-fluent' -r 'https://raw.githubusercontent.com/ckan/ckanext-fluent/master/requirements.txt'
+            pip install -e 'git+https://github.com/open-data/[email protected]#egg=ckanext-recombinant' -r 'https://raw.githubusercontent.com/open-data/ckanext-recombinant/canada-v2.9/requirements.txt'
+            pip install -e 'git+https://github.com/ckan/ckanext-scheming.git#egg=ckanext-scheming'
+            pip install -e 'git+https://github.com/open-data/ckanext-validation.git@canada#egg=ckanext-validation' -r 'https://raw.githubusercontent.com/open-data/ckanext-validation/canada/requirements.txt' -r 'https://raw.githubusercontent.com/open-data/ckanext-validation/canada/dev-requirements.txt'
+            pip install -e 'git+https://github.com/open-data/ckanext-xloader.git#egg=ckanext-xloader' -r 'https://raw.githubusercontent.com/open-data/ckanext-xloader/master/requirements.txt' -r 'https://raw.githubusercontent.com/open-data/ckanext-xloader/master/dev-requirements.txt'
+            pip install -e 'git+https://github.com/ckan/ckantoolkit.git#egg=ckantoolkit' -r 'https://raw.githubusercontent.com/ckan/ckantoolkit/master/requirements.txt'
+            pip install -e 'git+https://github.com/open-data/goodtables.git@canada#egg=goodtables' -r 'https://raw.githubusercontent.com/open-data/goodtables/canada/requirements.txt'
+            pip install -e 'git+https://github.com/open-data/ckanext-wet-boew.git#egg=ckanext-wet-boew' -r 'https://raw.githubusercontent.com/open-data/ckanext-wet-boew/master/requirements.txt'
+            pip install -e 'git+https://github.com/open-data/[email protected]#egg=ckanext-security' -r 'https://raw.githubusercontent.com/open-data/ckanext-security/canada-v2.9/requirements-py2.txt'
+            find \/srv\/app\/ -name '*.pyc' -delete
+      - run:
+          name: Setup extension (CKAN 2.9)
+          command: |
+            ln -s \/srv\/app\/src\/ckan\/test-core.ini .\/links\/test-core.ini
+            ln -s \/srv\/app\/src\/ckan\/who.ini .\/links\/who.ini
+            mkdir -p .\/links\/ckanext\/datastore\/tests\/ && ln -s \/srv\/app\/src\/ckan\/ckanext\/datastore\/tests\/allowed_functions.txt .\/links\/ckanext\/datastore\/tests\/allowed_functions.txt
+            mkdir -p .\/links\/ckan\/bin\/postgres_init\/ && ln -s \/srv\/app\/src\/ckan\/bin\/postgres_init\/1_create_ckan_db.sh .\/links\/ckan\/bin\/postgres_init\/1_create_ckan_db.sh && ln -s \/srv\/app\/src\/ckan\/bin\/postgres_init\/2_create_ckan_datastore_db.sh .\/links\/ckan\/bin\/postgres_init\/2_create_ckan_datastore_db.sh
+            python2 setup.py develop
+            . .\/links\/ckan\/bin\/postgres_init\/1_create_ckan_db.sh
+            . .\/links\/ckan\/bin\/postgres_init\/2_create_ckan_datastore_db.sh
+            ckan -c test-core.ini db init
+            ckan -c test-core.ini datastore set-permissions | psql -U postgres --set ON_ERROR_STOP=1
+            paster --plugin=ckanext-canada canada update-triggers -c test-core.ini
+            paster --plugin=ckanext-recombinant recombinant create-triggers -a -c test-core.ini
+            python2 .\/bin\/download_country.py
+      - run:
+          name: Run all tests
+          command: |
+            pytest --ckan-ini=test-core.ini ckanext/canada/tests
+      - slack/notify:
+          event: fail
+          template: basic_fail_1
+      - slack/notify:
+          event: pass
+          template: basic_success_1
+
+workflows:
+  build_and_test:
+    when: << pipeline.parameters.GHA_Action >>
+    jobs:
+      - test:
+          context:
+            - Slack

.github/workflows/circleci.yml

@@ -0,0 +1,11 @@
+name: Trigger CircleCI
+on: [push, pull_request, workflow_dispatch]
+jobs:
+  trigger-circleci:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Trigger CircleCI Build
+        id: trigger_circleci_build
+        uses: CircleCI-Public/[email protected]
+        env:
+          CCI_TOKEN: ${{ secrets.CCI_TOKEN }}

README.md

@@ -1,6 +1,6 @@
 # ckanext-canada
 
-[![Tests](https://github.com/open-data/ckanext-canada/workflows/Tests/badge.svg?branch=master&event=push)](https://github.com/open-data/ckanext-canada/actions?query=name%3ATests)
+[![CircleCI](https://dl.circleci.com/status-badge/img/gh/open-data/ckanext-canada/tree/master.svg?style=svg)](https://dl.circleci.com/status-badge/redirect/gh/open-data/ckanext-canada/tree/master)
 
 Government of Canada CKAN Extension - Extension à CKAN du Gouvernement du Canada
 
@@ -11,7 +11,7 @@ Features:
 Installation:
 
 * Use [open-data fork of CKAN](https://github.com/open-data/ckan),
-  branch canada-v2.8
+  branch canada-v2.9
 
 From a clean database you must run:
 
@@ -40,6 +40,10 @@ before loading any data.
   package processing between CKAN and Solr (requires
   `Scheming extension`, see below)
 
+`canada_security`
+  extra security processing (requires
+  `Security extension`, see below)
+
 ## Related Open Source Repos
 
 Project | Github group/repo | Our Contribution
@@ -64,14 +68,14 @@ The CKAN ini file needs the following settings for the registry server:
 ```ini
 ckan.plugins = dcat dcat_json_interface googleanalytics canada_forms canada_internal
         canada_public datastore recombinant
-        canada_datasets scheming_organizations fluent
+        canada_datasets scheming_organizations canada_security fluent
 ```
 
 For the public server use only:
 
 ```ini
 ckan.plugins = dcat dcat_json_interface googleanalytics canada_forms
-        canada_public canada_datasets scheming_organizations fluent
+        canada_public canada_datasets scheming_organizations canada_security fluent
 
 canada.portal_url = http://myserver.com
 
@@ -149,17 +153,36 @@ You will need to rebuild your search index using:
 ckan search-index rebuild
 ```
 
-## Compiling the updated French localization strings
+## Localization strings
+
+To update strings in the translation files:
+
+```bash
+python setup.py extract_messages
+```
+
+Extract messages will gather `gettext` calls in Python, JS, and Jinja2 files. It will also use th custom PD extractor to get specific strings for the Recombinant YAML files.
+
+To update the English and French catalog files:
+
+```bash
+python setup.py update_catalog
+```
+
+This will update both English and French PO files. You will need to confirm that there are NO `fuzzy` translations in either of the PO files.
+
+After updating the PO files and ensuring that there are no fuzzies, you may commit the two PO files along with the POT file.
+
+### Compiling localization strings
 
 Each time you install or update this extension you need to install the
 updated translations by running:
 
 ```bash
-bin/build-combined-ckan-mo.sh
+python setup.py compile_catalog
 ```
 
-This script overwrites the ckan French translations by combining it with
-ours.
+
 
 ## Migrating Proactive Disclosure (recombinant) data
 

ckanext/canada/extract.py

@@ -0,0 +1,73 @@
+from yaml import load
+from yaml.loader import SafeLoader
+from six import string_types
+
+
+class SafeLineLoader(SafeLoader):
+    def __init__(self, stream):
+        super(SafeLineLoader, self).__init__(stream)
+        self.line_numbers = {}
+
+    def construct_scalar(self, node):
+        if node.value in self.line_numbers:
+            self.line_numbers[node.value].append(node.start_mark.line + 1)
+        else:
+            self.line_numbers[node.value] = [node.start_mark.line + 1]
+        return super(SafeLineLoader, self).construct_scalar(node)
+
+    def get_single_data(self):
+        data = super(SafeLineLoader, self).get_single_data()
+        return self.line_numbers, data
+
+
+def extract_pd(fileobj, keywords, comment_tags, options):
+    """Extract messages from XXX files.
+
+    :param fileobj: the file-like object the messages should be extracted
+                    from
+    :param keywords: a list of keywords (i.e. function names) that should
+                     be recognized as translation functions
+    :param comment_tags: a list of translator tags to search for and
+                         include in the results
+    :param options: a dictionary of additional options (optional)
+    :return: an iterator over ``(lineno, funcname, message, comments)``
+             tuples
+    :rtype: ``iterator``
+    """
+    encoding = options.get('encoding', 'utf-8')
+    line_numbers, chromo = load(fileobj.read().decode(encoding), Loader=SafeLineLoader)
+
+    pd_type = chromo.get('dataset_type', 'unknown')
+
+    # PD Type Title
+    title = chromo.get('title')
+    if isinstance(title, string_types):
+        for line_number in line_numbers.get(title, [0]):
+            yield (line_number, '', title, ['Title for PD Type: %s' % pd_type])
+
+    # PD Type Short Label
+    label = chromo.get('shortname')
+    if isinstance(label, string_types):
+        for line_number in line_numbers.get(label, [0]):
+            yield (line_number, '', label, ['Label for PD Type: %s' % pd_type])
+
+    # PD Type Description
+    notes = chromo.get('notes')
+    if isinstance(notes, string_types):
+        for line_number in line_numbers.get(notes, [0]):
+            yield (line_number, '', notes, ['Description for PD Type: %s' % pd_type])
+
+    # PD Type Resources
+    resources = chromo.get('resources', [])
+    for resource in resources:
+        resource_title = resource.get('title')  # resource title
+        if isinstance(resource_title, string_types):
+            for line_number in line_numbers.get(resource_title, [0]):
+                yield (line_number, '', resource_title, ['Resource Title for PD Type: %s' % pd_type])
+
+        resource_trigger_strings = resource.get('trigger_strings')  # resource sql error messages
+        if resource_trigger_strings:
+            for k, v in resource_trigger_strings.items():
+                if isinstance(v, string_types):
+                    for line_number in line_numbers.get(v, [0]):
+                        yield (line_number, '', v, ['SQL Trigger String for PD Type: %s' % pd_type])