feat(cli): finished click implementation;

- Finalized `iClick` implementation.
- Deprecated old paster code.

ckanext/canada/cli.py

@@ -1,13 +1,241 @@
 import click
 
-from ckanext.canada.triggers import update_triggers
+from ckan.logic import get_action
+
+from ckanext.canada import utils, triggers
+
+
+def _get_user(user):
+    if user is not None:
+        return user
+    return get_action('get_site_user')({'ignore_auth': True}).get('name')
+
 
 def get_commands():
     return [canada]
 
 
-@click.group()
+@click.group(short_help="Canada management commands")
 def canada():
     """Canada management commands.
     """
     pass
+
+
+@canada.command(short_help="Updates Portal records with Registry records.")
+@click.argument("portal_ini")
+@click.option(
+    "-u",
+    "--ckan-user",
+    default=None,
+    help="Sets the owner of packages created, default: ckan system user",
+)
+@click.argument("last_activity_date", required=False)
+@click.option(
+    "-p",
+    "--processes",
+    default=1,
+    help="Sets the number of worker processes, default: 1",
+)
+@click.option(
+    "-m",
+    "--mirror",
+    is_flag=True,
+    help="Copy all datasets, default is to treat unreleased datasets as deleted",
+)
+@click.option(
+    "-l",
+    "--log",
+    default=None,
+    help="Write log of actions to log filename",
+)
+@click.option(
+    "-t",
+    "--tries",
+    default=1,
+    help="Try <num> times, set > 1 to retry on failures, default: 1",
+)
+@click.option(
+    "-d",
+    "--delay",
+    default=60,
+    help="Delay between retries, default: 60",
+)
+def portal_update(portal_ini,
+                  ckan_user,
+                  last_activity_date=None,
+                  processes=1,
+                  mirror=False,
+                  log=None,
+                  tries=1,
+                  delay=60):
+    """
+    Collect batches of packages modified at local CKAN since activity_date
+    and apply the package updates to the portal instance for all
+    packages with published_date set to any time in the past.
+
+    Full Usage:\n
+        canada portal-update <portal.ini> -u <user>\n
+                             [<last activity date> | [<k>d][<k>h][<k>m]]\n
+                             [-p <num>] [-m] [-l <log file>]\n
+                             [-t <num> [-d <seconds>]]
+
+    <last activity date>: Last date for reading activites, default: 7 days ago\n
+    <k> number of hours/minutes/seconds in the past for reading activities
+    """
+    utils.PortalUpdater(portal_ini,
+                        ckan_user,
+                        last_activity_date,
+                        processes,
+                        mirror,
+                        log,
+                        tries,
+                        delay).portal_update()
+
+
+@canada.command(short_help="Copy records from another source.")
+@click.option(
+    "-m",
+    "--mirror",
+    is_flag=True,
+    help="Copy all datasets, default is to treat unreleased datasets as deleted",
+)
+@click.option(
+    "-u",
+    "--ckan-user",
+    default=None,
+    help="Sets the owner of packages created, default: ckan system user",
+)
+@click.option(
+    "-o",
+    "--source",
+    default=None,
+    help="Source datastore url to copy datastore records",
+)
+def copy_datasets(mirror=False, ckan_user=None, source=None):
+    """
+    A process that accepts packages on stdin which are compared
+    to the local version of the same package.  The local package is
+    then created, updated, deleted or left unchanged.  This process
+    outputs that action as a string 'created', 'updated', 'deleted'
+    or 'unchanged'
+
+    Full Usage:\n
+        canada copy-datasets [-m] [-o <source url>]
+    """
+    utils.copy_datasets(source,
+                        _get_user(ckan_user),
+                        mirror)
+
+
+
+@canada.command(short_help="Lists changed records.")
+@click.argument("since_date")
+@click.option(
+    "-s",
+    "--server",
+    default=None,
+    help="Retrieve from <remote server>",
+)
+@click.option(
+    "-b",
+    "--brief",
+    is_flag=True,
+    help="Don't output requested dates",
+)
+def changed_datasets(since_date, server=None, brief=False):
+    """
+    Produce a list of dataset ids and requested dates. Each package
+    id will appear at most once, showing the activity date closest
+    to since_date. Requested dates are preceeded with a "#"
+
+    Full Usage:\n
+        canada changed-datasets [<since date>] [-s <remote server>] [-b]
+    """
+    utils.changed_datasets(since_date,
+                           server,
+                           brief)
+
+
+@canada.command(short_help="Load Suggested Datasets from a CSV file.")
+@click.argument("suggested_datasets_csv")
+@click.option(
+    "--use-created-date",
+    is_flag=True,
+    help="Use date_created field for date forwarded to data owner and other statuses instead of today's date",
+)
+def load_suggested(suggested_datasets_csv, use_created_date=False):
+    """
+    A process that loads suggested datasets from Drupal into CKAN
+
+    Full Usage:\n
+        canada load-suggested [--use-created-date] <suggested-datasets.csv>
+    """
+    utils.load_suggested(use_created_date,
+                         suggested_datasets_csv)
+
+
+@canada.command(short_help="Updates/creates database triggers.")
+def update_triggers():
+    """
+    Create/update triggers used by PD tables
+    """
+    triggers.update_triggers()
+
+
+@canada.command(short_help="Load Inventory Votes from a CSV file.")
+@click.argument("votes_json")
+def update_inventory_votes(votes_json):
+    """
+
+    Full Usage:\n
+        canada update-inventory-votes <votes.json>
+    """
+    utils.update_inventory_votes(votes_json)
+
+
+@canada.command(short_help="Tries to update resource sizes from a CSV file.")
+@click.argument("resource_sizes_csv")
+def resource_size_update(resource_sizes_csv):
+    """
+    Tries to update resource sizes from a CSV file.
+
+    Full Usage:\n
+        canada resource-size-update <resource_sizes.csv>
+    """
+    utils.resource_size_update(resource_sizes_csv)
+
+
+@canada.command(short_help="Tries to replace resource URLs from http to https.")
+@click.argument("https_report")
+@click.argument("https_alt_report")
+def update_resource_url_https(https_report, https_alt_report):
+    """
+    This function updates all broken http links into https links.
+    https_report: the report with all of the links (a .json file)
+    ex. https://github.com/open-data/opengov-orgs-http/blob/main/orgs_http_data.json.
+    https_alt_report: the report with links where alternates exist (a .json file)
+    ex. https://github.com/open-data/opengov-orgs-http/blob/main/https_alternative_count.json.
+    For more specifications about the files in use please visit,
+    https://github.com/open-data/opengov-orgs-http.
+
+    Full Usage:\n
+        canada update-resource-url-https <https_report> <https_alt_report>
+    """
+    utils.resource_https_update(https_report,
+                                https_alt_report)
+
+
+@canada.command(short_help="Runs ckanext-validation for all supported resources.")
+def bulk_validate():
+    """
+    Use this command to bulk validate the resources. Any resources which
+    are already in datastore but not validated will be removed.
+
+    Requires stdin
+
+    Full Usage:\n
+         ckanapi search datasets include_private=true -c $CONFIG_INI |\n
+         ckan -c $CONFIG_INI canada bulk-validate
+    """
+    utils.bulk_validate()

ckanext/canada/commands.py

@@ -1,127 +1,12 @@
-#!/usr/bin/env python
-from __future__ import print_function
-
-from docopt import docopt
-
 from ckan.lib.cli import CkanCommand
 
-import ckanext.canada.utils as utils
-from ckanext.canada.triggers import update_triggers
-
-
-#TODO: move all of this to ckanext.canda.cli and deprecate in the usage here...
-
-USAGE = """
-
-DEPRECATED: use `ckan [-c/--c=<config>] canada` instead.
-
-Commands:
-    - portal-update               Updates Portal records with Registry records.
-    - copy-datasets               Copy records from another source.
-    - changed-datasets            Lists changed records.
-    - load-suggested              Load Suggested Datasets from a CSV file.
-    - update-triggers             Updates/creates database triggers.
-    - update-inventory-votes      Load Inventory Votes from a CSV file.
-    - resource-size-update         Tries to update resource sizes from a CSV file.
-    - update-resource-url-https   Tries to replace resource URLs from http to https.
-    - bulk-validate               Runs ckanext-validation for all supported resources.
-
-Usage:
-    canada portal-update <portal.ini> -u <user>
-                         [<last activity date> | [<k>d][<k>h][<k>m]]
-                         [-p <num>] [-m] [-l <log file>]
-                         [-t <num> [-d <seconds>]] [--c=<config>]
-    canada copy-datasets [-m] [-o <source url>] [--c=<config>]
-    canada changed-datasets [<since date>] [-s <remote server>]
-                            [-b] [--c=<config>]
-    canada load-suggested [--use-created-date] <suggested-datasets.csv>
-                          [--c=<config>]
-    canada update-triggers [--c=<config>]
-    canada update-inventory-votes <votes.json> [--c=<config>]
-    canada resource-size-update <resource_sizes.csv> [--c=<config>]
-    canada update-resource-url-https <https_report> <https_alt_report>
-                                     [--c=<config>]
-    canada bulk-validate [--c=<config>]
-
-    <last activity date> for reading activites, default: 7 days ago
-    <k> number of hours/minutes/seconds in the past for reading activities
-"""
-
 
 class CanadaCommand(CkanCommand):
-    summary = 'ckanext-canada command line utilities.'
-    usage = USAGE
-
-
-    def __init__(self, name):
-        super(CanadaCommand, self).__init__(name)
-        self.parser.add_option(
-            '-p',
-            '--processes',
-            dest='processes',
-            default=1,
-            type='int',
-            help='sets the number of worker processes, default: 1'
-        )
-        self.parser.add_option(
-            '-u',
-            '--ckan-user',
-            dest='ckan_user',
-            default=None,
-            help='sets the owner of packages created, default: ckan system user'
-        )
-        self.parser.add_option('-l', '--log', dest='log', default=None, help='write log of actions to log filename')
-        self.parser.add_option('-m', '--mirror', dest='mirror', action='store_true', help='copy all datasets, default is to treat unreleased datasets as deleted')
-        self.parser.add_option(
-            '-a',
-            '--push-apikey',
-            dest='push_apikey',
-            default=None,
-            help='push to <remote server> using apikey'
-        )
-        self.parser.add_option('-s', '--server', dest='server', default=None, help='retrieve from <remote server>')
-        self.parser.add_option('-b', '--brief', dest='brief', action='store_true', help='don\'t output requested dates')
-        self.parser.add_option('-t', '--tries', dest='tries', default=1, type='int', help='try <num> times, set > 1 to retry on failures, default: 1')
-        self.parser.add_option('-d', '--delay', dest='delay', default=60, type='float', help='delay between retries, default: 60')
-        self.parser.add_option('--portal', dest='portal', action='store_true', help='don\'t filter record types')
-        self.parser.add_option('-o', '--source', dest='src_ds_url', default=None, help='source datastore url to copy datastore records')
-        self.parser.add_option('--use-created-date', dest='use_created_date', action='store_true', help='use date_created field for date forwarded to data owner and other statuses instead of today\'s date')
-
+    summary = "\nDEPRECATED: use `ckan [-c/--c=<config>] canada` instead.\n"
+    usage = "\nDEPRECATED: use `ckan [-c/--c=<config>] canada` instead.\n"
 
     def command(self):
-        '''
-        Parse command line arguments and call appropriate method.
-        '''
+        """
+        \nDEPRECATED: use `ckan [-c/--c=<config>] canada` instead.\n
+        """
         return
-        self._load_config()
-        args = docopt(USAGE, argv=self.args)
-
-        if args['portal-update']:
-            utils.portal_update(args[1], *args[2:])
-
-        elif args['copy-datasets']:
-            with utils._quiet_int_pipe():
-                utils.copy_datasets(source=self.options.src_ds_url,
-                                    user=self.options.ckan_user,
-                                    mirror=self.options.mirror)
-
-        elif args['changed-datasets']:
-            utils.changed_datasets(*self.args[1:])
-
-        elif args['update-triggers']:
-            update_triggers()
-
-        elif args['update-inventory-votes']:
-            utils.update_inventory_votes(*self.args[1:])
-
-        elif args['resource-size-update']:
-            utils.resource_size_update(*self.args[1:])
-
-        elif args['load-suggested']:
-            utils.load_suggested(self.options.use_created_date, *self.args[1:])
-
-        elif args['update-resource-url-https']:
-            utils.resource_https_update(*self.args[1:])
-
-        elif args['bulk-validate']:
-            utils.bulk_validate()