diff --git a/.gitignore b/.gitignore index ed4562890..f824f2cf8 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,7 @@ dist development.ini .DS_Store .idea +.eggs/** *.prefs ckan.log diff --git a/ckanext/canada/cli.py b/ckanext/canada/cli.py index 85bd733a9..11e0c2ae7 100644 --- a/ckanext/canada/cli.py +++ b/ckanext/canada/cli.py @@ -12,7 +12,7 @@ def _get_user(user): def get_commands(): - return [canada] + return canada @click.group(short_help="Canada management commands") diff --git a/ckanext/canada/pd.py b/ckanext/canada/pd.py index c7729efcd..04526df32 100644 --- a/ckanext/canada/pd.py +++ b/ckanext/canada/pd.py @@ -1,13 +1,10 @@ -# -*- coding: utf-8 -*- +import click import os import hashlib import calendar import time from babel.numbers import format_currency, format_decimal -import paste.script -from ckan.lib.cli import CkanCommand - from ckanapi import LocalCKAN, NotFound from ckanext.recombinant.tables import ( @@ -25,133 +22,80 @@ safe_for_solr) -class PDCommand(CkanCommand): - """ - Manage the Proactive Disclosures SOLR indexes + data files - - Usage:: - - paster clear - rebuild [--lenient] [-f ] [-s ] +def get_commands(): + return pd - Options:: - -f/--csv-file use specified CSV files as contracts input, - instead of the (default) CKAN database - -s/--solr-url use specified solr URL as output, - instead of default from ini file. - --lenient allow rebuild from csv files without checking - that columns match expected columns +@click.group(short_help="Proactive Disclosure/Publication management commands") +def pd(): + """Proactive Disclosure/Publication management commands. """ - summary = __doc__.split('\n')[0] - usage = __doc__ - - parser = paste.script.command.Command.standard_parser(verbose=True) - parser.add_option('-c', '--config', dest='config', - default='development.ini', help='Config file to use.') - parser.add_option( - '-f', - '--csv', - dest='csv_file', - help='CSV file to use as input (or default CKAN DB)') - parser.add_option( - '-s', - '--solr-url', - dest='solr_url', - help='Solr URL for output') - parser.add_option( - '--lenient', - action='store_false', - dest='strict', - default=True) - - def command(self): - if not self.args or self.args[0] in ['--help', '-h', 'help']: - print(self.__doc__) - return - - cmd = self.args[0] - self._load_config() - - if cmd == 'clear': - return clear_index(self.command_name) - elif cmd == 'rebuild': - return rebuild( - self.command_name, - [self.options.csv_file] if self.options.csv_file else None, - self.options.solr_url, - self.options.strict, - ) - - -class PDNilCommand(CkanCommand): + pass + +#TODO: Get chromos and make either the group ot subcommands dynamic based on schemas... + +@pd.command(short_help="Clear all SOLR records.") +@click.argument("pd_type") +def clear(pd_type): + clear_index(pd_type) + + +@pd.command(short_help="Rebuilds and reindexes all SOLR records.") +@click.argument("pd_type") +@click.option( + "--lenient", + is_flag=True, + help="Allow rebuild from csv files without checking hat columns match expected columns.", +) +@click.option( + "-f", + "--file", + default=None, + multiple=True, + help="CSV file to use as input (or default CKAN DB). Use PD and PD-nil files for NIL types.", +) +@click.option( + "-s", + "--solr-url", + default=None, + help="Solr URL for output.", +) +@click.option( + "-n", + "--is-nil", + is_flag=True, + help="If the PD Type is a NIL type.", +) +def rebuild(pd_type, files=None, solr_url=None, lenient=False, is_nil=False): """ - Manage the Proactive Disclosures SOLR indexes + data files + Rebuilds and reindexes all SOLR records. - Usage:: + Full Usage:\n + pd rebuild [--lenient] [-f ] [-s ] - paster clear - rebuild [-f ] [-s ] - - Options:: - - -f/--csv-file use specified CSV files as PD and - PD-nil input, instead of the - (default) CKAN database - -s/--solr-url use specified solr URL as output, - instead of default from ini file. - --lenient allow rebuild from csv files without checking - that columns match expected columns + For NIL:\n + pd rebuild [--lenient] [-f ] [-s ] """ - summary = __doc__.split('\n')[0] - usage = __doc__ - - parser = paste.script.command.Command.standard_parser(verbose=True) - parser.add_option('-c', '--config', dest='config', - default='development.ini', help='Config file to use.') - parser.add_option( - '-f', - '--csv', - nargs=2, - dest='csv_files', - help='CSV files to use as input (or default CKAN DB)') - parser.add_option( - '-s', - '--solr-url', - dest='solr_url', - help='Solr URL for output') - parser.add_option( - '--lenient', - action='store_false', - dest='strict', - default=True) - - def command(self): - if not self.args or self.args[0] in ['--help', '-h', 'help']: - print(self.__doc__) - return - - cmd = self.args[0] - self._load_config() - - if cmd == 'clear': - return clear_index(self.command_name) - elif cmd == 'rebuild': - return rebuild( - self.command_name, - self.options.csv_files, - self.options.solr_url, - self.options.strict, - ) - - -def clear_index(command_name, solr_url=None, commit=True): - conn = solr_connection(command_name, solr_url) + if files: + if not is_nil and len(files) >= 2: + raise ValueError('You may only supply one file for non NIL types') + if len(files) > 2: + raise ValueError('You may only supply up to two files') + strict = True + if lenient: + strict = False + rebuild(pd_type, + files, + solr_url, + strict) + + +def clear_index(pd_type, solr_url=None, commit=True): + conn = solr_connection(pd_type, solr_url) conn.delete(q="*:*", commit=commit) - -def rebuild(command_name, csv_files=None, solr_url=None, strict=True): +def rebuild(pd_type, csv_files=None, solr_url=None, strict=True): """ Implement rebuild command @@ -161,9 +105,9 @@ def rebuild(command_name, csv_files=None, solr_url=None, strict=True): :return: Nothing :rtype: None """ - clear_index(command_name, solr_url, False) + clear_index(pd_type, solr_url, False) - conn = solr_connection(command_name, solr_url) + conn = solr_connection(pd_type, solr_url) lc = LocalCKAN() if csv_files: for csv_file in csv_files: @@ -194,7 +138,7 @@ def rebuild(command_name, csv_files=None, solr_url=None, strict=True): count = 0 org_detail = lc.action.organization_show(id=org) unmatched = None - for resource_name, records in data_batch(org_detail['id'], lc, command_name): + for resource_name, records in data_batch(org_detail['id'], lc, pd_type): unmatched = _update_records( records, org_detail, conn, resource_name, unmatched) count += len(records) diff --git a/ckanext/canada/plugins.py b/ckanext/canada/plugins.py index ced7047bc..150c4b5d2 100755 --- a/ckanext/canada/plugins.py +++ b/ckanext/canada/plugins.py @@ -28,6 +28,7 @@ from ckanext.canada import auth from ckanext.canada import helpers from ckanext.canada import cli +from ckanext.canada.pd import get_commands as get_pd_commands from ckanext.canada import activity as act from ckanext.xloader.interfaces import IXloader import json @@ -621,7 +622,7 @@ def make_middleware(self, app, config): # IClick def get_commands(self): - return cli.get_commands() + return [cli.get_commands(), get_pd_commands()] diff --git a/requirements.txt b/requirements.txt index 11fb46f33..77b07d5f8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,3 +12,5 @@ backports.ssl-match-hostname>=3.5.0.1 azure-storage==0.33.0 configparser>=3.5.0 openpyxl==2.6.4 +Paste==3.6.0 +PasteScript==3.3.0