diff --git a/ckanext/xloader/interfaces.py b/ckanext/xloader/interfaces.py index cdecf52a..d32a0b16 100644 --- a/ckanext/xloader/interfaces.py +++ b/ckanext/xloader/interfaces.py @@ -10,6 +10,19 @@ class IXloader(Interface): The before_submit function, when implemented """ + def modify_download_request(self, url, resource, api_key, headers): + """ Can be used to modify the http download request. + The headers parameter is a dict which should be modified directly. + + Return value should be the modified (or unmodified) url. + + :param url: The download url + :param resource: The dict representation of the resource to be downloaded + :param api_key: The CKAN api key, in case authorization header needs to be modified + :param headers: The http request headers dict, to be modified directly + """ + return url + def can_upload(self, resource_id): """ This call when implemented can be used to stop the processing of the xloader submit function. This method will not be called if diff --git a/ckanext/xloader/jobs.py b/ckanext/xloader/jobs.py index 199cace1..1a44f341 100644 --- a/ckanext/xloader/jobs.py +++ b/ckanext/xloader/jobs.py @@ -14,6 +14,7 @@ import sqlalchemy as sa from ckan.plugins.toolkit import get_action +import ckan.plugins as p try: from ckan.plugins.toolkit import config except ImportError: @@ -22,6 +23,7 @@ import loader import db +import interfaces as xloader_interfaces from job_exceptions import JobError, HTTPError if config.get('ckanext.xloader.ssl_verify') in ['False', 'FALSE', '0', False, 0]: @@ -35,7 +37,6 @@ CHUNK_SIZE = 16 * 1024 # 16kb DOWNLOAD_TIMEOUT = 30 - # 'api_key': user['apikey'], # 'job_type': 'push_to_datastore', # 'result_url': callback_url, @@ -167,8 +168,12 @@ def xloader_data_into_datastore_(input, job_dict): headers['Authorization'] = api_key def get_url(): + _url = url + for plugin in p.PluginImplementations(xloader_interfaces.IXloader): + if hasattr(plugin, 'modify_download_request'): + _url = plugin.modify_download_request(_url, resource, api_key, headers) return requests.get( - url, + _url, headers=headers, timeout=DOWNLOAD_TIMEOUT, verify=SSL_VERIFY,