From 4081f09d23894721c422160bd66f7cb79a301d9e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Aicardi?= Date: Thu, 7 Feb 2019 18:10:24 +0100 Subject: [PATCH 01/25] add support for file parameters when a job expects a file to be uploaded, the following syntax can be used: =@ --- uws/UWS/connection.py | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/uws/UWS/connection.py b/uws/UWS/connection.py index 41b28cc..07a6a63 100644 --- a/uws/UWS/connection.py +++ b/uws/UWS/connection.py @@ -12,6 +12,8 @@ import urllib.parse as urllib from urllib.parse import urlparse import base64 +import mimetypes +import re class Connection(object): @@ -97,14 +99,35 @@ def get(self, path, params=None): return response def post(self, path, args): - params = urllib.urlencode(args) + # prepare multipart/form-data request + limit = '--------UWS_Client_Separator' + crlf = '\r\n' + params_list = [] + for key in args: + params_list.append('--' + limit) + if re.match('^@', args[key]): + dummy = 'Content-Disposition: form-data; ' + dummy += 'name="%s"; filename="%s"' % (key, args[key][1:]) + params_list.append(dummy) + dummy = 'Content-Type: ' + dummy += (mimetypes.guess_type(args[key])[0] or 'application/octet-stream') + params_list.append(dummy) + params_list.append('') + params_list.append(open(args[key][1:]).read()) + else: + params_list.append('Content-Disposition: form-data; name="%s"'%key) + params_list.append('') + params_list.append(args[key]) + params_list.append('--' + limit + '--') + params_list.append('') + params = crlf.join(params_list) if path: destination_url = self.base_path + "/" + path else: destination_url = self.base_path - self.headers['Content-type'] = "application/x-www-form-urlencoded" + self.headers['Content-type'] = "multipart/form-data; boundary=%s"%limit self.connection.request("POST", destination_url, body=params, headers=self.headers) response = self.connection.getresponse() response.read() # read body of request so we can send another From 80f8440f9a511ebb9efaafc810eb54c3086456fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Aicardi?= Date: Thu, 14 Mar 2019 15:56:12 +0100 Subject: [PATCH 02/25] python 3 support for cli --- uws/UWS/client.py | 2 +- uws/cli/main.py | 58 ++++++++++++++-------------- uws/lib/terminalsize/terminalsize.py | 4 +- 3 files changed, 32 insertions(+), 32 deletions(-) diff --git a/uws/UWS/client.py b/uws/UWS/client.py index 340562a..bb5d742 100644 --- a/uws/UWS/client.py +++ b/uws/UWS/client.py @@ -145,7 +145,7 @@ def get_phase(self, id): raise UWSError(str(e)) raw = response.read() - result = raw + result = str(raw, 'utf-8') return result diff --git a/uws/cli/main.py b/uws/cli/main.py index 79a7bc3..024ae9a 100755 --- a/uws/cli/main.py +++ b/uws/cli/main.py @@ -5,7 +5,7 @@ import texttable as tt from uws.lib.terminalsize import terminalsize as console -import cli_parser +from . import cli_parser from uws import UWS debug = False @@ -18,11 +18,11 @@ def handle(self, *args, **kwargs): return handler(self, *args, **kwargs) except UWS.UWSError as e: if not debug: - print "An error occurred:\n %s" % e.msg + print("An error occurred:\n %s" % e.msg) return else: - print "An error occurred:\n %s" % e.msg - print e.raw + print("An error occurred:\n %s" % e.msg) + print(e.raw) raise return handle @@ -98,9 +98,9 @@ def list_jobs(url, user_name, password, phases, after=None, last=None): table.set_cols_dtype(dtypes) # ['t', 't', 't', 't', 't']) table.add_rows(rows) - print "List of jobs on UWS service for user: '%s'" % user_name - print table.draw() - print "%d jobs listed." % (len(rows) - 1) + print("List of jobs on UWS service for user: '%s'" % user_name) + print(table.draw()) + print("%d jobs listed." % (len(rows) - 1)) def _register_job_reference_for_table(rows, jobref): @@ -147,7 +147,7 @@ def show_job(url, user_name, password, id, wait, phase): job = uws_client.get_job(id, wait, phase) if wait and job.version != "1.1": - print "Warning: Wait keyword is (probably) not supported by the server's UWS version %s (need 1.1). Server will probably ignore wait and return immediately." % job.version + print("Warning: Wait keyword is (probably) not supported by the server's UWS version %s (need 1.1). Server will probably ignore wait and return immediately." % job.version) _print_job(job) @@ -175,12 +175,12 @@ def new_job(url, user_name, password, parameters={}, run=False): _print_job(job) - print "\n" - print "*" * (console_width - 1) - print "You can access this job with the id:\n" - print "Job ID: %s" % job.job_id - print "Command: uws -H %s --user %s --password YOUR_PASSWORD_HERE job show %s" % (url, user_name, job.job_id) - print "*" * (console_width - 1) + print("\n") + print("*" * (console_width - 1)) + print("You can access this job with the id:\n") + print("Job ID: %s" % job.job_id) + print("Command: uws -H %s --user %s --password YOUR_PASSWORD_HERE job show %s" % (url, user_name, job.job_id)) + print("*" * (console_width - 1)) @handle_error @@ -220,7 +220,7 @@ def delete_job(url, user_name, password, id): success = uws_client.delete_job(id) if success: - print "Job %s successfully deleted!" % (id) + print("Job %s successfully deleted!" % (id)) @handle_error @@ -240,10 +240,10 @@ def print_progress(total_size, current): # if there are multiple result sets returned: force user to decide which ones to use? # or maybe rather let the service define a standard result? but how? if len(job.results) > 1 and not result_id: - print 'There are multiple results for this job, all of them are downloaded now.' - print 'If this is not what you intended, please specify the id of your desired result like this: ' - print '\nuws job results ID RESULTID\n' - print 'For RESULTID, you can choose from: ', ','.join([result.id for result in job.results]) + print('There are multiple results for this job, all of them are downloaded now.') + print('If this is not what you intended, please specify the id of your desired result like this: ') + print('\nuws job results ID RESULTID\n') + print('For RESULTID, you can choose from: ', ','.join([result.id for result in job.results])) # set file base name to job_id or tablename (if available) or user provided file_base file_base = job.job_id @@ -263,18 +263,18 @@ def print_progress(total_size, current): url = str(result.reference) - print "Downloading %s into file '%s'" % (result.id, filename) + print("Downloading %s into file '%s'" % (result.id, filename)) uws_client.connection.download_file(url, user_name, password, filename, callback=print_progress) - print "" - print "Finished downloading file '%s'\n" % (filename) + print("") + print("Finished downloading file '%s'\n" % (filename)) retrieved = True if not retrieved: if result_id: - print "Result Id '%s' not available. Use 'uws job show %s' for a list of available results." % (result_id, job.job_id) + print("Result Id '%s' not available. Use 'uws job show %s' for a list of available results." % (result_id, job.job_id)) else: - print "The job with id '%s' has no results." % (job.job_id) - print "Check with 'uws job show %s' the details, the job results may have been deleted." % (job.job_id) + print("The job with id '%s' has no results." % (job.job_id)) + print("Check with 'uws job show %s' the details, the job results may have been deleted." % (job.job_id)) def _print_job(job): @@ -313,7 +313,7 @@ def _print_job(job): pass for info in job.job_info: - rows.append(["Job info", unicode(info)]) + rows.append(["Job info", info]) (console_width, console_height) = console.get_terminal_size() @@ -325,7 +325,7 @@ def _print_job(job): table.set_cols_dtype(['t', 't']) table.set_cols_width([max_field_len, console_width - max_field_len - 4]) table.add_rows(rows) - print table.draw() + print(table.draw()) # check validity of wait and phases: @@ -392,7 +392,7 @@ def main(): if arguments.P: if arguments.password: - print "Error: You cannot use -P and --password together!" + print("Error: You cannot use -P and --password together!") sys.exit(1) arguments.password = getpass.getpass("Enter password: ") @@ -455,7 +455,7 @@ def main(): elif arguments.job_command == "results": results_job(arguments.host, arguments.user, arguments.password, arguments.id, arguments.result_id, arguments.file_base) else: - print "Error: Unknown command %s\n" % (arguments.job_command) + print("Error: Unknown command %s\n" % (arguments.job_command)) if __name__ == '__main__': main() diff --git a/uws/lib/terminalsize/terminalsize.py b/uws/lib/terminalsize/terminalsize.py index 570ede0..3d423c0 100644 --- a/uws/lib/terminalsize/terminalsize.py +++ b/uws/lib/terminalsize/terminalsize.py @@ -25,7 +25,7 @@ def get_terminal_size(): if current_os in ['Linux', 'Darwin'] or current_os.startswith('CYGWIN'): tuple_xy = _get_terminal_size_linux() if tuple_xy is None: - print "default" + print("default") tuple_xy = (80, 25) # default value return tuple_xy @@ -88,4 +88,4 @@ def ioctl_GWINSZ(fd): if __name__ == "__main__": sizex, sizey = get_terminal_size() - print 'width =', sizex, 'height =', sizey \ No newline at end of file + print('width =', sizex, 'height =', sizey) From edd0fe20238a97ad129ac2c8ab9a84b5f3c18a31 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Aicardi?= Date: Thu, 30 Jan 2020 17:27:49 +0100 Subject: [PATCH 03/25] Fix unicode --- uws/UWS/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uws/UWS/models.py b/uws/UWS/models.py index 6ac8acb..f18ced3 100644 --- a/uws/UWS/models.py +++ b/uws/UWS/models.py @@ -88,7 +88,7 @@ def __init__(self, xml=None): if xml is not None: # parse xml - parsed = et.fromstring(xml.encode('utf-8')) + parsed = et.fromstring(xml.decode()) uws_flavour = UWS1Flavour(parsed.nsmap) From 8f5f998594ee10d6a950a650e251c77366860554 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Aicardi?= Date: Thu, 30 Jan 2020 17:30:15 +0100 Subject: [PATCH 04/25] Fixes python version for Travis --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index f72f2c6..3bf3039 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,7 +4,7 @@ cache: directories: - $HOME/virtualenv python: -- 2.7 +- 3.7 install: - pip install -r requirements.pip - pip install -r requirements-dev.pip From 0e325ff2103cea0ba388b8d5428e223e319cea63 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Aicardi?= Date: Thu, 30 Jan 2020 17:39:08 +0100 Subject: [PATCH 05/25] Check type of xml xml is a string in the tests, but it is a bytes when talking to the server --- uws/UWS/models.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/uws/UWS/models.py b/uws/UWS/models.py index f18ced3..224c8f7 100644 --- a/uws/UWS/models.py +++ b/uws/UWS/models.py @@ -88,7 +88,10 @@ def __init__(self, xml=None): if xml is not None: # parse xml - parsed = et.fromstring(xml.decode()) + if isinstance(xml, bytes): + parsed = et.fromstring(xml.decode()) + else: + parsed = et.fromstring(xml) uws_flavour = UWS1Flavour(parsed.nsmap) From 4861c3912f1b267807f0f18203019c258df2caf0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Aicardi?= Date: Thu, 30 Jan 2020 18:08:05 +0100 Subject: [PATCH 06/25] Fix for travis ? --- setup.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index 5426976..dfc9a7d 100644 --- a/setup.py +++ b/setup.py @@ -15,8 +15,8 @@ setup( name='uws-client', version=version, - url='http://github.com/aipescience/uws-client/', - download_url='http://github.com/aipescience/uws-client/archive/%s.tar.gz' % version, + url='http://github.com/aicardi-obspm/uws-client/', + download_url='http://github.com/aicardi-obspm/uws-client/archive/%s.tar.gz' % version, packages=find_packages(), license=u'Apache License (2.0)', author=u'Adrian M. Partl', @@ -24,7 +24,7 @@ maintainer=u'AIP E-Science', maintainer_email=u'escience@aip.de', description=u'A command line client for IVOA UWS services, plus models for development', - long_description='This is a client for IVOA Virtual Observatroy UWS services. It can be used to access UWS services directly or through Basic Authentication. Please visit https://github.com/aipescience/uws-client/blob/master/README.md for how to use the software.', + long_description='This is a client for IVOA Virtual Observatroy UWS services. It can be used to access UWS services directly or through Basic Authentication. Please visit https://github.com/aicardi-obspm/uws-client/blob/master/README.md for how to use the software.', include_package_data=True, install_requires=install_requires, entry_points={ From 299f4efc5ec8790f82b589ed27b9113dc7b5aeec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Aicardi?= Date: Thu, 30 Jan 2020 18:45:09 +0100 Subject: [PATCH 07/25] debug nosetests on travis --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index 3bf3039..5dc35c9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,6 +10,8 @@ install: - pip install -r requirements-dev.pip - pip install coveralls script: +- nosetests -V +- which nosetests - nosetests --with-coverage --cover-tests --cover-html --cover-package=uws after_success: - coveralls From 4337be48ee257e67bf0c3faa49e10e64d9bfeb5c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Aicardi?= Date: Thu, 30 Jan 2020 18:53:23 +0100 Subject: [PATCH 08/25] ends debug on travis, should fix nosetests --- .travis.yml | 2 -- uws/UWS/models.py | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 5dc35c9..3bf3039 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,8 +10,6 @@ install: - pip install -r requirements-dev.pip - pip install coveralls script: -- nosetests -V -- which nosetests - nosetests --with-coverage --cover-tests --cover-html --cover-package=uws after_success: - coveralls diff --git a/uws/UWS/models.py b/uws/UWS/models.py index 224c8f7..ac18e58 100644 --- a/uws/UWS/models.py +++ b/uws/UWS/models.py @@ -91,7 +91,7 @@ def __init__(self, xml=None): if isinstance(xml, bytes): parsed = et.fromstring(xml.decode()) else: - parsed = et.fromstring(xml) + parsed = et.fromstring(xml.encode('utf-8')) uws_flavour = UWS1Flavour(parsed.nsmap) From 6abc171a7f546c95d1b991d3b18ae97479ece64f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Aicardi?= Date: Fri, 31 Jan 2020 11:26:05 +0100 Subject: [PATCH 09/25] pylint first pass --- uws/UWS/connection.py | 29 ++++++++---------- uws/UWS/models.py | 4 ++- uws/UWS/tests/test_base.py | 4 +-- uws/cli/main.py | 46 ++++++++++++++++++---------- uws/lib/terminalsize/terminalsize.py | 18 +++++------ 5 files changed, 55 insertions(+), 46 deletions(-) diff --git a/uws/UWS/connection.py b/uws/UWS/connection.py index 07a6a63..ba3ab5c 100644 --- a/uws/UWS/connection.py +++ b/uws/UWS/connection.py @@ -1,16 +1,8 @@ # -*- coding: utf-8 -*- -try: - import httplib -except ImportError: - import http.client as httplib -try: - import urllib2 - import urllib - from urlparse import urlparse -except ImportError: - import urllib.request as urllib2 - import urllib.parse as urllib - from urllib.parse import urlparse +import http.client as httplib +import urllib.request as urllib2 +import urllib.parse as urllib +from urllib.parse import urlparse import base64 import mimetypes import re @@ -21,7 +13,7 @@ def __init__(self, url, user=None, password=None): self._set_url(url) if user is not None and password is not None: - try: + try: user = user.decode('utf-8') password = password.decode('utf-8') except AttributeError: @@ -30,7 +22,7 @@ def __init__(self, url, user=None, password=None): self.auth_string = base64.encodestring(auth_string.encode()) self.headers = {"Authorization": ("Basic %s" % - self.auth_string.decode('utf-8')).strip('\n')} + self.auth_string.decode('utf-8')).strip('\n')} else: self.headers = {} @@ -94,7 +86,8 @@ def get(self, path, params=None): raise RuntimeError('Resource does not exist') if response.status != 200: - raise RuntimeError('Error with connection to server: Got response: %s %s' % (response.status, response.reason)) + raise RuntimeError('Error with connection to server: ' + + 'Got response: %s %s' % (response.status, response.reason)) return response @@ -163,7 +156,8 @@ def post(self, path, args): raise RuntimeError('Resource does not exist') if response.status != 200: - raise RuntimeError('Error with connection to server: Got response: %s %s' % (response.status, response.reason)) + raise RuntimeError('Error with connection to server: ' + + 'Got response: %s %s' % (response.status, response.reason)) return response @@ -204,7 +198,8 @@ def delete(self, path): raise RuntimeError('Resource does not exist') if response.status != 200: - raise RuntimeError('Error with connection to server: Got response: %s %s' % (response.status, response.reason)) + raise RuntimeError('Error with connection to server: ' + + 'Got response: %s %s' % (response.status, response.reason)) return response diff --git a/uws/UWS/models.py b/uws/UWS/models.py index ac18e58..1920eb5 100644 --- a/uws/UWS/models.py +++ b/uws/UWS/models.py @@ -1,8 +1,10 @@ # -*- coding: utf-8 -*- from __future__ import print_function +import six from lxml import etree as et + uws_1_namespace = "http://www.ivoa.net/xml/UWS/v1.0" #uws_2_namespace = "http://www.ivoa.net/xml/UWS/v2.0" xlink_namespace = "http://www.w3.org/1999/xlink" @@ -151,7 +153,7 @@ def __init__(self, id=None, phase=None, reference=None, xml_node=None, xml_names elif id is not None and phase is not None and reference is not None: self.id = id - if isinstance(phase, basestring): + if isinstance(phase, six.string_types): self.phase = [phase] else: self.phase = phase diff --git a/uws/UWS/tests/test_base.py b/uws/UWS/tests/test_base.py index 9d9d240..9d7ca45 100644 --- a/uws/UWS/tests/test_base.py +++ b/uws/UWS/tests/test_base.py @@ -96,7 +96,7 @@ def testValidateAndParseAfterLastFilter(self): params = UWS.client.Client("/")._validate_and_parse_filters(filters) self.assertEqual(params, [('AFTER','2015-09-10T10:01:02.135000'), - ('LAST', 100)]) + ('LAST', 100)]) def testValidateAndParseAfterLastPhaseFilter(self): filters = { @@ -108,7 +108,7 @@ def testValidateAndParseAfterLastPhaseFilter(self): params = UWS.client.Client("/")._validate_and_parse_filters(filters) self.assertEqual(params, [ ('PHASE','PENDING'), ('PHASE','ERROR'), - ('AFTER','2015-09-10T10:01:02.135000'), ('LAST', 100)]) + ('AFTER','2015-09-10T10:01:02.135000'), ('LAST', 100)]) def testValidateAndParseWaitNegative(self): wait = '-1' diff --git a/uws/cli/main.py b/uws/cli/main.py index 024ae9a..7e7b89c 100755 --- a/uws/cli/main.py +++ b/uws/cli/main.py @@ -147,7 +147,8 @@ def show_job(url, user_name, password, id, wait, phase): job = uws_client.get_job(id, wait, phase) if wait and job.version != "1.1": - print("Warning: Wait keyword is (probably) not supported by the server's UWS version %s (need 1.1). Server will probably ignore wait and return immediately." % job.version) + print("Warning: Wait keyword is (probably) not supported by the server's UWS version " + + f"{job.version} (need 1.1). Server will probably ignore wait and return immediately.") _print_job(job) @@ -178,8 +179,9 @@ def new_job(url, user_name, password, parameters={}, run=False): print("\n") print("*" * (console_width - 1)) print("You can access this job with the id:\n") - print("Job ID: %s" % job.job_id) - print("Command: uws -H %s --user %s --password YOUR_PASSWORD_HERE job show %s" % (url, user_name, job.job_id)) + print(f"Job ID: {job.job_id}") + print(f"Command: uws -H {url} --user {user_name} --password YOUR_PASSWORD_HERE" + + f"job show {job.job_id}") print("*" * (console_width - 1)) @@ -230,7 +232,7 @@ def print_progress(total_size, current): sys.stdout.write("\r%d bytes" % current) # or print >> sys.stdout, "\r%d%%" %i, sys.stdout.flush() else: - sys.stdout.write("\rDownloaded %d bytes" % current) # or print >> sys.stdout, "\r%d%%" %i, + sys.stdout.write(f"\rDownloaded {current} bytes") # or print >> sys.stdout, "\r%d%%" %i, sys.stdout.flush() uws_client = UWS.client.Client(url=url, user=user_name, password=password) @@ -241,9 +243,11 @@ def print_progress(total_size, current): # or maybe rather let the service define a standard result? but how? if len(job.results) > 1 and not result_id: print('There are multiple results for this job, all of them are downloaded now.') - print('If this is not what you intended, please specify the id of your desired result like this: ') + print('If this is not what you intended, please specify the id of your desired', end='') + print(' result like this: ') print('\nuws job results ID RESULTID\n') - print('For RESULTID, you can choose from: ', ','.join([result.id for result in job.results])) + print('For RESULTID, you can choose from: ', + ','.join([result.id for result in job.results])) # set file base name to job_id or tablename (if available) or user provided file_base file_base = job.job_id @@ -264,17 +268,20 @@ def print_progress(total_size, current): url = str(result.reference) print("Downloading %s into file '%s'" % (result.id, filename)) - uws_client.connection.download_file(url, user_name, password, filename, callback=print_progress) + uws_client.connection.download_file(url, user_name, password, + filename, callback=print_progress) print("") print("Finished downloading file '%s'\n" % (filename)) retrieved = True if not retrieved: if result_id: - print("Result Id '%s' not available. Use 'uws job show %s' for a list of available results." % (result_id, job.job_id)) + print(f"Result Id '{result_id}' not available. ", end="") + print(f"Use 'uws job show {job.job_id}' for a list of available results.") else: - print("The job with id '%s' has no results." % (job.job_id)) - print("Check with 'uws job show %s' the details, the job results may have been deleted." % (job.job_id)) + print(f"The job with id '{job.job_id}' has no results.") + print(f"Check with 'uws job show {job.job_id}' the details, ", end="") + print("the job results may have been deleted.") def _print_job(job): @@ -283,7 +290,7 @@ def _print_job(job): rows.append(["Job id", job.job_id]) rows.append(["Run id", job.run_id]) - + if(job.owner_id): rows.append(["Owner id", job.owner_id]) @@ -291,7 +298,7 @@ def _print_job(job): if(job.quote): rows.append(["Quote", job.quote]) - + if(job.creation_time): rows.append(["Creation time", job.creation_time]) @@ -335,14 +342,16 @@ def _check_job_wait_args(arguments): phase = arguments.phase if wait is None and phase is not None: - raise RuntimeError("Additional phase for 'job show' only allowed in combination with 'wait'-keyword.") + raise RuntimeError("Additional phase for 'job show' only allowed in combination with" + + "'wait'-keyword.") if phase: if phase.upper() in UWS.models.JobPhases.active_phases: phase = phase.upper() else: active_phases = ', '.join(UWS.models.JobPhases.active_phases) - raise RuntimeError("Phase '" + phase + "' is not supported with WAIT keyword, choose one of the active phases: " + active_phases) + raise RuntimeError("Phase '" + phase + "' is not supported with WAIT keyword, " + + "choose one of the active phases: " + active_phases) return wait, phase @@ -440,12 +449,14 @@ def main(): # parse the job parameters and store in argument list job_parameters = _check_job_parameter_args(arguments.job_parameters) - new_job(arguments.host, arguments.user, arguments.password, job_parameters, arguments.run) + new_job(arguments.host, arguments.user, arguments.password, + job_parameters, arguments.run) elif arguments.job_command == "set": # parse the job parameters and store in argument list job_parameters = _check_job_parameter_args(arguments.job_parameters) - set_parameters_job(arguments.host, arguments.user, arguments.password, arguments.id, job_parameters) + set_parameters_job(arguments.host, arguments.user, arguments.password, + arguments.id, job_parameters) elif arguments.job_command == "run": run_job(arguments.host, arguments.user, arguments.password, arguments.id) elif arguments.job_command == "abort": @@ -453,7 +464,8 @@ def main(): elif arguments.job_command == "delete": delete_job(arguments.host, arguments.user, arguments.password, arguments.id) elif arguments.job_command == "results": - results_job(arguments.host, arguments.user, arguments.password, arguments.id, arguments.result_id, arguments.file_base) + results_job(arguments.host, arguments.user, arguments.password, arguments.id, + arguments.result_id, arguments.file_base) else: print("Error: Unknown command %s\n" % (arguments.job_command)) diff --git a/uws/lib/terminalsize/terminalsize.py b/uws/lib/terminalsize/terminalsize.py index 3d423c0..f02dc43 100644 --- a/uws/lib/terminalsize/terminalsize.py +++ b/uws/lib/terminalsize/terminalsize.py @@ -6,8 +6,8 @@ import struct import platform import subprocess - - + + def get_terminal_size(): """ getTerminalSize() - get width and height of console @@ -28,8 +28,8 @@ def get_terminal_size(): print("default") tuple_xy = (80, 25) # default value return tuple_xy - - + + def _get_terminal_size_windows(): try: from ctypes import windll, create_string_buffer @@ -48,8 +48,8 @@ def _get_terminal_size_windows(): return sizex, sizey except: pass - - + + def _get_terminal_size_tput(): # get terminal width # src: http://stackoverflow.com/questions/263890/how-do-i-find-the-width-height-of-a-terminal-window @@ -59,8 +59,8 @@ def _get_terminal_size_tput(): return (cols, rows) except: pass - - + + def _get_terminal_size_linux(): def ioctl_GWINSZ(fd): try: @@ -85,7 +85,7 @@ def ioctl_GWINSZ(fd): except: return None return int(cr[1]), int(cr[0]) - + if __name__ == "__main__": sizex, sizey = get_terminal_size() print('width =', sizex, 'height =', sizey) From 1fe23388213fa5ddef61d13d055cc44e5d92066e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Aicardi?= Date: Fri, 31 Jan 2020 11:27:13 +0100 Subject: [PATCH 10/25] python3 way of getting terminal size no need of a 9+ years old code in python2 --- uws/cli/main.py | 4 +- uws/lib/terminalsize/terminalsize.py | 91 ---------------------------- 2 files changed, 2 insertions(+), 93 deletions(-) delete mode 100644 uws/lib/terminalsize/terminalsize.py diff --git a/uws/cli/main.py b/uws/cli/main.py index 7e7b89c..7f7a6e3 100755 --- a/uws/cli/main.py +++ b/uws/cli/main.py @@ -3,7 +3,7 @@ from functools import wraps import texttable as tt -from uws.lib.terminalsize import terminalsize as console +import shutil from . import cli_parser from uws import UWS @@ -72,7 +72,7 @@ def list_jobs(url, user_name, password, phases, after=None, last=None): # add ARCHIVED phase as well for services with version 1.0 that already support this if job_phases.ARCHIVED in phases and job_phases.ARCHIVED in job.phase: _register_job_reference_for_table(rows, job) - (console_width, console_height) = console.get_terminal_size() + (console_width, console_height) = shutil.get_terminal_size() # Now we have the rows all stored. Check if all columns exist and remove # empty columns for a more friendly output. diff --git a/uws/lib/terminalsize/terminalsize.py b/uws/lib/terminalsize/terminalsize.py deleted file mode 100644 index f02dc43..0000000 --- a/uws/lib/terminalsize/terminalsize.py +++ /dev/null @@ -1,91 +0,0 @@ -#!/usr/bin/env python - -#source: https://gist.github.com/jtriley/1108174 -import os -import shlex -import struct -import platform -import subprocess - - -def get_terminal_size(): - """ getTerminalSize() - - get width and height of console - - works on linux,os x,windows,cygwin(windows) - originally retrieved from: - http://stackoverflow.com/questions/566746/how-to-get-console-window-width-in-python - """ - current_os = platform.system() - tuple_xy = None - if current_os == 'Windows': - tuple_xy = _get_terminal_size_windows() - if tuple_xy is None: - tuple_xy = _get_terminal_size_tput() - # needed for window's python in cygwin's xterm! - if current_os in ['Linux', 'Darwin'] or current_os.startswith('CYGWIN'): - tuple_xy = _get_terminal_size_linux() - if tuple_xy is None: - print("default") - tuple_xy = (80, 25) # default value - return tuple_xy - - -def _get_terminal_size_windows(): - try: - from ctypes import windll, create_string_buffer - # stdin handle is -10 - # stdout handle is -11 - # stderr handle is -12 - h = windll.kernel32.GetStdHandle(-12) - csbi = create_string_buffer(22) - res = windll.kernel32.GetConsoleScreenBufferInfo(h, csbi) - if res: - (bufx, bufy, curx, cury, wattr, - left, top, right, bottom, - maxx, maxy) = struct.unpack("hhhhHhhhhhh", csbi.raw) - sizex = right - left + 1 - sizey = bottom - top + 1 - return sizex, sizey - except: - pass - - -def _get_terminal_size_tput(): - # get terminal width - # src: http://stackoverflow.com/questions/263890/how-do-i-find-the-width-height-of-a-terminal-window - try: - cols = int(subprocess.check_call(shlex.split('tput cols'))) - rows = int(subprocess.check_call(shlex.split('tput lines'))) - return (cols, rows) - except: - pass - - -def _get_terminal_size_linux(): - def ioctl_GWINSZ(fd): - try: - import fcntl - import termios - cr = struct.unpack('hh', - fcntl.ioctl(fd, termios.TIOCGWINSZ, '1234')) - return cr - except: - pass - cr = ioctl_GWINSZ(0) or ioctl_GWINSZ(1) or ioctl_GWINSZ(2) - if not cr: - try: - fd = os.open(os.ctermid(), os.O_RDONLY) - cr = ioctl_GWINSZ(fd) - os.close(fd) - except: - pass - if not cr: - try: - cr = (os.environ['LINES'], os.environ['COLUMNS']) - except: - return None - return int(cr[1]), int(cr[0]) - -if __name__ == "__main__": - sizex, sizey = get_terminal_size() - print('width =', sizex, 'height =', sizey) From e9ee193b38d3ba6cc66f379c68392a1433b63565 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Aicardi?= Date: Fri, 31 Jan 2020 11:31:16 +0100 Subject: [PATCH 11/25] full supression of terminalsize --- uws/cli/main.py | 4 ++-- uws/lib/terminalsize/__init__.py | 0 2 files changed, 2 insertions(+), 2 deletions(-) delete mode 100644 uws/lib/terminalsize/__init__.py diff --git a/uws/cli/main.py b/uws/cli/main.py index 7f7a6e3..85dc2e7 100755 --- a/uws/cli/main.py +++ b/uws/cli/main.py @@ -172,7 +172,7 @@ def new_job(url, user_name, password, parameters={}, run=False): # execute the job job = uws_client.run_job(job.job_id) - (console_width, console_height) = console.get_terminal_size() + (console_width, console_height) = shutil.get_terminal_size() _print_job(job) @@ -322,7 +322,7 @@ def _print_job(job): for info in job.job_info: rows.append(["Job info", info]) - (console_width, console_height) = console.get_terminal_size() + (console_width, console_height) = shutil.get_terminal_size() fields = [row[0] for row in rows] max_field_len = len(max(fields, key=len)) diff --git a/uws/lib/terminalsize/__init__.py b/uws/lib/terminalsize/__init__.py deleted file mode 100644 index e69de29..0000000 From 5bcee76660c0d885c151335baa78125e16ddbf43 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Aicardi?= Date: Fri, 31 Jan 2020 13:46:28 +0100 Subject: [PATCH 12/25] pylint : long lines --- uws/UWS/client.py | 37 +++++---- uws/UWS/models.py | 29 ++++--- uws/UWS/tests/test_models.py | 143 ++++++++++++++++++++++++++++++----- uws/cli/cli_parser.py | 65 ++++++++++------ 4 files changed, 213 insertions(+), 61 deletions(-) diff --git a/uws/UWS/client.py b/uws/UWS/client.py index bb5d742..c324689 100644 --- a/uws/UWS/client.py +++ b/uws/UWS/client.py @@ -41,7 +41,8 @@ def get_job_list(self, filters=None): try: job_list = models.Jobs(raw) except XMLSyntaxError as e: - raise UWSError("Malformatted response. Are you sure the host you specified is a IVOA UWS service?", raw) + raise UWSError("Malformatted response. Are you sure the host you specified is a " + "IVOA UWS service?", raw) except Exception as e: raise e @@ -66,7 +67,8 @@ def _validate_and_parse_filters(self, filters): if after: # TODO: Allow to provide local time and convert here to UTC? - # TODO: We may encounter more troubles with microseconds, if ',' used instead of '.'(e.g. German systems) + # TODO: We may encounter more troubles with microseconds, + # if ',' used instead of '.'(e.g. German systems) try: date = dateutil.parser.parse(after) @@ -74,10 +76,12 @@ def _validate_and_parse_filters(self, filters): # given (e.g. '2010-09'->'2010-09-06'). # Let's tell the user how the given value was interpreted: if str(date) != str(after): - print("Note: Changed value for keyword 'after' from '%s' to '%s'." % (after, str(date))) + print(f"Note: Changed value for keyword 'after' from '{after}' " + f"to '{str(date)}'.") except: - raise UWSError("Date time format could not be parsed, expecting UTC in ISO 8601:2004 format or compatible: %s" % (str(after))) + raise UWSError("Date time format could not be parsed, expecting UTC in " + "ISO 8601:2004 format or compatible: %s" % (str(after))) # Convert from given time (with attached timezone information) to UTC time if date.utcoffset() is not None: @@ -92,10 +96,10 @@ def _validate_and_parse_filters(self, filters): try: last = int(last) except: - raise UWSError("Value for 'last' argument must be a positive integer: %s" % (str(last))) + raise UWSError(f"Value for 'last' argument must be a positive integer: {str(last)}") if last < 1: - raise UWSError("Value for 'last' argument must be a positive integer: %s" % (str(last))) + raise UWSError(f"Value for 'last' argument must be a positive integer: {str(last)}") params.append(("LAST", last)) return params @@ -111,7 +115,8 @@ def _validate_and_parse_wait(self, wait, phase=None): if phase: if phase not in models.JobPhases.active_phases: - raise UWSError("Given phase '%s' is not an active phase, 'wait' with this phase is not supported." % phase) + raise UWSError(f"Given phase '{phase}' is not an active phase, " + "'wait' with this phase is not supported.") params.append(("PHASE", phase)) return params @@ -132,7 +137,8 @@ def get_job(self, id, wait=None, phase=None): try: result = models.Job(raw) except XMLSyntaxError as e: - raise UWSError("Malformatted response. Are you sure the host you specified is a IVOA UWS service?", raw) + raise UWSError("Malformatted response. Are you sure the host you specified is a " + "IVOA UWS service?", raw) except Exception as e: raise e @@ -159,7 +165,8 @@ def new_job(self, args={}): try: result = models.Job(raw) except XMLSyntaxError as e: - raise UWSError("Malformatted response. Are you sure the host you specified is a IVOA UWS service?", raw) + raise UWSError("Malformatted response. Are you sure the host you specified is a " + "IVOA UWS service?", raw) except Exception as e: raise @@ -175,7 +182,8 @@ def set_parameters_job(self, id, args={}): try: result = models.Job(raw) except XMLSyntaxError as e: - raise UWSError("Malformatted response. Are you sure the host you specified is a IVOA UWS service?", raw) + raise UWSError("Malformatted response. Are you sure the host you specified is a " + "IVOA UWS service?", raw) except Exception as e: raise e @@ -191,7 +199,8 @@ def run_job(self, id): try: result = models.Job(raw) except XMLSyntaxError as e: - raise UWSError("Malformatted response. Are you sure the host you specified is a IVOA UWS service?", raw) + raise UWSError("Malformatted response. Are you sure the host you specified is a " + "IVOA UWS service?", raw) except Exception as e: raise e @@ -207,7 +216,8 @@ def abort_job(self, id): try: result = models.Job(raw) except XMLSyntaxError as e: - raise UWSError("Malformatted response. Are you sure the host you specified is a IVOA UWS service?", raw) + raise UWSError("Malformatted response. Are you sure the host you specified is a IVOA " + "UWS service?", raw) except Exception as e: raise e @@ -223,7 +233,8 @@ def delete_job(self, id): try: result = True except XMLSyntaxError as e: - raise UWSError("Malformatted response. Are you sure the host you specified is a IVOA UWS service?", raw) + raise UWSError("Malformatted response. Are you sure the host you specified is a IVOA " + "UWS service?", raw) except Exception as e: raise e diff --git a/uws/UWS/models.py b/uws/UWS/models.py index 1920eb5..9a190d3 100644 --- a/uws/UWS/models.py +++ b/uws/UWS/models.py @@ -14,7 +14,8 @@ class UWS1Flavour(object): def __init__(self, namespaces=None): if uws_1_namespace not in namespaces.values(): - raise RuntimeError("No supported UWS namespace found in xml-response, cannot parse xml.") + raise RuntimeError("No supported UWS namespace found in xml-response, " + "cannot parse xml.") # prepend each element's name with the correct uws-namespace # for this version @@ -131,7 +132,8 @@ def add_job(self, id=None, href=None, phase=None, job=None): class JobRef(BaseUWSModel): - def __init__(self, id=None, phase=None, reference=None, xml_node=None, xml_namespace=None, uws_flavour=None): + def __init__(self, id=None, phase=None, reference=None, xml_node=None, xml_namespace=None, + uws_flavour=None): super(JobRef, self).__init__() self.id = None @@ -168,9 +170,12 @@ def set_phase(self, new_phase): def __unicode__(self): if self.creationTime is not None: - return "Job '%s' in phase '%s' created at '%s' - %s" % (self.id, ', '.join(self.phase), self.creationTime, str(self.reference)) + return "Job '%s' in phase '%s' created at '%s' - %s" % (self.id, ', '.join(self.phase), + self.creationTime, + str(self.reference)) else: - return "Job '%s' in phase '%s' - %s" % (self.id, ', '.join(self.phase), str(self.reference)) + return "Job '%s' in phase '%s' - %s" % (self.id, ', '.join(self.phase), + str(self.reference)) def __str__(self): return str(self.__unicode__()) @@ -186,7 +191,8 @@ def __init__(self, href=None, type=None, xml_node=None, xml_namespace=None): if xml_node is not None: # check that namespace for xlink really exists if xlink_namespace not in xml_namespace.values(): - raise RuntimeError("No supported xlink namespace found in xml-response, cannot parse xml.") + raise RuntimeError("No supported xlink namespace found in xml-response, " + "cannot parse xml.") qualifiedname_type = et.QName(xlink_namespace, "type") qualifiedname_href = et.QName(xlink_namespace, "href") @@ -245,7 +251,8 @@ def __init__(self, xml=None): self.creation_time = self._get_optional(parsed, uws_flavour.creationTime) self.start_time = self._get_mandatory(parsed, uws_flavour.startTime) self.end_time = self._get_mandatory(parsed, uws_flavour.endTime) - self.execution_duration = int(self._get_mandatory(parsed, uws_flavour.executionDuration)) + self.execution_duration = int(self._get_mandatory(parsed, + uws_flavour.executionDuration)) self.destruction = self._get_mandatory(parsed, uws_flavour.destruction) self.parameters = [] @@ -333,7 +340,8 @@ def _get_mandatory(self, parsed, element_name): element = parsed.find(element_name) if element is None: - raise RuntimeError("Mandatory element ", element_name.text, " could not be found in xml-response.") + raise RuntimeError("Mandatory element ", element_name.text, + " could not be found in xml-response.") else: return element.text @@ -359,7 +367,8 @@ def __init__(self, id=None, by_reference=False, is_post=False, value=None, xml_n self.value = value def __unicode__(self): - return "Parameter id '%s' byRef: %s is_post: %s - value: %s" % (self.id, self.by_reference, self.is_post, self.value) + return "Parameter id '%s' byRef: %s is_post: %s - value: %s" % (self.id, self.by_reference, + self.is_post, self.value) def __str__(self): return str(self.__unicode__()) @@ -415,7 +424,9 @@ def __init__(self, type="transient", has_detail=False, messages=None, self.messages = messages def __unicode__(self): - return "Error Summary - type '%s' hasDetail: %s - message: %s" % (self.type, self.has_detail, "\n".join(self.messages)) + return "Error Summary - type '%s' hasDetail: %s - message: %s" % (self.type, + self.has_detail, + "\n".join(self.messages)) def __str__(self): return str(self.__unicode__()) diff --git a/uws/UWS/tests/test_models.py b/uws/UWS/tests/test_models.py index b0a6d56..e9193e1 100644 --- a/uws/UWS/tests/test_models.py +++ b/uws/UWS/tests/test_models.py @@ -32,38 +32,52 @@ def test(self): self.assertEqual(len(job_list.job_reference), 5) - job_list_str = "Job '2014-06-03T15:33:29:4235' in phase 'COMPLETED' - https://www.cosmosim.org/uws/query/335912448787925\nJob '2014-06-02T10:14:25:1677' in phase 'ABORTED' - https://www.cosmosim.org/uws/query/308893189727250\nJob '2014-05-28T10:46:39:2755' in phase 'COMPLETED' - https://www.cosmosim.org/uws/query/198796714554760\nJob '2014-05-09T15:13:50:6896' in phase 'ERROR' - https://www.cosmosim.org/uws/query/1177277256137938\nJob 'rndSamp2' in phase 'COMPLETED' - https://www.cosmosim.org/uws/query/356246647522833857\n" + job_list_str = ("Job '2014-06-03T15:33:29:4235' in phase 'COMPLETED' - " + "https://www.cosmosim.org/uws/query/335912448787925\nJob " + "'2014-06-02T10:14:25:1677' in phase 'ABORTED' - " + "https://www.cosmosim.org/uws/query/308893189727250\nJob " + "'2014-05-28T10:46:39:2755' in phase 'COMPLETED' - " + "https://www.cosmosim.org/uws/query/198796714554760\nJob " + "'2014-05-09T15:13:50:6896' in phase 'ERROR' - " + "https://www.cosmosim.org/uws/query/1177277256137938\nJob " + "'rndSamp2' in phase 'COMPLETED' - " + "https://www.cosmosim.org/uws/query/356246647522833857\n") self.assertEqual(str(job_list), job_list_str) job1 = job_list.job_reference[0] self.assertEqual(job1.id, '2014-06-03T15:33:29:4235') self.assertEqual(job1.phase, ['COMPLETED']) self.assertEqual(job1.reference.type, "simple") - self.assertEqual(job1.reference.href, "https://www.cosmosim.org/uws/query/335912448787925") + self.assertEqual(job1.reference.href, + "https://www.cosmosim.org/uws/query/335912448787925") job2 = job_list.job_reference[1] self.assertEqual(job2.id, '2014-06-02T10:14:25:1677') self.assertEqual(job2.phase, ['ABORTED']) self.assertEqual(job2.reference.type, "simple") - self.assertEqual(job2.reference.href, "https://www.cosmosim.org/uws/query/308893189727250") + self.assertEqual(job2.reference.href, + "https://www.cosmosim.org/uws/query/308893189727250") job3 = job_list.job_reference[2] self.assertEqual(job3.id, '2014-05-28T10:46:39:2755') self.assertEqual(job3.phase, ['COMPLETED']) self.assertEqual(job3.reference.type, "simple") - self.assertEqual(job3.reference.href, "https://www.cosmosim.org/uws/query/198796714554760") + self.assertEqual(job3.reference.href, + "https://www.cosmosim.org/uws/query/198796714554760") job4 = job_list.job_reference[3] self.assertEqual(job4.id, '2014-05-09T15:13:50:6896') self.assertEqual(job4.phase, ['ERROR']) self.assertEqual(job4.reference.type, "simple") - self.assertEqual(job4.reference.href, "https://www.cosmosim.org/uws/query/1177277256137938") + self.assertEqual(job4.reference.href, + "https://www.cosmosim.org/uws/query/1177277256137938") job5 = job_list.job_reference[4] self.assertEqual(job5.id, u'rndSamp2') self.assertEqual(job5.phase, ['COMPLETED']) self.assertEqual(job5.reference.type, "simple") - self.assertEqual(job5.reference.href, "https://www.cosmosim.org/uws/query/356246647522833857") + self.assertEqual(job5.reference.href, + "https://www.cosmosim.org/uws/query/356246647522833857") class CompletedJobTest(unittest.TestCase): @@ -110,7 +124,36 @@ def setUp(self): def test(self): job = UWS.models.Job(self.xml) - job_str = "JobId : '335912448787925'\nRunId : 'None'\nOwnerId : 'adrian'\nPhase : 'COMPLETED'\nQuote : 'None'\nCreationTime : 'None'\nStartTime : '2014-06-03T15:33:30+02:00'\nEndTime : '2014-06-03T15:33:31+02:00'\nExecutionDuration : '30'\nDestruction : '2999-12-31T00:00:00+01:00'\nParameters :\nParameter id 'database' byRef: False is_post: False - value: cosmosim_user_adrian\nParameter id 'table' byRef: False is_post: False - value: 2014-06-03T15:33:29:4235\nParameter id 'query' byRef: False is_post: False - value: SELECT 0.25*(0.5+FLOOR(LOG10(Mvir)/0.25)) AS log_mass, COUNT(*) AS num\r\nFROM MDR1.BDMV\r\nWHERE snapnum=85 \r\nGROUP BY FLOOR(LOG10(Mvir)/0.25)\r\nORDER BY log_mass\n-- The query plan used to run this query: --\n--------------------------------------------\n--\n-- CALL paquExec('SELECT 0.25 * ( 0.5 + FLOOR( LOG10( `Mvir` ) / 0.25 ) ) AS `log_mass`,COUNT(*) AS `num`,FLOOR( LOG10( `Mvir` ) / 0.25 ) AS `_FLOOR_LOG10_Mvir_/_0__25_` FROM MDR1.BDMV WHERE ( `snapnum` = 85 ) GROUP BY FLOOR( LOG10( Mvir ) / 0.25 ) ', 'aggregation_tmp_75797262')\n-- USE spider_tmp_shard\n-- SET @i=0\n-- CREATE TABLE cosmosim_user_adrian.`2014-06-03T15:33:29:4235` ENGINE=MyISAM SELECT @i:=@i+1 AS `row_id`, `log_mass`,SUM(`num`) AS `num`\r FROM `aggregation_tmp_75797262` GROUP BY `_FLOOR_LOG10_Mvir_/_0__25_` ORDER BY `log_mass` ASC \n-- CALL paquDropTmp('aggregation_tmp_75797262')\n\nParameter id 'queue' byRef: False is_post: False - value: short\nResults :\nResult id 'csv' reference: https://www.cosmosim.org/query/download/stream/table/2014-06-03T15%3A33%3A29%3A4235/format/csv\nResult id 'votable.xml' reference: https://www.cosmosim.org/query/download/stream/table/2014-06-03T15%3A33%3A29%3A4235/format/votable\nResult id 'votableB1.xml' reference: https://www.cosmosim.org/query/download/stream/table/2014-06-03T15%3A33%3A29%3A4235/format/votableB1\nResult id 'votableB2.xml' reference: https://www.cosmosim.org/query/download/stream/table/2014-06-03T15%3A33%3A29%3A4235/format/votableB2\nerrorSummary :\n False\njobInfo :\n" + job_str = ("JobId : '335912448787925'\nRunId : 'None'\nOwnerId : 'adrian'\n" + "Phase : 'COMPLETED'\nQuote : 'None'\nCreationTime : 'None'\n" + "StartTime : '2014-06-03T15:33:30+02:00'\nEndTime : '2014-06-03T15:33:31+02:00'" + "\nExecutionDuration : '30'\nDestruction : '2999-12-31T00:00:00+01:00'\n" + "Parameters :\nParameter id 'database' byRef: False is_post: False - " + "value: cosmosim_user_adrian\nParameter id 'table' byRef: False is_post: False -" + " value: 2014-06-03T15:33:29:4235\nParameter id 'query' byRef: False " + "is_post: False - value: SELECT 0.25*(0.5+FLOOR(LOG10(Mvir)/0.25)) AS log_mass, " + "COUNT(*) AS num\r\nFROM MDR1.BDMV\r\nWHERE snapnum=85 \r\n" + "GROUP BY FLOOR(LOG10(Mvir)/0.25)\r\nORDER BY log_mass\n-- The query plan used " + "to run this query: --\n--------------------------------------------\n--\n-- " + "CALL paquExec('SELECT 0.25 * ( 0.5 + FLOOR( LOG10( `Mvir` ) / 0.25 ) ) AS " + "`log_mass`,COUNT(*) AS `num`,FLOOR( LOG10( `Mvir` ) / 0.25 ) AS " + "`_FLOOR_LOG10_Mvir_/_0__25_` FROM MDR1.BDMV WHERE ( `snapnum` = 85 ) " + "GROUP BY FLOOR( LOG10( Mvir ) / 0.25 ) ', 'aggregation_tmp_75797262')\n-- " + "USE spider_tmp_shard\n-- SET @i=0\n-- " + "CREATE TABLE cosmosim_user_adrian.`2014-06-03T15:33:29:4235` ENGINE=MyISAM " + "SELECT @i:=@i+1 AS `row_id`, `log_mass`,SUM(`num`) AS `num`\r FROM " + "`aggregation_tmp_75797262` GROUP BY `_FLOOR_LOG10_Mvir_/_0__25_` ORDER BY " + "`log_mass` ASC \n-- CALL paquDropTmp('aggregation_tmp_75797262')\n\n" + "Parameter id 'queue' byRef: False is_post: False - value: short\nResults :\n" + "Result id 'csv' reference: https://www.cosmosim.org/query/download/stream/" + "table/2014-06-03T15%3A33%3A29%3A4235/format/csv\nResult id 'votable.xml' " + "reference: https://www.cosmosim.org/query/download/stream/table/" + "2014-06-03T15%3A33%3A29%3A4235/format/votable\nResult id 'votableB1.xml' " + "reference: https://www.cosmosim.org/query/download/stream/table/" + "2014-06-03T15%3A33%3A29%3A4235/format/votableB1\nResult id 'votableB2.xml' " + "reference: https://www.cosmosim.org/query/download/stream/table/" + "2014-06-03T15%3A33%3A29%3A4235/format/votableB2\nerrorSummary :\n False\n" + "jobInfo :\n") self.assertEqual(str(job), job_str) self.assertEqual(job.job_id, '335912448787925') @@ -140,7 +183,18 @@ def test(self): self.assertEqual(param.id, 'query') self.assertEqual(param.by_reference, False) self.assertEqual(param.is_post, False) - query_value = "SELECT 0.25*(0.5+FLOOR(LOG10(Mvir)/0.25)) AS log_mass, COUNT(*) AS num\r\nFROM MDR1.BDMV\r\nWHERE snapnum=85 \r\nGROUP BY FLOOR(LOG10(Mvir)/0.25)\r\nORDER BY log_mass\n-- The query plan used to run this query: --\n--------------------------------------------\n--\n-- CALL paquExec('SELECT 0.25 * ( 0.5 + FLOOR( LOG10( `Mvir` ) / 0.25 ) ) AS `log_mass`,COUNT(*) AS `num`,FLOOR( LOG10( `Mvir` ) / 0.25 ) AS `_FLOOR_LOG10_Mvir_/_0__25_` FROM MDR1.BDMV WHERE ( `snapnum` = 85 ) GROUP BY FLOOR( LOG10( Mvir ) / 0.25 ) ', 'aggregation_tmp_75797262')\n-- USE spider_tmp_shard\n-- SET @i=0\n-- CREATE TABLE cosmosim_user_adrian.`2014-06-03T15:33:29:4235` ENGINE=MyISAM SELECT @i:=@i+1 AS `row_id`, `log_mass`,SUM(`num`) AS `num`\r FROM `aggregation_tmp_75797262` GROUP BY `_FLOOR_LOG10_Mvir_/_0__25_` ORDER BY `log_mass` ASC \n-- CALL paquDropTmp('aggregation_tmp_75797262')\n" + query_value = ("SELECT 0.25*(0.5+FLOOR(LOG10(Mvir)/0.25)) AS log_mass, COUNT(*) AS num\r\n" + "FROM MDR1.BDMV\r\nWHERE snapnum=85 \r\nGROUP BY FLOOR(LOG10(Mvir)/0.25)\r" + "\nORDER BY log_mass\n-- The query plan used to run this query: --\n------" + "--------------------------------------\n--\n-- CALL paquExec('SELECT 0.25 " + "* ( 0.5 + FLOOR( LOG10( `Mvir` ) / 0.25 ) ) AS `log_mass`,COUNT(*) AS " + "`num`,FLOOR( LOG10( `Mvir` ) / 0.25 ) AS `_FLOOR_LOG10_Mvir_/_0__25_` FROM " + "MDR1.BDMV WHERE ( `snapnum` = 85 ) GROUP BY FLOOR( LOG10( Mvir ) / 0.25 ) " + " ', 'aggregation_tmp_75797262')\n-- USE spider_tmp_shard\n-- SET @i=0\n-- " + "CREATE TABLE cosmosim_user_adrian.`2014-06-03T15:33:29:4235` ENGINE=MyISAM " + "SELECT @i:=@i+1 AS `row_id`, `log_mass`,SUM(`num`) AS `num`\r FROM " + "`aggregation_tmp_75797262` GROUP BY `_FLOOR_LOG10_Mvir_/_0__25_` ORDER BY " + "`log_mass` ASC \n-- CALL paquDropTmp('aggregation_tmp_75797262')\n") self.assertEqual(param.value, query_value) param = job.parameters[3] @@ -153,22 +207,29 @@ def test(self): result = job.results[0] self.assertEqual(result.id, 'csv') self.assertEqual(result.reference.type, 'simple') - self.assertEqual(result.reference.href, 'https://www.cosmosim.org/query/download/stream/table/2014-06-03T15%3A33%3A29%3A4235/format/csv') + self.assertEqual(result.reference.href, ('https://www.cosmosim.org/query/download/stream/' + 'table/2014-06-03T15%3A33%3A29%3A4235/format/csv')) result = job.results[1] self.assertEqual(result.id, 'votable.xml') self.assertEqual(result.reference.type, 'simple') - self.assertEqual(result.reference.href, 'https://www.cosmosim.org/query/download/stream/table/2014-06-03T15%3A33%3A29%3A4235/format/votable') + self.assertEqual(result.reference.href, ('https://www.cosmosim.org/query/download/stream/' + 'table/2014-06-03T15%3A33%3A29%3A4235/' + 'format/votable')) result = job.results[2] self.assertEqual(result.id, 'votableB1.xml') self.assertEqual(result.reference.type, 'simple') - self.assertEqual(result.reference.href, 'https://www.cosmosim.org/query/download/stream/table/2014-06-03T15%3A33%3A29%3A4235/format/votableB1') + self.assertEqual(result.reference.href, ('https://www.cosmosim.org/query/download/stream/' + 'table/2014-06-03T15%3A33%3A29%3A4235/format/' + 'votableB1')) result = job.results[3] self.assertEqual(result.id, 'votableB2.xml') self.assertEqual(result.reference.type, 'simple') - self.assertEqual(result.reference.href, 'https://www.cosmosim.org/query/download/stream/table/2014-06-03T15%3A33%3A29%3A4235/format/votableB2') + self.assertEqual(result.reference.href, ('https://www.cosmosim.org/query/download/stream/' + 'table/2014-06-03T15%3A33%3A29%3A4235/format/' + 'votableB2')) self.assertEqual(job.error_summary, False) self.assertEqual(job.job_info, []) @@ -209,7 +270,22 @@ def setUp(self): def test(self): job = UWS.models.Job(self.xml) - job_str = "JobId : '308893189727250'\nRunId : 'None'\nOwnerId : 'adrian'\nPhase : 'ABORTED'\nQuote : 'None'\nCreationTime : 'None'\nStartTime : '2014-06-02T10:14:25+02:00'\nEndTime : '2014-06-02T10:14:55+02:00'\nExecutionDuration : '30'\nDestruction : '2999-12-31T00:00:00+01:00'\nParameters :\nParameter id 'database' byRef: False is_post: False - value: cosmosim_user_adrian\nParameter id 'table' byRef: False is_post: False - value: 2014-06-02T10:14:25:1677\nParameter id 'query' byRef: False is_post: False - value: select count(*) from MDR1.Particles85 where x < 1\n-- The query plan used to run this query: --\n--------------------------------------------\n--\n-- CALL paquExec('SELECT COUNT(*) AS `_count_*_` FROM MDR1.Particles85 WHERE ( `x` < 1 ) ', 'aggregation_tmp_49645551')\n-- USE spider_tmp_shard\n-- SET @i=0\n-- CREATE TABLE cosmosim_user_adrian.`2014-06-02T10:14:25:1677` ENGINE=MyISAM SELECT @i:=@i+1 AS `row_id`, SUM(`_count_*_`) AS `_count_*_`\r FROM `aggregation_tmp_49645551` \n-- CALL paquDropTmp('aggregation_tmp_49645551')\n\nParameter id 'queue' byRef: False is_post: False - value: short\nResults :\nerrorSummary :\n False\njobInfo :\n" + job_str = ("JobId : '308893189727250'\nRunId : 'None'\nOwnerId : 'adrian'\nPhase : " + "'ABORTED'\nQuote : 'None'\nCreationTime : 'None'\nStartTime : " + "'2014-06-02T10:14:25+02:00'\nEndTime : '2014-06-02T10:14:55+02:00'\n" + "ExecutionDuration : '30'\nDestruction : '2999-12-31T00:00:00+01:00'\n" + "Parameters :\nParameter id 'database' byRef: False is_post: False - value: " + "cosmosim_user_adrian\nParameter id 'table' byRef: False is_post: False - " + "value: 2014-06-02T10:14:25:1677\nParameter id 'query' byRef: False is_post: " + "False - value: select count(*) from MDR1.Particles85 where x < 1\n-- The query " + "plan used to run this query: --\n--------------------------------------------\n" + "--\n-- CALL paquExec('SELECT COUNT(*) AS `_count_*_` FROM MDR1.Particles85 " + "WHERE ( `x` < 1 ) ', 'aggregation_tmp_49645551')\n-- USE spider_tmp_shard\n" + "-- SET @i=0\n-- CREATE TABLE cosmosim_user_adrian.`2014-06-02T10:14:25:1677` " + "ENGINE=MyISAM SELECT @i:=@i+1 AS `row_id`, SUM(`_count_*_`) AS `_count_*_`\r " + "FROM `aggregation_tmp_49645551` \n-- CALL " + "paquDropTmp('aggregation_tmp_49645551')\n\nParameter id 'queue' byRef: False " + "is_post: False - value: short\nResults :\nerrorSummary :\n False\njobInfo :\n") self.assertEqual(str(job), job_str) self.assertEqual(job.job_id, '308893189727250') @@ -239,7 +315,14 @@ def test(self): self.assertEqual(param.id, 'query') self.assertEqual(param.by_reference, False) self.assertEqual(param.is_post, False) - query_value = "select count(*) from MDR1.Particles85 where x < 1\n-- The query plan used to run this query: --\n--------------------------------------------\n--\n-- CALL paquExec('SELECT COUNT(*) AS `_count_*_` FROM MDR1.Particles85 WHERE ( `x` < 1 ) ', 'aggregation_tmp_49645551')\n-- USE spider_tmp_shard\n-- SET @i=0\n-- CREATE TABLE cosmosim_user_adrian.`2014-06-02T10:14:25:1677` ENGINE=MyISAM SELECT @i:=@i+1 AS `row_id`, SUM(`_count_*_`) AS `_count_*_`\r FROM `aggregation_tmp_49645551` \n-- CALL paquDropTmp('aggregation_tmp_49645551')\n" + query_value = ("select count(*) from MDR1.Particles85 where x < 1\n-- The query plan used " + "to run this query: --\n--------------------------------------------\n" + "--\n-- CALL paquExec('SELECT COUNT(*) AS `_count_*_` FROM MDR1.Particles85 " + "WHERE ( `x` < 1 ) ', 'aggregation_tmp_49645551')\n-- USE spider_tmp_shard" + "\n-- SET @i=0\n-- CREATE TABLE cosmosim_user_adrian." + "`2014-06-02T10:14:25:1677` ENGINE=MyISAM SELECT @i:=@i+1 AS `row_id`, " + "SUM(`_count_*_`) AS `_count_*_`\r FROM `aggregation_tmp_49645551` \n-- " + "CALL paquDropTmp('aggregation_tmp_49645551')\n") self.assertEqual(param.value, query_value) param = job.parameters[3] @@ -291,7 +374,24 @@ def setUp(self): def test(self): job = UWS.models.Job(self.xml) - job_str = "JobId : '1177277256137938'\nRunId : 'None'\nOwnerId : 'adrian'\nPhase : 'ERROR'\nQuote : 'None'\nCreationTime : 'None'\nStartTime : '2014-05-09T15:13:48+02:00'\nEndTime : '2014-05-09T15:13:48+02:00'\nExecutionDuration : '30'\nDestruction : '2999-12-31T00:00:00+01:00'\nParameters :\nParameter id 'database' byRef: False is_post: False - value: cosmosim_user_adrian\nParameter id 'table' byRef: False is_post: False - value: 2014-05-09T15:13:50:6896\nParameter id 'query' byRef: False is_post: False - value: select avg(x) from `MDPL`.`Particles88tmp`;\n-- The query plan used to run this query: --\n--------------------------------------------\n--\n-- CALL paquExec('SELECT COUNT(x) AS `cnt_avg(x)`, SUM(x) AS `sum_avg(x)` FROM `MDPL`.`Particles88tmp` ', 'aggregation_tmp_9424512')\n-- USE spider_tmp_shard\n-- SET @i=0\n-- CREATE TABLE cosmosim_user_adrian.`2014-05-09T15:13:50:6896` ENGINE=MyISAM SELECT @i:=@i+1 AS `row_id`, (SUM(`sum_avg(x)`) / SUM(`cnt_avg(x)`)) AS `avg(x)`\r FROM `aggregation_tmp_9424512` \n-- CALL paquDropTmp('aggregation_tmp_9424512')\n\nParameter id 'queue' byRef: False is_post: False - value: short\nResults :\nerrorSummary :\n Error Summary - type 'transient' hasDetail: False - message: Remote MySQL server has gone away\njobInfo :\n" + job_str = ("JobId : '1177277256137938'\nRunId : 'None'\nOwnerId : 'adrian'\nPhase : " + "'ERROR'\nQuote : 'None'\nCreationTime : 'None'\nStartTime : " + "'2014-05-09T15:13:48+02:00'\nEndTime : '2014-05-09T15:13:48+02:00'\n" + "ExecutionDuration : '30'\nDestruction : '2999-12-31T00:00:00+01:00'\n" + "Parameters :\nParameter id 'database' byRef: False is_post: False - value: " + "cosmosim_user_adrian\nParameter id 'table' byRef: False is_post: False - " + "value: 2014-05-09T15:13:50:6896\nParameter id 'query' byRef: False is_post: " + "False - value: select avg(x) from `MDPL`.`Particles88tmp`;\n-- The query plan " + "used to run this query: --\n--------------------------------------------\n--" + "\n-- CALL paquExec('SELECT COUNT(x) AS `cnt_avg(x)`, SUM(x) AS `sum_avg(x)` " + "FROM `MDPL`.`Particles88tmp` ', 'aggregation_tmp_9424512')\n-- USE " + "spider_tmp_shard\n-- SET @i=0\n-- CREATE TABLE cosmosim_user_adrian." + "`2014-05-09T15:13:50:6896` ENGINE=MyISAM SELECT @i:=@i+1 AS `row_id`, " + "(SUM(`sum_avg(x)`) / SUM(`cnt_avg(x)`)) AS `avg(x)`\r FROM " + "`aggregation_tmp_9424512` \n-- CALL paquDropTmp('aggregation_tmp_9424512')\n" + "\nParameter id 'queue' byRef: False is_post: False - value: short\nResults :\n" + "errorSummary :\n Error Summary - type 'transient' hasDetail: False - message: " + "Remote MySQL server has gone away\njobInfo :\n") self.assertEqual(str(job), job_str) self.assertEqual(job.job_id, '1177277256137938') @@ -321,7 +421,15 @@ def test(self): self.assertEqual(param.id, 'query') self.assertEqual(param.by_reference, False) self.assertEqual(param.is_post, False) - query_value = "select avg(x) from `MDPL`.`Particles88tmp`;\n-- The query plan used to run this query: --\n--------------------------------------------\n--\n-- CALL paquExec('SELECT COUNT(x) AS `cnt_avg(x)`, SUM(x) AS `sum_avg(x)` FROM `MDPL`.`Particles88tmp` ', 'aggregation_tmp_9424512')\n-- USE spider_tmp_shard\n-- SET @i=0\n-- CREATE TABLE cosmosim_user_adrian.`2014-05-09T15:13:50:6896` ENGINE=MyISAM SELECT @i:=@i+1 AS `row_id`, (SUM(`sum_avg(x)`) / SUM(`cnt_avg(x)`)) AS `avg(x)`\r FROM `aggregation_tmp_9424512` \n-- CALL paquDropTmp('aggregation_tmp_9424512')\n" + query_value = ("select avg(x) from `MDPL`.`Particles88tmp`;\n-- The query plan used to " + "run this query: --\n--------------------------------------------\n--\n-- " + "CALL paquExec('SELECT COUNT(x) AS `cnt_avg(x)`, SUM(x) AS `sum_avg(x)` " + "FROM `MDPL`.`Particles88tmp` ', 'aggregation_tmp_9424512')\n-- USE " + "spider_tmp_shard\n-- SET @i=0\n-- CREATE TABLE cosmosim_user_adrian." + "`2014-05-09T15:13:50:6896` ENGINE=MyISAM SELECT @i:=@i+1 AS `row_id`, " + "(SUM(`sum_avg(x)`) / SUM(`cnt_avg(x)`)) AS `avg(x)`\r FROM " + "`aggregation_tmp_9424512` \n-- CALL " + "paquDropTmp('aggregation_tmp_9424512')\n") self.assertEqual(param.value, query_value) param = job.parameters[3] @@ -357,7 +465,8 @@ def test(self): self.assertEqual(len(job_list.job_reference), 1) - job_list_str = "Job '335912448787925' in phase 'COMPLETED' - https://www.cosmosim.org/uws/query/335912448787925\n" + job_list_str = ("Job '335912448787925' in phase 'COMPLETED' - " + "https://www.cosmosim.org/uws/query/335912448787925\n") self.assertEqual(str(job_list), job_list_str) job1 = job_list.job_reference[0] diff --git a/uws/cli/cli_parser.py b/uws/cli/cli_parser.py index 2eab50a..c2082a1 100644 --- a/uws/cli/cli_parser.py +++ b/uws/cli/cli_parser.py @@ -25,12 +25,17 @@ def build_list_argparse(subparsers): parser_list.add_argument('-e', '--executing', action='store_true', help='show executing jobs') parser_list.add_argument('-E', '--error', action='store_true', help='show jobs with errors') parser_list.add_argument('-a', '--aborted', action='store_true', help='show aborted jobs') - parser_list.add_argument('-A', '--archived', action='store_true', help='[UWS1.1] show (deleted) jobs archived on the server') + parser_list.add_argument('-A', '--archived', action='store_true', + help='[UWS1.1] show (deleted) jobs archived on the server') parser_list.add_argument('--unknown', action='store_true', help='show unknown state jobs') parser_list.add_argument('--held', action='store_true', help='show held jobs') parser_list.add_argument('--suspended', action='store_true', help='show suspended jobs') - parser_list.add_argument('--after', action='store', help='[UWS1.1] show only jobs started after given UTC time or local time + timezone') - parser_list.add_argument('--last', action='store', help='[UWS1.1] show only most recently started jobs') + parser_list.add_argument('--after', action='store', + help=('[UWS1.1] show only jobs started after given UTC time or ' + 'local time + timezone') + ) + parser_list.add_argument('--last', action='store', + help='[UWS1.1] show only most recently started jobs') return parser_list @@ -38,43 +43,59 @@ def build_list_argparse(subparsers): def build_job_argparse(subparsers): parser_job = subparsers.add_parser('job', help='access a given job on the UWS service') - job_subparsers = parser_job.add_subparsers(dest='job_command', help='commands for manipulating jobs') + job_subparsers = parser_job.add_subparsers(dest='job_command', + help='commands for manipulating jobs') parser_job_show = job_subparsers.add_parser('show', help='show the specific job') parser_job_show.add_argument('id', help='job id') - parser_job_show.add_argument('-w', '--wait', nargs='?', const='-1', default=None, help='[UWS1.1] wait for phase change before returning, but at most the specified amount of seconds or infinitely, if no value is given') + parser_job_show.add_argument('-w', '--wait', nargs='?', const='-1', default=None, + help=('[UWS1.1] wait for phase change before returning, ' + 'but at most the specified amount of seconds or ' + 'infinitely, if no value is given')) parser_job_show.add_argument('-s', '--phase', help='[UWS1.1] required phase while waiting') parser_job_phase = job_subparsers.add_parser('phase', help='show the phase of specific job') parser_job_phase.add_argument('id', help='job id') parser_job_new = job_subparsers.add_parser('new', help='create a new job') - parser_job_new.add_argument('-r', '--run', action='store_true', help='immediately submits the job on creation') - parser_job_new.add_argument('job_parameters', nargs='*', help='unspecified list of UWS service parameters in the form' + - ' "=" - ' + - 'Default parameters are: ' + - 'destruction (Destruction time of the job), ' + - 'executionDuration (Execution duration of the job in seconds)') + parser_job_new.add_argument('-r', '--run', action='store_true', + help='immediately submits the job on creation') + parser_job_new.add_argument('job_parameters', nargs='*', + help=('unspecified list of UWS service parameters in the form' + ' "=" - Default parameters are: ' + 'destruction (Destruction time of the job), ' + 'executionDuration (Execution duration of the job in seconds)' + ) + ) parser_job_set = job_subparsers.add_parser('set', help='set parameters for the specific job') parser_job_set.add_argument('id', help='job id') - parser_job_set.add_argument('job_parameters', nargs='*', help='unspecified list of UWS service parameters in the form' + - ' "=" - ' + - 'Default parameters are: ' + - 'destruction (Destruction time of the job), ' + - 'executionDuration (Execution duration of the job in seconds)') - - parser_job_run = job_subparsers.add_parser('run', help="run the specific job if its state is pending") + parser_job_set.add_argument('job_parameters', nargs='*', + help=('unspecified list of UWS service parameters in the form' + ' "=" - Default parameters are: ' + 'destruction (Destruction time of the job), ' + 'executionDuration (Execution duration of the job in seconds)' + ) + ) + + parser_job_run = job_subparsers.add_parser('run', + help="run the specific job if its state is pending") parser_job_run.add_argument('id', help='job id') - parser_job_abort = job_subparsers.add_parser('abort', help="aborts the execution of a specific job") + parser_job_abort = job_subparsers.add_parser('abort', + help="aborts the execution of a specific job") parser_job_abort.add_argument('id', help='job id') parser_job_abort = job_subparsers.add_parser('delete', help="delete a specific job") parser_job_abort.add_argument('id', help='job id') - parser_job_results = job_subparsers.add_parser('results', help="download results of a specific job") + parser_job_results = job_subparsers.add_parser('results', + help="download results of a specific job") parser_job_results.add_argument('id', help='job id') - parser_job_results.add_argument('result_id', nargs='?', help='result id (e.g. for specifying the format, optional)') - parser_job_results.add_argument('-f', '--file_base', help='basename of output file (optional), will be appended with result_id') + parser_job_results.add_argument('result_id', nargs='?', + help='result id (e.g. for specifying the format, optional)') + parser_job_results.add_argument('-f', '--file_base', + help=('basename of output file (optional), ' + 'will be appended with result_id') + ) return parser_job_results From d5c3d617990f1b550f20cfc2e8e8055dae15dc6d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Aicardi?= Date: Fri, 31 Jan 2020 13:49:17 +0100 Subject: [PATCH 13/25] pylint : fix imports --- uws/UWS/client.py | 6 +++--- uws/cli/main.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/uws/UWS/client.py b/uws/UWS/client.py index c324689..2dbcdf9 100644 --- a/uws/UWS/client.py +++ b/uws/UWS/client.py @@ -1,5 +1,8 @@ # -*- coding: utf-8 -*- +from datetime import datetime from lxml.etree import XMLSyntaxError as XMLSyntaxError +import dateutil.parser +import pytz #import connection as UWSConnection #import models @@ -8,9 +11,6 @@ from . import connection as UWSConnection from . import models from .errors import UWSError -from datetime import datetime -import dateutil.parser -import pytz class Client(object): diff --git a/uws/cli/main.py b/uws/cli/main.py index 85dc2e7..839caa9 100755 --- a/uws/cli/main.py +++ b/uws/cli/main.py @@ -2,11 +2,11 @@ import sys from functools import wraps -import texttable as tt import shutil +import texttable as tt -from . import cli_parser from uws import UWS +from . import cli_parser debug = False From 226811f07b56a74636bce3d0c42e0cd9cd8c6981 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Aicardi?= Date: Fri, 31 Jan 2020 13:56:58 +0100 Subject: [PATCH 14/25] pylint : fixes spaces --- uws/UWS/__init__.py | 1 - uws/UWS/client.py | 2 +- uws/UWS/models.py | 16 ++++++++-------- uws/UWS/tests/test_base.py | 12 ++++++------ uws/UWS/tests/test_models.py | 1 - uws/cli/cli_parser.py | 2 +- uws/cli/main.py | 2 +- 7 files changed, 17 insertions(+), 19 deletions(-) diff --git a/uws/UWS/__init__.py b/uws/UWS/__init__.py index dd3761c..9398323 100644 --- a/uws/UWS/__init__.py +++ b/uws/UWS/__init__.py @@ -8,4 +8,3 @@ from . import connection from . import models from .errors import UWSError - diff --git a/uws/UWS/client.py b/uws/UWS/client.py index 2dbcdf9..de23266 100644 --- a/uws/UWS/client.py +++ b/uws/UWS/client.py @@ -67,7 +67,7 @@ def _validate_and_parse_filters(self, filters): if after: # TODO: Allow to provide local time and convert here to UTC? - # TODO: We may encounter more troubles with microseconds, + # TODO: We may encounter more troubles with microseconds, # if ',' used instead of '.'(e.g. German systems) try: diff --git a/uws/UWS/models.py b/uws/UWS/models.py index 9a190d3..88cc972 100644 --- a/uws/UWS/models.py +++ b/uws/UWS/models.py @@ -243,17 +243,17 @@ def __init__(self, xml=None): if parsed.get("version"): self.version = parsed.get("version") - self.job_id = self._get_mandatory(parsed, uws_flavour.jobId) - self.run_id = self._get_optional(parsed, uws_flavour.runId) - self.owner_id = self._get_optional(parsed, uws_flavour.ownerId) - self.phase = [self._get_mandatory(parsed, uws_flavour.phase)] - self.quote = self._get_optional(parsed, uws_flavour.quote) + self.job_id = self._get_mandatory(parsed, uws_flavour.jobId) + self.run_id = self._get_optional(parsed, uws_flavour.runId) + self.owner_id = self._get_optional(parsed, uws_flavour.ownerId) + self.phase = [self._get_mandatory(parsed, uws_flavour.phase)] + self.quote = self._get_optional(parsed, uws_flavour.quote) self.creation_time = self._get_optional(parsed, uws_flavour.creationTime) - self.start_time = self._get_mandatory(parsed, uws_flavour.startTime) - self.end_time = self._get_mandatory(parsed, uws_flavour.endTime) + self.start_time = self._get_mandatory(parsed, uws_flavour.startTime) + self.end_time = self._get_mandatory(parsed, uws_flavour.endTime) self.execution_duration = int(self._get_mandatory(parsed, uws_flavour.executionDuration)) - self.destruction = self._get_mandatory(parsed, uws_flavour.destruction) + self.destruction = self._get_mandatory(parsed, uws_flavour.destruction) self.parameters = [] tmp = parsed.find(uws_flavour.parameters) diff --git a/uws/UWS/tests/test_base.py b/uws/UWS/tests/test_base.py index 9d7ca45..e929f68 100644 --- a/uws/UWS/tests/test_base.py +++ b/uws/UWS/tests/test_base.py @@ -12,7 +12,7 @@ def testValidateAndParsePhaseFilter(self): params = UWS.client.Client("/")._validate_and_parse_filters(filters) - self.assertEqual(params, [('PHASE','COMPLETED'), ('PHASE','PENDING')]) + self.assertEqual(params, [('PHASE', 'COMPLETED'), ('PHASE', 'PENDING')]) def testValidateAndParsePhaseFilterInvalidPhase(self): filters = { @@ -32,7 +32,7 @@ def testValidateAndParseAfterFilter(self): params = UWS.client.Client("/")._validate_and_parse_filters(filters) - self.assertEqual(params, [('AFTER','2015-09-10T10:01:02.135000')]) + self.assertEqual(params, [('AFTER', '2015-09-10T10:01:02.135000')]) def testValidateAndParseAfterFilterInvalidDate(self): filters = { @@ -63,7 +63,7 @@ def testValidateAndParseLastFilter(self): params = UWS.client.Client("/")._validate_and_parse_filters(filters) - self.assertEqual(params, [('LAST',1000)]) + self.assertEqual(params, [('LAST', 1000)]) def testValidateAndParseLastFilterFloatValue(self): filters = { @@ -95,7 +95,7 @@ def testValidateAndParseAfterLastFilter(self): params = UWS.client.Client("/")._validate_and_parse_filters(filters) - self.assertEqual(params, [('AFTER','2015-09-10T10:01:02.135000'), + self.assertEqual(params, [('AFTER', '2015-09-10T10:01:02.135000'), ('LAST', 100)]) def testValidateAndParseAfterLastPhaseFilter(self): @@ -107,8 +107,8 @@ def testValidateAndParseAfterLastPhaseFilter(self): params = UWS.client.Client("/")._validate_and_parse_filters(filters) - self.assertEqual(params, [ ('PHASE','PENDING'), ('PHASE','ERROR'), - ('AFTER','2015-09-10T10:01:02.135000'), ('LAST', 100)]) + self.assertEqual(params, [('PHASE', 'PENDING'), ('PHASE', 'ERROR'), + ('AFTER', '2015-09-10T10:01:02.135000'), ('LAST', 100)]) def testValidateAndParseWaitNegative(self): wait = '-1' diff --git a/uws/UWS/tests/test_models.py b/uws/UWS/tests/test_models.py index e9193e1..dfa75f8 100644 --- a/uws/UWS/tests/test_models.py +++ b/uws/UWS/tests/test_models.py @@ -564,4 +564,3 @@ def setUp(self): '''[1:] - diff --git a/uws/cli/cli_parser.py b/uws/cli/cli_parser.py index c2082a1..40075b4 100644 --- a/uws/cli/cli_parser.py +++ b/uws/cli/cli_parser.py @@ -30,7 +30,7 @@ def build_list_argparse(subparsers): parser_list.add_argument('--unknown', action='store_true', help='show unknown state jobs') parser_list.add_argument('--held', action='store_true', help='show held jobs') parser_list.add_argument('--suspended', action='store_true', help='show suspended jobs') - parser_list.add_argument('--after', action='store', + parser_list.add_argument('--after', action='store', help=('[UWS1.1] show only jobs started after given UTC time or ' 'local time + timezone') ) diff --git a/uws/cli/main.py b/uws/cli/main.py index 839caa9..0e3a37f 100755 --- a/uws/cli/main.py +++ b/uws/cli/main.py @@ -91,7 +91,7 @@ def list_jobs(url, user_name, password, phases, after=None, last=None): dtypes = ['t']*ncols # remove empty cols (in-place removal) - rows[:] = [ [ col for i, col in enumerate(row) if existing_col[i] == 1 ] for row in rows ] + rows[:] = [[col for i, col in enumerate(row) if existing_col[i] == 1] for row in rows] table = tt.Texttable(max_width=console_width) table.set_deco(tt.Texttable.HEADER) From 9393a761577535f699259e63ab6c618971cd1c0e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Aicardi?= Date: Fri, 31 Jan 2020 14:02:54 +0100 Subject: [PATCH 15/25] pylint : fixes unnecessary parens after if --- uws/cli/main.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/uws/cli/main.py b/uws/cli/main.py index 0e3a37f..c702ceb 100755 --- a/uws/cli/main.py +++ b/uws/cli/main.py @@ -114,23 +114,23 @@ def _register_job_reference_for_table(rows, jobref): cols = [jobref.id] - if (jobref.reference.href is not None): + if jobref.reference.href is not None: href_jobid = jobref.reference.href.rsplit("/", 1)[1] if href_jobid != jobref.id: # replace id with href_jobid cols[0] = href_jobid - if (jobref.runId is not None): + if jobref.runId is not None: cols.append(jobref.runId) else: cols.append('') - if (jobref.ownerId is not None): + if jobref.ownerId is not None: cols.append(jobref.ownerId) else: cols.append('') - if (jobref.creationTime is not None): + if jobref.creationTime is not None: cols.append(jobref.creationTime) else: cols.append('') @@ -291,15 +291,15 @@ def _print_job(job): rows.append(["Run id", job.run_id]) - if(job.owner_id): + if job.owner_id: rows.append(["Owner id", job.owner_id]) rows.append(["Phase", ", ".join(job.phase)]) - if(job.quote): + if job.quote: rows.append(["Quote", job.quote]) - if(job.creation_time): + if job.creation_time: rows.append(["Creation time", job.creation_time]) rows.append(["Start time", job.start_time]) @@ -314,7 +314,7 @@ def _print_job(job): rows.append(["Result " + result.id, result.reference]) try: - if(job.error_summary): + if job.error_summary: rows.append(["Errors", "; ".join(job.error_summary.messages)]) except: pass From c48375f2fa80b5e23a667c656b67208e2818ad9c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Aicardi?= Date: Fri, 31 Jan 2020 14:26:17 +0100 Subject: [PATCH 16/25] pylint : fixes redefined-builtin --- uws/UWS/client.py | 24 +++++++++++----------- uws/UWS/models.py | 46 ++++++++++++++++++++--------------------- uws/cli/main.py | 52 +++++++++++++++++++++++------------------------ 3 files changed, 61 insertions(+), 61 deletions(-) diff --git a/uws/UWS/client.py b/uws/UWS/client.py index de23266..0cf965b 100644 --- a/uws/UWS/client.py +++ b/uws/UWS/client.py @@ -121,13 +121,13 @@ def _validate_and_parse_wait(self, wait, phase=None): return params - def get_job(self, id, wait=None, phase=None): + def get_job(self, jid, wait=None, phase=None): params = None if wait: params = self._validate_and_parse_wait(wait, phase) try: - response = self.connection.get(id, params) + response = self.connection.get(jid, params) except Exception as e: # Do not make a second request without params, throw error # immediately @@ -144,9 +144,9 @@ def get_job(self, id, wait=None, phase=None): return result - def get_phase(self, id): + def get_phase(self, jid): try: - response = self.connection.get(id + '/phase') + response = self.connection.get(jid + '/phase') except Exception as e: raise UWSError(str(e)) @@ -172,9 +172,9 @@ def new_job(self, args={}): return result - def set_parameters_job(self, id, args={}): + def set_parameters_job(self, jid, args={}): try: - response = self.connection.post(id, args) + response = self.connection.post(jid, args) except Exception as e: raise UWSError(str(e)) @@ -189,9 +189,9 @@ def set_parameters_job(self, id, args={}): return result - def run_job(self, id): + def run_job(self, jid): try: - response = self.connection.post(id + '/phase', {"PHASE": "RUN"}) + response = self.connection.post(jid + '/phase', {"PHASE": "RUN"}) except Exception as e: raise UWSError(str(e)) @@ -206,9 +206,9 @@ def run_job(self, id): return result - def abort_job(self, id): + def abort_job(self, jid): try: - response = self.connection.post(id, {"PHASE": "ABORT"}) + response = self.connection.post(jid, {"PHASE": "ABORT"}) except Exception as e: raise UWSError(str(e)) @@ -223,9 +223,9 @@ def abort_job(self, id): return result - def delete_job(self, id): + def delete_job(self, jid): try: - response = self.connection.delete(id) + response = self.connection.delete(jid) except Exception as e: raise UWSError(str(e)) diff --git a/uws/UWS/models.py b/uws/UWS/models.py index 88cc972..dfa2264 100644 --- a/uws/UWS/models.py +++ b/uws/UWS/models.py @@ -122,17 +122,17 @@ def __str__(self): return str(self.__unicode__()) # return unicode(self).encode('utf-8') - def add_job(self, id=None, href=None, phase=None, job=None): + def add_job(self, jid=None, href=None, phase=None, job=None): if job is not None: self.job_reference.append(job) else: - reference = Reference(href=href, type="simple") - job_reference = JobRef(id=id, phase=phase, reference=reference) + reference = Reference(href=href, rtype="simple") + job_reference = JobRef(jid=jid, phase=phase, reference=reference) self.job_reference.append(job_reference) class JobRef(BaseUWSModel): - def __init__(self, id=None, phase=None, reference=None, xml_node=None, xml_namespace=None, + def __init__(self, jid=None, phase=None, reference=None, xml_node=None, xml_namespace=None, uws_flavour=None): super(JobRef, self).__init__() @@ -152,8 +152,8 @@ def __init__(self, id=None, phase=None, reference=None, xml_node=None, xml_names self.ownerId = xml_node.get('ownerId') self.creationTime = xml_node.get('creationTime') - elif id is not None and phase is not None and reference is not None: - self.id = id + elif jid is not None and phase is not None and reference is not None: + self.id = jid if isinstance(phase, six.string_types): self.phase = [phase] @@ -163,7 +163,7 @@ def __init__(self, id=None, phase=None, reference=None, xml_node=None, xml_names if isinstance(reference, Reference): self.reference = reference else: - raise RuntimeError("Malformated reference given in jobref id: %s" % id) + raise RuntimeError("Malformated reference given in jobref id: %s" % jid) def set_phase(self, new_phase): self.phase = [new_phase] @@ -182,7 +182,7 @@ def __str__(self): class Reference(BaseUWSModel): - def __init__(self, href=None, type=None, xml_node=None, xml_namespace=None): + def __init__(self, href=None, rtype=None, xml_node=None, xml_namespace=None): super(Reference, self).__init__() self.type = "simple" @@ -198,8 +198,8 @@ def __init__(self, href=None, type=None, xml_node=None, xml_namespace=None): qualifiedname_href = et.QName(xlink_namespace, "href") self.type = xml_node.get(qualifiedname_type) self.href = xml_node.get(qualifiedname_href) - elif href is not None and type is not None: - self.type = type + elif href is not None and rtype is not None: + self.type = rtype self.href = href def __unicode__(self): @@ -311,16 +311,16 @@ def __str__(self): return str(self.__unicode__()) # return unicode(self).encode('utf-8') - def add_parameter(self, id=None, by_reference=False, is_post=False, value=None, parameter=None): + def add_parameter(self, pid=None, by_reference=False, is_post=False, value=None, parameter=None): if not parameter: - parameter = Parameter(id=id, by_reference=by_reference, is_post=is_post, value=value) + parameter = Parameter(pid=pid, by_reference=by_reference, is_post=is_post, value=value) self.parameters.append(parameter) - def add_result(self, id=None, href=None, result=None): + def add_result(self, rid=None, href=None, result=None): if not result: - reference = Reference(href=href, type="simple") - result = Result(id=id, reference=reference) + reference = Reference(href=href, rtype="simple") + result = Result(rid=rid, reference=reference) self.results.append(result) @@ -347,7 +347,7 @@ def _get_mandatory(self, parsed, element_name): class Parameter(BaseUWSModel): - def __init__(self, id=None, by_reference=False, is_post=False, value=None, xml_node=None): + def __init__(self, pid=None, by_reference=False, is_post=False, value=None, xml_node=None): super(Parameter, self).__init__() self.id = None @@ -360,8 +360,8 @@ def __init__(self, id=None, by_reference=False, is_post=False, value=None, xml_n self.by_reference = self._parse_bool(xml_node.get('by_reference', default=False)) self.is_post = self._parse_bool(xml_node.get('is_post', default=False)) self.value = xml_node.text - elif id is not None and value is not None: - self.id = id + elif pid is not None and value is not None: + self.id = pid self.by_reference = by_reference self.is_post = is_post self.value = value @@ -375,7 +375,7 @@ def __str__(self): class Result(BaseUWSModel): - def __init__(self, id=None, reference=None, xml_node=None, xml_namespace=None): + def __init__(self, rid=None, reference=None, xml_node=None, xml_namespace=None): super(Result, self).__init__() self.id = None @@ -384,8 +384,8 @@ def __init__(self, id=None, reference=None, xml_node=None, xml_namespace=None): if xml_node is not None: self.id = xml_node.get('id') self.reference = Reference(xml_node=xml_node, xml_namespace=xml_namespace) - elif id is not None and reference is not None: - self.id = id + elif rid is not None and reference is not None: + self.id = rid if isinstance(reference, Reference): self.reference = reference @@ -400,7 +400,7 @@ def __str__(self): class ErrorSummary(BaseUWSModel): - def __init__(self, type="transient", has_detail=False, messages=None, + def __init__(self, etype="transient", has_detail=False, messages=None, xml_node=None, uws_flavour=None): super(ErrorSummary, self).__init__() @@ -419,7 +419,7 @@ def __init__(self, type="transient", has_detail=False, messages=None, self.messages.append(message.text) elif messages is not None: - self.type = type + self.type = etype self.has_detail = has_detail self.messages = messages diff --git a/uws/cli/main.py b/uws/cli/main.py index c702ceb..78aef66 100755 --- a/uws/cli/main.py +++ b/uws/cli/main.py @@ -52,26 +52,26 @@ def list_jobs(url, user_name, password, phases, after=None, last=None): _register_job_reference_for_table(rows, job) else: if job_phases.COMPLETED in phases and job_phases.COMPLETED in job.phase: - _register_job_reference_for_table(rows, job) + _register_job_reference_for_table(rows, job) if job_phases.PENDING in phases and job_phases.PENDING in job.phase: - _register_job_reference_for_table(rows, job) + _register_job_reference_for_table(rows, job) if job_phases.QUEUED in phases and job_phases.QUEUED in job.phase: - _register_job_reference_for_table(rows, job) + _register_job_reference_for_table(rows, job) if job_phases.EXECUTING in phases and job_phases.EXECUTING in job.phase: - _register_job_reference_for_table(rows, job) + _register_job_reference_for_table(rows, job) if job_phases.ERROR in phases and job_phases.ERROR in job.phase: - _register_job_reference_for_table(rows, job) + _register_job_reference_for_table(rows, job) if job_phases.ABORTED in phases and job_phases.ABORTED in job.phase: - _register_job_reference_for_table(rows, job) + _register_job_reference_for_table(rows, job) if job_phases.UNKNOWN in phases and job_phases.UNKNOWN in job.phase: - _register_job_reference_for_table(rows, job) + _register_job_reference_for_table(rows, job) if job_phases.HELD in phases and job_phases.HELD in job.phase: - _register_job_reference_for_table(rows, job) + _register_job_reference_for_table(rows, job) if job_phases.SUSPENDED in phases and job_phases.SUSPENDED in job.phase: - _register_job_reference_for_table(rows, job) + _register_job_reference_for_table(rows, job) # add ARCHIVED phase as well for services with version 1.0 that already support this if job_phases.ARCHIVED in phases and job_phases.ARCHIVED in job.phase: - _register_job_reference_for_table(rows, job) + _register_job_reference_for_table(rows, job) (console_width, console_height) = shutil.get_terminal_size() # Now we have the rows all stored. Check if all columns exist and remove @@ -141,10 +141,10 @@ def _register_job_reference_for_table(rows, jobref): @handle_error -def show_job(url, user_name, password, id, wait, phase): +def show_job(url, user_name, password, jid, wait, phase): uws_client = UWS.client.Client(url=url, user=user_name, password=password) - job = uws_client.get_job(id, wait, phase) + job = uws_client.get_job(jid, wait, phase) if wait and job.version != "1.1": print("Warning: Wait keyword is (probably) not supported by the server's UWS version " + @@ -154,10 +154,10 @@ def show_job(url, user_name, password, id, wait, phase): @handle_error -def show_phase(url, user_name, password, id): +def show_phase(url, user_name, password, jid): uws_client = UWS.client.Client(url=url, user=user_name, password=password) - phase = uws_client.get_phase(id) + phase = uws_client.get_phase(jid) print(phase) @@ -186,47 +186,47 @@ def new_job(url, user_name, password, parameters={}, run=False): @handle_error -def set_parameters_job(url, user_name, password, id, parameters={}): +def set_parameters_job(url, user_name, password, jid, parameters={}): uws_client = UWS.client.Client(url=url, user=user_name, password=password) if len(parameters) == 0: - job = uws_client.get_job(id) + job = uws_client.get_job(jid) else: - job = uws_client.set_parameters_job(id, parameters) + job = uws_client.set_parameters_job(jid, parameters) _print_job(job) @handle_error -def run_job(url, user_name, password, id): +def run_job(url, user_name, password, jid): uws_client = UWS.client.Client(url=url, user=user_name, password=password) - job = uws_client.run_job(id) + job = uws_client.run_job(jid) _print_job(job) @handle_error -def abort_job(url, user_name, password, id): +def abort_job(url, user_name, password, jid): uws_client = UWS.client.Client(url=url, user=user_name, password=password) - job = uws_client.abort_job(id) + job = uws_client.abort_job(jid) _print_job(job) @handle_error -def delete_job(url, user_name, password, id): +def delete_job(url, user_name, password, jid): uws_client = UWS.client.Client(url=url, user=user_name, password=password) - success = uws_client.delete_job(id) + success = uws_client.delete_job(jid) if success: - print("Job %s successfully deleted!" % (id)) + print("Job %s successfully deleted!" % (jid)) @handle_error -def results_job(url, user_name, password, id, result_id, user_file_base): +def results_job(url, user_name, password, jid, result_id, user_file_base): def print_progress(total_size, current): if total_size: sys.stdout.write("\r%d bytes" % current) # or print >> sys.stdout, "\r%d%%" %i, @@ -237,7 +237,7 @@ def print_progress(total_size, current): uws_client = UWS.client.Client(url=url, user=user_name, password=password) - job = uws_client.get_job(id) + job = uws_client.get_job(jid) # if there are multiple result sets returned: force user to decide which ones to use? # or maybe rather let the service define a standard result? but how? From ff5a37fb89cd87a79ea02199a8b679a64d122a4f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Aicardi?= Date: Fri, 31 Jan 2020 15:26:57 +0100 Subject: [PATCH 17/25] pylint : fixes dangerous-default-value --- uws/UWS/client.py | 10 ++++++---- uws/cli/main.py | 6 ++++-- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/uws/UWS/client.py b/uws/UWS/client.py index 0cf965b..3620b31 100644 --- a/uws/UWS/client.py +++ b/uws/UWS/client.py @@ -55,14 +55,14 @@ def _validate_and_parse_filters(self, filters): last = filters_copy.pop('last', None) if filters_copy: - raise UWSError("Unknown filter properties %s", filters_copy.keys()) + raise UWSError(f"Unknown filter properties {filters_copy.keys()}") params = [] if phases: for phase in phases: if phase not in models.JobPhases.phases: - raise UWSError("Unknown phase %s in filter", phase) + raise UWSError(f"Unknown phase {phase} in filter") params.append(("PHASE", phase)) if after: @@ -155,7 +155,8 @@ def get_phase(self, jid): return result - def new_job(self, args={}): + def new_job(self, args=None): + args = args or {} try: response = self.connection.post('', args) except Exception as e: @@ -172,7 +173,8 @@ def new_job(self, args={}): return result - def set_parameters_job(self, jid, args={}): + def set_parameters_job(self, jid, args=None): + args = args or {} try: response = self.connection.post(jid, args) except Exception as e: diff --git a/uws/cli/main.py b/uws/cli/main.py index 78aef66..ac67379 100755 --- a/uws/cli/main.py +++ b/uws/cli/main.py @@ -163,7 +163,8 @@ def show_phase(url, user_name, password, jid): @handle_error -def new_job(url, user_name, password, parameters={}, run=False): +def new_job(url, user_name, password, parameters=None, run=False): + parameters = parameters or {} uws_client = UWS.client.Client(url=url, user=user_name, password=password) job = uws_client.new_job(parameters) @@ -186,7 +187,8 @@ def new_job(url, user_name, password, parameters={}, run=False): @handle_error -def set_parameters_job(url, user_name, password, jid, parameters={}): +def set_parameters_job(url, user_name, password, jid, parameters=None): + parameters = parameters or {} uws_client = UWS.client.Client(url=url, user=user_name, password=password) if len(parameters) == 0: From 85969a5664a729ad0594c15044e818751cc4fdd7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Aicardi?= Date: Fri, 31 Jan 2020 15:50:28 +0100 Subject: [PATCH 18/25] pylint : fixes bare-except --- uws/UWS/client.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/uws/UWS/client.py b/uws/UWS/client.py index 3620b31..cbdb66f 100644 --- a/uws/UWS/client.py +++ b/uws/UWS/client.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from datetime import datetime from lxml.etree import XMLSyntaxError as XMLSyntaxError import dateutil.parser import pytz @@ -169,7 +168,7 @@ def new_job(self, args=None): raise UWSError("Malformatted response. Are you sure the host you specified is a " "IVOA UWS service?", raw) except Exception as e: - raise + raise e return result From 561b5339c2231f75bc85529ac6a1c22f7f11fe8c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Aicardi?= Date: Fri, 31 Jan 2020 15:51:19 +0100 Subject: [PATCH 19/25] pylint: fixes deprecated-method --- uws/UWS/connection.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uws/UWS/connection.py b/uws/UWS/connection.py index ba3ab5c..1afa4b7 100644 --- a/uws/UWS/connection.py +++ b/uws/UWS/connection.py @@ -19,7 +19,7 @@ def __init__(self, url, user=None, password=None): except AttributeError: pass auth_string = '%s:%s' % (user, password) - self.auth_string = base64.encodestring(auth_string.encode()) + self.auth_string = base64.encodebytes(auth_string.encode()) self.headers = {"Authorization": ("Basic %s" % self.auth_string.decode('utf-8')).strip('\n')} From 3d86a8f5f8f7d62d27037b4499c437cb8e253ed5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Aicardi?= Date: Fri, 31 Jan 2020 15:53:02 +0100 Subject: [PATCH 20/25] pylint: fixes useless-object-inheritance --- uws/UWS/client.py | 2 +- uws/UWS/connection.py | 2 +- uws/UWS/models.py | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/uws/UWS/client.py b/uws/UWS/client.py index cbdb66f..f7b784c 100644 --- a/uws/UWS/client.py +++ b/uws/UWS/client.py @@ -12,7 +12,7 @@ from .errors import UWSError -class Client(object): +class Client: def __init__(self, url=None, user=None, password=None, connection=None): if connection: self.connection = connection diff --git a/uws/UWS/connection.py b/uws/UWS/connection.py index 1afa4b7..c4b57cf 100644 --- a/uws/UWS/connection.py +++ b/uws/UWS/connection.py @@ -8,7 +8,7 @@ import re -class Connection(object): +class Connection: def __init__(self, url, user=None, password=None): self._set_url(url) diff --git a/uws/UWS/models.py b/uws/UWS/models.py index dfa2264..58baeaf 100644 --- a/uws/UWS/models.py +++ b/uws/UWS/models.py @@ -10,7 +10,7 @@ xlink_namespace = "http://www.w3.org/1999/xlink" -class UWS1Flavour(object): +class UWS1Flavour: def __init__(self, namespaces=None): if uws_1_namespace not in namespaces.values(): @@ -39,7 +39,7 @@ def __init__(self, namespaces=None): self.jobInfo = et.QName(self.uws_namespace, "jobInfo") -class JobPhases(object): +class JobPhases: COMPLETED = 'COMPLETED' PENDING = 'PENDING' QUEUED = 'QUEUED' @@ -72,7 +72,7 @@ class JobPhases(object): } -class BaseUWSModel(object): +class BaseUWSModel: def __init__(self): self.version = "1.0" From d148c152bdba7e579bcfaa0a54bfc66c46da37f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Aicardi?= Date: Fri, 31 Jan 2020 16:01:06 +0100 Subject: [PATCH 21/25] pylint : fixes no-self-use --- uws/UWS/client.py | 6 ++++-- uws/UWS/models.py | 9 ++++++--- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/uws/UWS/client.py b/uws/UWS/client.py index f7b784c..5e1d087 100644 --- a/uws/UWS/client.py +++ b/uws/UWS/client.py @@ -47,7 +47,8 @@ def get_job_list(self, filters=None): return job_list - def _validate_and_parse_filters(self, filters): + @staticmethod + def _validate_and_parse_filters(filters): filters_copy = filters.copy() phases = filters_copy.pop('phases', None) after = filters_copy.pop('after', None) @@ -103,7 +104,8 @@ def _validate_and_parse_filters(self, filters): return params - def _validate_and_parse_wait(self, wait, phase=None): + @staticmethod + def _validate_and_parse_wait(wait, phase=None): # wait must be positive integer or -1 if wait.isdigit() or wait == '-1': duration = int(wait) diff --git a/uws/UWS/models.py b/uws/UWS/models.py index 58baeaf..ebab4ec 100644 --- a/uws/UWS/models.py +++ b/uws/UWS/models.py @@ -76,7 +76,8 @@ class BaseUWSModel: def __init__(self): self.version = "1.0" - def _parse_bool(self, value): + @staticmethod + def _parse_bool(value): if isinstance(value, str): if value.lower() == 'true': return True @@ -324,7 +325,8 @@ def add_result(self, rid=None, href=None, result=None): self.results.append(result) - def _get_optional(self, parsed, element_name): + @staticmethod + def _get_optional(parsed, element_name): """Returns the text value of element_name within the parsed elementTree. If element_name doesn't exist, return None. @@ -335,7 +337,8 @@ def _get_optional(self, parsed, element_name): else: return option.text - def _get_mandatory(self, parsed, element_name): + @staticmethod + def _get_mandatory(parsed, element_name): """Check if the element exists, return text or error""" element = parsed.find(element_name) From 8d6cd0b989c73039c1f35bf0331a31b198f18b2f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Aicardi?= Date: Fri, 31 Jan 2020 16:20:09 +0100 Subject: [PATCH 22/25] pylint: fixes inconsistent-return-statements --- uws/cli/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uws/cli/main.py b/uws/cli/main.py index ac67379..8a7142e 100755 --- a/uws/cli/main.py +++ b/uws/cli/main.py @@ -19,7 +19,7 @@ def handle(self, *args, **kwargs): except UWS.UWSError as e: if not debug: print("An error occurred:\n %s" % e.msg) - return + raise else: print("An error occurred:\n %s" % e.msg) print(e.raw) From b3cc4dae204fc95b2b4bd312e38c4107dd185f86 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Aicardi?= Date: Fri, 31 Jan 2020 16:24:07 +0100 Subject: [PATCH 23/25] pylint : fixes import-outside-toplevel and drops python2.7 support --- uws/UWS/tests/test_connection.py | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/uws/UWS/tests/test_connection.py b/uws/UWS/tests/test_connection.py index 77eb9dd..bf10099 100644 --- a/uws/UWS/tests/test_connection.py +++ b/uws/UWS/tests/test_connection.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- import unittest - +import http.client as httplib from uws import UWS @@ -21,11 +21,6 @@ def testSetAuthHeader(self): self.assertDictEqual(connection.headers, {'Authorization': 'Basic YWRtaW46YWRtaW4='}) def testSetURLHTTP(self): - try: - import httplib - except ImportError: - import http.client as httplib - connection = UWS.connection.Connection( "http://www.example.com/uws/", user="admin", @@ -38,11 +33,6 @@ def testSetURLHTTP(self): self.assertIsInstance(connection.connection, httplib.HTTPConnection) def testSetURLHTTPS(self): - try: - import httplib - except ImportError: - import http.client as httplib - connection = UWS.connection.Connection( "https://www.example.com/uws/", user="admin", From e63b1eb22019db2b25693e94b452aad620912b99 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Aicardi?= Date: Fri, 31 Jan 2020 16:35:59 +0100 Subject: [PATCH 24/25] pylint --- uws/UWS/client.py | 2 +- uws/UWS/models.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/uws/UWS/client.py b/uws/UWS/client.py index 5e1d087..e43abfb 100644 --- a/uws/UWS/client.py +++ b/uws/UWS/client.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -from lxml.etree import XMLSyntaxError as XMLSyntaxError +from lxml.etree import XMLSyntaxError import dateutil.parser import pytz diff --git a/uws/UWS/models.py b/uws/UWS/models.py index ebab4ec..79bd3f2 100644 --- a/uws/UWS/models.py +++ b/uws/UWS/models.py @@ -312,7 +312,8 @@ def __str__(self): return str(self.__unicode__()) # return unicode(self).encode('utf-8') - def add_parameter(self, pid=None, by_reference=False, is_post=False, value=None, parameter=None): + def add_parameter(self, pid=None, by_reference=False, is_post=False, value=None, + parameter=None): if not parameter: parameter = Parameter(pid=pid, by_reference=by_reference, is_post=is_post, value=value) From 1342e27ef8c0cd12730943d6a087e63de1db2b25 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Aicardi?= Date: Fri, 31 Jan 2020 17:25:40 +0100 Subject: [PATCH 25/25] pylint: fixes invalid-name --- uws/UWS/client.py | 74 ++++++++++++------------ uws/UWS/models.py | 99 ++++++++++++++++---------------- uws/UWS/tests/test_base.py | 32 +++++------ uws/UWS/tests/test_connection.py | 6 +- uws/UWS/tests/test_models.py | 44 +++++++------- uws/cli/main.py | 46 +++++++-------- 6 files changed, 151 insertions(+), 150 deletions(-) diff --git a/uws/UWS/client.py b/uws/UWS/client.py index e43abfb..e8f77fa 100644 --- a/uws/UWS/client.py +++ b/uws/UWS/client.py @@ -27,23 +27,23 @@ def get_job_list(self, filters=None): try: response = self.connection.get('', params) - except Exception as e: + except Exception as exc: # Do not try to make a second request without parameters here, # because cannot call self.connection.get() a second time and reusing the connection # without calling a getresponse() or close() or something beforehand. # (This would lead to a httplib CannotSendRequest() error!) # Let's just raise the error immediately. - raise UWSError(str(e)) + raise UWSError(str(exc)) raw = response.read() try: job_list = models.Jobs(raw) - except XMLSyntaxError as e: + except XMLSyntaxError as exc: raise UWSError("Malformatted response. Are you sure the host you specified is a " "IVOA UWS service?", raw) - except Exception as e: - raise e + except Exception as exc: + raise exc return job_list @@ -129,27 +129,27 @@ def get_job(self, jid, wait=None, phase=None): try: response = self.connection.get(jid, params) - except Exception as e: + except Exception as exc: # Do not make a second request without params, throw error # immediately - raise UWSError(str(e)) + raise UWSError(str(exc)) raw = response.read() try: result = models.Job(raw) - except XMLSyntaxError as e: + except XMLSyntaxError as exc: raise UWSError("Malformatted response. Are you sure the host you specified is a " "IVOA UWS service?", raw) - except Exception as e: - raise e + except Exception as exc: + raise exc return result def get_phase(self, jid): try: response = self.connection.get(jid + '/phase') - except Exception as e: - raise UWSError(str(e)) + except Exception as exc: + raise UWSError(str(exc)) raw = response.read() result = str(raw, 'utf-8') @@ -160,17 +160,17 @@ def new_job(self, args=None): args = args or {} try: response = self.connection.post('', args) - except Exception as e: - raise UWSError(str(e)) + except Exception as exc: + raise UWSError(str(exc)) raw = response.read() try: result = models.Job(raw) - except XMLSyntaxError as e: + except XMLSyntaxError as exc: raise UWSError("Malformatted response. Are you sure the host you specified is a " "IVOA UWS service?", raw) - except Exception as e: - raise e + except Exception as exc: + raise exc return result @@ -178,67 +178,67 @@ def set_parameters_job(self, jid, args=None): args = args or {} try: response = self.connection.post(jid, args) - except Exception as e: - raise UWSError(str(e)) + except Exception as exc: + raise UWSError(str(exc)) raw = response.read() try: result = models.Job(raw) - except XMLSyntaxError as e: + except XMLSyntaxError as exc: raise UWSError("Malformatted response. Are you sure the host you specified is a " "IVOA UWS service?", raw) - except Exception as e: - raise e + except Exception as exc: + raise exc return result def run_job(self, jid): try: response = self.connection.post(jid + '/phase', {"PHASE": "RUN"}) - except Exception as e: - raise UWSError(str(e)) + except Exception as exc: + raise UWSError(str(exc)) raw = response.read() try: result = models.Job(raw) - except XMLSyntaxError as e: + except XMLSyntaxError as exc: raise UWSError("Malformatted response. Are you sure the host you specified is a " "IVOA UWS service?", raw) - except Exception as e: - raise e + except Exception as exc: + raise exc return result def abort_job(self, jid): try: response = self.connection.post(jid, {"PHASE": "ABORT"}) - except Exception as e: - raise UWSError(str(e)) + except Exception as exc: + raise UWSError(str(exc)) raw = response.read() try: result = models.Job(raw) - except XMLSyntaxError as e: + except XMLSyntaxError as exc: raise UWSError("Malformatted response. Are you sure the host you specified is a IVOA " "UWS service?", raw) - except Exception as e: - raise e + except Exception as exc: + raise exc return result def delete_job(self, jid): try: response = self.connection.delete(jid) - except Exception as e: - raise UWSError(str(e)) + except Exception as exc: + raise UWSError(str(exc)) raw = response.read() try: result = True - except XMLSyntaxError as e: + except XMLSyntaxError as exc: raise UWSError("Malformatted response. Are you sure the host you specified is a IVOA " "UWS service?", raw) - except Exception as e: - raise e + except Exception as exc: + raise exc return result diff --git a/uws/UWS/models.py b/uws/UWS/models.py index 79bd3f2..d1f5d1d 100644 --- a/uws/UWS/models.py +++ b/uws/UWS/models.py @@ -5,38 +5,38 @@ from lxml import etree as et -uws_1_namespace = "http://www.ivoa.net/xml/UWS/v1.0" -#uws_2_namespace = "http://www.ivoa.net/xml/UWS/v2.0" -xlink_namespace = "http://www.w3.org/1999/xlink" +UWS_1_NAMESPACE = "http://www.ivoa.net/xml/UWS/v1.0" +#UWS_2_NAMESPACE = "http://www.ivoa.net/xml/UWS/v2.0" +XLINK_NAMESPACE = "http://www.w3.org/1999/xlink" class UWS1Flavour: def __init__(self, namespaces=None): - if uws_1_namespace not in namespaces.values(): + if UWS_1_NAMESPACE not in namespaces.values(): raise RuntimeError("No supported UWS namespace found in xml-response, " "cannot parse xml.") # prepend each element's name with the correct uws-namespace # for this version - self.uws_namespace = uws_1_namespace + self.uws_namespace = UWS_1_NAMESPACE self.jobs = et.QName(self.uws_namespace, "jobs") self.jobref = et.QName(self.uws_namespace, "jobref") self.phase = et.QName(self.uws_namespace, "phase") - self.jobId = et.QName(self.uws_namespace, "jobId") - self.runId = et.QName(self.uws_namespace, "runId") - self.ownerId = et.QName(self.uws_namespace, "ownerId") + self.job_id = et.QName(self.uws_namespace, "jobId") + self.run_id = et.QName(self.uws_namespace, "runId") + self.owner_id = et.QName(self.uws_namespace, "ownerId") self.quote = et.QName(self.uws_namespace, "quote") - self.creationTime = et.QName(self.uws_namespace, "creationTime") - self.startTime = et.QName(self.uws_namespace, "startTime") - self.endTime = et.QName(self.uws_namespace, "endTime") - self.executionDuration = et.QName(self.uws_namespace, "executionDuration") + self.creation_time = et.QName(self.uws_namespace, "creationTime") + self.start_time = et.QName(self.uws_namespace, "startTime") + self.end_time = et.QName(self.uws_namespace, "endTime") + self.execution_duration = et.QName(self.uws_namespace, "executionDuration") self.destruction = et.QName(self.uws_namespace, "destruction") self.parameters = et.QName(self.uws_namespace, "parameters") self.results = et.QName(self.uws_namespace, "results") - self.errorSummary = et.QName(self.uws_namespace, "errorSummary") + self.error_summary = et.QName(self.uws_namespace, "errorSummary") self.message = et.QName(self.uws_namespace, "message") - self.jobInfo = et.QName(self.uws_namespace, "jobInfo") + self.job_info = et.QName(self.uws_namespace, "jobInfo") class JobPhases: @@ -106,10 +106,10 @@ def __init__(self, xml=None): self.job_reference = [] - for xmlJob in xml_jobs: - self.add_job( - job=JobRef(xml_node=xmlJob, xml_namespace=parsed.nsmap, uws_flavour=uws_flavour) - ) + for xml_job in xml_jobs: + self.add_job(job=JobRef(xml_node=xml_job, + xml_namespace=parsed.nsmap, + uws_flavour=uws_flavour)) else: self.job_reference = [] @@ -137,24 +137,24 @@ def __init__(self, jid=None, phase=None, reference=None, xml_node=None, xml_name uws_flavour=None): super(JobRef, self).__init__() - self.id = None + self.jid = None self.reference = Reference() self.phase = [] if xml_node is not None: # When should this ever be None????? - self.id = xml_node.get('id') + self.jid = xml_node.get('id') # UWS standard defines array, therefore treat phase as array # (... actually it does not, but keep it anyway like this, maybe at # some point in the future all phases of a job are provided as list) self.phase = [elm.text for elm in xml_node.findall(uws_flavour.phase)] self.reference = Reference(xml_node=xml_node, xml_namespace=xml_namespace) - self.runId = xml_node.get('runId') - self.ownerId = xml_node.get('ownerId') - self.creationTime = xml_node.get('creationTime') + self.run_id = xml_node.get('runId') + self.owner_id = xml_node.get('ownerId') + self.creation_time = xml_node.get('creationTime') elif jid is not None and phase is not None and reference is not None: - self.id = jid + self.jid = jid if isinstance(phase, six.string_types): self.phase = [phase] @@ -170,12 +170,13 @@ def set_phase(self, new_phase): self.phase = [new_phase] def __unicode__(self): - if self.creationTime is not None: - return "Job '%s' in phase '%s' created at '%s' - %s" % (self.id, ', '.join(self.phase), - self.creationTime, + if self.creation_time is not None: + return "Job '%s' in phase '%s' created at '%s' - %s" % (self.jid, + ', '.join(self.phase), + self.creation_time, str(self.reference)) else: - return "Job '%s' in phase '%s' - %s" % (self.id, ', '.join(self.phase), + return "Job '%s' in phase '%s' - %s" % (self.jid, ', '.join(self.phase), str(self.reference)) def __str__(self): @@ -191,12 +192,12 @@ def __init__(self, href=None, rtype=None, xml_node=None, xml_namespace=None): if xml_node is not None: # check that namespace for xlink really exists - if xlink_namespace not in xml_namespace.values(): + if XLINK_NAMESPACE not in xml_namespace.values(): raise RuntimeError("No supported xlink namespace found in xml-response, " "cannot parse xml.") - qualifiedname_type = et.QName(xlink_namespace, "type") - qualifiedname_href = et.QName(xlink_namespace, "href") + qualifiedname_type = et.QName(XLINK_NAMESPACE, "type") + qualifiedname_href = et.QName(XLINK_NAMESPACE, "href") self.type = xml_node.get(qualifiedname_type) self.href = xml_node.get(qualifiedname_href) elif href is not None and rtype is not None: @@ -244,16 +245,16 @@ def __init__(self, xml=None): if parsed.get("version"): self.version = parsed.get("version") - self.job_id = self._get_mandatory(parsed, uws_flavour.jobId) - self.run_id = self._get_optional(parsed, uws_flavour.runId) - self.owner_id = self._get_optional(parsed, uws_flavour.ownerId) + self.job_id = self._get_mandatory(parsed, uws_flavour.job_id) + self.run_id = self._get_optional(parsed, uws_flavour.run_id) + self.owner_id = self._get_optional(parsed, uws_flavour.owner_id) self.phase = [self._get_mandatory(parsed, uws_flavour.phase)] self.quote = self._get_optional(parsed, uws_flavour.quote) - self.creation_time = self._get_optional(parsed, uws_flavour.creationTime) - self.start_time = self._get_mandatory(parsed, uws_flavour.startTime) - self.end_time = self._get_mandatory(parsed, uws_flavour.endTime) + self.creation_time = self._get_optional(parsed, uws_flavour.creation_time) + self.start_time = self._get_mandatory(parsed, uws_flavour.start_time) + self.end_time = self._get_mandatory(parsed, uws_flavour.end_time) self.execution_duration = int(self._get_mandatory(parsed, - uws_flavour.executionDuration)) + uws_flavour.execution_duration)) self.destruction = self._get_mandatory(parsed, uws_flavour.destruction) self.parameters = [] @@ -271,12 +272,12 @@ def __init__(self, xml=None): self.add_result(result=Result(xml_node=res, xml_namespace=parsed.nsmap)) self.error_summary = False - tmp = parsed.find(uws_flavour.errorSummary) + tmp = parsed.find(uws_flavour.error_summary) if tmp is not None: self.error_summary = ErrorSummary(xml_node=tmp, uws_flavour=uws_flavour) self.job_info = [] - tmp = parsed.find(uws_flavour.jobInfo) + tmp = parsed.find(uws_flavour.job_info) if tmp is not None: self.job_info = list(tmp) @@ -354,24 +355,24 @@ class Parameter(BaseUWSModel): def __init__(self, pid=None, by_reference=False, is_post=False, value=None, xml_node=None): super(Parameter, self).__init__() - self.id = None + self.pid = None self.by_reference = False self.is_post = False self.value = None if xml_node is not None: - self.id = xml_node.get('id') + self.pid = xml_node.get('id') self.by_reference = self._parse_bool(xml_node.get('by_reference', default=False)) self.is_post = self._parse_bool(xml_node.get('is_post', default=False)) self.value = xml_node.text elif pid is not None and value is not None: - self.id = pid + self.pid = pid self.by_reference = by_reference self.is_post = is_post self.value = value def __unicode__(self): - return "Parameter id '%s' byRef: %s is_post: %s - value: %s" % (self.id, self.by_reference, + return "Parameter id '%s' byRef: %s is_post: %s - value: %s" % (self.pid, self.by_reference, self.is_post, self.value) def __str__(self): @@ -382,22 +383,22 @@ class Result(BaseUWSModel): def __init__(self, rid=None, reference=None, xml_node=None, xml_namespace=None): super(Result, self).__init__() - self.id = None + self.rid = None self.reference = Reference() if xml_node is not None: - self.id = xml_node.get('id') + self.rid = xml_node.get('id') self.reference = Reference(xml_node=xml_node, xml_namespace=xml_namespace) elif rid is not None and reference is not None: - self.id = rid + self.rid = rid if isinstance(reference, Reference): self.reference = reference else: - raise RuntimeError("Malformated reference given in result id: %s" % id) + raise RuntimeError("Malformated reference given in result id: %s" % rid) def __unicode__(self): - return "Result id '%s' reference: %s" % (self.id, str(self.reference)) + return "Result id '%s' reference: %s" % (self.rid, str(self.reference)) def __str__(self): return str(self.__unicode__()) diff --git a/uws/UWS/tests/test_base.py b/uws/UWS/tests/test_base.py index e929f68..6102e0c 100644 --- a/uws/UWS/tests/test_base.py +++ b/uws/UWS/tests/test_base.py @@ -5,7 +5,7 @@ class BaseTest(unittest.TestCase): - def testValidateAndParsePhaseFilter(self): + def test_validate_and_parse_phase_filter(self): filters = { 'phases': ['COMPLETED', 'PENDING'] } @@ -14,7 +14,7 @@ def testValidateAndParsePhaseFilter(self): self.assertEqual(params, [('PHASE', 'COMPLETED'), ('PHASE', 'PENDING')]) - def testValidateAndParsePhaseFilterInvalidPhase(self): + def test_validate_and_parse_phase_filter_invalid_phase(self): filters = { 'phases': ['FOO', 'PENDING'] } @@ -25,7 +25,7 @@ def testValidateAndParsePhaseFilterInvalidPhase(self): filters ) - def testValidateAndParseAfterFilter(self): + def test_validate_and_parse_after_filter(self): filters = { 'after': '2015-09-10T10:01:02.135' } @@ -34,7 +34,7 @@ def testValidateAndParseAfterFilter(self): self.assertEqual(params, [('AFTER', '2015-09-10T10:01:02.135000')]) - def testValidateAndParseAfterFilterInvalidDate(self): + def test_validate_and_parse_after_filter_invalid_date(self): filters = { 'after': '2010-4--' } @@ -45,7 +45,7 @@ def testValidateAndParseAfterFilterInvalidDate(self): filters ) - def testValidateAndParseAfterFilterTimeZone(self): + def test_validate_and_parse_after_filter_time_zone(self): filters = { 'after': '2015-10-03T01:12+2:00' } @@ -56,7 +56,7 @@ def testValidateAndParseAfterFilterTimeZone(self): - def testValidateAndParseLastFilter(self): + def test_validate_and_parse_last_filter(self): filters = { 'last': '1000' } @@ -65,7 +65,7 @@ def testValidateAndParseLastFilter(self): self.assertEqual(params, [('LAST', 1000)]) - def testValidateAndParseLastFilterFloatValue(self): + def test_validate_and_parse_last_filter_float_value(self): filters = { 'last': '100.0' } @@ -76,7 +76,7 @@ def testValidateAndParseLastFilterFloatValue(self): filters ) - def testValidateAndParseLastFilterNegativeValue(self): + def test_validate_and_parse_last_filter_negative_value(self): filters = { 'last': '-100' } @@ -87,7 +87,7 @@ def testValidateAndParseLastFilterNegativeValue(self): filters ) - def testValidateAndParseAfterLastFilter(self): + def test_validate_and_parse_after_last_filter(self): filters = { 'after': '2015-09-10T10:01:02.135', 'last': '100' @@ -98,7 +98,7 @@ def testValidateAndParseAfterLastFilter(self): self.assertEqual(params, [('AFTER', '2015-09-10T10:01:02.135000'), ('LAST', 100)]) - def testValidateAndParseAfterLastPhaseFilter(self): + def test_validate_and_parse_after_last_phase_filter(self): filters = { 'after': '2015-09-10T10:01:02.135', 'last': '100', @@ -110,19 +110,19 @@ def testValidateAndParseAfterLastPhaseFilter(self): self.assertEqual(params, [('PHASE', 'PENDING'), ('PHASE', 'ERROR'), ('AFTER', '2015-09-10T10:01:02.135000'), ('LAST', 100)]) - def testValidateAndParseWaitNegative(self): + def test_validate_and_parse_wait_negative(self): wait = '-1' params = UWS.client.Client("/")._validate_and_parse_wait(wait) self.assertEqual(params, [('WAIT', -1)]) - def testValidateAndParseWait(self): + def test_validate_and_parse_wait(self): wait = '30' params = UWS.client.Client("/")._validate_and_parse_wait(wait) self.assertEqual(params, [('WAIT', 30)]) - def testValidateAndParseWaitInvalidWait(self): + def test_validate_and_parse_wait_invalid_wait(self): wait = '30.587' self.assertRaises( @@ -131,7 +131,7 @@ def testValidateAndParseWaitInvalidWait(self): wait ) - def testValidateAndParseWaitInvalidWaitNegative(self): + def test_validate_and_parse_wait_invalid_wait_negative(self): wait = '-30' self.assertRaises( @@ -140,7 +140,7 @@ def testValidateAndParseWaitInvalidWaitNegative(self): wait ) - def testValidateAndParseWaitPhase(self): + def test_validate_and_parse_wait_phase(self): wait = '30' phase = 'EXECUTING' @@ -148,7 +148,7 @@ def testValidateAndParseWaitPhase(self): self.assertEqual(params, [('WAIT', 30), ('PHASE', 'EXECUTING')]) - def testValidateAndParseWaitInvalidPhase(self): + def test_validate_and_parse_wait_invalid_phase(self): wait = '15' phase = 'COMPLETED' diff --git a/uws/UWS/tests/test_connection.py b/uws/UWS/tests/test_connection.py index bf10099..cc1b579 100644 --- a/uws/UWS/tests/test_connection.py +++ b/uws/UWS/tests/test_connection.py @@ -5,7 +5,7 @@ class ConnectionTest(unittest.TestCase): - def testSetAuthHeader(self): + def test_set_auth_header(self): connection = UWS.connection.Connection( "http://www.example.com/uws", user="admin", @@ -20,7 +20,7 @@ def testSetAuthHeader(self): self.assertEqual(auth_string, "YWRtaW46YWRtaW4=") self.assertDictEqual(connection.headers, {'Authorization': 'Basic YWRtaW46YWRtaW4='}) - def testSetURLHTTP(self): + def test_set_url_http(self): connection = UWS.connection.Connection( "http://www.example.com/uws/", user="admin", @@ -32,7 +32,7 @@ def testSetURLHTTP(self): self.assertEqual(connection.base_path, "/uws") self.assertIsInstance(connection.connection, httplib.HTTPConnection) - def testSetURLHTTPS(self): + def test_set_url_https(self): connection = UWS.connection.Connection( "https://www.example.com/uws/", user="admin", diff --git a/uws/UWS/tests/test_models.py b/uws/UWS/tests/test_models.py index dfa75f8..9f8e330 100644 --- a/uws/UWS/tests/test_models.py +++ b/uws/UWS/tests/test_models.py @@ -45,35 +45,35 @@ def test(self): self.assertEqual(str(job_list), job_list_str) job1 = job_list.job_reference[0] - self.assertEqual(job1.id, '2014-06-03T15:33:29:4235') + self.assertEqual(job1.jid, '2014-06-03T15:33:29:4235') self.assertEqual(job1.phase, ['COMPLETED']) self.assertEqual(job1.reference.type, "simple") self.assertEqual(job1.reference.href, "https://www.cosmosim.org/uws/query/335912448787925") job2 = job_list.job_reference[1] - self.assertEqual(job2.id, '2014-06-02T10:14:25:1677') + self.assertEqual(job2.jid, '2014-06-02T10:14:25:1677') self.assertEqual(job2.phase, ['ABORTED']) self.assertEqual(job2.reference.type, "simple") self.assertEqual(job2.reference.href, "https://www.cosmosim.org/uws/query/308893189727250") job3 = job_list.job_reference[2] - self.assertEqual(job3.id, '2014-05-28T10:46:39:2755') + self.assertEqual(job3.jid, '2014-05-28T10:46:39:2755') self.assertEqual(job3.phase, ['COMPLETED']) self.assertEqual(job3.reference.type, "simple") self.assertEqual(job3.reference.href, "https://www.cosmosim.org/uws/query/198796714554760") job4 = job_list.job_reference[3] - self.assertEqual(job4.id, '2014-05-09T15:13:50:6896') + self.assertEqual(job4.jid, '2014-05-09T15:13:50:6896') self.assertEqual(job4.phase, ['ERROR']) self.assertEqual(job4.reference.type, "simple") self.assertEqual(job4.reference.href, "https://www.cosmosim.org/uws/query/1177277256137938") job5 = job_list.job_reference[4] - self.assertEqual(job5.id, u'rndSamp2') + self.assertEqual(job5.jid, u'rndSamp2') self.assertEqual(job5.phase, ['COMPLETED']) self.assertEqual(job5.reference.type, "simple") self.assertEqual(job5.reference.href, @@ -168,19 +168,19 @@ def test(self): # check parameters param = job.parameters[0] - self.assertEqual(param.id, 'database') + self.assertEqual(param.pid, 'database') self.assertEqual(param.by_reference, False) self.assertEqual(param.is_post, False) self.assertEqual(param.value, 'cosmosim_user_adrian') param = job.parameters[1] - self.assertEqual(param.id, 'table') + self.assertEqual(param.pid, 'table') self.assertEqual(param.by_reference, False) self.assertEqual(param.is_post, False) self.assertEqual(param.value, '2014-06-03T15:33:29:4235') param = job.parameters[2] - self.assertEqual(param.id, 'query') + self.assertEqual(param.pid, 'query') self.assertEqual(param.by_reference, False) self.assertEqual(param.is_post, False) query_value = ("SELECT 0.25*(0.5+FLOOR(LOG10(Mvir)/0.25)) AS log_mass, COUNT(*) AS num\r\n" @@ -198,34 +198,34 @@ def test(self): self.assertEqual(param.value, query_value) param = job.parameters[3] - self.assertEqual(param.id, 'queue') + self.assertEqual(param.pid, 'queue') self.assertEqual(param.by_reference, False) self.assertEqual(param.is_post, False) self.assertEqual(param.value, 'short') # check results result = job.results[0] - self.assertEqual(result.id, 'csv') + self.assertEqual(result.rid, 'csv') self.assertEqual(result.reference.type, 'simple') self.assertEqual(result.reference.href, ('https://www.cosmosim.org/query/download/stream/' 'table/2014-06-03T15%3A33%3A29%3A4235/format/csv')) result = job.results[1] - self.assertEqual(result.id, 'votable.xml') + self.assertEqual(result.rid, 'votable.xml') self.assertEqual(result.reference.type, 'simple') self.assertEqual(result.reference.href, ('https://www.cosmosim.org/query/download/stream/' 'table/2014-06-03T15%3A33%3A29%3A4235/' 'format/votable')) result = job.results[2] - self.assertEqual(result.id, 'votableB1.xml') + self.assertEqual(result.rid, 'votableB1.xml') self.assertEqual(result.reference.type, 'simple') self.assertEqual(result.reference.href, ('https://www.cosmosim.org/query/download/stream/' 'table/2014-06-03T15%3A33%3A29%3A4235/format/' 'votableB1')) result = job.results[3] - self.assertEqual(result.id, 'votableB2.xml') + self.assertEqual(result.rid, 'votableB2.xml') self.assertEqual(result.reference.type, 'simple') self.assertEqual(result.reference.href, ('https://www.cosmosim.org/query/download/stream/' 'table/2014-06-03T15%3A33%3A29%3A4235/format/' @@ -300,19 +300,19 @@ def test(self): # check parameters param = job.parameters[0] - self.assertEqual(param.id, 'database') + self.assertEqual(param.pid, 'database') self.assertEqual(param.by_reference, False) self.assertEqual(param.is_post, False) self.assertEqual(param.value, 'cosmosim_user_adrian') param = job.parameters[1] - self.assertEqual(param.id, 'table') + self.assertEqual(param.pid, 'table') self.assertEqual(param.by_reference, False) self.assertEqual(param.is_post, False) self.assertEqual(param.value, '2014-06-02T10:14:25:1677') param = job.parameters[2] - self.assertEqual(param.id, 'query') + self.assertEqual(param.pid, 'query') self.assertEqual(param.by_reference, False) self.assertEqual(param.is_post, False) query_value = ("select count(*) from MDR1.Particles85 where x < 1\n-- The query plan used " @@ -326,7 +326,7 @@ def test(self): self.assertEqual(param.value, query_value) param = job.parameters[3] - self.assertEqual(param.id, 'queue') + self.assertEqual(param.pid, 'queue') self.assertEqual(param.by_reference, False) self.assertEqual(param.is_post, False) self.assertEqual(param.value, 'short') @@ -406,19 +406,19 @@ def test(self): # check parameters param = job.parameters[0] - self.assertEqual(param.id, 'database') + self.assertEqual(param.pid, 'database') self.assertEqual(param.by_reference, False) self.assertEqual(param.is_post, False) self.assertEqual(param.value, 'cosmosim_user_adrian') param = job.parameters[1] - self.assertEqual(param.id, 'table') + self.assertEqual(param.pid, 'table') self.assertEqual(param.by_reference, False) self.assertEqual(param.is_post, False) self.assertEqual(param.value, '2014-05-09T15:13:50:6896') param = job.parameters[2] - self.assertEqual(param.id, 'query') + self.assertEqual(param.pid, 'query') self.assertEqual(param.by_reference, False) self.assertEqual(param.is_post, False) query_value = ("select avg(x) from `MDPL`.`Particles88tmp`;\n-- The query plan used to " @@ -433,7 +433,7 @@ def test(self): self.assertEqual(param.value, query_value) param = job.parameters[3] - self.assertEqual(param.id, 'queue') + self.assertEqual(param.pid, 'queue') self.assertEqual(param.by_reference, False) self.assertEqual(param.is_post, False) self.assertEqual(param.value, 'short') @@ -470,7 +470,7 @@ def test(self): self.assertEqual(str(job_list), job_list_str) job1 = job_list.job_reference[0] - self.assertEqual(job1.id, '335912448787925') + self.assertEqual(job1.jid, '335912448787925') self.assertEqual(job1.phase, ['COMPLETED']) self.assertEqual(job1.reference.type, "simple") self.assertEqual(job1.reference.href, "https://www.cosmosim.org/uws/query/335912448787925") diff --git a/uws/cli/main.py b/uws/cli/main.py index 8a7142e..60d3311 100755 --- a/uws/cli/main.py +++ b/uws/cli/main.py @@ -8,7 +8,7 @@ from uws import UWS from . import cli_parser -debug = False +DEBUG = False def handle_error(handler): @@ -16,13 +16,13 @@ def handle_error(handler): def handle(self, *args, **kwargs): try: return handler(self, *args, **kwargs) - except UWS.UWSError as e: - if not debug: - print("An error occurred:\n %s" % e.msg) + except UWS.UWSError as exc: + if not DEBUG: + print("An error occurred:\n %s" % exc.msg) raise else: - print("An error occurred:\n %s" % e.msg) - print(e.raw) + print("An error occurred:\n %s" % exc.msg) + print(exc.raw) raise return handle @@ -112,26 +112,26 @@ def _register_job_reference_for_table(rows, jobref): # only the href-id, if they differ. Because this really MUST be the # correct jobId. - cols = [jobref.id] + cols = [jobref.jid] if jobref.reference.href is not None: href_jobid = jobref.reference.href.rsplit("/", 1)[1] - if href_jobid != jobref.id: + if href_jobid != jobref.jid: # replace id with href_jobid cols[0] = href_jobid - if jobref.runId is not None: - cols.append(jobref.runId) + if jobref.run_id is not None: + cols.append(jobref.run_id) else: cols.append('') - if jobref.ownerId is not None: - cols.append(jobref.ownerId) + if jobref.owner_id is not None: + cols.append(jobref.owner_id) else: cols.append('') - if jobref.creationTime is not None: - cols.append(jobref.creationTime) + if jobref.creation_time is not None: + cols.append(jobref.creation_time) else: cols.append('') @@ -249,12 +249,12 @@ def print_progress(total_size, current): print(' result like this: ') print('\nuws job results ID RESULTID\n') print('For RESULTID, you can choose from: ', - ','.join([result.id for result in job.results])) + ','.join([result.rid for result in job.results])) # set file base name to job_id or tablename (if available) or user provided file_base file_base = job.job_id for parameter in job.parameters: - if parameter.id == 'table': + if parameter.pid == 'table': file_base = parameter.value break @@ -263,13 +263,13 @@ def print_progress(total_size, current): retrieved = False for result in job.results: - if not result_id or result_id == result.id: + if not result_id or result_id == result.rid: - filename = file_base + '.' + result.id + filename = file_base + '.' + result.rid url = str(result.reference) - print("Downloading %s into file '%s'" % (result.id, filename)) + print("Downloading %s into file '%s'" % (result.rid, filename)) uws_client.connection.download_file(url, user_name, password, filename, callback=print_progress) print("") @@ -310,10 +310,10 @@ def _print_job(job): rows.append(["Destruction time", job.destruction]) for param in job.parameters: - rows.append(["Parameter " + param.id, param.value]) + rows.append(["Parameter " + param.pid, param.value]) for result in job.results: - rows.append(["Result " + result.id, result.reference]) + rows.append(["Result " + result.rid, result.reference]) try: if job.error_summary: @@ -394,12 +394,12 @@ def _check_joblist_last(argument): def main(): - global debug + global DEBUG parser = cli_parser.build_argparse() arguments = parser.parse_args() if arguments.dbg: - debug = True + DEBUG = True if arguments.P: if arguments.password: