diff --git a/.travis.yml b/.travis.yml index f72f2c6..3bf3039 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,7 +4,7 @@ cache: directories: - $HOME/virtualenv python: -- 2.7 +- 3.7 install: - pip install -r requirements.pip - pip install -r requirements-dev.pip diff --git a/setup.py b/setup.py index 5426976..dfc9a7d 100644 --- a/setup.py +++ b/setup.py @@ -15,8 +15,8 @@ setup( name='uws-client', version=version, - url='http://github.com/aipescience/uws-client/', - download_url='http://github.com/aipescience/uws-client/archive/%s.tar.gz' % version, + url='http://github.com/aicardi-obspm/uws-client/', + download_url='http://github.com/aicardi-obspm/uws-client/archive/%s.tar.gz' % version, packages=find_packages(), license=u'Apache License (2.0)', author=u'Adrian M. Partl', @@ -24,7 +24,7 @@ maintainer=u'AIP E-Science', maintainer_email=u'escience@aip.de', description=u'A command line client for IVOA UWS services, plus models for development', - long_description='This is a client for IVOA Virtual Observatroy UWS services. It can be used to access UWS services directly or through Basic Authentication. Please visit https://github.com/aipescience/uws-client/blob/master/README.md for how to use the software.', + long_description='This is a client for IVOA Virtual Observatroy UWS services. It can be used to access UWS services directly or through Basic Authentication. Please visit https://github.com/aicardi-obspm/uws-client/blob/master/README.md for how to use the software.', include_package_data=True, install_requires=install_requires, entry_points={ diff --git a/uws/UWS/__init__.py b/uws/UWS/__init__.py index dd3761c..9398323 100644 --- a/uws/UWS/__init__.py +++ b/uws/UWS/__init__.py @@ -8,4 +8,3 @@ from . import connection from . import models from .errors import UWSError - diff --git a/uws/UWS/client.py b/uws/UWS/client.py index 340562a..e8f77fa 100644 --- a/uws/UWS/client.py +++ b/uws/UWS/client.py @@ -1,5 +1,7 @@ # -*- coding: utf-8 -*- -from lxml.etree import XMLSyntaxError as XMLSyntaxError +from lxml.etree import XMLSyntaxError +import dateutil.parser +import pytz #import connection as UWSConnection #import models @@ -8,12 +10,9 @@ from . import connection as UWSConnection from . import models from .errors import UWSError -from datetime import datetime -import dateutil.parser -import pytz -class Client(object): +class Client: def __init__(self, url=None, user=None, password=None, connection=None): if connection: self.connection = connection @@ -28,45 +27,48 @@ def get_job_list(self, filters=None): try: response = self.connection.get('', params) - except Exception as e: + except Exception as exc: # Do not try to make a second request without parameters here, # because cannot call self.connection.get() a second time and reusing the connection # without calling a getresponse() or close() or something beforehand. # (This would lead to a httplib CannotSendRequest() error!) # Let's just raise the error immediately. - raise UWSError(str(e)) + raise UWSError(str(exc)) raw = response.read() try: job_list = models.Jobs(raw) - except XMLSyntaxError as e: - raise UWSError("Malformatted response. Are you sure the host you specified is a IVOA UWS service?", raw) - except Exception as e: - raise e + except XMLSyntaxError as exc: + raise UWSError("Malformatted response. Are you sure the host you specified is a " + "IVOA UWS service?", raw) + except Exception as exc: + raise exc return job_list - def _validate_and_parse_filters(self, filters): + @staticmethod + def _validate_and_parse_filters(filters): filters_copy = filters.copy() phases = filters_copy.pop('phases', None) after = filters_copy.pop('after', None) last = filters_copy.pop('last', None) if filters_copy: - raise UWSError("Unknown filter properties %s", filters_copy.keys()) + raise UWSError(f"Unknown filter properties {filters_copy.keys()}") params = [] if phases: for phase in phases: if phase not in models.JobPhases.phases: - raise UWSError("Unknown phase %s in filter", phase) + raise UWSError(f"Unknown phase {phase} in filter") params.append(("PHASE", phase)) if after: # TODO: Allow to provide local time and convert here to UTC? - # TODO: We may encounter more troubles with microseconds, if ',' used instead of '.'(e.g. German systems) + # TODO: We may encounter more troubles with microseconds, + # if ',' used instead of '.'(e.g. German systems) try: date = dateutil.parser.parse(after) @@ -74,10 +76,12 @@ def _validate_and_parse_filters(self, filters): # given (e.g. '2010-09'->'2010-09-06'). # Let's tell the user how the given value was interpreted: if str(date) != str(after): - print("Note: Changed value for keyword 'after' from '%s' to '%s'." % (after, str(date))) + print(f"Note: Changed value for keyword 'after' from '{after}' " + f"to '{str(date)}'.") except: - raise UWSError("Date time format could not be parsed, expecting UTC in ISO 8601:2004 format or compatible: %s" % (str(after))) + raise UWSError("Date time format could not be parsed, expecting UTC in " + "ISO 8601:2004 format or compatible: %s" % (str(after))) # Convert from given time (with attached timezone information) to UTC time if date.utcoffset() is not None: @@ -92,15 +96,16 @@ def _validate_and_parse_filters(self, filters): try: last = int(last) except: - raise UWSError("Value for 'last' argument must be a positive integer: %s" % (str(last))) + raise UWSError(f"Value for 'last' argument must be a positive integer: {str(last)}") if last < 1: - raise UWSError("Value for 'last' argument must be a positive integer: %s" % (str(last))) + raise UWSError(f"Value for 'last' argument must be a positive integer: {str(last)}") params.append(("LAST", last)) return params - def _validate_and_parse_wait(self, wait, phase=None): + @staticmethod + def _validate_and_parse_wait(wait, phase=None): # wait must be positive integer or -1 if wait.isdigit() or wait == '-1': duration = int(wait) @@ -111,120 +116,129 @@ def _validate_and_parse_wait(self, wait, phase=None): if phase: if phase not in models.JobPhases.active_phases: - raise UWSError("Given phase '%s' is not an active phase, 'wait' with this phase is not supported." % phase) + raise UWSError(f"Given phase '{phase}' is not an active phase, " + "'wait' with this phase is not supported.") params.append(("PHASE", phase)) return params - def get_job(self, id, wait=None, phase=None): + def get_job(self, jid, wait=None, phase=None): params = None if wait: params = self._validate_and_parse_wait(wait, phase) try: - response = self.connection.get(id, params) - except Exception as e: + response = self.connection.get(jid, params) + except Exception as exc: # Do not make a second request without params, throw error # immediately - raise UWSError(str(e)) + raise UWSError(str(exc)) raw = response.read() try: result = models.Job(raw) - except XMLSyntaxError as e: - raise UWSError("Malformatted response. Are you sure the host you specified is a IVOA UWS service?", raw) - except Exception as e: - raise e + except XMLSyntaxError as exc: + raise UWSError("Malformatted response. Are you sure the host you specified is a " + "IVOA UWS service?", raw) + except Exception as exc: + raise exc return result - def get_phase(self, id): + def get_phase(self, jid): try: - response = self.connection.get(id + '/phase') - except Exception as e: - raise UWSError(str(e)) + response = self.connection.get(jid + '/phase') + except Exception as exc: + raise UWSError(str(exc)) raw = response.read() - result = raw + result = str(raw, 'utf-8') return result - def new_job(self, args={}): + def new_job(self, args=None): + args = args or {} try: response = self.connection.post('', args) - except Exception as e: - raise UWSError(str(e)) + except Exception as exc: + raise UWSError(str(exc)) raw = response.read() try: result = models.Job(raw) - except XMLSyntaxError as e: - raise UWSError("Malformatted response. Are you sure the host you specified is a IVOA UWS service?", raw) - except Exception as e: - raise + except XMLSyntaxError as exc: + raise UWSError("Malformatted response. Are you sure the host you specified is a " + "IVOA UWS service?", raw) + except Exception as exc: + raise exc return result - def set_parameters_job(self, id, args={}): + def set_parameters_job(self, jid, args=None): + args = args or {} try: - response = self.connection.post(id, args) - except Exception as e: - raise UWSError(str(e)) + response = self.connection.post(jid, args) + except Exception as exc: + raise UWSError(str(exc)) raw = response.read() try: result = models.Job(raw) - except XMLSyntaxError as e: - raise UWSError("Malformatted response. Are you sure the host you specified is a IVOA UWS service?", raw) - except Exception as e: - raise e + except XMLSyntaxError as exc: + raise UWSError("Malformatted response. Are you sure the host you specified is a " + "IVOA UWS service?", raw) + except Exception as exc: + raise exc return result - def run_job(self, id): + def run_job(self, jid): try: - response = self.connection.post(id + '/phase', {"PHASE": "RUN"}) - except Exception as e: - raise UWSError(str(e)) + response = self.connection.post(jid + '/phase', {"PHASE": "RUN"}) + except Exception as exc: + raise UWSError(str(exc)) raw = response.read() try: result = models.Job(raw) - except XMLSyntaxError as e: - raise UWSError("Malformatted response. Are you sure the host you specified is a IVOA UWS service?", raw) - except Exception as e: - raise e + except XMLSyntaxError as exc: + raise UWSError("Malformatted response. Are you sure the host you specified is a " + "IVOA UWS service?", raw) + except Exception as exc: + raise exc return result - def abort_job(self, id): + def abort_job(self, jid): try: - response = self.connection.post(id, {"PHASE": "ABORT"}) - except Exception as e: - raise UWSError(str(e)) + response = self.connection.post(jid, {"PHASE": "ABORT"}) + except Exception as exc: + raise UWSError(str(exc)) raw = response.read() try: result = models.Job(raw) - except XMLSyntaxError as e: - raise UWSError("Malformatted response. Are you sure the host you specified is a IVOA UWS service?", raw) - except Exception as e: - raise e + except XMLSyntaxError as exc: + raise UWSError("Malformatted response. Are you sure the host you specified is a IVOA " + "UWS service?", raw) + except Exception as exc: + raise exc return result - def delete_job(self, id): + def delete_job(self, jid): try: - response = self.connection.delete(id) - except Exception as e: - raise UWSError(str(e)) + response = self.connection.delete(jid) + except Exception as exc: + raise UWSError(str(exc)) raw = response.read() try: result = True - except XMLSyntaxError as e: - raise UWSError("Malformatted response. Are you sure the host you specified is a IVOA UWS service?", raw) - except Exception as e: - raise e + except XMLSyntaxError as exc: + raise UWSError("Malformatted response. Are you sure the host you specified is a IVOA " + "UWS service?", raw) + except Exception as exc: + raise exc return result diff --git a/uws/UWS/connection.py b/uws/UWS/connection.py index 41b28cc..c4b57cf 100644 --- a/uws/UWS/connection.py +++ b/uws/UWS/connection.py @@ -1,34 +1,28 @@ # -*- coding: utf-8 -*- -try: - import httplib -except ImportError: - import http.client as httplib -try: - import urllib2 - import urllib - from urlparse import urlparse -except ImportError: - import urllib.request as urllib2 - import urllib.parse as urllib - from urllib.parse import urlparse +import http.client as httplib +import urllib.request as urllib2 +import urllib.parse as urllib +from urllib.parse import urlparse import base64 +import mimetypes +import re -class Connection(object): +class Connection: def __init__(self, url, user=None, password=None): self._set_url(url) if user is not None and password is not None: - try: + try: user = user.decode('utf-8') password = password.decode('utf-8') except AttributeError: pass auth_string = '%s:%s' % (user, password) - self.auth_string = base64.encodestring(auth_string.encode()) + self.auth_string = base64.encodebytes(auth_string.encode()) self.headers = {"Authorization": ("Basic %s" % - self.auth_string.decode('utf-8')).strip('\n')} + self.auth_string.decode('utf-8')).strip('\n')} else: self.headers = {} @@ -92,19 +86,41 @@ def get(self, path, params=None): raise RuntimeError('Resource does not exist') if response.status != 200: - raise RuntimeError('Error with connection to server: Got response: %s %s' % (response.status, response.reason)) + raise RuntimeError('Error with connection to server: ' + + 'Got response: %s %s' % (response.status, response.reason)) return response def post(self, path, args): - params = urllib.urlencode(args) + # prepare multipart/form-data request + limit = '--------UWS_Client_Separator' + crlf = '\r\n' + params_list = [] + for key in args: + params_list.append('--' + limit) + if re.match('^@', args[key]): + dummy = 'Content-Disposition: form-data; ' + dummy += 'name="%s"; filename="%s"' % (key, args[key][1:]) + params_list.append(dummy) + dummy = 'Content-Type: ' + dummy += (mimetypes.guess_type(args[key])[0] or 'application/octet-stream') + params_list.append(dummy) + params_list.append('') + params_list.append(open(args[key][1:]).read()) + else: + params_list.append('Content-Disposition: form-data; name="%s"'%key) + params_list.append('') + params_list.append(args[key]) + params_list.append('--' + limit + '--') + params_list.append('') + params = crlf.join(params_list) if path: destination_url = self.base_path + "/" + path else: destination_url = self.base_path - self.headers['Content-type'] = "application/x-www-form-urlencoded" + self.headers['Content-type'] = "multipart/form-data; boundary=%s"%limit self.connection.request("POST", destination_url, body=params, headers=self.headers) response = self.connection.getresponse() response.read() # read body of request so we can send another @@ -140,7 +156,8 @@ def post(self, path, args): raise RuntimeError('Resource does not exist') if response.status != 200: - raise RuntimeError('Error with connection to server: Got response: %s %s' % (response.status, response.reason)) + raise RuntimeError('Error with connection to server: ' + + 'Got response: %s %s' % (response.status, response.reason)) return response @@ -181,7 +198,8 @@ def delete(self, path): raise RuntimeError('Resource does not exist') if response.status != 200: - raise RuntimeError('Error with connection to server: Got response: %s %s' % (response.status, response.reason)) + raise RuntimeError('Error with connection to server: ' + + 'Got response: %s %s' % (response.status, response.reason)) return response diff --git a/uws/UWS/models.py b/uws/UWS/models.py index 6ac8acb..d1f5d1d 100644 --- a/uws/UWS/models.py +++ b/uws/UWS/models.py @@ -1,42 +1,45 @@ # -*- coding: utf-8 -*- from __future__ import print_function +import six from lxml import etree as et -uws_1_namespace = "http://www.ivoa.net/xml/UWS/v1.0" -#uws_2_namespace = "http://www.ivoa.net/xml/UWS/v2.0" -xlink_namespace = "http://www.w3.org/1999/xlink" +UWS_1_NAMESPACE = "http://www.ivoa.net/xml/UWS/v1.0" +#UWS_2_NAMESPACE = "http://www.ivoa.net/xml/UWS/v2.0" +XLINK_NAMESPACE = "http://www.w3.org/1999/xlink" -class UWS1Flavour(object): + +class UWS1Flavour: def __init__(self, namespaces=None): - if uws_1_namespace not in namespaces.values(): - raise RuntimeError("No supported UWS namespace found in xml-response, cannot parse xml.") + if UWS_1_NAMESPACE not in namespaces.values(): + raise RuntimeError("No supported UWS namespace found in xml-response, " + "cannot parse xml.") # prepend each element's name with the correct uws-namespace # for this version - self.uws_namespace = uws_1_namespace + self.uws_namespace = UWS_1_NAMESPACE self.jobs = et.QName(self.uws_namespace, "jobs") self.jobref = et.QName(self.uws_namespace, "jobref") self.phase = et.QName(self.uws_namespace, "phase") - self.jobId = et.QName(self.uws_namespace, "jobId") - self.runId = et.QName(self.uws_namespace, "runId") - self.ownerId = et.QName(self.uws_namespace, "ownerId") + self.job_id = et.QName(self.uws_namespace, "jobId") + self.run_id = et.QName(self.uws_namespace, "runId") + self.owner_id = et.QName(self.uws_namespace, "ownerId") self.quote = et.QName(self.uws_namespace, "quote") - self.creationTime = et.QName(self.uws_namespace, "creationTime") - self.startTime = et.QName(self.uws_namespace, "startTime") - self.endTime = et.QName(self.uws_namespace, "endTime") - self.executionDuration = et.QName(self.uws_namespace, "executionDuration") + self.creation_time = et.QName(self.uws_namespace, "creationTime") + self.start_time = et.QName(self.uws_namespace, "startTime") + self.end_time = et.QName(self.uws_namespace, "endTime") + self.execution_duration = et.QName(self.uws_namespace, "executionDuration") self.destruction = et.QName(self.uws_namespace, "destruction") self.parameters = et.QName(self.uws_namespace, "parameters") self.results = et.QName(self.uws_namespace, "results") - self.errorSummary = et.QName(self.uws_namespace, "errorSummary") + self.error_summary = et.QName(self.uws_namespace, "errorSummary") self.message = et.QName(self.uws_namespace, "message") - self.jobInfo = et.QName(self.uws_namespace, "jobInfo") + self.job_info = et.QName(self.uws_namespace, "jobInfo") -class JobPhases(object): +class JobPhases: COMPLETED = 'COMPLETED' PENDING = 'PENDING' QUEUED = 'QUEUED' @@ -69,11 +72,12 @@ class JobPhases(object): } -class BaseUWSModel(object): +class BaseUWSModel: def __init__(self): self.version = "1.0" - def _parse_bool(self, value): + @staticmethod + def _parse_bool(value): if isinstance(value, str): if value.lower() == 'true': return True @@ -88,7 +92,10 @@ def __init__(self, xml=None): if xml is not None: # parse xml - parsed = et.fromstring(xml.encode('utf-8')) + if isinstance(xml, bytes): + parsed = et.fromstring(xml.decode()) + else: + parsed = et.fromstring(xml.encode('utf-8')) uws_flavour = UWS1Flavour(parsed.nsmap) @@ -99,10 +106,10 @@ def __init__(self, xml=None): self.job_reference = [] - for xmlJob in xml_jobs: - self.add_job( - job=JobRef(xml_node=xmlJob, xml_namespace=parsed.nsmap, uws_flavour=uws_flavour) - ) + for xml_job in xml_jobs: + self.add_job(job=JobRef(xml_node=xml_job, + xml_namespace=parsed.nsmap, + uws_flavour=uws_flavour)) else: self.job_reference = [] @@ -116,39 +123,40 @@ def __str__(self): return str(self.__unicode__()) # return unicode(self).encode('utf-8') - def add_job(self, id=None, href=None, phase=None, job=None): + def add_job(self, jid=None, href=None, phase=None, job=None): if job is not None: self.job_reference.append(job) else: - reference = Reference(href=href, type="simple") - job_reference = JobRef(id=id, phase=phase, reference=reference) + reference = Reference(href=href, rtype="simple") + job_reference = JobRef(jid=jid, phase=phase, reference=reference) self.job_reference.append(job_reference) class JobRef(BaseUWSModel): - def __init__(self, id=None, phase=None, reference=None, xml_node=None, xml_namespace=None, uws_flavour=None): + def __init__(self, jid=None, phase=None, reference=None, xml_node=None, xml_namespace=None, + uws_flavour=None): super(JobRef, self).__init__() - self.id = None + self.jid = None self.reference = Reference() self.phase = [] if xml_node is not None: # When should this ever be None????? - self.id = xml_node.get('id') + self.jid = xml_node.get('id') # UWS standard defines array, therefore treat phase as array # (... actually it does not, but keep it anyway like this, maybe at # some point in the future all phases of a job are provided as list) self.phase = [elm.text for elm in xml_node.findall(uws_flavour.phase)] self.reference = Reference(xml_node=xml_node, xml_namespace=xml_namespace) - self.runId = xml_node.get('runId') - self.ownerId = xml_node.get('ownerId') - self.creationTime = xml_node.get('creationTime') + self.run_id = xml_node.get('runId') + self.owner_id = xml_node.get('ownerId') + self.creation_time = xml_node.get('creationTime') - elif id is not None and phase is not None and reference is not None: - self.id = id + elif jid is not None and phase is not None and reference is not None: + self.jid = jid - if isinstance(phase, basestring): + if isinstance(phase, six.string_types): self.phase = [phase] else: self.phase = phase @@ -156,23 +164,27 @@ def __init__(self, id=None, phase=None, reference=None, xml_node=None, xml_names if isinstance(reference, Reference): self.reference = reference else: - raise RuntimeError("Malformated reference given in jobref id: %s" % id) + raise RuntimeError("Malformated reference given in jobref id: %s" % jid) def set_phase(self, new_phase): self.phase = [new_phase] def __unicode__(self): - if self.creationTime is not None: - return "Job '%s' in phase '%s' created at '%s' - %s" % (self.id, ', '.join(self.phase), self.creationTime, str(self.reference)) + if self.creation_time is not None: + return "Job '%s' in phase '%s' created at '%s' - %s" % (self.jid, + ', '.join(self.phase), + self.creation_time, + str(self.reference)) else: - return "Job '%s' in phase '%s' - %s" % (self.id, ', '.join(self.phase), str(self.reference)) + return "Job '%s' in phase '%s' - %s" % (self.jid, ', '.join(self.phase), + str(self.reference)) def __str__(self): return str(self.__unicode__()) class Reference(BaseUWSModel): - def __init__(self, href=None, type=None, xml_node=None, xml_namespace=None): + def __init__(self, href=None, rtype=None, xml_node=None, xml_namespace=None): super(Reference, self).__init__() self.type = "simple" @@ -180,15 +192,16 @@ def __init__(self, href=None, type=None, xml_node=None, xml_namespace=None): if xml_node is not None: # check that namespace for xlink really exists - if xlink_namespace not in xml_namespace.values(): - raise RuntimeError("No supported xlink namespace found in xml-response, cannot parse xml.") + if XLINK_NAMESPACE not in xml_namespace.values(): + raise RuntimeError("No supported xlink namespace found in xml-response, " + "cannot parse xml.") - qualifiedname_type = et.QName(xlink_namespace, "type") - qualifiedname_href = et.QName(xlink_namespace, "href") + qualifiedname_type = et.QName(XLINK_NAMESPACE, "type") + qualifiedname_href = et.QName(XLINK_NAMESPACE, "href") self.type = xml_node.get(qualifiedname_type) self.href = xml_node.get(qualifiedname_href) - elif href is not None and type is not None: - self.type = type + elif href is not None and rtype is not None: + self.type = rtype self.href = href def __unicode__(self): @@ -232,16 +245,17 @@ def __init__(self, xml=None): if parsed.get("version"): self.version = parsed.get("version") - self.job_id = self._get_mandatory(parsed, uws_flavour.jobId) - self.run_id = self._get_optional(parsed, uws_flavour.runId) - self.owner_id = self._get_optional(parsed, uws_flavour.ownerId) - self.phase = [self._get_mandatory(parsed, uws_flavour.phase)] - self.quote = self._get_optional(parsed, uws_flavour.quote) - self.creation_time = self._get_optional(parsed, uws_flavour.creationTime) - self.start_time = self._get_mandatory(parsed, uws_flavour.startTime) - self.end_time = self._get_mandatory(parsed, uws_flavour.endTime) - self.execution_duration = int(self._get_mandatory(parsed, uws_flavour.executionDuration)) - self.destruction = self._get_mandatory(parsed, uws_flavour.destruction) + self.job_id = self._get_mandatory(parsed, uws_flavour.job_id) + self.run_id = self._get_optional(parsed, uws_flavour.run_id) + self.owner_id = self._get_optional(parsed, uws_flavour.owner_id) + self.phase = [self._get_mandatory(parsed, uws_flavour.phase)] + self.quote = self._get_optional(parsed, uws_flavour.quote) + self.creation_time = self._get_optional(parsed, uws_flavour.creation_time) + self.start_time = self._get_mandatory(parsed, uws_flavour.start_time) + self.end_time = self._get_mandatory(parsed, uws_flavour.end_time) + self.execution_duration = int(self._get_mandatory(parsed, + uws_flavour.execution_duration)) + self.destruction = self._get_mandatory(parsed, uws_flavour.destruction) self.parameters = [] tmp = parsed.find(uws_flavour.parameters) @@ -258,12 +272,12 @@ def __init__(self, xml=None): self.add_result(result=Result(xml_node=res, xml_namespace=parsed.nsmap)) self.error_summary = False - tmp = parsed.find(uws_flavour.errorSummary) + tmp = parsed.find(uws_flavour.error_summary) if tmp is not None: self.error_summary = ErrorSummary(xml_node=tmp, uws_flavour=uws_flavour) self.job_info = [] - tmp = parsed.find(uws_flavour.jobInfo) + tmp = parsed.find(uws_flavour.job_info) if tmp is not None: self.job_info = list(tmp) @@ -299,20 +313,22 @@ def __str__(self): return str(self.__unicode__()) # return unicode(self).encode('utf-8') - def add_parameter(self, id=None, by_reference=False, is_post=False, value=None, parameter=None): + def add_parameter(self, pid=None, by_reference=False, is_post=False, value=None, + parameter=None): if not parameter: - parameter = Parameter(id=id, by_reference=by_reference, is_post=is_post, value=value) + parameter = Parameter(pid=pid, by_reference=by_reference, is_post=is_post, value=value) self.parameters.append(parameter) - def add_result(self, id=None, href=None, result=None): + def add_result(self, rid=None, href=None, result=None): if not result: - reference = Reference(href=href, type="simple") - result = Result(id=id, reference=reference) + reference = Reference(href=href, rtype="simple") + result = Result(rid=rid, reference=reference) self.results.append(result) - def _get_optional(self, parsed, element_name): + @staticmethod + def _get_optional(parsed, element_name): """Returns the text value of element_name within the parsed elementTree. If element_name doesn't exist, return None. @@ -323,70 +339,73 @@ def _get_optional(self, parsed, element_name): else: return option.text - def _get_mandatory(self, parsed, element_name): + @staticmethod + def _get_mandatory(parsed, element_name): """Check if the element exists, return text or error""" element = parsed.find(element_name) if element is None: - raise RuntimeError("Mandatory element ", element_name.text, " could not be found in xml-response.") + raise RuntimeError("Mandatory element ", element_name.text, + " could not be found in xml-response.") else: return element.text class Parameter(BaseUWSModel): - def __init__(self, id=None, by_reference=False, is_post=False, value=None, xml_node=None): + def __init__(self, pid=None, by_reference=False, is_post=False, value=None, xml_node=None): super(Parameter, self).__init__() - self.id = None + self.pid = None self.by_reference = False self.is_post = False self.value = None if xml_node is not None: - self.id = xml_node.get('id') + self.pid = xml_node.get('id') self.by_reference = self._parse_bool(xml_node.get('by_reference', default=False)) self.is_post = self._parse_bool(xml_node.get('is_post', default=False)) self.value = xml_node.text - elif id is not None and value is not None: - self.id = id + elif pid is not None and value is not None: + self.pid = pid self.by_reference = by_reference self.is_post = is_post self.value = value def __unicode__(self): - return "Parameter id '%s' byRef: %s is_post: %s - value: %s" % (self.id, self.by_reference, self.is_post, self.value) + return "Parameter id '%s' byRef: %s is_post: %s - value: %s" % (self.pid, self.by_reference, + self.is_post, self.value) def __str__(self): return str(self.__unicode__()) class Result(BaseUWSModel): - def __init__(self, id=None, reference=None, xml_node=None, xml_namespace=None): + def __init__(self, rid=None, reference=None, xml_node=None, xml_namespace=None): super(Result, self).__init__() - self.id = None + self.rid = None self.reference = Reference() if xml_node is not None: - self.id = xml_node.get('id') + self.rid = xml_node.get('id') self.reference = Reference(xml_node=xml_node, xml_namespace=xml_namespace) - elif id is not None and reference is not None: - self.id = id + elif rid is not None and reference is not None: + self.rid = rid if isinstance(reference, Reference): self.reference = reference else: - raise RuntimeError("Malformated reference given in result id: %s" % id) + raise RuntimeError("Malformated reference given in result id: %s" % rid) def __unicode__(self): - return "Result id '%s' reference: %s" % (self.id, str(self.reference)) + return "Result id '%s' reference: %s" % (self.rid, str(self.reference)) def __str__(self): return str(self.__unicode__()) class ErrorSummary(BaseUWSModel): - def __init__(self, type="transient", has_detail=False, messages=None, + def __init__(self, etype="transient", has_detail=False, messages=None, xml_node=None, uws_flavour=None): super(ErrorSummary, self).__init__() @@ -405,12 +424,14 @@ def __init__(self, type="transient", has_detail=False, messages=None, self.messages.append(message.text) elif messages is not None: - self.type = type + self.type = etype self.has_detail = has_detail self.messages = messages def __unicode__(self): - return "Error Summary - type '%s' hasDetail: %s - message: %s" % (self.type, self.has_detail, "\n".join(self.messages)) + return "Error Summary - type '%s' hasDetail: %s - message: %s" % (self.type, + self.has_detail, + "\n".join(self.messages)) def __str__(self): return str(self.__unicode__()) diff --git a/uws/UWS/tests/test_base.py b/uws/UWS/tests/test_base.py index 9d9d240..6102e0c 100644 --- a/uws/UWS/tests/test_base.py +++ b/uws/UWS/tests/test_base.py @@ -5,16 +5,16 @@ class BaseTest(unittest.TestCase): - def testValidateAndParsePhaseFilter(self): + def test_validate_and_parse_phase_filter(self): filters = { 'phases': ['COMPLETED', 'PENDING'] } params = UWS.client.Client("/")._validate_and_parse_filters(filters) - self.assertEqual(params, [('PHASE','COMPLETED'), ('PHASE','PENDING')]) + self.assertEqual(params, [('PHASE', 'COMPLETED'), ('PHASE', 'PENDING')]) - def testValidateAndParsePhaseFilterInvalidPhase(self): + def test_validate_and_parse_phase_filter_invalid_phase(self): filters = { 'phases': ['FOO', 'PENDING'] } @@ -25,16 +25,16 @@ def testValidateAndParsePhaseFilterInvalidPhase(self): filters ) - def testValidateAndParseAfterFilter(self): + def test_validate_and_parse_after_filter(self): filters = { 'after': '2015-09-10T10:01:02.135' } params = UWS.client.Client("/")._validate_and_parse_filters(filters) - self.assertEqual(params, [('AFTER','2015-09-10T10:01:02.135000')]) + self.assertEqual(params, [('AFTER', '2015-09-10T10:01:02.135000')]) - def testValidateAndParseAfterFilterInvalidDate(self): + def test_validate_and_parse_after_filter_invalid_date(self): filters = { 'after': '2010-4--' } @@ -45,7 +45,7 @@ def testValidateAndParseAfterFilterInvalidDate(self): filters ) - def testValidateAndParseAfterFilterTimeZone(self): + def test_validate_and_parse_after_filter_time_zone(self): filters = { 'after': '2015-10-03T01:12+2:00' } @@ -56,16 +56,16 @@ def testValidateAndParseAfterFilterTimeZone(self): - def testValidateAndParseLastFilter(self): + def test_validate_and_parse_last_filter(self): filters = { 'last': '1000' } params = UWS.client.Client("/")._validate_and_parse_filters(filters) - self.assertEqual(params, [('LAST',1000)]) + self.assertEqual(params, [('LAST', 1000)]) - def testValidateAndParseLastFilterFloatValue(self): + def test_validate_and_parse_last_filter_float_value(self): filters = { 'last': '100.0' } @@ -76,7 +76,7 @@ def testValidateAndParseLastFilterFloatValue(self): filters ) - def testValidateAndParseLastFilterNegativeValue(self): + def test_validate_and_parse_last_filter_negative_value(self): filters = { 'last': '-100' } @@ -87,7 +87,7 @@ def testValidateAndParseLastFilterNegativeValue(self): filters ) - def testValidateAndParseAfterLastFilter(self): + def test_validate_and_parse_after_last_filter(self): filters = { 'after': '2015-09-10T10:01:02.135', 'last': '100' @@ -95,10 +95,10 @@ def testValidateAndParseAfterLastFilter(self): params = UWS.client.Client("/")._validate_and_parse_filters(filters) - self.assertEqual(params, [('AFTER','2015-09-10T10:01:02.135000'), - ('LAST', 100)]) + self.assertEqual(params, [('AFTER', '2015-09-10T10:01:02.135000'), + ('LAST', 100)]) - def testValidateAndParseAfterLastPhaseFilter(self): + def test_validate_and_parse_after_last_phase_filter(self): filters = { 'after': '2015-09-10T10:01:02.135', 'last': '100', @@ -107,22 +107,22 @@ def testValidateAndParseAfterLastPhaseFilter(self): params = UWS.client.Client("/")._validate_and_parse_filters(filters) - self.assertEqual(params, [ ('PHASE','PENDING'), ('PHASE','ERROR'), - ('AFTER','2015-09-10T10:01:02.135000'), ('LAST', 100)]) + self.assertEqual(params, [('PHASE', 'PENDING'), ('PHASE', 'ERROR'), + ('AFTER', '2015-09-10T10:01:02.135000'), ('LAST', 100)]) - def testValidateAndParseWaitNegative(self): + def test_validate_and_parse_wait_negative(self): wait = '-1' params = UWS.client.Client("/")._validate_and_parse_wait(wait) self.assertEqual(params, [('WAIT', -1)]) - def testValidateAndParseWait(self): + def test_validate_and_parse_wait(self): wait = '30' params = UWS.client.Client("/")._validate_and_parse_wait(wait) self.assertEqual(params, [('WAIT', 30)]) - def testValidateAndParseWaitInvalidWait(self): + def test_validate_and_parse_wait_invalid_wait(self): wait = '30.587' self.assertRaises( @@ -131,7 +131,7 @@ def testValidateAndParseWaitInvalidWait(self): wait ) - def testValidateAndParseWaitInvalidWaitNegative(self): + def test_validate_and_parse_wait_invalid_wait_negative(self): wait = '-30' self.assertRaises( @@ -140,7 +140,7 @@ def testValidateAndParseWaitInvalidWaitNegative(self): wait ) - def testValidateAndParseWaitPhase(self): + def test_validate_and_parse_wait_phase(self): wait = '30' phase = 'EXECUTING' @@ -148,7 +148,7 @@ def testValidateAndParseWaitPhase(self): self.assertEqual(params, [('WAIT', 30), ('PHASE', 'EXECUTING')]) - def testValidateAndParseWaitInvalidPhase(self): + def test_validate_and_parse_wait_invalid_phase(self): wait = '15' phase = 'COMPLETED' diff --git a/uws/UWS/tests/test_connection.py b/uws/UWS/tests/test_connection.py index 77eb9dd..cc1b579 100644 --- a/uws/UWS/tests/test_connection.py +++ b/uws/UWS/tests/test_connection.py @@ -1,11 +1,11 @@ # -*- coding: utf-8 -*- import unittest - +import http.client as httplib from uws import UWS class ConnectionTest(unittest.TestCase): - def testSetAuthHeader(self): + def test_set_auth_header(self): connection = UWS.connection.Connection( "http://www.example.com/uws", user="admin", @@ -20,12 +20,7 @@ def testSetAuthHeader(self): self.assertEqual(auth_string, "YWRtaW46YWRtaW4=") self.assertDictEqual(connection.headers, {'Authorization': 'Basic YWRtaW46YWRtaW4='}) - def testSetURLHTTP(self): - try: - import httplib - except ImportError: - import http.client as httplib - + def test_set_url_http(self): connection = UWS.connection.Connection( "http://www.example.com/uws/", user="admin", @@ -37,12 +32,7 @@ def testSetURLHTTP(self): self.assertEqual(connection.base_path, "/uws") self.assertIsInstance(connection.connection, httplib.HTTPConnection) - def testSetURLHTTPS(self): - try: - import httplib - except ImportError: - import http.client as httplib - + def test_set_url_https(self): connection = UWS.connection.Connection( "https://www.example.com/uws/", user="admin", diff --git a/uws/UWS/tests/test_models.py b/uws/UWS/tests/test_models.py index b0a6d56..9f8e330 100644 --- a/uws/UWS/tests/test_models.py +++ b/uws/UWS/tests/test_models.py @@ -32,38 +32,52 @@ def test(self): self.assertEqual(len(job_list.job_reference), 5) - job_list_str = "Job '2014-06-03T15:33:29:4235' in phase 'COMPLETED' - https://www.cosmosim.org/uws/query/335912448787925\nJob '2014-06-02T10:14:25:1677' in phase 'ABORTED' - https://www.cosmosim.org/uws/query/308893189727250\nJob '2014-05-28T10:46:39:2755' in phase 'COMPLETED' - https://www.cosmosim.org/uws/query/198796714554760\nJob '2014-05-09T15:13:50:6896' in phase 'ERROR' - https://www.cosmosim.org/uws/query/1177277256137938\nJob 'rndSamp2' in phase 'COMPLETED' - https://www.cosmosim.org/uws/query/356246647522833857\n" + job_list_str = ("Job '2014-06-03T15:33:29:4235' in phase 'COMPLETED' - " + "https://www.cosmosim.org/uws/query/335912448787925\nJob " + "'2014-06-02T10:14:25:1677' in phase 'ABORTED' - " + "https://www.cosmosim.org/uws/query/308893189727250\nJob " + "'2014-05-28T10:46:39:2755' in phase 'COMPLETED' - " + "https://www.cosmosim.org/uws/query/198796714554760\nJob " + "'2014-05-09T15:13:50:6896' in phase 'ERROR' - " + "https://www.cosmosim.org/uws/query/1177277256137938\nJob " + "'rndSamp2' in phase 'COMPLETED' - " + "https://www.cosmosim.org/uws/query/356246647522833857\n") self.assertEqual(str(job_list), job_list_str) job1 = job_list.job_reference[0] - self.assertEqual(job1.id, '2014-06-03T15:33:29:4235') + self.assertEqual(job1.jid, '2014-06-03T15:33:29:4235') self.assertEqual(job1.phase, ['COMPLETED']) self.assertEqual(job1.reference.type, "simple") - self.assertEqual(job1.reference.href, "https://www.cosmosim.org/uws/query/335912448787925") + self.assertEqual(job1.reference.href, + "https://www.cosmosim.org/uws/query/335912448787925") job2 = job_list.job_reference[1] - self.assertEqual(job2.id, '2014-06-02T10:14:25:1677') + self.assertEqual(job2.jid, '2014-06-02T10:14:25:1677') self.assertEqual(job2.phase, ['ABORTED']) self.assertEqual(job2.reference.type, "simple") - self.assertEqual(job2.reference.href, "https://www.cosmosim.org/uws/query/308893189727250") + self.assertEqual(job2.reference.href, + "https://www.cosmosim.org/uws/query/308893189727250") job3 = job_list.job_reference[2] - self.assertEqual(job3.id, '2014-05-28T10:46:39:2755') + self.assertEqual(job3.jid, '2014-05-28T10:46:39:2755') self.assertEqual(job3.phase, ['COMPLETED']) self.assertEqual(job3.reference.type, "simple") - self.assertEqual(job3.reference.href, "https://www.cosmosim.org/uws/query/198796714554760") + self.assertEqual(job3.reference.href, + "https://www.cosmosim.org/uws/query/198796714554760") job4 = job_list.job_reference[3] - self.assertEqual(job4.id, '2014-05-09T15:13:50:6896') + self.assertEqual(job4.jid, '2014-05-09T15:13:50:6896') self.assertEqual(job4.phase, ['ERROR']) self.assertEqual(job4.reference.type, "simple") - self.assertEqual(job4.reference.href, "https://www.cosmosim.org/uws/query/1177277256137938") + self.assertEqual(job4.reference.href, + "https://www.cosmosim.org/uws/query/1177277256137938") job5 = job_list.job_reference[4] - self.assertEqual(job5.id, u'rndSamp2') + self.assertEqual(job5.jid, u'rndSamp2') self.assertEqual(job5.phase, ['COMPLETED']) self.assertEqual(job5.reference.type, "simple") - self.assertEqual(job5.reference.href, "https://www.cosmosim.org/uws/query/356246647522833857") + self.assertEqual(job5.reference.href, + "https://www.cosmosim.org/uws/query/356246647522833857") class CompletedJobTest(unittest.TestCase): @@ -110,7 +124,36 @@ def setUp(self): def test(self): job = UWS.models.Job(self.xml) - job_str = "JobId : '335912448787925'\nRunId : 'None'\nOwnerId : 'adrian'\nPhase : 'COMPLETED'\nQuote : 'None'\nCreationTime : 'None'\nStartTime : '2014-06-03T15:33:30+02:00'\nEndTime : '2014-06-03T15:33:31+02:00'\nExecutionDuration : '30'\nDestruction : '2999-12-31T00:00:00+01:00'\nParameters :\nParameter id 'database' byRef: False is_post: False - value: cosmosim_user_adrian\nParameter id 'table' byRef: False is_post: False - value: 2014-06-03T15:33:29:4235\nParameter id 'query' byRef: False is_post: False - value: SELECT 0.25*(0.5+FLOOR(LOG10(Mvir)/0.25)) AS log_mass, COUNT(*) AS num\r\nFROM MDR1.BDMV\r\nWHERE snapnum=85 \r\nGROUP BY FLOOR(LOG10(Mvir)/0.25)\r\nORDER BY log_mass\n-- The query plan used to run this query: --\n--------------------------------------------\n--\n-- CALL paquExec('SELECT 0.25 * ( 0.5 + FLOOR( LOG10( `Mvir` ) / 0.25 ) ) AS `log_mass`,COUNT(*) AS `num`,FLOOR( LOG10( `Mvir` ) / 0.25 ) AS `_FLOOR_LOG10_Mvir_/_0__25_` FROM MDR1.BDMV WHERE ( `snapnum` = 85 ) GROUP BY FLOOR( LOG10( Mvir ) / 0.25 ) ', 'aggregation_tmp_75797262')\n-- USE spider_tmp_shard\n-- SET @i=0\n-- CREATE TABLE cosmosim_user_adrian.`2014-06-03T15:33:29:4235` ENGINE=MyISAM SELECT @i:=@i+1 AS `row_id`, `log_mass`,SUM(`num`) AS `num`\r FROM `aggregation_tmp_75797262` GROUP BY `_FLOOR_LOG10_Mvir_/_0__25_` ORDER BY `log_mass` ASC \n-- CALL paquDropTmp('aggregation_tmp_75797262')\n\nParameter id 'queue' byRef: False is_post: False - value: short\nResults :\nResult id 'csv' reference: https://www.cosmosim.org/query/download/stream/table/2014-06-03T15%3A33%3A29%3A4235/format/csv\nResult id 'votable.xml' reference: https://www.cosmosim.org/query/download/stream/table/2014-06-03T15%3A33%3A29%3A4235/format/votable\nResult id 'votableB1.xml' reference: https://www.cosmosim.org/query/download/stream/table/2014-06-03T15%3A33%3A29%3A4235/format/votableB1\nResult id 'votableB2.xml' reference: https://www.cosmosim.org/query/download/stream/table/2014-06-03T15%3A33%3A29%3A4235/format/votableB2\nerrorSummary :\n False\njobInfo :\n" + job_str = ("JobId : '335912448787925'\nRunId : 'None'\nOwnerId : 'adrian'\n" + "Phase : 'COMPLETED'\nQuote : 'None'\nCreationTime : 'None'\n" + "StartTime : '2014-06-03T15:33:30+02:00'\nEndTime : '2014-06-03T15:33:31+02:00'" + "\nExecutionDuration : '30'\nDestruction : '2999-12-31T00:00:00+01:00'\n" + "Parameters :\nParameter id 'database' byRef: False is_post: False - " + "value: cosmosim_user_adrian\nParameter id 'table' byRef: False is_post: False -" + " value: 2014-06-03T15:33:29:4235\nParameter id 'query' byRef: False " + "is_post: False - value: SELECT 0.25*(0.5+FLOOR(LOG10(Mvir)/0.25)) AS log_mass, " + "COUNT(*) AS num\r\nFROM MDR1.BDMV\r\nWHERE snapnum=85 \r\n" + "GROUP BY FLOOR(LOG10(Mvir)/0.25)\r\nORDER BY log_mass\n-- The query plan used " + "to run this query: --\n--------------------------------------------\n--\n-- " + "CALL paquExec('SELECT 0.25 * ( 0.5 + FLOOR( LOG10( `Mvir` ) / 0.25 ) ) AS " + "`log_mass`,COUNT(*) AS `num`,FLOOR( LOG10( `Mvir` ) / 0.25 ) AS " + "`_FLOOR_LOG10_Mvir_/_0__25_` FROM MDR1.BDMV WHERE ( `snapnum` = 85 ) " + "GROUP BY FLOOR( LOG10( Mvir ) / 0.25 ) ', 'aggregation_tmp_75797262')\n-- " + "USE spider_tmp_shard\n-- SET @i=0\n-- " + "CREATE TABLE cosmosim_user_adrian.`2014-06-03T15:33:29:4235` ENGINE=MyISAM " + "SELECT @i:=@i+1 AS `row_id`, `log_mass`,SUM(`num`) AS `num`\r FROM " + "`aggregation_tmp_75797262` GROUP BY `_FLOOR_LOG10_Mvir_/_0__25_` ORDER BY " + "`log_mass` ASC \n-- CALL paquDropTmp('aggregation_tmp_75797262')\n\n" + "Parameter id 'queue' byRef: False is_post: False - value: short\nResults :\n" + "Result id 'csv' reference: https://www.cosmosim.org/query/download/stream/" + "table/2014-06-03T15%3A33%3A29%3A4235/format/csv\nResult id 'votable.xml' " + "reference: https://www.cosmosim.org/query/download/stream/table/" + "2014-06-03T15%3A33%3A29%3A4235/format/votable\nResult id 'votableB1.xml' " + "reference: https://www.cosmosim.org/query/download/stream/table/" + "2014-06-03T15%3A33%3A29%3A4235/format/votableB1\nResult id 'votableB2.xml' " + "reference: https://www.cosmosim.org/query/download/stream/table/" + "2014-06-03T15%3A33%3A29%3A4235/format/votableB2\nerrorSummary :\n False\n" + "jobInfo :\n") self.assertEqual(str(job), job_str) self.assertEqual(job.job_id, '335912448787925') @@ -125,50 +168,68 @@ def test(self): # check parameters param = job.parameters[0] - self.assertEqual(param.id, 'database') + self.assertEqual(param.pid, 'database') self.assertEqual(param.by_reference, False) self.assertEqual(param.is_post, False) self.assertEqual(param.value, 'cosmosim_user_adrian') param = job.parameters[1] - self.assertEqual(param.id, 'table') + self.assertEqual(param.pid, 'table') self.assertEqual(param.by_reference, False) self.assertEqual(param.is_post, False) self.assertEqual(param.value, '2014-06-03T15:33:29:4235') param = job.parameters[2] - self.assertEqual(param.id, 'query') + self.assertEqual(param.pid, 'query') self.assertEqual(param.by_reference, False) self.assertEqual(param.is_post, False) - query_value = "SELECT 0.25*(0.5+FLOOR(LOG10(Mvir)/0.25)) AS log_mass, COUNT(*) AS num\r\nFROM MDR1.BDMV\r\nWHERE snapnum=85 \r\nGROUP BY FLOOR(LOG10(Mvir)/0.25)\r\nORDER BY log_mass\n-- The query plan used to run this query: --\n--------------------------------------------\n--\n-- CALL paquExec('SELECT 0.25 * ( 0.5 + FLOOR( LOG10( `Mvir` ) / 0.25 ) ) AS `log_mass`,COUNT(*) AS `num`,FLOOR( LOG10( `Mvir` ) / 0.25 ) AS `_FLOOR_LOG10_Mvir_/_0__25_` FROM MDR1.BDMV WHERE ( `snapnum` = 85 ) GROUP BY FLOOR( LOG10( Mvir ) / 0.25 ) ', 'aggregation_tmp_75797262')\n-- USE spider_tmp_shard\n-- SET @i=0\n-- CREATE TABLE cosmosim_user_adrian.`2014-06-03T15:33:29:4235` ENGINE=MyISAM SELECT @i:=@i+1 AS `row_id`, `log_mass`,SUM(`num`) AS `num`\r FROM `aggregation_tmp_75797262` GROUP BY `_FLOOR_LOG10_Mvir_/_0__25_` ORDER BY `log_mass` ASC \n-- CALL paquDropTmp('aggregation_tmp_75797262')\n" + query_value = ("SELECT 0.25*(0.5+FLOOR(LOG10(Mvir)/0.25)) AS log_mass, COUNT(*) AS num\r\n" + "FROM MDR1.BDMV\r\nWHERE snapnum=85 \r\nGROUP BY FLOOR(LOG10(Mvir)/0.25)\r" + "\nORDER BY log_mass\n-- The query plan used to run this query: --\n------" + "--------------------------------------\n--\n-- CALL paquExec('SELECT 0.25 " + "* ( 0.5 + FLOOR( LOG10( `Mvir` ) / 0.25 ) ) AS `log_mass`,COUNT(*) AS " + "`num`,FLOOR( LOG10( `Mvir` ) / 0.25 ) AS `_FLOOR_LOG10_Mvir_/_0__25_` FROM " + "MDR1.BDMV WHERE ( `snapnum` = 85 ) GROUP BY FLOOR( LOG10( Mvir ) / 0.25 ) " + " ', 'aggregation_tmp_75797262')\n-- USE spider_tmp_shard\n-- SET @i=0\n-- " + "CREATE TABLE cosmosim_user_adrian.`2014-06-03T15:33:29:4235` ENGINE=MyISAM " + "SELECT @i:=@i+1 AS `row_id`, `log_mass`,SUM(`num`) AS `num`\r FROM " + "`aggregation_tmp_75797262` GROUP BY `_FLOOR_LOG10_Mvir_/_0__25_` ORDER BY " + "`log_mass` ASC \n-- CALL paquDropTmp('aggregation_tmp_75797262')\n") self.assertEqual(param.value, query_value) param = job.parameters[3] - self.assertEqual(param.id, 'queue') + self.assertEqual(param.pid, 'queue') self.assertEqual(param.by_reference, False) self.assertEqual(param.is_post, False) self.assertEqual(param.value, 'short') # check results result = job.results[0] - self.assertEqual(result.id, 'csv') + self.assertEqual(result.rid, 'csv') self.assertEqual(result.reference.type, 'simple') - self.assertEqual(result.reference.href, 'https://www.cosmosim.org/query/download/stream/table/2014-06-03T15%3A33%3A29%3A4235/format/csv') + self.assertEqual(result.reference.href, ('https://www.cosmosim.org/query/download/stream/' + 'table/2014-06-03T15%3A33%3A29%3A4235/format/csv')) result = job.results[1] - self.assertEqual(result.id, 'votable.xml') + self.assertEqual(result.rid, 'votable.xml') self.assertEqual(result.reference.type, 'simple') - self.assertEqual(result.reference.href, 'https://www.cosmosim.org/query/download/stream/table/2014-06-03T15%3A33%3A29%3A4235/format/votable') + self.assertEqual(result.reference.href, ('https://www.cosmosim.org/query/download/stream/' + 'table/2014-06-03T15%3A33%3A29%3A4235/' + 'format/votable')) result = job.results[2] - self.assertEqual(result.id, 'votableB1.xml') + self.assertEqual(result.rid, 'votableB1.xml') self.assertEqual(result.reference.type, 'simple') - self.assertEqual(result.reference.href, 'https://www.cosmosim.org/query/download/stream/table/2014-06-03T15%3A33%3A29%3A4235/format/votableB1') + self.assertEqual(result.reference.href, ('https://www.cosmosim.org/query/download/stream/' + 'table/2014-06-03T15%3A33%3A29%3A4235/format/' + 'votableB1')) result = job.results[3] - self.assertEqual(result.id, 'votableB2.xml') + self.assertEqual(result.rid, 'votableB2.xml') self.assertEqual(result.reference.type, 'simple') - self.assertEqual(result.reference.href, 'https://www.cosmosim.org/query/download/stream/table/2014-06-03T15%3A33%3A29%3A4235/format/votableB2') + self.assertEqual(result.reference.href, ('https://www.cosmosim.org/query/download/stream/' + 'table/2014-06-03T15%3A33%3A29%3A4235/format/' + 'votableB2')) self.assertEqual(job.error_summary, False) self.assertEqual(job.job_info, []) @@ -209,7 +270,22 @@ def setUp(self): def test(self): job = UWS.models.Job(self.xml) - job_str = "JobId : '308893189727250'\nRunId : 'None'\nOwnerId : 'adrian'\nPhase : 'ABORTED'\nQuote : 'None'\nCreationTime : 'None'\nStartTime : '2014-06-02T10:14:25+02:00'\nEndTime : '2014-06-02T10:14:55+02:00'\nExecutionDuration : '30'\nDestruction : '2999-12-31T00:00:00+01:00'\nParameters :\nParameter id 'database' byRef: False is_post: False - value: cosmosim_user_adrian\nParameter id 'table' byRef: False is_post: False - value: 2014-06-02T10:14:25:1677\nParameter id 'query' byRef: False is_post: False - value: select count(*) from MDR1.Particles85 where x < 1\n-- The query plan used to run this query: --\n--------------------------------------------\n--\n-- CALL paquExec('SELECT COUNT(*) AS `_count_*_` FROM MDR1.Particles85 WHERE ( `x` < 1 ) ', 'aggregation_tmp_49645551')\n-- USE spider_tmp_shard\n-- SET @i=0\n-- CREATE TABLE cosmosim_user_adrian.`2014-06-02T10:14:25:1677` ENGINE=MyISAM SELECT @i:=@i+1 AS `row_id`, SUM(`_count_*_`) AS `_count_*_`\r FROM `aggregation_tmp_49645551` \n-- CALL paquDropTmp('aggregation_tmp_49645551')\n\nParameter id 'queue' byRef: False is_post: False - value: short\nResults :\nerrorSummary :\n False\njobInfo :\n" + job_str = ("JobId : '308893189727250'\nRunId : 'None'\nOwnerId : 'adrian'\nPhase : " + "'ABORTED'\nQuote : 'None'\nCreationTime : 'None'\nStartTime : " + "'2014-06-02T10:14:25+02:00'\nEndTime : '2014-06-02T10:14:55+02:00'\n" + "ExecutionDuration : '30'\nDestruction : '2999-12-31T00:00:00+01:00'\n" + "Parameters :\nParameter id 'database' byRef: False is_post: False - value: " + "cosmosim_user_adrian\nParameter id 'table' byRef: False is_post: False - " + "value: 2014-06-02T10:14:25:1677\nParameter id 'query' byRef: False is_post: " + "False - value: select count(*) from MDR1.Particles85 where x < 1\n-- The query " + "plan used to run this query: --\n--------------------------------------------\n" + "--\n-- CALL paquExec('SELECT COUNT(*) AS `_count_*_` FROM MDR1.Particles85 " + "WHERE ( `x` < 1 ) ', 'aggregation_tmp_49645551')\n-- USE spider_tmp_shard\n" + "-- SET @i=0\n-- CREATE TABLE cosmosim_user_adrian.`2014-06-02T10:14:25:1677` " + "ENGINE=MyISAM SELECT @i:=@i+1 AS `row_id`, SUM(`_count_*_`) AS `_count_*_`\r " + "FROM `aggregation_tmp_49645551` \n-- CALL " + "paquDropTmp('aggregation_tmp_49645551')\n\nParameter id 'queue' byRef: False " + "is_post: False - value: short\nResults :\nerrorSummary :\n False\njobInfo :\n") self.assertEqual(str(job), job_str) self.assertEqual(job.job_id, '308893189727250') @@ -224,26 +300,33 @@ def test(self): # check parameters param = job.parameters[0] - self.assertEqual(param.id, 'database') + self.assertEqual(param.pid, 'database') self.assertEqual(param.by_reference, False) self.assertEqual(param.is_post, False) self.assertEqual(param.value, 'cosmosim_user_adrian') param = job.parameters[1] - self.assertEqual(param.id, 'table') + self.assertEqual(param.pid, 'table') self.assertEqual(param.by_reference, False) self.assertEqual(param.is_post, False) self.assertEqual(param.value, '2014-06-02T10:14:25:1677') param = job.parameters[2] - self.assertEqual(param.id, 'query') + self.assertEqual(param.pid, 'query') self.assertEqual(param.by_reference, False) self.assertEqual(param.is_post, False) - query_value = "select count(*) from MDR1.Particles85 where x < 1\n-- The query plan used to run this query: --\n--------------------------------------------\n--\n-- CALL paquExec('SELECT COUNT(*) AS `_count_*_` FROM MDR1.Particles85 WHERE ( `x` < 1 ) ', 'aggregation_tmp_49645551')\n-- USE spider_tmp_shard\n-- SET @i=0\n-- CREATE TABLE cosmosim_user_adrian.`2014-06-02T10:14:25:1677` ENGINE=MyISAM SELECT @i:=@i+1 AS `row_id`, SUM(`_count_*_`) AS `_count_*_`\r FROM `aggregation_tmp_49645551` \n-- CALL paquDropTmp('aggregation_tmp_49645551')\n" + query_value = ("select count(*) from MDR1.Particles85 where x < 1\n-- The query plan used " + "to run this query: --\n--------------------------------------------\n" + "--\n-- CALL paquExec('SELECT COUNT(*) AS `_count_*_` FROM MDR1.Particles85 " + "WHERE ( `x` < 1 ) ', 'aggregation_tmp_49645551')\n-- USE spider_tmp_shard" + "\n-- SET @i=0\n-- CREATE TABLE cosmosim_user_adrian." + "`2014-06-02T10:14:25:1677` ENGINE=MyISAM SELECT @i:=@i+1 AS `row_id`, " + "SUM(`_count_*_`) AS `_count_*_`\r FROM `aggregation_tmp_49645551` \n-- " + "CALL paquDropTmp('aggregation_tmp_49645551')\n") self.assertEqual(param.value, query_value) param = job.parameters[3] - self.assertEqual(param.id, 'queue') + self.assertEqual(param.pid, 'queue') self.assertEqual(param.by_reference, False) self.assertEqual(param.is_post, False) self.assertEqual(param.value, 'short') @@ -291,7 +374,24 @@ def setUp(self): def test(self): job = UWS.models.Job(self.xml) - job_str = "JobId : '1177277256137938'\nRunId : 'None'\nOwnerId : 'adrian'\nPhase : 'ERROR'\nQuote : 'None'\nCreationTime : 'None'\nStartTime : '2014-05-09T15:13:48+02:00'\nEndTime : '2014-05-09T15:13:48+02:00'\nExecutionDuration : '30'\nDestruction : '2999-12-31T00:00:00+01:00'\nParameters :\nParameter id 'database' byRef: False is_post: False - value: cosmosim_user_adrian\nParameter id 'table' byRef: False is_post: False - value: 2014-05-09T15:13:50:6896\nParameter id 'query' byRef: False is_post: False - value: select avg(x) from `MDPL`.`Particles88tmp`;\n-- The query plan used to run this query: --\n--------------------------------------------\n--\n-- CALL paquExec('SELECT COUNT(x) AS `cnt_avg(x)`, SUM(x) AS `sum_avg(x)` FROM `MDPL`.`Particles88tmp` ', 'aggregation_tmp_9424512')\n-- USE spider_tmp_shard\n-- SET @i=0\n-- CREATE TABLE cosmosim_user_adrian.`2014-05-09T15:13:50:6896` ENGINE=MyISAM SELECT @i:=@i+1 AS `row_id`, (SUM(`sum_avg(x)`) / SUM(`cnt_avg(x)`)) AS `avg(x)`\r FROM `aggregation_tmp_9424512` \n-- CALL paquDropTmp('aggregation_tmp_9424512')\n\nParameter id 'queue' byRef: False is_post: False - value: short\nResults :\nerrorSummary :\n Error Summary - type 'transient' hasDetail: False - message: Remote MySQL server has gone away\njobInfo :\n" + job_str = ("JobId : '1177277256137938'\nRunId : 'None'\nOwnerId : 'adrian'\nPhase : " + "'ERROR'\nQuote : 'None'\nCreationTime : 'None'\nStartTime : " + "'2014-05-09T15:13:48+02:00'\nEndTime : '2014-05-09T15:13:48+02:00'\n" + "ExecutionDuration : '30'\nDestruction : '2999-12-31T00:00:00+01:00'\n" + "Parameters :\nParameter id 'database' byRef: False is_post: False - value: " + "cosmosim_user_adrian\nParameter id 'table' byRef: False is_post: False - " + "value: 2014-05-09T15:13:50:6896\nParameter id 'query' byRef: False is_post: " + "False - value: select avg(x) from `MDPL`.`Particles88tmp`;\n-- The query plan " + "used to run this query: --\n--------------------------------------------\n--" + "\n-- CALL paquExec('SELECT COUNT(x) AS `cnt_avg(x)`, SUM(x) AS `sum_avg(x)` " + "FROM `MDPL`.`Particles88tmp` ', 'aggregation_tmp_9424512')\n-- USE " + "spider_tmp_shard\n-- SET @i=0\n-- CREATE TABLE cosmosim_user_adrian." + "`2014-05-09T15:13:50:6896` ENGINE=MyISAM SELECT @i:=@i+1 AS `row_id`, " + "(SUM(`sum_avg(x)`) / SUM(`cnt_avg(x)`)) AS `avg(x)`\r FROM " + "`aggregation_tmp_9424512` \n-- CALL paquDropTmp('aggregation_tmp_9424512')\n" + "\nParameter id 'queue' byRef: False is_post: False - value: short\nResults :\n" + "errorSummary :\n Error Summary - type 'transient' hasDetail: False - message: " + "Remote MySQL server has gone away\njobInfo :\n") self.assertEqual(str(job), job_str) self.assertEqual(job.job_id, '1177277256137938') @@ -306,26 +406,34 @@ def test(self): # check parameters param = job.parameters[0] - self.assertEqual(param.id, 'database') + self.assertEqual(param.pid, 'database') self.assertEqual(param.by_reference, False) self.assertEqual(param.is_post, False) self.assertEqual(param.value, 'cosmosim_user_adrian') param = job.parameters[1] - self.assertEqual(param.id, 'table') + self.assertEqual(param.pid, 'table') self.assertEqual(param.by_reference, False) self.assertEqual(param.is_post, False) self.assertEqual(param.value, '2014-05-09T15:13:50:6896') param = job.parameters[2] - self.assertEqual(param.id, 'query') + self.assertEqual(param.pid, 'query') self.assertEqual(param.by_reference, False) self.assertEqual(param.is_post, False) - query_value = "select avg(x) from `MDPL`.`Particles88tmp`;\n-- The query plan used to run this query: --\n--------------------------------------------\n--\n-- CALL paquExec('SELECT COUNT(x) AS `cnt_avg(x)`, SUM(x) AS `sum_avg(x)` FROM `MDPL`.`Particles88tmp` ', 'aggregation_tmp_9424512')\n-- USE spider_tmp_shard\n-- SET @i=0\n-- CREATE TABLE cosmosim_user_adrian.`2014-05-09T15:13:50:6896` ENGINE=MyISAM SELECT @i:=@i+1 AS `row_id`, (SUM(`sum_avg(x)`) / SUM(`cnt_avg(x)`)) AS `avg(x)`\r FROM `aggregation_tmp_9424512` \n-- CALL paquDropTmp('aggregation_tmp_9424512')\n" + query_value = ("select avg(x) from `MDPL`.`Particles88tmp`;\n-- The query plan used to " + "run this query: --\n--------------------------------------------\n--\n-- " + "CALL paquExec('SELECT COUNT(x) AS `cnt_avg(x)`, SUM(x) AS `sum_avg(x)` " + "FROM `MDPL`.`Particles88tmp` ', 'aggregation_tmp_9424512')\n-- USE " + "spider_tmp_shard\n-- SET @i=0\n-- CREATE TABLE cosmosim_user_adrian." + "`2014-05-09T15:13:50:6896` ENGINE=MyISAM SELECT @i:=@i+1 AS `row_id`, " + "(SUM(`sum_avg(x)`) / SUM(`cnt_avg(x)`)) AS `avg(x)`\r FROM " + "`aggregation_tmp_9424512` \n-- CALL " + "paquDropTmp('aggregation_tmp_9424512')\n") self.assertEqual(param.value, query_value) param = job.parameters[3] - self.assertEqual(param.id, 'queue') + self.assertEqual(param.pid, 'queue') self.assertEqual(param.by_reference, False) self.assertEqual(param.is_post, False) self.assertEqual(param.value, 'short') @@ -357,11 +465,12 @@ def test(self): self.assertEqual(len(job_list.job_reference), 1) - job_list_str = "Job '335912448787925' in phase 'COMPLETED' - https://www.cosmosim.org/uws/query/335912448787925\n" + job_list_str = ("Job '335912448787925' in phase 'COMPLETED' - " + "https://www.cosmosim.org/uws/query/335912448787925\n") self.assertEqual(str(job_list), job_list_str) job1 = job_list.job_reference[0] - self.assertEqual(job1.id, '335912448787925') + self.assertEqual(job1.jid, '335912448787925') self.assertEqual(job1.phase, ['COMPLETED']) self.assertEqual(job1.reference.type, "simple") self.assertEqual(job1.reference.href, "https://www.cosmosim.org/uws/query/335912448787925") @@ -455,4 +564,3 @@ def setUp(self): '''[1:] - diff --git a/uws/cli/cli_parser.py b/uws/cli/cli_parser.py index 2eab50a..40075b4 100644 --- a/uws/cli/cli_parser.py +++ b/uws/cli/cli_parser.py @@ -25,12 +25,17 @@ def build_list_argparse(subparsers): parser_list.add_argument('-e', '--executing', action='store_true', help='show executing jobs') parser_list.add_argument('-E', '--error', action='store_true', help='show jobs with errors') parser_list.add_argument('-a', '--aborted', action='store_true', help='show aborted jobs') - parser_list.add_argument('-A', '--archived', action='store_true', help='[UWS1.1] show (deleted) jobs archived on the server') + parser_list.add_argument('-A', '--archived', action='store_true', + help='[UWS1.1] show (deleted) jobs archived on the server') parser_list.add_argument('--unknown', action='store_true', help='show unknown state jobs') parser_list.add_argument('--held', action='store_true', help='show held jobs') parser_list.add_argument('--suspended', action='store_true', help='show suspended jobs') - parser_list.add_argument('--after', action='store', help='[UWS1.1] show only jobs started after given UTC time or local time + timezone') - parser_list.add_argument('--last', action='store', help='[UWS1.1] show only most recently started jobs') + parser_list.add_argument('--after', action='store', + help=('[UWS1.1] show only jobs started after given UTC time or ' + 'local time + timezone') + ) + parser_list.add_argument('--last', action='store', + help='[UWS1.1] show only most recently started jobs') return parser_list @@ -38,43 +43,59 @@ def build_list_argparse(subparsers): def build_job_argparse(subparsers): parser_job = subparsers.add_parser('job', help='access a given job on the UWS service') - job_subparsers = parser_job.add_subparsers(dest='job_command', help='commands for manipulating jobs') + job_subparsers = parser_job.add_subparsers(dest='job_command', + help='commands for manipulating jobs') parser_job_show = job_subparsers.add_parser('show', help='show the specific job') parser_job_show.add_argument('id', help='job id') - parser_job_show.add_argument('-w', '--wait', nargs='?', const='-1', default=None, help='[UWS1.1] wait for phase change before returning, but at most the specified amount of seconds or infinitely, if no value is given') + parser_job_show.add_argument('-w', '--wait', nargs='?', const='-1', default=None, + help=('[UWS1.1] wait for phase change before returning, ' + 'but at most the specified amount of seconds or ' + 'infinitely, if no value is given')) parser_job_show.add_argument('-s', '--phase', help='[UWS1.1] required phase while waiting') parser_job_phase = job_subparsers.add_parser('phase', help='show the phase of specific job') parser_job_phase.add_argument('id', help='job id') parser_job_new = job_subparsers.add_parser('new', help='create a new job') - parser_job_new.add_argument('-r', '--run', action='store_true', help='immediately submits the job on creation') - parser_job_new.add_argument('job_parameters', nargs='*', help='unspecified list of UWS service parameters in the form' + - ' "=" - ' + - 'Default parameters are: ' + - 'destruction (Destruction time of the job), ' + - 'executionDuration (Execution duration of the job in seconds)') + parser_job_new.add_argument('-r', '--run', action='store_true', + help='immediately submits the job on creation') + parser_job_new.add_argument('job_parameters', nargs='*', + help=('unspecified list of UWS service parameters in the form' + ' "=" - Default parameters are: ' + 'destruction (Destruction time of the job), ' + 'executionDuration (Execution duration of the job in seconds)' + ) + ) parser_job_set = job_subparsers.add_parser('set', help='set parameters for the specific job') parser_job_set.add_argument('id', help='job id') - parser_job_set.add_argument('job_parameters', nargs='*', help='unspecified list of UWS service parameters in the form' + - ' "=" - ' + - 'Default parameters are: ' + - 'destruction (Destruction time of the job), ' + - 'executionDuration (Execution duration of the job in seconds)') - - parser_job_run = job_subparsers.add_parser('run', help="run the specific job if its state is pending") + parser_job_set.add_argument('job_parameters', nargs='*', + help=('unspecified list of UWS service parameters in the form' + ' "=" - Default parameters are: ' + 'destruction (Destruction time of the job), ' + 'executionDuration (Execution duration of the job in seconds)' + ) + ) + + parser_job_run = job_subparsers.add_parser('run', + help="run the specific job if its state is pending") parser_job_run.add_argument('id', help='job id') - parser_job_abort = job_subparsers.add_parser('abort', help="aborts the execution of a specific job") + parser_job_abort = job_subparsers.add_parser('abort', + help="aborts the execution of a specific job") parser_job_abort.add_argument('id', help='job id') parser_job_abort = job_subparsers.add_parser('delete', help="delete a specific job") parser_job_abort.add_argument('id', help='job id') - parser_job_results = job_subparsers.add_parser('results', help="download results of a specific job") + parser_job_results = job_subparsers.add_parser('results', + help="download results of a specific job") parser_job_results.add_argument('id', help='job id') - parser_job_results.add_argument('result_id', nargs='?', help='result id (e.g. for specifying the format, optional)') - parser_job_results.add_argument('-f', '--file_base', help='basename of output file (optional), will be appended with result_id') + parser_job_results.add_argument('result_id', nargs='?', + help='result id (e.g. for specifying the format, optional)') + parser_job_results.add_argument('-f', '--file_base', + help=('basename of output file (optional), ' + 'will be appended with result_id') + ) return parser_job_results diff --git a/uws/cli/main.py b/uws/cli/main.py index 79a7bc3..60d3311 100755 --- a/uws/cli/main.py +++ b/uws/cli/main.py @@ -2,13 +2,13 @@ import sys from functools import wraps +import shutil import texttable as tt -from uws.lib.terminalsize import terminalsize as console -import cli_parser from uws import UWS +from . import cli_parser -debug = False +DEBUG = False def handle_error(handler): @@ -16,13 +16,13 @@ def handle_error(handler): def handle(self, *args, **kwargs): try: return handler(self, *args, **kwargs) - except UWS.UWSError as e: - if not debug: - print "An error occurred:\n %s" % e.msg - return + except UWS.UWSError as exc: + if not DEBUG: + print("An error occurred:\n %s" % exc.msg) + raise else: - print "An error occurred:\n %s" % e.msg - print e.raw + print("An error occurred:\n %s" % exc.msg) + print(exc.raw) raise return handle @@ -52,27 +52,27 @@ def list_jobs(url, user_name, password, phases, after=None, last=None): _register_job_reference_for_table(rows, job) else: if job_phases.COMPLETED in phases and job_phases.COMPLETED in job.phase: - _register_job_reference_for_table(rows, job) + _register_job_reference_for_table(rows, job) if job_phases.PENDING in phases and job_phases.PENDING in job.phase: - _register_job_reference_for_table(rows, job) + _register_job_reference_for_table(rows, job) if job_phases.QUEUED in phases and job_phases.QUEUED in job.phase: - _register_job_reference_for_table(rows, job) + _register_job_reference_for_table(rows, job) if job_phases.EXECUTING in phases and job_phases.EXECUTING in job.phase: - _register_job_reference_for_table(rows, job) + _register_job_reference_for_table(rows, job) if job_phases.ERROR in phases and job_phases.ERROR in job.phase: - _register_job_reference_for_table(rows, job) + _register_job_reference_for_table(rows, job) if job_phases.ABORTED in phases and job_phases.ABORTED in job.phase: - _register_job_reference_for_table(rows, job) + _register_job_reference_for_table(rows, job) if job_phases.UNKNOWN in phases and job_phases.UNKNOWN in job.phase: - _register_job_reference_for_table(rows, job) + _register_job_reference_for_table(rows, job) if job_phases.HELD in phases and job_phases.HELD in job.phase: - _register_job_reference_for_table(rows, job) + _register_job_reference_for_table(rows, job) if job_phases.SUSPENDED in phases and job_phases.SUSPENDED in job.phase: - _register_job_reference_for_table(rows, job) + _register_job_reference_for_table(rows, job) # add ARCHIVED phase as well for services with version 1.0 that already support this if job_phases.ARCHIVED in phases and job_phases.ARCHIVED in job.phase: - _register_job_reference_for_table(rows, job) - (console_width, console_height) = console.get_terminal_size() + _register_job_reference_for_table(rows, job) + (console_width, console_height) = shutil.get_terminal_size() # Now we have the rows all stored. Check if all columns exist and remove # empty columns for a more friendly output. @@ -91,16 +91,16 @@ def list_jobs(url, user_name, password, phases, after=None, last=None): dtypes = ['t']*ncols # remove empty cols (in-place removal) - rows[:] = [ [ col for i, col in enumerate(row) if existing_col[i] == 1 ] for row in rows ] + rows[:] = [[col for i, col in enumerate(row) if existing_col[i] == 1] for row in rows] table = tt.Texttable(max_width=console_width) table.set_deco(tt.Texttable.HEADER) table.set_cols_dtype(dtypes) # ['t', 't', 't', 't', 't']) table.add_rows(rows) - print "List of jobs on UWS service for user: '%s'" % user_name - print table.draw() - print "%d jobs listed." % (len(rows) - 1) + print("List of jobs on UWS service for user: '%s'" % user_name) + print(table.draw()) + print("%d jobs listed." % (len(rows) - 1)) def _register_job_reference_for_table(rows, jobref): @@ -112,26 +112,26 @@ def _register_job_reference_for_table(rows, jobref): # only the href-id, if they differ. Because this really MUST be the # correct jobId. - cols = [jobref.id] + cols = [jobref.jid] - if (jobref.reference.href is not None): + if jobref.reference.href is not None: href_jobid = jobref.reference.href.rsplit("/", 1)[1] - if href_jobid != jobref.id: + if href_jobid != jobref.jid: # replace id with href_jobid cols[0] = href_jobid - if (jobref.runId is not None): - cols.append(jobref.runId) + if jobref.run_id is not None: + cols.append(jobref.run_id) else: cols.append('') - if (jobref.ownerId is not None): - cols.append(jobref.ownerId) + if jobref.owner_id is not None: + cols.append(jobref.owner_id) else: cols.append('') - if (jobref.creationTime is not None): - cols.append(jobref.creationTime) + if jobref.creation_time is not None: + cols.append(jobref.creation_time) else: cols.append('') @@ -141,28 +141,30 @@ def _register_job_reference_for_table(rows, jobref): @handle_error -def show_job(url, user_name, password, id, wait, phase): +def show_job(url, user_name, password, jid, wait, phase): uws_client = UWS.client.Client(url=url, user=user_name, password=password) - job = uws_client.get_job(id, wait, phase) + job = uws_client.get_job(jid, wait, phase) if wait and job.version != "1.1": - print "Warning: Wait keyword is (probably) not supported by the server's UWS version %s (need 1.1). Server will probably ignore wait and return immediately." % job.version + print("Warning: Wait keyword is (probably) not supported by the server's UWS version " + + f"{job.version} (need 1.1). Server will probably ignore wait and return immediately.") _print_job(job) @handle_error -def show_phase(url, user_name, password, id): +def show_phase(url, user_name, password, jid): uws_client = UWS.client.Client(url=url, user=user_name, password=password) - phase = uws_client.get_phase(id) + phase = uws_client.get_phase(jid) print(phase) @handle_error -def new_job(url, user_name, password, parameters={}, run=False): +def new_job(url, user_name, password, parameters=None, run=False): + parameters = parameters or {} uws_client = UWS.client.Client(url=url, user=user_name, password=password) job = uws_client.new_job(parameters) @@ -171,84 +173,88 @@ def new_job(url, user_name, password, parameters={}, run=False): # execute the job job = uws_client.run_job(job.job_id) - (console_width, console_height) = console.get_terminal_size() + (console_width, console_height) = shutil.get_terminal_size() _print_job(job) - print "\n" - print "*" * (console_width - 1) - print "You can access this job with the id:\n" - print "Job ID: %s" % job.job_id - print "Command: uws -H %s --user %s --password YOUR_PASSWORD_HERE job show %s" % (url, user_name, job.job_id) - print "*" * (console_width - 1) + print("\n") + print("*" * (console_width - 1)) + print("You can access this job with the id:\n") + print(f"Job ID: {job.job_id}") + print(f"Command: uws -H {url} --user {user_name} --password YOUR_PASSWORD_HERE" + + f"job show {job.job_id}") + print("*" * (console_width - 1)) @handle_error -def set_parameters_job(url, user_name, password, id, parameters={}): +def set_parameters_job(url, user_name, password, jid, parameters=None): + parameters = parameters or {} uws_client = UWS.client.Client(url=url, user=user_name, password=password) if len(parameters) == 0: - job = uws_client.get_job(id) + job = uws_client.get_job(jid) else: - job = uws_client.set_parameters_job(id, parameters) + job = uws_client.set_parameters_job(jid, parameters) _print_job(job) @handle_error -def run_job(url, user_name, password, id): +def run_job(url, user_name, password, jid): uws_client = UWS.client.Client(url=url, user=user_name, password=password) - job = uws_client.run_job(id) + job = uws_client.run_job(jid) _print_job(job) @handle_error -def abort_job(url, user_name, password, id): +def abort_job(url, user_name, password, jid): uws_client = UWS.client.Client(url=url, user=user_name, password=password) - job = uws_client.abort_job(id) + job = uws_client.abort_job(jid) _print_job(job) @handle_error -def delete_job(url, user_name, password, id): +def delete_job(url, user_name, password, jid): uws_client = UWS.client.Client(url=url, user=user_name, password=password) - success = uws_client.delete_job(id) + success = uws_client.delete_job(jid) if success: - print "Job %s successfully deleted!" % (id) + print("Job %s successfully deleted!" % (jid)) @handle_error -def results_job(url, user_name, password, id, result_id, user_file_base): +def results_job(url, user_name, password, jid, result_id, user_file_base): def print_progress(total_size, current): if total_size: sys.stdout.write("\r%d bytes" % current) # or print >> sys.stdout, "\r%d%%" %i, sys.stdout.flush() else: - sys.stdout.write("\rDownloaded %d bytes" % current) # or print >> sys.stdout, "\r%d%%" %i, + sys.stdout.write(f"\rDownloaded {current} bytes") # or print >> sys.stdout, "\r%d%%" %i, sys.stdout.flush() uws_client = UWS.client.Client(url=url, user=user_name, password=password) - job = uws_client.get_job(id) + job = uws_client.get_job(jid) # if there are multiple result sets returned: force user to decide which ones to use? # or maybe rather let the service define a standard result? but how? if len(job.results) > 1 and not result_id: - print 'There are multiple results for this job, all of them are downloaded now.' - print 'If this is not what you intended, please specify the id of your desired result like this: ' - print '\nuws job results ID RESULTID\n' - print 'For RESULTID, you can choose from: ', ','.join([result.id for result in job.results]) + print('There are multiple results for this job, all of them are downloaded now.') + print('If this is not what you intended, please specify the id of your desired', end='') + print(' result like this: ') + print('\nuws job results ID RESULTID\n') + print('For RESULTID, you can choose from: ', + ','.join([result.rid for result in job.results])) # set file base name to job_id or tablename (if available) or user provided file_base file_base = job.job_id for parameter in job.parameters: - if parameter.id == 'table': + if parameter.pid == 'table': file_base = parameter.value break @@ -257,24 +263,27 @@ def print_progress(total_size, current): retrieved = False for result in job.results: - if not result_id or result_id == result.id: + if not result_id or result_id == result.rid: - filename = file_base + '.' + result.id + filename = file_base + '.' + result.rid url = str(result.reference) - print "Downloading %s into file '%s'" % (result.id, filename) - uws_client.connection.download_file(url, user_name, password, filename, callback=print_progress) - print "" - print "Finished downloading file '%s'\n" % (filename) + print("Downloading %s into file '%s'" % (result.rid, filename)) + uws_client.connection.download_file(url, user_name, password, + filename, callback=print_progress) + print("") + print("Finished downloading file '%s'\n" % (filename)) retrieved = True if not retrieved: if result_id: - print "Result Id '%s' not available. Use 'uws job show %s' for a list of available results." % (result_id, job.job_id) + print(f"Result Id '{result_id}' not available. ", end="") + print(f"Use 'uws job show {job.job_id}' for a list of available results.") else: - print "The job with id '%s' has no results." % (job.job_id) - print "Check with 'uws job show %s' the details, the job results may have been deleted." % (job.job_id) + print(f"The job with id '{job.job_id}' has no results.") + print(f"Check with 'uws job show {job.job_id}' the details, ", end="") + print("the job results may have been deleted.") def _print_job(job): @@ -283,16 +292,16 @@ def _print_job(job): rows.append(["Job id", job.job_id]) rows.append(["Run id", job.run_id]) - - if(job.owner_id): + + if job.owner_id: rows.append(["Owner id", job.owner_id]) rows.append(["Phase", ", ".join(job.phase)]) - if(job.quote): + if job.quote: rows.append(["Quote", job.quote]) - - if(job.creation_time): + + if job.creation_time: rows.append(["Creation time", job.creation_time]) rows.append(["Start time", job.start_time]) @@ -301,21 +310,21 @@ def _print_job(job): rows.append(["Destruction time", job.destruction]) for param in job.parameters: - rows.append(["Parameter " + param.id, param.value]) + rows.append(["Parameter " + param.pid, param.value]) for result in job.results: - rows.append(["Result " + result.id, result.reference]) + rows.append(["Result " + result.rid, result.reference]) try: - if(job.error_summary): + if job.error_summary: rows.append(["Errors", "; ".join(job.error_summary.messages)]) except: pass for info in job.job_info: - rows.append(["Job info", unicode(info)]) + rows.append(["Job info", info]) - (console_width, console_height) = console.get_terminal_size() + (console_width, console_height) = shutil.get_terminal_size() fields = [row[0] for row in rows] max_field_len = len(max(fields, key=len)) @@ -325,7 +334,7 @@ def _print_job(job): table.set_cols_dtype(['t', 't']) table.set_cols_width([max_field_len, console_width - max_field_len - 4]) table.add_rows(rows) - print table.draw() + print(table.draw()) # check validity of wait and phases: @@ -335,14 +344,16 @@ def _check_job_wait_args(arguments): phase = arguments.phase if wait is None and phase is not None: - raise RuntimeError("Additional phase for 'job show' only allowed in combination with 'wait'-keyword.") + raise RuntimeError("Additional phase for 'job show' only allowed in combination with" + + "'wait'-keyword.") if phase: if phase.upper() in UWS.models.JobPhases.active_phases: phase = phase.upper() else: active_phases = ', '.join(UWS.models.JobPhases.active_phases) - raise RuntimeError("Phase '" + phase + "' is not supported with WAIT keyword, choose one of the active phases: " + active_phases) + raise RuntimeError("Phase '" + phase + "' is not supported with WAIT keyword, " + + "choose one of the active phases: " + active_phases) return wait, phase @@ -383,16 +394,16 @@ def _check_joblist_last(argument): def main(): - global debug + global DEBUG parser = cli_parser.build_argparse() arguments = parser.parse_args() if arguments.dbg: - debug = True + DEBUG = True if arguments.P: if arguments.password: - print "Error: You cannot use -P and --password together!" + print("Error: You cannot use -P and --password together!") sys.exit(1) arguments.password = getpass.getpass("Enter password: ") @@ -440,12 +451,14 @@ def main(): # parse the job parameters and store in argument list job_parameters = _check_job_parameter_args(arguments.job_parameters) - new_job(arguments.host, arguments.user, arguments.password, job_parameters, arguments.run) + new_job(arguments.host, arguments.user, arguments.password, + job_parameters, arguments.run) elif arguments.job_command == "set": # parse the job parameters and store in argument list job_parameters = _check_job_parameter_args(arguments.job_parameters) - set_parameters_job(arguments.host, arguments.user, arguments.password, arguments.id, job_parameters) + set_parameters_job(arguments.host, arguments.user, arguments.password, + arguments.id, job_parameters) elif arguments.job_command == "run": run_job(arguments.host, arguments.user, arguments.password, arguments.id) elif arguments.job_command == "abort": @@ -453,9 +466,10 @@ def main(): elif arguments.job_command == "delete": delete_job(arguments.host, arguments.user, arguments.password, arguments.id) elif arguments.job_command == "results": - results_job(arguments.host, arguments.user, arguments.password, arguments.id, arguments.result_id, arguments.file_base) + results_job(arguments.host, arguments.user, arguments.password, arguments.id, + arguments.result_id, arguments.file_base) else: - print "Error: Unknown command %s\n" % (arguments.job_command) + print("Error: Unknown command %s\n" % (arguments.job_command)) if __name__ == '__main__': main() diff --git a/uws/lib/terminalsize/__init__.py b/uws/lib/terminalsize/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/uws/lib/terminalsize/terminalsize.py b/uws/lib/terminalsize/terminalsize.py deleted file mode 100644 index 570ede0..0000000 --- a/uws/lib/terminalsize/terminalsize.py +++ /dev/null @@ -1,91 +0,0 @@ -#!/usr/bin/env python - -#source: https://gist.github.com/jtriley/1108174 -import os -import shlex -import struct -import platform -import subprocess - - -def get_terminal_size(): - """ getTerminalSize() - - get width and height of console - - works on linux,os x,windows,cygwin(windows) - originally retrieved from: - http://stackoverflow.com/questions/566746/how-to-get-console-window-width-in-python - """ - current_os = platform.system() - tuple_xy = None - if current_os == 'Windows': - tuple_xy = _get_terminal_size_windows() - if tuple_xy is None: - tuple_xy = _get_terminal_size_tput() - # needed for window's python in cygwin's xterm! - if current_os in ['Linux', 'Darwin'] or current_os.startswith('CYGWIN'): - tuple_xy = _get_terminal_size_linux() - if tuple_xy is None: - print "default" - tuple_xy = (80, 25) # default value - return tuple_xy - - -def _get_terminal_size_windows(): - try: - from ctypes import windll, create_string_buffer - # stdin handle is -10 - # stdout handle is -11 - # stderr handle is -12 - h = windll.kernel32.GetStdHandle(-12) - csbi = create_string_buffer(22) - res = windll.kernel32.GetConsoleScreenBufferInfo(h, csbi) - if res: - (bufx, bufy, curx, cury, wattr, - left, top, right, bottom, - maxx, maxy) = struct.unpack("hhhhHhhhhhh", csbi.raw) - sizex = right - left + 1 - sizey = bottom - top + 1 - return sizex, sizey - except: - pass - - -def _get_terminal_size_tput(): - # get terminal width - # src: http://stackoverflow.com/questions/263890/how-do-i-find-the-width-height-of-a-terminal-window - try: - cols = int(subprocess.check_call(shlex.split('tput cols'))) - rows = int(subprocess.check_call(shlex.split('tput lines'))) - return (cols, rows) - except: - pass - - -def _get_terminal_size_linux(): - def ioctl_GWINSZ(fd): - try: - import fcntl - import termios - cr = struct.unpack('hh', - fcntl.ioctl(fd, termios.TIOCGWINSZ, '1234')) - return cr - except: - pass - cr = ioctl_GWINSZ(0) or ioctl_GWINSZ(1) or ioctl_GWINSZ(2) - if not cr: - try: - fd = os.open(os.ctermid(), os.O_RDONLY) - cr = ioctl_GWINSZ(fd) - os.close(fd) - except: - pass - if not cr: - try: - cr = (os.environ['LINES'], os.environ['COLUMNS']) - except: - return None - return int(cr[1]), int(cr[0]) - -if __name__ == "__main__": - sizex, sizey = get_terminal_size() - print 'width =', sizex, 'height =', sizey \ No newline at end of file