From 07bc852e6425d4879c0065591b68de11161036d1 Mon Sep 17 00:00:00 2001 From: Stefan Oderbolz Date: Thu, 21 May 2020 00:08:13 +0200 Subject: [PATCH 1/2] Fix linting errors --- .github/workflows/lint_python.yml | 2 +- setup.cfg | 2 ++ sruthi/__init__.py | 4 +-- sruthi/client.py | 15 ++++++--- sruthi/errors.py | 9 ++++-- sruthi/metadata.py | 53 ++++++++++++++----------------- sruthi/sru.py | 2 ++ sruthi/xmlparse.py | 1 + tests/client_test.py | 12 +++++-- tests/sru_test.py | 6 ++-- tests/sruthi_test.py | 3 +- 11 files changed, 63 insertions(+), 46 deletions(-) create mode 100644 setup.cfg diff --git a/.github/workflows/lint_python.yml b/.github/workflows/lint_python.yml index b4ea755..1b4443f 100644 --- a/.github/workflows/lint_python.yml +++ b/.github/workflows/lint_python.yml @@ -25,5 +25,5 @@ jobs: pip install -r requirements.txt pip install -r test-requirements.txt - - run: flake8 sruthi --count --show-source --statistics + - run: flake8 . --count --show-source --statistics - run: pytest diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..aec1659 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,2 @@ +[flake8] +max-complexity = 10 diff --git a/sruthi/__init__.py b/sruthi/__init__.py index 92e6642..87af0a1 100644 --- a/sruthi/__init__.py +++ b/sruthi/__init__.py @@ -1,4 +1,4 @@ __version__ = '0.0.1' -from .errors import (SruthiError, ServerIncompatibleError, SruError, NoMoreRecordsError) -from .sru import searchretrieve, explain +from .errors import (SruthiError, ServerIncompatibleError, SruError, NoMoreRecordsError) # noqa +from .sru import searchretrieve, explain # noqa diff --git a/sruthi/client.py b/sruthi/client.py index bdeba86..456cd5f 100644 --- a/sruthi/client.py +++ b/sruthi/client.py @@ -5,6 +5,7 @@ from . import xmlparse from . import metadata + class Client(object): def __init__(self, url=None, page_size=100): self.session = requests.Session() @@ -34,7 +35,6 @@ def explain(self): params = { 'operation': 'explain', 'version': '1.2', - 'query': query, } content = self._get_content(self.url, params) return content @@ -101,9 +101,14 @@ def _get_content(self, url, params): def _check_errors(self, xml): config = self.OPERATIONS[self.operation] if not xml.tag == config['response']: - raise errors.ServerIncompatibleError('Server response did not contain a searchRetrieveResponse tag') + raise errors.ServerIncompatibleError( + 'Server response did not contain a searchRetrieveResponse tag' + ) - diagnostics = xmlparse.find(xml, '%sdiagnostics/%sdiagnostic' % (self.sru, self.sru)) + diagnostics = xmlparse.find( + xml, + f'{self.sru}diagnostics/{self.sru}diagnostic' + ) if diagnostics: - error_msg = " ".join([d.find(detail).text for d in diagnostics]) - raise errors.SruError(error_msg) + error_msg = " ".join([d.find('detail').text for d in diagnostics]) + raise errors.SruError(error_msg) diff --git a/sruthi/errors.py b/sruthi/errors.py index 7925b82..266834a 100644 --- a/sruthi/errors.py +++ b/sruthi/errors.py @@ -3,18 +3,23 @@ class SruthiError(Exception): General sruthi error class to provide a superclass for all other errors """ + class ServerIncompatibleError(SruthiError): """ The error raised from sru.search/sru.explain when the server doesn't behave like a SRU endpoint. """ + class SruError(SruthiError): """ - The error raised from sru.search/sru.explain when the SRU response contains an error + The error raised from sru.search/sru.explain when the SRU response contains + an error """ + class NoMoreRecordsError(SruthiError): """ - This error is raised if all records have been loaded (or no records are present) + This error is raised if all records have been loaded (or no records are + present) """ diff --git a/sruthi/metadata.py b/sruthi/metadata.py index 1a5974d..e19cb25 100644 --- a/sruthi/metadata.py +++ b/sruthi/metadata.py @@ -5,13 +5,14 @@ from . import xmlparse from . import errors + class SruData(object): - def __init__(self, records=[], sru_version=None, count=0, data_loader=None): + def __init__(self, records=[], sru_version=None, count=0, data_loader=None): # noqa self.records = records self.sru_version = sru_version self.count = count self._data_loader = data_loader - + def __repr__(self): return ( 'SruData(' @@ -29,7 +30,7 @@ def __length_hint__(self): def __iter__(self): # use while loop since self.records could grow while iterating - i = 0 + i = 0 while True: # load new data when near end if i == len(self.records): @@ -38,38 +39,32 @@ def __iter__(self): except errors.NoMoreRecordsError: break yield self.records[i] - i += 1 + i += 1 def __getitem__(self, key): if isinstance(key, slice): limit = max(key.start or 0, key.stop or self.count) - while limit > len(self.records): - try: - self._load_new_data() - except errors.NoMoreRecordsError: - pass - return [self.records[k] for k in range(*key.indices(len(self.records)))] + self._load_new_data_until(limit) + count = len(self.records) + return [self.records[k] for k in range(*key.indices(count))] + if not isinstance(key, int): - raise IndexError("Index must be an integer") - if key >= self.count: - raise IndexError("Index %s is out of range" % key) - if key < 0: - # load all data - while True: - try: - self._load_new_data() - except errors.NoMoreRecordsError: - break - else: - while key >= len(self.records): - try: - self._load_new_data() - except errors.NoMoreRecordsError: - if key >= len(self.records): - raise IndexError("Index %s not found" % key) + raise TypeError("Index must be an integer or slice") + limit = key + if limit < 0: + # if we get a negative index, load all data + limit = self.count + self._load_new_data_until(limit) return self.records[key] + def _load_new_data_until(self, limit): + while limit >= len(self.records): + try: + self._load_new_data() + except errors.NoMoreRecordsError: + break + def _load_new_data(self): result = self._data_loader() self.records.extend(result['records']) @@ -84,7 +79,7 @@ def extract_records(xml): record['schema'] = xmlparse.find(xml_rec, './sru:recordSchema').text record_data = xmlparse.find(xml_rec, './sru:recordData') extra_data = xmlparse.find(xml_rec, './sru:extraRecordData') - + for elem in record_data.iter(): record = tag_data(record, elem) @@ -106,6 +101,6 @@ def tag_data(record, elem): tag_name = ns_pattern.sub('', elem.tag) if elem.text and elem.text.strip(): record[tag_name] = elem.text.strip() - elif len(list(elem)) == 0: # leaf element + elif len(list(elem)) == 0: # leaf element record[tag_name] = None return record diff --git a/sruthi/sru.py b/sruthi/sru.py index 26d5d13..9210a02 100644 --- a/sruthi/sru.py +++ b/sruthi/sru.py @@ -2,10 +2,12 @@ from . import client + def searchretrieve(url, query): c = client.Client(url) return c.searchretrieve(query) + def explain(url): c = client.Client(url) return c.explain() diff --git a/sruthi/xmlparse.py b/sruthi/xmlparse.py index 97a81f5..7f27a1d 100644 --- a/sruthi/xmlparse.py +++ b/sruthi/xmlparse.py @@ -8,6 +8,7 @@ 'ap': 'http://www.archivportal.ch/srw/extension/', } + class XMLNone(object): def __nonzero__(self): return False diff --git a/tests/client_test.py b/tests/client_test.py index 802249e..f983e4a 100644 --- a/tests/client_test.py +++ b/tests/client_test.py @@ -18,8 +18,16 @@ def test_searchretrieve(self, session_mock): for rec in r: self.assertIsInstance(rec, dict) self.assertEqual(rec['schema'], 'isad') - - session_mock.return_value.get.assert_any_call('http://test.com/sru', params={'startRecord': 1, 'query': 'Test-Query', 'operation': 'searchretrieve', 'version': '1.2'}) + + session_mock.return_value.get.assert_any_call( + 'http://test.com/sru', + params={ + 'startRecord': 1, + 'query': 'Test-Query', + 'operation': 'searchretrieve', + 'version': '1.2' + } + ) # add test for getitem with slices etc. # print("-3") diff --git a/tests/sru_test.py b/tests/sru_test.py index d1122c3..b905a51 100644 --- a/tests/sru_test.py +++ b/tests/sru_test.py @@ -21,15 +21,15 @@ def test_searchretrieve(self, session_mock): 'reference': 'VII.335.:2.34.8.', 'extra': { 'score': '0.38', - 'link': 'https://amsquery.stadt-zuerich.ch/detail.aspx?Id=410130', + 'link': 'https://amsquery.stadt-zuerich.ch/detail.aspx?Id=410130', # noqa 'hasDigitizedItems': '0', 'endDateISO': '1998-12-31', - 'beginDateISO': '1998-01-01', + 'beginDateISO': '1998-01-01', 'beginApprox': '0', 'endApprox': '0' }, 'descriptionlevel': 'Dossier', - 'title': u'Podium "Frauen und Politik" beim Jubil\xe4umsanlass "Frauenrechte-Menschenrechte" des Bundes Schweizerischer Frauenorganisationen BSF zu 150 Jahre Bundesstaat, 50 Jahre UNO-Menschenrechtserkl\xe4rung und 27 Jahre politische Gleichberechtigung im Nationalratssaal in Bern vom 4. April 1998', + 'title': u'Podium "Frauen und Politik" beim Jubil\xe4umsanlass "Frauenrechte-Menschenrechte" des Bundes Schweizerischer Frauenorganisationen BSF zu 150 Jahre Bundesstaat, 50 Jahre UNO-Menschenrechtserkl\xe4rung und 27 Jahre politische Gleichberechtigung im Nationalratssaal in Bern vom 4. April 1998', # noqa 'extent': None, 'date': '1998', 'creator': None, diff --git a/tests/sruthi_test.py b/tests/sruthi_test.py index 7fe058a..66b3f28 100644 --- a/tests/sruthi_test.py +++ b/tests/sruthi_test.py @@ -20,5 +20,4 @@ def _session_mock(self, session_mock, filename=None): filename ) with open(path) as file: - session_mock.return_value.get.return_value = mock.MagicMock(content=file.read()) - + session_mock.return_value.get.return_value = mock.MagicMock(content=file.read()) # noqa From 853081942362e73f6c336659470e662d73a02b97 Mon Sep 17 00:00:00 2001 From: Stefan Oderbolz Date: Sat, 23 May 2020 12:53:47 +0200 Subject: [PATCH 2/2] Install package for tests --- .github/workflows/lint_python.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/lint_python.yml b/.github/workflows/lint_python.yml index 1b4443f..8050a26 100644 --- a/.github/workflows/lint_python.yml +++ b/.github/workflows/lint_python.yml @@ -24,6 +24,7 @@ jobs: python -m pip install --upgrade pip pip install -r requirements.txt pip install -r test-requirements.txt + pip install -e . - run: flake8 . --count --show-source --statistics - run: pytest