From 7f5b97cd16f6bd4ec278b2e796dc66aed654f2a2 Mon Sep 17 00:00:00 2001 From: Krafpy Date: Sat, 2 Sep 2023 22:57:48 +0200 Subject: [PATCH 1/7] Added option to include raw data in result The `include_raw` parameter in the `query`, `parse_json` and `parse_xml` methods allow to keep a copy of the raw data (str or bytes) returned by the query in a `raw` field in the parsed result. --- overpy/__init__.py | 38 +++++++++++++++++++++++++++++--------- tests/test_json.py | 14 ++++++++++++++ tests/test_xml.py | 14 ++++++++++++++ 3 files changed, 57 insertions(+), 9 deletions(-) diff --git a/overpy/__init__.py b/overpy/__init__.py index a85bb01..9683d3f 100644 --- a/overpy/__init__.py +++ b/overpy/__init__.py @@ -111,11 +111,12 @@ def _handle_remark_msg(msg: str) -> NoReturn: raise exception.OverpassRuntimeRemark(msg=msg) raise exception.OverpassUnknownError(msg=msg) - def query(self, query: Union[bytes, str]) -> "Result": + def query(self, query: Union[bytes, str], include_raw: bool = False) -> "Result": """ Query the Overpass API :param query: The query string in Overpass QL + :param include_raw: True to store the raw data along with the parsed result :return: The parsed result """ if not isinstance(query, bytes): @@ -146,10 +147,10 @@ def query(self, query: Union[bytes, str]) -> "Result": content_type = f.getheader("Content-Type") if content_type == "application/json": - return self.parse_json(response) + return self.parse_json(response, include_raw=include_raw) if content_type == "application/osm3s+xml": - return self.parse_xml(response) + return self.parse_xml(response, include_raw=include_raw) current_exception = exception.OverpassUnknownContentType(content_type) if not do_retry: @@ -198,12 +199,17 @@ def query(self, query: Union[bytes, str]) -> "Result": raise exception.MaxRetriesReached(retry_count=retry_num, exceptions=retry_exceptions) - def parse_json(self, data: Union[bytes, str], encoding: str = "utf-8") -> "Result": + def parse_json( + self, + data: Union[bytes, str], + encoding: str = "utf-8", + include_raw: bool = False) -> "Result": """ Parse raw response from Overpass service. :param data: Raw JSON Data :param encoding: Encoding to decode byte string + :param include_raw: True to store the data along with the parsed result :return: Result object """ if isinstance(data, bytes): @@ -211,14 +217,23 @@ def parse_json(self, data: Union[bytes, str], encoding: str = "utf-8") -> "Resul data_parsed: dict = json.loads(data, parse_float=Decimal) if "remark" in data_parsed: self._handle_remark_msg(msg=data_parsed.get("remark")) - return Result.from_json(data_parsed, api=self) + result = Result.from_json(data_parsed, api=self) + if include_raw: + result.raw = data + return result - def parse_xml(self, data: Union[bytes, str], encoding: str = "utf-8", parser: Optional[int] = None): + def parse_xml( + self, + data: Union[bytes, str], + encoding: str = "utf-8", + parser: Optional[int] = None, + include_raw: bool = False) -> "Result": """ :param data: Raw XML Data :param encoding: Encoding to decode byte string :param parser: The XML parser to use + :param include_raw: True to store the data along with the parsed result :return: Result object """ if parser is None: @@ -230,8 +245,10 @@ def parse_xml(self, data: Union[bytes, str], encoding: str = "utf-8", parser: Op m = re.compile("(?P[^<>]*)").search(data) if m: self._handle_remark_msg(m.group("msg")) - - return Result.from_xml(data, api=self, parser=parser) + result = Result.from_xml(data, api=self, parser=parser) + if include_raw: + result.raw = data + return result class Result: @@ -242,11 +259,13 @@ class Result: def __init__( self, elements: Optional[List[Union["Area", "Node", "Relation", "Way"]]] = None, - api: Optional[Overpass] = None): + api: Optional[Overpass] = None, + raw: Optional[Union[bytes, str]] = None): """ :param elements: List of elements to initialize the result with :param api: The API object to load additional resources and elements + :param raw: The raw data corresponding to these elements """ if elements is None: elements = [] @@ -269,6 +288,7 @@ def __init__( Area: self._areas } self.api = api + self.raw = raw def expand(self, other: "Result"): """ diff --git a/tests/test_json.py b/tests/test_json.py index 9a96d0e..001e83b 100644 --- a/tests/test_json.py +++ b/tests/test_json.py @@ -116,3 +116,17 @@ def test_remark_unknown(self): api = overpy.Overpass() with pytest.raises(overpy.exception.OverpassUnknownError): api.parse_json(read_file("json/remark-unknown-01.json")) + + +class TestIncludeRawJSON: + def test_include_raw_json(self): + api = overpy.Overpass() + data = read_file("json/area-01.json") + result = api.parse_json(data, include_raw=True) + assert result.raw == data + + def test_not_include_raw(self): + api = overpy.Overpass() + data = read_file("json/area-01.json") + result = api.parse_json(data, include_raw=False) + assert result.raw is None \ No newline at end of file diff --git a/tests/test_xml.py b/tests/test_xml.py index 8a00b76..1e16b4f 100644 --- a/tests/test_xml.py +++ b/tests/test_xml.py @@ -209,3 +209,17 @@ def test_remark_unknown(self): api = overpy.Overpass() with pytest.raises(overpy.exception.OverpassUnknownError): api.parse_xml(read_file("xml/remark-unknown-01.xml")) + + +class TestIncludeRawXML: + def test_include_raw_xml(self): + api = overpy.Overpass() + data = read_file("xml/area-01.xml") + result = api.parse_xml(data, include_raw=True) + assert result.raw == data + + def test_not_include_raw(self): + api = overpy.Overpass() + data = read_file("xml/area-01.xml") + result = api.parse_xml(data, include_raw=False) + assert result.raw is None \ No newline at end of file From 0665abe2f92ddb4ad499aa8818f8057c54ba60bf Mon Sep 17 00:00:00 2001 From: Krafpy Date: Sat, 2 Sep 2023 23:23:14 +0200 Subject: [PATCH 2/7] Added raw option in query If `raw` is set to True when calling `query`, the raw str or bytes of the response are returned. --- overpy/__init__.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/overpy/__init__.py b/overpy/__init__.py index 9683d3f..3d2476e 100644 --- a/overpy/__init__.py +++ b/overpy/__init__.py @@ -111,11 +111,17 @@ def _handle_remark_msg(msg: str) -> NoReturn: raise exception.OverpassRuntimeRemark(msg=msg) raise exception.OverpassUnknownError(msg=msg) - def query(self, query: Union[bytes, str], include_raw: bool = False) -> "Result": + def query( + self, + query: Union[bytes, str], + *, + raw: bool = False, + include_raw: bool = False) -> Union["Result", bytes, str]: """ Query the Overpass API :param query: The query string in Overpass QL + :param raw: True to return the raw response without parsing :param include_raw: True to store the raw data along with the parsed result :return: The parsed result """ @@ -144,6 +150,9 @@ def query(self, query: Union[bytes, str], include_raw: bool = False) -> "Result" current_exception: exception.OverPyException if f.code == 200: + if raw: + return response + content_type = f.getheader("Content-Type") if content_type == "application/json": From 8c2ac8d51039086d5593661a75961b6124dbda77 Mon Sep 17 00:00:00 2001 From: Krafpy Date: Mon, 11 Sep 2023 10:35:28 +0200 Subject: [PATCH 3/7] Moved raw query to dedicated query_raw method Removed `raw` argument from the `query` method, which only returns the parsed result (as originally). The `query_raw` is used to return the raw bytes or string, along with the response's content type. This should fix the typing errors. --- overpy/__init__.py | 45 ++++++++++++++++++++++++++------------------- 1 file changed, 26 insertions(+), 19 deletions(-) diff --git a/overpy/__init__.py b/overpy/__init__.py index 3d2476e..7a31af8 100644 --- a/overpy/__init__.py +++ b/overpy/__init__.py @@ -110,20 +110,13 @@ def _handle_remark_msg(msg: str) -> NoReturn: elif msg.startswith("runtime remark:"): raise exception.OverpassRuntimeRemark(msg=msg) raise exception.OverpassUnknownError(msg=msg) - - def query( - self, - query: Union[bytes, str], - *, - raw: bool = False, - include_raw: bool = False) -> Union["Result", bytes, str]: + + def query_raw(self, query: Union[bytes, str]) -> Tuple[Union[bytes, str], str]: """ - Query the Overpass API + Query the Overpass API and returns the raw response :param query: The query string in Overpass QL - :param raw: True to return the raw response without parsing - :param include_raw: True to store the raw data along with the parsed result - :return: The parsed result + :return: A tuple made of the raw response and the response content type """ if not isinstance(query, bytes): query = query.encode("utf-8") @@ -150,16 +143,10 @@ def query( current_exception: exception.OverPyException if f.code == 200: - if raw: - return response - content_type = f.getheader("Content-Type") - if content_type == "application/json": - return self.parse_json(response, include_raw=include_raw) - - if content_type == "application/osm3s+xml": - return self.parse_xml(response, include_raw=include_raw) + if content_type in ("application/json", "application/osm3s+xml"): + return response, content_type current_exception = exception.OverpassUnknownContentType(content_type) if not do_retry: @@ -208,6 +195,26 @@ def query( raise exception.MaxRetriesReached(retry_count=retry_num, exceptions=retry_exceptions) + def query( + self, + query: Union[bytes, str], + *, + include_raw: bool = False) -> "Result": + """ + Query the Overpass API and returns parsed result + + :param query: The query string in Overpass QL + :param include_raw: True to store the raw data along with the parsed result + :return: The parsed result + """ + + response, content_type = self.query_raw(query) + if content_type == "application/json": + return self.parse_json(response, include_raw=include_raw) + + if content_type == "application/osm3s+xml": + return self.parse_xml(response, include_raw=include_raw) + def parse_json( self, data: Union[bytes, str], From 02b707a43f6694e0a7425d6e89aaa77ac25c1524 Mon Sep 17 00:00:00 2001 From: Krafpy Date: Mon, 11 Sep 2023 10:54:06 +0200 Subject: [PATCH 4/7] Enforced return in query --- overpy/__init__.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/overpy/__init__.py b/overpy/__init__.py index 7a31af8..15b499b 100644 --- a/overpy/__init__.py +++ b/overpy/__init__.py @@ -211,8 +211,7 @@ def query( response, content_type = self.query_raw(query) if content_type == "application/json": return self.parse_json(response, include_raw=include_raw) - - if content_type == "application/osm3s+xml": + else: # "application/osm3s+xml": return self.parse_xml(response, include_raw=include_raw) def parse_json( From ec5826f9048be678337d01d230ff9f755d3d116b Mon Sep 17 00:00:00 2001 From: Krafpy Date: Mon, 11 Sep 2023 11:03:07 +0200 Subject: [PATCH 5/7] Fixed writing convention errors --- overpy/__init__.py | 2 +- tests/test_json.py | 2 +- tests/test_xml.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/overpy/__init__.py b/overpy/__init__.py index 15b499b..9f65f20 100644 --- a/overpy/__init__.py +++ b/overpy/__init__.py @@ -211,7 +211,7 @@ def query( response, content_type = self.query_raw(query) if content_type == "application/json": return self.parse_json(response, include_raw=include_raw) - else: # "application/osm3s+xml": + else: # "application/osm3s+xml": return self.parse_xml(response, include_raw=include_raw) def parse_json( diff --git a/tests/test_json.py b/tests/test_json.py index 001e83b..001d70d 100644 --- a/tests/test_json.py +++ b/tests/test_json.py @@ -129,4 +129,4 @@ def test_not_include_raw(self): api = overpy.Overpass() data = read_file("json/area-01.json") result = api.parse_json(data, include_raw=False) - assert result.raw is None \ No newline at end of file + assert result.raw is None diff --git a/tests/test_xml.py b/tests/test_xml.py index 1e16b4f..08e027a 100644 --- a/tests/test_xml.py +++ b/tests/test_xml.py @@ -222,4 +222,4 @@ def test_not_include_raw(self): api = overpy.Overpass() data = read_file("xml/area-01.xml") result = api.parse_xml(data, include_raw=False) - assert result.raw is None \ No newline at end of file + assert result.raw is None From af104c9726f999163a33ab763cc71833c145d6a6 Mon Sep 17 00:00:00 2001 From: Krafpy Date: Mon, 11 Sep 2023 11:08:38 +0200 Subject: [PATCH 6/7] Removed trailing whitespaces --- overpy/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/overpy/__init__.py b/overpy/__init__.py index 9f65f20..3c6d744 100644 --- a/overpy/__init__.py +++ b/overpy/__init__.py @@ -110,7 +110,7 @@ def _handle_remark_msg(msg: str) -> NoReturn: elif msg.startswith("runtime remark:"): raise exception.OverpassRuntimeRemark(msg=msg) raise exception.OverpassUnknownError(msg=msg) - + def query_raw(self, query: Union[bytes, str]) -> Tuple[Union[bytes, str], str]: """ Query the Overpass API and returns the raw response @@ -211,7 +211,7 @@ def query( response, content_type = self.query_raw(query) if content_type == "application/json": return self.parse_json(response, include_raw=include_raw) - else: # "application/osm3s+xml": + else: # "application/osm3s+xml" return self.parse_xml(response, include_raw=include_raw) def parse_json( From af795c62994dc529661f88299e92659c5715333f Mon Sep 17 00:00:00 2001 From: Krafpy Date: Mon, 11 Sep 2023 11:11:43 +0200 Subject: [PATCH 7/7] Removed more trailing whitespaces --- tests/test_json.py | 2 +- tests/test_xml.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_json.py b/tests/test_json.py index 001d70d..97f35a2 100644 --- a/tests/test_json.py +++ b/tests/test_json.py @@ -124,7 +124,7 @@ def test_include_raw_json(self): data = read_file("json/area-01.json") result = api.parse_json(data, include_raw=True) assert result.raw == data - + def test_not_include_raw(self): api = overpy.Overpass() data = read_file("json/area-01.json") diff --git a/tests/test_xml.py b/tests/test_xml.py index 08e027a..a3db45d 100644 --- a/tests/test_xml.py +++ b/tests/test_xml.py @@ -217,7 +217,7 @@ def test_include_raw_xml(self): data = read_file("xml/area-01.xml") result = api.parse_xml(data, include_raw=True) assert result.raw == data - + def test_not_include_raw(self): api = overpy.Overpass() data = read_file("xml/area-01.xml")