From db8654b2c0242cf49fa05b98a0bde8900c304b0d Mon Sep 17 00:00:00 2001 From: Flavio Garcia Date: Fri, 29 Mar 2024 15:36:22 -0400 Subject: [PATCH 01/13] refactor(client): break client in modules Fixes: #8 --- peasant/client/__init__.py | 0 peasant/client/protocol.py | 63 ++++++++++++++++++ peasant/{client.py => client/tornado.py} | 83 +----------------------- peasant/client/transport.py | 57 ++++++++++++++++ tests/runtests.py | 1 - tests/tornado_test.py | 2 +- 6 files changed, 123 insertions(+), 83 deletions(-) create mode 100644 peasant/client/__init__.py create mode 100644 peasant/client/protocol.py rename peasant/{client.py => client/tornado.py} (72%) create mode 100644 peasant/client/transport.py diff --git a/peasant/client/__init__.py b/peasant/client/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/peasant/client/protocol.py b/peasant/client/protocol.py new file mode 100644 index 0000000..95194bb --- /dev/null +++ b/peasant/client/protocol.py @@ -0,0 +1,63 @@ +# Copyright 2020-2024 Flavio Garcia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from peasant.client.transport import Transport +import logging + +logger = logging.getLogger(__name__) + + +class Peasant(object): + + _transport: Transport + + def __init__(self, transport): + self._directory_cache = None + self._transport = transport + self._transport.peasant = self + + @property + def directory_cache(self): + return self._directory_cache + + @directory_cache.setter + def directory_cache(self, directory_cache): + self._directory_cache = directory_cache + + @property + def transport(self): + return self._transport + + def directory(self): + if self.directory_cache is None: + self.transport.set_directory() + return self.directory_cache + + def new_nonce(self): + return self.transport.new_nonce() + + +class AsyncPeasant(Peasant): + + def __init__(self, transport): + super(AsyncPeasant, self).__init__(transport) + + async def directory(self): + if self._directory_cache is None: + future = self.transport.set_directory() + if future is not None: + logger.debug("Running transport set directory cache " + "asynchronously.") + await future + return self._directory_cache diff --git a/peasant/client.py b/peasant/client/tornado.py similarity index 72% rename from peasant/client.py rename to peasant/client/tornado.py index 47a94ab..8c7b5a2 100644 --- a/peasant/client.py +++ b/peasant/client/tornado.py @@ -15,90 +15,11 @@ import copy import logging from peasant import get_version +from peasant.client.transport import Transport from urllib.parse import urlencode logger = logging.getLogger(__name__) - -class PeasantTransport: - - _peasant: "Peasant" - - @property - def peasant(self): - return self._peasant - - @peasant.setter - def peasant(self, peasant: "Peasant"): - self._peasant = peasant - - def get(self, path, **kwargs): - raise NotImplementedError - - def head(self, path, **kwargs): - raise NotImplementedError - - def post(self, path, **kwargs): - raise NotImplementedError - - def post_as_get(self, path, **kwargs): - raise NotImplementedError - - def set_directory(self): - raise NotImplementedError - - def new_nonce(self): - raise NotImplementedError - - def is_registered(self): - raise NotImplementedError - - -class Peasant(object): - - _transport: PeasantTransport - - def __init__(self, transport): - self._directory_cache = None - self._transport = transport - self._transport.peasant = self - - @property - def directory_cache(self): - return self._directory_cache - - @directory_cache.setter - def directory_cache(self, directory_cache): - self._directory_cache = directory_cache - - @property - def transport(self): - return self._transport - - def directory(self): - if self.directory_cache is None: - self.transport.set_directory() - return self.directory_cache - - def new_nonce(self): - return self.transport.new_nonce() - - -class AsyncPeasant(Peasant): - - def __init__(self, transport): - super(AsyncPeasant, self).__init__(transport) - - async def directory(self): - if self._directory_cache is None: - future = self.transport.set_directory() - if future is not None: - logger.debug("Running transport set directory cache " - "asynchronously.") - await future - return self._directory_cache - - tornado_installed = False try: from tornado.httpclient import HTTPRequest @@ -139,7 +60,7 @@ def get_tornado_request(url, **kwargs): pass -class TornadoTransport(PeasantTransport): +class TornadoTransport(Transport): def __init__(self, bastion_address): super().__init__() diff --git a/peasant/client/transport.py b/peasant/client/transport.py new file mode 100644 index 0000000..9049b85 --- /dev/null +++ b/peasant/client/transport.py @@ -0,0 +1,57 @@ +# Copyright 2020-2024 Flavio Garcia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import logging +import typing + +if typing.TYPE_CHECKING: + from peasant.client.protocol import Peasant + +logger = logging.getLogger(__name__) + + +class Transport: + + _peasant: Peasant + + @property + def peasant(self) -> Peasant: + return self._peasant + + @peasant.setter + def peasant(self, peasant: Peasant): + self._peasant = peasant + + def get(self, path, **kwargs): + raise NotImplementedError + + def head(self, path, **kwargs): + raise NotImplementedError + + def post(self, path, **kwargs): + raise NotImplementedError + + def post_as_get(self, path, **kwargs): + raise NotImplementedError + + def set_directory(self): + raise NotImplementedError + + def new_nonce(self): + raise NotImplementedError + + def is_registered(self): + raise NotImplementedError diff --git a/tests/runtests.py b/tests/runtests.py index f31bd96..1e089de 100644 --- a/tests/runtests.py +++ b/tests/runtests.py @@ -14,7 +14,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -# print current directory import unittest from tests import tornado_test diff --git a/tests/tornado_test.py b/tests/tornado_test.py index 1e491f1..d6b8dfc 100644 --- a/tests/tornado_test.py +++ b/tests/tornado_test.py @@ -14,7 +14,7 @@ from firenado.testing import TornadoAsyncTestCase from firenado.launcher import ProcessLauncher -from peasant.client import TornadoTransport +from peasant.client.tornado import TornadoTransport from tests import chdir_fixture_app, PROJECT_ROOT from tornado.testing import gen_test From 0ad5f5ac094e764887e722ad77fc7c42bbe444ce Mon Sep 17 00:00:00 2001 From: Flavio Garcia Date: Fri, 29 Mar 2024 16:14:54 -0400 Subject: [PATCH 02/13] refactor(client): rename tornado transport module Refs: #8 --- peasant/client/{tornado.py => transport_tornado.py} | 0 tests/tornado_test.py | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename peasant/client/{tornado.py => transport_tornado.py} (100%) diff --git a/peasant/client/tornado.py b/peasant/client/transport_tornado.py similarity index 100% rename from peasant/client/tornado.py rename to peasant/client/transport_tornado.py diff --git a/tests/tornado_test.py b/tests/tornado_test.py index d6b8dfc..0b61606 100644 --- a/tests/tornado_test.py +++ b/tests/tornado_test.py @@ -14,7 +14,7 @@ from firenado.testing import TornadoAsyncTestCase from firenado.launcher import ProcessLauncher -from peasant.client.tornado import TornadoTransport +from peasant.client.transport_tornado import TornadoTransport from tests import chdir_fixture_app, PROJECT_ROOT from tornado.testing import gen_test From d708cc436622cb78d165965eba190abda6c2031b Mon Sep 17 00:00:00 2001 From: Flavio Garcia Date: Fri, 29 Mar 2024 23:22:51 -0400 Subject: [PATCH 03/13] feat(transport): add fix address function Refs: #10 --- peasant/client/transport.py | 8 ++++++++ tests/runtests.py | 3 ++- tests/transport_test.py | 35 +++++++++++++++++++++++++++++++++++ 3 files changed, 45 insertions(+), 1 deletion(-) create mode 100644 tests/transport_test.py diff --git a/peasant/client/transport.py b/peasant/client/transport.py index 9049b85..e1f8b90 100644 --- a/peasant/client/transport.py +++ b/peasant/client/transport.py @@ -16,6 +16,7 @@ import logging import typing +from urllib.parse import urlparse if typing.TYPE_CHECKING: from peasant.client.protocol import Peasant @@ -23,6 +24,13 @@ logger = logging.getLogger(__name__) +def fix_address(address): + parsed_address = urlparse(address) + if parsed_address.path.endswith("/"): + parsed_address = parsed_address._replace(path=parsed_address.path[:-1]) + return parsed_address.geturl() + + class Transport: _peasant: Peasant diff --git a/tests/runtests.py b/tests/runtests.py index 1e089de..ac67bca 100644 --- a/tests/runtests.py +++ b/tests/runtests.py @@ -15,13 +15,14 @@ # limitations under the License. import unittest -from tests import tornado_test +from tests import tornado_test, transport_test def suite(): testLoader = unittest.TestLoader() alltests = unittest.TestSuite() alltests.addTests(testLoader.loadTestsFromModule(tornado_test)) + alltests.addTests(testLoader.loadTestsFromModule(transport_test)) return alltests diff --git a/tests/transport_test.py b/tests/transport_test.py new file mode 100644 index 0000000..6b1da21 --- /dev/null +++ b/tests/transport_test.py @@ -0,0 +1,35 @@ +# Copyright 2020-2024 Flavio Garcia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from peasant.client.transport import fix_address +from unittest import TestCase + + +class TransportTestCase(TestCase): + + def test_fix_address(self): + address = "http://localhost" + expected_address = "http://localhost" + fixed_address = fix_address(address) + self.assertEqual(expected_address, fixed_address) + + address = "https://localhost/" + expected_address = "https://localhost" + fixed_address = fix_address(address) + self.assertEqual(expected_address, fixed_address) + + address = "http://localhost/a/path/" + expected_address = "http://localhost/a/path" + fixed_address = fix_address(address) + self.assertEqual(expected_address, fixed_address) From c6645083268d5715bcb463097ee478e1ea6327a8 Mon Sep 17 00:00:00 2001 From: Flavio Garcia Date: Fri, 29 Mar 2024 23:36:41 -0400 Subject: [PATCH 04/13] fix(transport): fix address resolution in the tornado transport Fixes: #10 --- peasant/client/transport_tornado.py | 45 ++++++++++++++------------ tests/fixtures/bastiontest/app.py | 4 ++- tests/fixtures/bastiontest/handlers.py | 27 ++++++++++++++-- tests/tornado_test.py | 21 +++++++++++- 4 files changed, 73 insertions(+), 24 deletions(-) diff --git a/peasant/client/transport_tornado.py b/peasant/client/transport_tornado.py index 8c7b5a2..c89d516 100644 --- a/peasant/client/transport_tornado.py +++ b/peasant/client/transport_tornado.py @@ -15,7 +15,7 @@ import copy import logging from peasant import get_version -from peasant.client.transport import Transport +from peasant.client.transport import fix_address, Transport from urllib.parse import urlencode logger = logging.getLogger(__name__) @@ -47,10 +47,10 @@ def get_tornado_request(url, **kwargs): method = kwargs.get("method", "GET") path = kwargs.get("path", None) form_urlencoded = kwargs.get("form_urlencoded", False) - if not url.endswith("/"): - url = f"{url}/" - if path is not None and path != "/": - url = f"{url}{path}" % () + if path is not None: + if not path.startswith("/"): + path = f"/{path}" + url = f"{url}{path}" request = HTTPRequest(url, method=method) if form_urlencoded: request.headers.add("Content-Type", @@ -62,7 +62,7 @@ def get_tornado_request(url, **kwargs): class TornadoTransport(Transport): - def __init__(self, bastion_address): + def __init__(self, bastion_address) -> None: super().__init__() if not tornado_installed: logger.warn("TornadoTransport cannot be used without tornado " @@ -72,7 +72,7 @@ def __init__(self, bastion_address): "\n\nInstalling tornado manually will also work.\n") raise NotImplementedError self._client = AsyncHTTPClient() - self._bastion_address = bastion_address + self._bastion_address = fix_address(bastion_address) self._directory = None self.user_agent = (f"Peasant/{get_version()}" f"Tornado/{tornado_version}") @@ -80,12 +80,13 @@ def __init__(self, bastion_address): 'User-Agent': self.user_agent } - def _get_path(self, path, **kwargs): + def get_path(self, path, **kwargs) -> str: query_string = kwargs.get('query_string') if query_string: path = url_concat(path, query_string) + return path - def _get_headers(self, **kwargs): + def get_headers(self, **kwargs): headers = copy.deepcopy(self._basic_headers) _headers = kwargs.get('headers') if _headers: @@ -93,10 +94,13 @@ def _get_headers(self, **kwargs): return headers async def get(self, path, **kwargs): - path = self._get_path(path, **kwargs) + path = self.get_path(path, **kwargs) + body = kwargs.get("body") request = get_tornado_request(self._bastion_address, path=path) - headers = self._get_headers(**kwargs) + headers = self.get_headers(**kwargs) request.headers.update(headers) + if body: + request.body = body try: result = await self._client.fetch(request) except HTTPClientError as error: @@ -104,9 +108,10 @@ async def get(self, path, **kwargs): return result async def head(self, path, **kwargs): - path = self._get_path(path, **kwargs) - request = get_tornado_request(path, method="HEAD") - headers = self._get_headers(**kwargs) + path = self.get_path(path, **kwargs) + request = get_tornado_request(self._bastion_address, path=path, + method="HEAD") + headers = self.get_headers(**kwargs) request.headers.update(headers) try: result = await self._client.fetch(request) @@ -115,13 +120,13 @@ async def head(self, path, **kwargs): return result async def post(self, path, **kwargs): - path = self._get_path(path, **kwargs) - form_data = kwargs.get("form_data", {}) - request = get_tornado_request(path, method="POST", - form_urlencoded=True) - headers = self._get_headers(**kwargs) + path = self.get_path(path, **kwargs) + request = get_tornado_request(self._bastion_address, path=path, + method="POST") + headers = self.get_headers(**kwargs) + body = kwargs.get("body", []) request.headers.update(headers) - request.body = urlencode(form_data) + request.body = urlencode(body) try: result = await self._client.fetch(request) except HTTPClientError as error: diff --git a/tests/fixtures/bastiontest/app.py b/tests/fixtures/bastiontest/app.py index e68e887..5f34890 100644 --- a/tests/fixtures/bastiontest/app.py +++ b/tests/fixtures/bastiontest/app.py @@ -6,5 +6,7 @@ class BastiontestComponent(tornadoweb.TornadoComponent): def get_handlers(self): return [ - (r"/", handlers.IndexHandler), + (r"/", handlers.GetHandler), + (r"/head", handlers.HeadHandler), + (r"/post", handlers.PostHandler), ] diff --git a/tests/fixtures/bastiontest/handlers.py b/tests/fixtures/bastiontest/handlers.py index 0d6ffc4..1236572 100644 --- a/tests/fixtures/bastiontest/handlers.py +++ b/tests/fixtures/bastiontest/handlers.py @@ -1,7 +1,30 @@ from firenado import tornadoweb +import logging +from tornado.web import HTTPError +logger = logging.getLogger(__name__) -class IndexHandler(tornadoweb.TornadoHandler): + +class HeadHandler(tornadoweb.TornadoHandler): + + def head(self): + user_agent = self.request.headers.get("user-agent") + if not user_agent: + raise HTTPError(status_code=400) + return + self.add_header("head-response", "Head method response") + self.add_header("user-agent", user_agent) + + +class GetHandler(tornadoweb.TornadoHandler): def get(self): - self.write("IndexHandler output") + self.write("Get method output") + + +class PostHandler(tornadoweb.TornadoHandler): + + def post(self): + body = self.request.body + self.add_header("request-body", body) + self.write("Post method output") diff --git a/tests/tornado_test.py b/tests/tornado_test.py index 0b61606..0ca23f3 100644 --- a/tests/tornado_test.py +++ b/tests/tornado_test.py @@ -32,10 +32,29 @@ def setUp(self) -> None: self.transport = TornadoTransport( f"http://localhost:{self.http_port()}") + @gen_test + async def test_head(self): + try: + response = await self.transport.head("/head") + except Exception as e: + raise e + self.assertEqual(response.headers.get("head-response"), + "Head method response") + self.assertEqual(response.headers.get("user-agent"), + self.transport.user_agent) + @gen_test async def test_get(self): try: response = await self.transport.get("/") except Exception as e: raise e - self.assertEqual(response.body, b"IndexHandler output") + self.assertEqual(response.body, b"Get method output") + + @gen_test + async def test_post(self): + try: + response = await self.transport.post("/post") + except Exception as e: + raise e + self.assertEqual(response.body, b"Post method output") From 6bc25d3d6e2dabbeeb0f02c37a0d2dee42739c49 Mon Sep 17 00:00:00 2001 From: Flavio Garcia Date: Fri, 29 Mar 2024 23:49:53 -0400 Subject: [PATCH 05/13] chore(transport): add get tornado request tests Refs: #6 --- tests/tornado_test.py | 31 +++++++++++++++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/tests/tornado_test.py b/tests/tornado_test.py index 0ca23f3..da15a3f 100644 --- a/tests/tornado_test.py +++ b/tests/tornado_test.py @@ -14,13 +14,40 @@ from firenado.testing import TornadoAsyncTestCase from firenado.launcher import ProcessLauncher -from peasant.client.transport_tornado import TornadoTransport +from peasant.client.transport import fix_address +from peasant.client.transport_tornado import (get_tornado_request, + TornadoTransport) from tests import chdir_fixture_app, PROJECT_ROOT from tornado.testing import gen_test +from unittest import TestCase + + +class GetTornadoRequestTestCase(TestCase): + + def test_get_tornado_request(self): + bastion_address = fix_address("http://bastion/") + request = get_tornado_request(bastion_address) + expected_url = "http://bastion" + self.assertEqual(expected_url, request.url) + + request = get_tornado_request(bastion_address, path="resource") + expected_url = "http://bastion/resource" + self.assertEqual(expected_url, request.url) + + request = get_tornado_request(bastion_address, path="/resource") + expected_url = "http://bastion/resource" + self.assertEqual(expected_url, request.url) + + request = get_tornado_request(bastion_address, path="resource/") + expected_url = "http://bastion/resource/" + self.assertEqual(expected_url, request.url) + + request = get_tornado_request(bastion_address, path="/resource/") + expected_url = "http://bastion/resource/" + self.assertEqual(expected_url, request.url) class TornadoTransportTestCase(TornadoAsyncTestCase): - """ Tornado based client test case. """ def get_launcher(self) -> ProcessLauncher: application_dir = chdir_fixture_app("bastiontest") From e82906ed023198cbce8842fdc3e301585e13a429 Mon Sep 17 00:00:00 2001 From: Flavio Garcia Date: Sun, 7 Apr 2024 14:34:25 -0400 Subject: [PATCH 06/13] feat(transport): add requests transport to the project Refs: #9 --- peasant/client/transport.py | 27 +++++++++- peasant/client/transport_requests.py | 73 ++++++++++++++++++++++++++++ requirements/requests.txt | 1 + setup.py | 1 + tests/requests_test.py | 41 ++++++++++++++++ tests/runtests.py | 3 +- 6 files changed, 144 insertions(+), 2 deletions(-) create mode 100644 peasant/client/transport_requests.py create mode 100644 requirements/requests.txt create mode 100644 tests/requests_test.py diff --git a/peasant/client/transport.py b/peasant/client/transport.py index e1f8b90..ec0ff4b 100644 --- a/peasant/client/transport.py +++ b/peasant/client/transport.py @@ -16,7 +16,7 @@ import logging import typing -from urllib.parse import urlparse +from urllib.parse import parse_qsl, urlparse if typing.TYPE_CHECKING: from peasant.client.protocol import Peasant @@ -24,6 +24,31 @@ logger = logging.getLogger(__name__) +def concat_url(url: str, **kwargs) -> str: + """ Concatenate a given url to a path, and query string if informed. + + :param str url: Base url + :param dict kwargs: + :key path: Path to be added to the returned url + :key query_string: Query string to be added to the returned url + """ + path = kwargs.get("path", None) + query_string = kwargs.get('query_string') + if query_string: + if isinstance(query_string, dict): + query_string = parse_qsl(query_string, keep_blank_values=True) + else: + err = (f"'query_string' parameter should be dict, or string. " + f"Not {type(query_string)}") + raise TypeError(err) + path = f"{path}?{path}" + if not url.endswith("/"): + url = f"{url}/" + if path is not None and path != "/": + url = f"{url}{path}" + return url + + def fix_address(address): parsed_address = urlparse(address) if parsed_address.path.endswith("/"): diff --git a/peasant/client/transport_requests.py b/peasant/client/transport_requests.py new file mode 100644 index 0000000..60f4b6e --- /dev/null +++ b/peasant/client/transport_requests.py @@ -0,0 +1,73 @@ +# Copyright 2020-2024 Flavio Garcia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import copy +import logging +from peasant import get_version +from peasant.client.transport import concat_url, Transport + +logger = logging.getLogger(__name__) + +requests_installed = False + +try: + import requests + requests_installed = True + +except ImportError: + pass + + +class RequestsTransport(Transport): + + basic_headers: dict + user_agent: str + + def __init__(self, bastion_address): + super().__init__() + if not requests_installed: + logger.warn("RequestsTransport cannot be used without requests " + "installed.\nIt is necessary to install peasant " + "with extras modifiers all or requests.\n\n Ex: pip " + "install peasant[all] or pip install peasant[requests]" + "\n\nInstalling requests manually will also work.\n") + raise NotImplementedError + self._bastion_address = bastion_address + self._directory = None + self.user_agent = (f"Peasant/{get_version()}" + f"Requests/{requests.__version__}") + self.basic_headers = { + 'User-Agent': self.user_agent + } + + def _get_path(self, path, **kwargs): + query_string = kwargs.get('query_string') + if query_string: + path = concat_url(path, query_string=query_string) + + def get_headers(self, **kwargs): + headers = copy.deepcopy(self.basic_headers) + _headers = kwargs.get('headers') + if _headers: + headers.update(_headers) + return headers + + async def get(self, path, **kwargs): + url = concat_url(self._bastion_address, **kwargs) + headers = self.get_headers(**kwargs) + try: + result = requests.get(url, headers=headers) + except requests.HTTPError as error: + result = error.response + return result diff --git a/requirements/requests.txt b/requirements/requests.txt new file mode 100644 index 0000000..2c24336 --- /dev/null +++ b/requirements/requests.txt @@ -0,0 +1 @@ +requests==2.31.0 diff --git a/setup.py b/setup.py index 5c7ec73..5033277 100644 --- a/setup.py +++ b/setup.py @@ -59,6 +59,7 @@ def resolve_requires(requirements_file): author_email=peasant.get_author_email(), extras_require={ 'all': resolve_requires("requirements/all.txt"), + 'requests': resolve_requires("requirements/requests.txt"), 'tornado': resolve_requires("requirements/tornado.txt"), }, classifiers=[ diff --git a/tests/requests_test.py b/tests/requests_test.py new file mode 100644 index 0000000..2deba1f --- /dev/null +++ b/tests/requests_test.py @@ -0,0 +1,41 @@ +# Copyright 2020-2024 Flavio Garcia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from firenado.testing import TornadoAsyncTestCase +from firenado.launcher import ProcessLauncher +from peasant.client.transport_requests import RequestsTransport +from tests import chdir_fixture_app, PROJECT_ROOT +from tornado.testing import gen_test + + +class RequestsTransportTestCase(TornadoAsyncTestCase): + """ Tornado based client test case. """ + + def get_launcher(self) -> ProcessLauncher: + application_dir = chdir_fixture_app("bastiontest") + return ProcessLauncher( + dir=application_dir, path=PROJECT_ROOT) + + def setUp(self) -> None: + super().setUp() + self.transport = RequestsTransport( + f"http://localhost:{self.http_port()}") + + @gen_test + async def test_get(self): + try: + response = await self.transport.get("/") + except Exception as e: + raise e + self.assertEqual(response.content, b"Get method output") diff --git a/tests/runtests.py b/tests/runtests.py index ac67bca..477b4b5 100644 --- a/tests/runtests.py +++ b/tests/runtests.py @@ -15,12 +15,13 @@ # limitations under the License. import unittest -from tests import tornado_test, transport_test +from tests import requests_test, tornado_test, transport_test def suite(): testLoader = unittest.TestLoader() alltests = unittest.TestSuite() + alltests.addTests(testLoader.loadTestsFromModule(requests_test)) alltests.addTests(testLoader.loadTestsFromModule(tornado_test)) alltests.addTests(testLoader.loadTestsFromModule(transport_test)) return alltests From c3e75d376be5bec122808ba11fbb14bd3bef7784 Mon Sep 17 00:00:00 2001 From: Flavio Garcia Date: Sun, 7 Apr 2024 14:40:54 -0400 Subject: [PATCH 07/13] build(requirements): add requests extras require all Refs: #9 --- requirements/all.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements/all.txt b/requirements/all.txt index 4c7e720..0fa6f1b 100644 --- a/requirements/all.txt +++ b/requirements/all.txt @@ -1,2 +1,3 @@ -r basic.txt +-r requests.txt -r tornado.txt From a6f5896cc3b69c2c7b09bdcf9fa49778754566d0 Mon Sep 17 00:00:00 2001 From: Flavio Garcia Date: Sun, 7 Apr 2024 17:37:19 -0400 Subject: [PATCH 08/13] refactor(transport): use same url handlering for both transports Refs: #6 #9 --- peasant/client/transport.py | 18 ++++++------ peasant/client/transport_tornado.py | 45 ++++++++++------------------- tests/tornado_test.py | 36 +++-------------------- tests/transport_test.py | 44 +++++++++++++++++++++++++++- 4 files changed, 71 insertions(+), 72 deletions(-) diff --git a/peasant/client/transport.py b/peasant/client/transport.py index ec0ff4b..cc3fdbf 100644 --- a/peasant/client/transport.py +++ b/peasant/client/transport.py @@ -16,7 +16,7 @@ import logging import typing -from urllib.parse import parse_qsl, urlparse +from urllib.parse import urlencode, urlparse if typing.TYPE_CHECKING: from peasant.client.protocol import Peasant @@ -24,7 +24,7 @@ logger = logging.getLogger(__name__) -def concat_url(url: str, **kwargs) -> str: +def concat_url(url: str, **kwargs: dict) -> str: """ Concatenate a given url to a path, and query string if informed. :param str url: Base url @@ -33,19 +33,19 @@ def concat_url(url: str, **kwargs) -> str: :key query_string: Query string to be added to the returned url """ path = kwargs.get("path", None) - query_string = kwargs.get('query_string') + query_string = kwargs.get("query_string", None) if query_string: if isinstance(query_string, dict): - query_string = parse_qsl(query_string, keep_blank_values=True) - else: + query_string = urlencode(query_string) + if not isinstance(query_string, str): err = (f"'query_string' parameter should be dict, or string. " f"Not {type(query_string)}") raise TypeError(err) - path = f"{path}?{path}" - if not url.endswith("/"): - url = f"{url}/" + path = f"{path}?{query_string}" if path is not None and path != "/": - url = f"{url}{path}" + if path.startswith("/"): + path = path[1:] + url = f"{url}/{path}" return url diff --git a/peasant/client/transport_tornado.py b/peasant/client/transport_tornado.py index c89d516..619b55b 100644 --- a/peasant/client/transport_tornado.py +++ b/peasant/client/transport_tornado.py @@ -15,8 +15,7 @@ import copy import logging from peasant import get_version -from peasant.client.transport import fix_address, Transport -from urllib.parse import urlencode +from peasant.client.transport import concat_url, fix_address, Transport logger = logging.getLogger(__name__) @@ -24,7 +23,6 @@ try: from tornado.httpclient import HTTPRequest from tornado import version as tornado_version - from tornado.httputil import url_concat from tornado.httpclient import AsyncHTTPClient, HTTPClientError tornado_installed = True @@ -45,13 +43,11 @@ def get_tornado_request(url, **kwargs): :return HTTPRequest: """ method = kwargs.get("method", "GET") - path = kwargs.get("path", None) form_urlencoded = kwargs.get("form_urlencoded", False) - if path is not None: - if not path.startswith("/"): - path = f"/{path}" - url = f"{url}{path}" request = HTTPRequest(url, method=method) + body = kwargs.get("body", None) + if body: + request.body = body if form_urlencoded: request.headers.add("Content-Type", "application/x-www-form-urlencoded") @@ -80,12 +76,6 @@ def __init__(self, bastion_address) -> None: 'User-Agent': self.user_agent } - def get_path(self, path, **kwargs) -> str: - query_string = kwargs.get('query_string') - if query_string: - path = url_concat(path, query_string) - return path - def get_headers(self, **kwargs): headers = copy.deepcopy(self._basic_headers) _headers = kwargs.get('headers') @@ -93,24 +83,21 @@ def get_headers(self, **kwargs): headers.update(_headers) return headers - async def get(self, path, **kwargs): - path = self.get_path(path, **kwargs) - body = kwargs.get("body") - request = get_tornado_request(self._bastion_address, path=path) + async def get(self, **kwargs): + url = concat_url(self._bastion_address, **kwargs) + request = get_tornado_request(url, **kwargs) headers = self.get_headers(**kwargs) request.headers.update(headers) - if body: - request.body = body try: result = await self._client.fetch(request) except HTTPClientError as error: result = error.response return result - async def head(self, path, **kwargs): - path = self.get_path(path, **kwargs) - request = get_tornado_request(self._bastion_address, path=path, - method="HEAD") + async def head(self, **kwargs): + url = concat_url(self._bastion_address, **kwargs) + kwargs["method"] = "HEAD" + request = get_tornado_request(url, **kwargs) headers = self.get_headers(**kwargs) request.headers.update(headers) try: @@ -119,14 +106,12 @@ async def head(self, path, **kwargs): result = error.response return result - async def post(self, path, **kwargs): - path = self.get_path(path, **kwargs) - request = get_tornado_request(self._bastion_address, path=path, - method="POST") + async def post(self, **kwargs): + url = concat_url(self._bastion_address, **kwargs) + kwargs["method"] = "POST" + request = get_tornado_request(url, **kwargs) headers = self.get_headers(**kwargs) - body = kwargs.get("body", []) request.headers.update(headers) - request.body = urlencode(body) try: result = await self._client.fetch(request) except HTTPClientError as error: diff --git a/tests/tornado_test.py b/tests/tornado_test.py index da15a3f..ca69ee6 100644 --- a/tests/tornado_test.py +++ b/tests/tornado_test.py @@ -14,37 +14,9 @@ from firenado.testing import TornadoAsyncTestCase from firenado.launcher import ProcessLauncher -from peasant.client.transport import fix_address -from peasant.client.transport_tornado import (get_tornado_request, - TornadoTransport) +from peasant.client.transport_tornado import TornadoTransport from tests import chdir_fixture_app, PROJECT_ROOT from tornado.testing import gen_test -from unittest import TestCase - - -class GetTornadoRequestTestCase(TestCase): - - def test_get_tornado_request(self): - bastion_address = fix_address("http://bastion/") - request = get_tornado_request(bastion_address) - expected_url = "http://bastion" - self.assertEqual(expected_url, request.url) - - request = get_tornado_request(bastion_address, path="resource") - expected_url = "http://bastion/resource" - self.assertEqual(expected_url, request.url) - - request = get_tornado_request(bastion_address, path="/resource") - expected_url = "http://bastion/resource" - self.assertEqual(expected_url, request.url) - - request = get_tornado_request(bastion_address, path="resource/") - expected_url = "http://bastion/resource/" - self.assertEqual(expected_url, request.url) - - request = get_tornado_request(bastion_address, path="/resource/") - expected_url = "http://bastion/resource/" - self.assertEqual(expected_url, request.url) class TornadoTransportTestCase(TornadoAsyncTestCase): @@ -62,7 +34,7 @@ def setUp(self) -> None: @gen_test async def test_head(self): try: - response = await self.transport.head("/head") + response = await self.transport.head(path="/head") except Exception as e: raise e self.assertEqual(response.headers.get("head-response"), @@ -73,7 +45,7 @@ async def test_head(self): @gen_test async def test_get(self): try: - response = await self.transport.get("/") + response = await self.transport.get(path="/") except Exception as e: raise e self.assertEqual(response.body, b"Get method output") @@ -81,7 +53,7 @@ async def test_get(self): @gen_test async def test_post(self): try: - response = await self.transport.post("/post") + response = await self.transport.post(path="/post", body="empty") except Exception as e: raise e self.assertEqual(response.body, b"Post method output") diff --git a/tests/transport_test.py b/tests/transport_test.py index 6b1da21..f2d9a7e 100644 --- a/tests/transport_test.py +++ b/tests/transport_test.py @@ -12,12 +12,54 @@ # See the License for the specific language governing permissions and # limitations under the License. -from peasant.client.transport import fix_address +from peasant.client.transport import concat_url, fix_address from unittest import TestCase class TransportTestCase(TestCase): + def test_concat_url(self): + bastion_address = fix_address("http://bastion/") + url = concat_url(bastion_address) + expected_url = "http://bastion" + self.assertEqual(expected_url, url) + + url = concat_url(bastion_address, path="resource") + expected_url = "http://bastion/resource" + self.assertEqual(expected_url, url) + + url = concat_url(bastion_address, path="/resource") + expected_url = "http://bastion/resource" + self.assertEqual(expected_url, url) + + url = concat_url(bastion_address, path="resource/") + expected_url = "http://bastion/resource/" + self.assertEqual(expected_url, url) + + url = concat_url(bastion_address, path="/resource/") + expected_url = "http://bastion/resource/" + self.assertEqual(expected_url, url) + + url = concat_url(bastion_address, path="/resource", + query_string={}) + expected_url = "http://bastion/resource" + self.assertEqual(expected_url, url) + + url = concat_url(bastion_address, path="/resource", + query_string="abc=1") + expected_url = "http://bastion/resource?abc=1" + self.assertEqual(expected_url, url) + + url = concat_url(bastion_address, path="/resource", + query_string={"abc": 1}) + expected_url = "http://bastion/resource?abc=1" + self.assertEqual(expected_url, url) + + url = concat_url(bastion_address, path="/resource", + query_string={"abc": 1, "def": 2}) + expected_url = "http://bastion/resource?abc=1&def=2" + self.assertEqual(expected_url, url) + def test_fix_address(self): address = "http://localhost" expected_address = "http://localhost" From 010c83d6d2a07b7a726fdee7f4c5b9052c2a19d5 Mon Sep 17 00:00:00 2001 From: Flavio Garcia Date: Mon, 8 Apr 2024 00:31:38 -0400 Subject: [PATCH 09/13] feat(transport): add head and post methods to requests transport Also renamed tranpost implementation tests files. Refs: #6 #9 --- peasant/client/transport.py | 4 +- peasant/client/transport_requests.py | 37 ++++++++++++++----- peasant/client/transport_tornado.py | 20 +++++----- tests/runtests.py | 7 ++-- ...sts_test.py => transport_requests_test.py} | 32 +++++++++++++++- ...nado_test.py => transport_tornado_test.py} | 8 ++-- 6 files changed, 80 insertions(+), 28 deletions(-) rename tests/{requests_test.py => transport_requests_test.py} (51%) rename tests/{tornado_test.py => transport_tornado_test.py} (86%) diff --git a/peasant/client/transport.py b/peasant/client/transport.py index cc3fdbf..2958a8f 100644 --- a/peasant/client/transport.py +++ b/peasant/client/transport.py @@ -24,15 +24,15 @@ logger = logging.getLogger(__name__) -def concat_url(url: str, **kwargs: dict) -> str: +def concat_url(url: str, path: str = None, **kwargs: dict) -> str: """ Concatenate a given url to a path, and query string if informed. :param str url: Base url + :param str path: Path to be added to the returned url :param dict kwargs: :key path: Path to be added to the returned url :key query_string: Query string to be added to the returned url """ - path = kwargs.get("path", None) query_string = kwargs.get("query_string", None) if query_string: if isinstance(query_string, dict): diff --git a/peasant/client/transport_requests.py b/peasant/client/transport_requests.py index 60f4b6e..0ab3c74 100644 --- a/peasant/client/transport_requests.py +++ b/peasant/client/transport_requests.py @@ -45,17 +45,12 @@ def __init__(self, bastion_address): raise NotImplementedError self._bastion_address = bastion_address self._directory = None - self.user_agent = (f"Peasant/{get_version()}" + self.user_agent = (f"Peasant/{get_version()} " f"Requests/{requests.__version__}") self.basic_headers = { 'User-Agent': self.user_agent } - def _get_path(self, path, **kwargs): - query_string = kwargs.get('query_string') - if query_string: - path = concat_url(path, query_string=query_string) - def get_headers(self, **kwargs): headers = copy.deepcopy(self.basic_headers) _headers = kwargs.get('headers') @@ -63,11 +58,35 @@ def get_headers(self, **kwargs): headers.update(_headers) return headers - async def get(self, path, **kwargs): - url = concat_url(self._bastion_address, **kwargs) + def head(self, path, **kwargs): + url = concat_url(self._bastion_address, path, **kwargs) + headers = self.get_headers(**kwargs) + kwargs['headers'] = headers + try: + with requests.head(url, **kwargs) as result: + result.raise_for_status() + except requests.HTTPError as error: + raise error + return result + + def get(self, path, **kwargs): + url = concat_url(self._bastion_address, path, **kwargs) headers = self.get_headers(**kwargs) + kwargs['headers'] = headers try: - result = requests.get(url, headers=headers) + result = requests.get(url, **kwargs) + result.raise_for_status() except requests.HTTPError as error: result = error.response return result + + def post(self, path, **kwargs): + url = concat_url(self._bastion_address, path, **kwargs) + headers = self.get_headers(**kwargs) + kwargs['headers'] = headers + try: + with requests.post(url, **kwargs) as result: + result.raise_for_status() + except requests.HTTPError as error: + raise error + return result diff --git a/peasant/client/transport_tornado.py b/peasant/client/transport_tornado.py index 619b55b..ae5409c 100644 --- a/peasant/client/transport_tornado.py +++ b/peasant/client/transport_tornado.py @@ -70,7 +70,7 @@ def __init__(self, bastion_address) -> None: self._client = AsyncHTTPClient() self._bastion_address = fix_address(bastion_address) self._directory = None - self.user_agent = (f"Peasant/{get_version()}" + self.user_agent = (f"Peasant/{get_version()} " f"Tornado/{tornado_version}") self._basic_headers = { 'User-Agent': self.user_agent @@ -83,19 +83,19 @@ def get_headers(self, **kwargs): headers.update(_headers) return headers - async def get(self, **kwargs): - url = concat_url(self._bastion_address, **kwargs) + async def get(self, path: str, **kwargs: dict): + url = concat_url(self._bastion_address, path, **kwargs) request = get_tornado_request(url, **kwargs) headers = self.get_headers(**kwargs) request.headers.update(headers) try: result = await self._client.fetch(request) except HTTPClientError as error: - result = error.response + raise error return result - async def head(self, **kwargs): - url = concat_url(self._bastion_address, **kwargs) + async def head(self, path: str, **kwargs: dict): + url = concat_url(self._bastion_address, path, **kwargs) kwargs["method"] = "HEAD" request = get_tornado_request(url, **kwargs) headers = self.get_headers(**kwargs) @@ -103,11 +103,11 @@ async def head(self, **kwargs): try: result = await self._client.fetch(request) except HTTPClientError as error: - result = error.response + raise error return result - async def post(self, **kwargs): - url = concat_url(self._bastion_address, **kwargs) + async def post(self, path: str, **kwargs: dict): + url = concat_url(self._bastion_address, path, **kwargs) kwargs["method"] = "POST" request = get_tornado_request(url, **kwargs) headers = self.get_headers(**kwargs) @@ -115,5 +115,5 @@ async def post(self, **kwargs): try: result = await self._client.fetch(request) except HTTPClientError as error: - result = error.response + raise error return result diff --git a/tests/runtests.py b/tests/runtests.py index 477b4b5..198aead 100644 --- a/tests/runtests.py +++ b/tests/runtests.py @@ -15,15 +15,16 @@ # limitations under the License. import unittest -from tests import requests_test, tornado_test, transport_test +from tests import (transport_requests_test, transport_test, + transport_tornado_test) def suite(): testLoader = unittest.TestLoader() alltests = unittest.TestSuite() - alltests.addTests(testLoader.loadTestsFromModule(requests_test)) - alltests.addTests(testLoader.loadTestsFromModule(tornado_test)) + alltests.addTests(testLoader.loadTestsFromModule(transport_requests_test)) alltests.addTests(testLoader.loadTestsFromModule(transport_test)) + alltests.addTests(testLoader.loadTestsFromModule(transport_tornado_test)) return alltests diff --git a/tests/requests_test.py b/tests/transport_requests_test.py similarity index 51% rename from tests/requests_test.py rename to tests/transport_requests_test.py index 2deba1f..f53c49c 100644 --- a/tests/requests_test.py +++ b/tests/transport_requests_test.py @@ -29,13 +29,43 @@ def get_launcher(self) -> ProcessLauncher: def setUp(self) -> None: super().setUp() + # setting tests simplefilter to ignore because requests uses a + # keep-alive model, not closing sockets explicitly in many cases. + # with that will cause the ResourceWarning warn be displayed in testing + # as unittests will set warnings.simplefilter to default. + # See: + # - https://github.com/psf/requests/issues/3912#issuecomment-284328247 + # - https://python.readthedocs.io/en/stable/library/warnings.html#updating-code-for-new-versions-of-python + import warnings + warnings.simplefilter("ignore") self.transport = RequestsTransport( f"http://localhost:{self.http_port()}") + @gen_test + async def test_head(self): + try: + response = self.transport.head("/head") + except Exception as e: + raise e + self.assertEqual(response.headers.get("head-response"), + "Head method response") + self.assertEqual(response.headers.get("user-agent"), + self.transport.user_agent) + @gen_test async def test_get(self): try: - response = await self.transport.get("/") + response = self.transport.get("/") except Exception as e: raise e self.assertEqual(response.content, b"Get method output") + + @gen_test + async def test_post(self): + expected_body = "da body" + try: + response = self.transport.post("/post", data="da body") + except Exception as e: + raise e + self.assertEqual(expected_body, response.headers.get("request-body")) + self.assertEqual(response.content, b"Post method output") diff --git a/tests/tornado_test.py b/tests/transport_tornado_test.py similarity index 86% rename from tests/tornado_test.py rename to tests/transport_tornado_test.py index ca69ee6..19c3d01 100644 --- a/tests/tornado_test.py +++ b/tests/transport_tornado_test.py @@ -34,7 +34,7 @@ def setUp(self) -> None: @gen_test async def test_head(self): try: - response = await self.transport.head(path="/head") + response = await self.transport.head("/head") except Exception as e: raise e self.assertEqual(response.headers.get("head-response"), @@ -45,15 +45,17 @@ async def test_head(self): @gen_test async def test_get(self): try: - response = await self.transport.get(path="/") + response = await self.transport.get("/") except Exception as e: raise e self.assertEqual(response.body, b"Get method output") @gen_test async def test_post(self): + expected_body = "da body" try: - response = await self.transport.post(path="/post", body="empty") + response = await self.transport.post("/post", body="da body") except Exception as e: raise e + self.assertEqual(expected_body, response.headers.get("request-body")) self.assertEqual(response.body, b"Post method output") From 0978ed00991b463dc56eece31e3bbc9f16a39ac3 Mon Sep 17 00:00:00 2001 From: Flavio Garcia Date: Mon, 8 Apr 2024 01:39:07 -0400 Subject: [PATCH 10/13] feat(transport): add delete, options, patch, and put methods Those were added to the transport and request transport. Refs: #9 --- peasant/client/transport.py | 26 +++++++++-- peasant/client/transport_requests.py | 58 +++++++++++++++++++++--- tests/fixtures/bastiontest/app.py | 4 ++ tests/fixtures/bastiontest/handlers.py | 32 +++++++++++++ tests/transport_requests_test.py | 63 ++++++++++++++++++++++---- 5 files changed, 164 insertions(+), 19 deletions(-) diff --git a/peasant/client/transport.py b/peasant/client/transport.py index 2958a8f..ca357d9 100644 --- a/peasant/client/transport.py +++ b/peasant/client/transport.py @@ -68,16 +68,34 @@ def peasant(self) -> Peasant: def peasant(self, peasant: Peasant): self._peasant = peasant - def get(self, path, **kwargs): + def get_url(self, path: str, **kwargs: dict): + if (path.lower().startswith("http://") or + path.lower().startswith("https://")): + return concat_url(path, "", **kwargs) + return concat_url(self._bastion_address, path, **kwargs) + + def delete(self, path: str, **kwargs: dict): + raise NotImplementedError + + def get(self, path: str, **kwargs: dict): + raise NotImplementedError + + def head(self, path: str, **kwargs: dict): + raise NotImplementedError + + def options(self, path: str, **kwargs: dict): + raise NotImplementedError + + def patch(self, path: str, **kwargs: dict): raise NotImplementedError - def head(self, path, **kwargs): + def post(self, path: str, **kwargs: dict): raise NotImplementedError - def post(self, path, **kwargs): + def post_as_get(self, path: str, **kwargs: dict): raise NotImplementedError - def post_as_get(self, path, **kwargs): + def put(self, path: str, **kwargs: dict): raise NotImplementedError def set_directory(self): diff --git a/peasant/client/transport_requests.py b/peasant/client/transport_requests.py index 0ab3c74..6d5f775 100644 --- a/peasant/client/transport_requests.py +++ b/peasant/client/transport_requests.py @@ -15,7 +15,7 @@ import copy import logging from peasant import get_version -from peasant.client.transport import concat_url, Transport +from peasant.client.transport import Transport logger = logging.getLogger(__name__) @@ -58,19 +58,19 @@ def get_headers(self, **kwargs): headers.update(_headers) return headers - def head(self, path, **kwargs): - url = concat_url(self._bastion_address, path, **kwargs) + def delete(self, path: str, **kwargs): + url = self.get_url(path, **kwargs) headers = self.get_headers(**kwargs) kwargs['headers'] = headers try: - with requests.head(url, **kwargs) as result: - result.raise_for_status() + result = requests.delete(url, **kwargs) + result.raise_for_status() except requests.HTTPError as error: raise error return result def get(self, path, **kwargs): - url = concat_url(self._bastion_address, path, **kwargs) + url = self.get_url(path, **kwargs) headers = self.get_headers(**kwargs) kwargs['headers'] = headers try: @@ -80,8 +80,41 @@ def get(self, path, **kwargs): result = error.response return result + def head(self, path, **kwargs): + url = self.get_url(path, **kwargs) + headers = self.get_headers(**kwargs) + kwargs['headers'] = headers + try: + result = requests.head(url, **kwargs) + result.raise_for_status() + except requests.HTTPError as error: + raise error + return result + + def options(self, path, **kwargs): + url = self.get_url(path, **kwargs) + headers = self.get_headers(**kwargs) + kwargs['headers'] = headers + try: + result = requests.options(url, **kwargs) + result.raise_for_status() + except requests.HTTPError as error: + raise error + return result + + def patch(self, path, **kwargs): + url = self.get_url(path, **kwargs) + headers = self.get_headers(**kwargs) + kwargs['headers'] = headers + try: + with requests.patch(url, **kwargs) as result: + result.raise_for_status() + except requests.HTTPError as error: + raise error + return result + def post(self, path, **kwargs): - url = concat_url(self._bastion_address, path, **kwargs) + url = self.get_url(path, **kwargs) headers = self.get_headers(**kwargs) kwargs['headers'] = headers try: @@ -90,3 +123,14 @@ def post(self, path, **kwargs): except requests.HTTPError as error: raise error return result + + def put(self, path, **kwargs): + url = self.get_url(path, **kwargs) + headers = self.get_headers(**kwargs) + kwargs['headers'] = headers + try: + with requests.put(url, **kwargs) as result: + result.raise_for_status() + except requests.HTTPError as error: + raise error + return result diff --git a/tests/fixtures/bastiontest/app.py b/tests/fixtures/bastiontest/app.py index 5f34890..ed9a83f 100644 --- a/tests/fixtures/bastiontest/app.py +++ b/tests/fixtures/bastiontest/app.py @@ -7,6 +7,10 @@ class BastiontestComponent(tornadoweb.TornadoComponent): def get_handlers(self): return [ (r"/", handlers.GetHandler), + (r"/delete", handlers.DeleteHandler), (r"/head", handlers.HeadHandler), + (r"/options", handlers.OptionsHandler), + (r"/patch", handlers.PatchHandler), (r"/post", handlers.PostHandler), + (r"/put", handlers.PutHandler), ] diff --git a/tests/fixtures/bastiontest/handlers.py b/tests/fixtures/bastiontest/handlers.py index 1236572..b2fd03d 100644 --- a/tests/fixtures/bastiontest/handlers.py +++ b/tests/fixtures/bastiontest/handlers.py @@ -16,15 +16,47 @@ def head(self): self.add_header("user-agent", user_agent) +class DeleteHandler(tornadoweb.TornadoHandler): + + def delete(self): + body = self.request.body + self.add_header("request-body", body) + self.write("Delete method output") + + class GetHandler(tornadoweb.TornadoHandler): def get(self): self.write("Get method output") +class OptionsHandler(tornadoweb.TornadoHandler): + + def options(self): + body = "da body" + self.add_header("request-body", body) + self.write("Options method output") + + +class PatchHandler(tornadoweb.TornadoHandler): + + def patch(self): + body = self.request.body + self.add_header("request-body", body) + self.write("Patch method output") + + class PostHandler(tornadoweb.TornadoHandler): def post(self): body = self.request.body self.add_header("request-body", body) self.write("Post method output") + + +class PutHandler(tornadoweb.TornadoHandler): + + def put(self): + body = self.request.body + self.add_header("request-body", body) + self.write("Put method output") diff --git a/tests/transport_requests_test.py b/tests/transport_requests_test.py index f53c49c..bb99415 100644 --- a/tests/transport_requests_test.py +++ b/tests/transport_requests_test.py @@ -42,30 +42,77 @@ def setUp(self) -> None: f"http://localhost:{self.http_port()}") @gen_test - async def test_head(self): + async def test_delete(self): + expected_body = "da body" + expected_content = b"Delete method output" try: - response = self.transport.head("/head") + response = self.transport.delete("/delete", data="da body") except Exception as e: raise e - self.assertEqual(response.headers.get("head-response"), - "Head method response") - self.assertEqual(response.headers.get("user-agent"), - self.transport.user_agent) + self.assertEqual(expected_body, response.headers.get("request-body")) + self.assertEqual(expected_content, response.content) @gen_test async def test_get(self): + expected_content = b"Get method output" try: response = self.transport.get("/") except Exception as e: raise e - self.assertEqual(response.content, b"Get method output") + self.assertEqual(expected_content, response.content) + + @gen_test + async def test_head(self): + expected_head_response = "Head method response" + try: + response = self.transport.head("/head") + except Exception as e: + raise e + self.assertEqual(expected_head_response, + response.headers.get("head-response")) + self.assertEqual(self.transport.user_agent, + response.headers.get("user-agent")) + + @gen_test + async def test_options(self): + expected_body = "da body" + expected_content = b"Options method output" + try: + response = self.transport.options("/options") + except Exception as e: + raise e + self.assertEqual(expected_body, response.headers.get("request-body")) + self.assertEqual(expected_content, response.content) + + @gen_test + async def test_patch(self): + expected_body = "da body" + expected_content = b"Patch method output" + try: + response = self.transport.patch("/patch", data="da body") + except Exception as e: + raise e + self.assertEqual(expected_body, response.headers.get("request-body")) + self.assertEqual(expected_content, response.content) @gen_test async def test_post(self): expected_body = "da body" + expected_content = b"Post method output" try: response = self.transport.post("/post", data="da body") except Exception as e: raise e self.assertEqual(expected_body, response.headers.get("request-body")) - self.assertEqual(response.content, b"Post method output") + self.assertEqual(expected_content, response.content) + + @gen_test + async def test_put(self): + expected_body = "da body" + expected_content = b"Put method output" + try: + response = self.transport.put("/put", data="da body") + except Exception as e: + raise e + self.assertEqual(expected_body, response.headers.get("request-body")) + self.assertEqual(expected_content, response.content) From 17a920fee8a8b74f8c955223b35a819f2db5afe3 Mon Sep 17 00:00:00 2001 From: Flavio Garcia Date: Mon, 8 Apr 2024 02:27:58 -0400 Subject: [PATCH 11/13] feat(transport): add missing methods to transport tornado Fixes: #6 --- peasant/client/transport_tornado.py | 109 ++++++++++++++++++++++++- tests/fixtures/bastiontest/handlers.py | 2 +- tests/transport_requests_test.py | 2 +- tests/transport_tornado_test.py | 44 ++++++++++ 4 files changed, 151 insertions(+), 6 deletions(-) diff --git a/peasant/client/transport_tornado.py b/peasant/client/transport_tornado.py index ae5409c..d03625d 100644 --- a/peasant/client/transport_tornado.py +++ b/peasant/client/transport_tornado.py @@ -43,8 +43,61 @@ def get_tornado_request(url, **kwargs): :return HTTPRequest: """ method = kwargs.get("method", "GET") + + auth_username = kwargs.get("auth_username") + auth_password = kwargs.get("auth_password") + auth_mode = kwargs.get("auth_mode") + connect_timeout = kwargs.get("connect_timeout") + request_timeout = kwargs.get("request_timeout") + if_modified_since = kwargs.get("if_modified_since") + follow_redirects = kwargs.get("follow_redirects") + max_redirects = kwargs.get("max_redirects") + user_agent = kwargs.get("user_agent") + use_gzip = kwargs.get("use_gzip") + network_interface = kwargs.get("network_interface") + streaming_callback = kwargs.get("streaming_callback") + header_callback = kwargs.get("header_callback") + prepare_curl_callback = kwargs.get("prepare_curl_callback") + proxy_host = kwargs.get("proxy_host") + proxy_port = kwargs.get("proxy_port") + proxy_username = kwargs.get("proxy_username") + proxy_password = kwargs.get("proxy_password") + proxy_auth_mode = kwargs.get("proxy_auth_mode") + allow_nonstandard_methods = kwargs.get("allow_nonstandard_methods") + validate_cert = kwargs.get("validate_cert") + ca_certs = kwargs.get("ca_certs") + allow_ipv6 = kwargs.get("allow_ipv6") + client_key = kwargs.get("client_key") + client_cert = kwargs.get("client_cert") + body_producer = kwargs.get("body_producer") + expect_100_continue = kwargs.get("expect_100_continue") + decompress_response = kwargs.get("decompress_response") + ssl_options = kwargs.get("ssl_options") + form_urlencoded = kwargs.get("form_urlencoded", False) - request = HTTPRequest(url, method=method) + request = HTTPRequest( + url, method=method, headers=None, body=None, + auth_username=auth_username, auth_password=auth_password, + auth_mode=auth_mode, connect_timeout=connect_timeout, + request_timeout=request_timeout, + if_modified_since=if_modified_since, + follow_redirects=follow_redirects, max_redirects=max_redirects, + user_agent=user_agent, use_gzip=use_gzip, + network_interface=network_interface, + streaming_callback=streaming_callback, + header_callback=header_callback, + prepare_curl_callback=prepare_curl_callback, + proxy_host=proxy_host, proxy_port=proxy_port, + proxy_username=proxy_username, proxy_password=proxy_password, + proxy_auth_mode=proxy_auth_mode, + allow_nonstandard_methods=allow_nonstandard_methods, + validate_cert=validate_cert, ca_certs=ca_certs, + allow_ipv6=allow_ipv6, client_key=client_key, + client_cert=client_cert, body_producer=body_producer, + expect_100_continue=expect_100_continue, + decompress_response=decompress_response, + ssl_options=ssl_options, + ) body = kwargs.get("body", None) if body: request.body = body @@ -83,8 +136,20 @@ def get_headers(self, **kwargs): headers.update(_headers) return headers + async def delete(self, path: str, **kwargs: dict): + url = self.get_url(path, **kwargs) + kwargs["method"] = "DELETE" + request = get_tornado_request(url, **kwargs) + headers = self.get_headers(**kwargs) + request.headers.update(headers) + try: + result = await self._client.fetch(request) + except HTTPClientError as error: + raise error + return result + async def get(self, path: str, **kwargs: dict): - url = concat_url(self._bastion_address, path, **kwargs) + url = self.get_url(path, **kwargs) request = get_tornado_request(url, **kwargs) headers = self.get_headers(**kwargs) request.headers.update(headers) @@ -95,7 +160,7 @@ async def get(self, path: str, **kwargs: dict): return result async def head(self, path: str, **kwargs: dict): - url = concat_url(self._bastion_address, path, **kwargs) + url = self.get_url(path, **kwargs) kwargs["method"] = "HEAD" request = get_tornado_request(url, **kwargs) headers = self.get_headers(**kwargs) @@ -106,8 +171,32 @@ async def head(self, path: str, **kwargs: dict): raise error return result + async def options(self, path: str, **kwargs: dict): + url = self.get_url(path, **kwargs) + kwargs["method"] = "OPTIONS" + request = get_tornado_request(url, **kwargs) + headers = self.get_headers(**kwargs) + request.headers.update(headers) + try: + result = await self._client.fetch(request) + except HTTPClientError as error: + raise error + return result + + async def patch(self, path: str, **kwargs: dict): + url = self.get_url(path, **kwargs) + kwargs["method"] = "PATCH" + request = get_tornado_request(url, **kwargs) + headers = self.get_headers(**kwargs) + request.headers.update(headers) + try: + result = await self._client.fetch(request) + except HTTPClientError as error: + raise error + return result + async def post(self, path: str, **kwargs: dict): - url = concat_url(self._bastion_address, path, **kwargs) + url = self.get_url(path, **kwargs) kwargs["method"] = "POST" request = get_tornado_request(url, **kwargs) headers = self.get_headers(**kwargs) @@ -117,3 +206,15 @@ async def post(self, path: str, **kwargs: dict): except HTTPClientError as error: raise error return result + + async def put(self, path: str, **kwargs: dict): + url = self.get_url(path, **kwargs) + kwargs["method"] = "PUT" + request = get_tornado_request(url, **kwargs) + headers = self.get_headers(**kwargs) + request.headers.update(headers) + try: + result = await self._client.fetch(request) + except HTTPClientError as error: + raise error + return result diff --git a/tests/fixtures/bastiontest/handlers.py b/tests/fixtures/bastiontest/handlers.py index b2fd03d..f332272 100644 --- a/tests/fixtures/bastiontest/handlers.py +++ b/tests/fixtures/bastiontest/handlers.py @@ -19,7 +19,7 @@ def head(self): class DeleteHandler(tornadoweb.TornadoHandler): def delete(self): - body = self.request.body + body = "da body" self.add_header("request-body", body) self.write("Delete method output") diff --git a/tests/transport_requests_test.py b/tests/transport_requests_test.py index bb99415..2fd5f0d 100644 --- a/tests/transport_requests_test.py +++ b/tests/transport_requests_test.py @@ -46,7 +46,7 @@ async def test_delete(self): expected_body = "da body" expected_content = b"Delete method output" try: - response = self.transport.delete("/delete", data="da body") + response = self.transport.delete("/delete") except Exception as e: raise e self.assertEqual(expected_body, response.headers.get("request-body")) diff --git a/tests/transport_tornado_test.py b/tests/transport_tornado_test.py index 19c3d01..aaf6be4 100644 --- a/tests/transport_tornado_test.py +++ b/tests/transport_tornado_test.py @@ -42,6 +42,17 @@ async def test_head(self): self.assertEqual(response.headers.get("user-agent"), self.transport.user_agent) + @gen_test + async def test_delete(self): + expected_body = "da body" + expected_content = b"Delete method output" + try: + response = await self.transport.delete("/delete") + except Exception as e: + raise e + self.assertEqual(expected_body, response.headers.get("request-body")) + self.assertEqual(expected_content, response.body) + @gen_test async def test_get(self): try: @@ -50,6 +61,28 @@ async def test_get(self): raise e self.assertEqual(response.body, b"Get method output") + @gen_test + async def test_options(self): + expected_body = "da body" + expected_content = b"Options method output" + try: + response = await self.transport.options("/options") + except Exception as e: + raise e + self.assertEqual(expected_body, response.headers.get("request-body")) + self.assertEqual(expected_content, response.body) + + @gen_test + async def test_patch(self): + expected_body = "da body" + expected_content = b"Patch method output" + try: + response = await self.transport.patch("/patch", body="da body") + except Exception as e: + raise e + self.assertEqual(expected_body, response.headers.get("request-body")) + self.assertEqual(expected_content, response.body) + @gen_test async def test_post(self): expected_body = "da body" @@ -59,3 +92,14 @@ async def test_post(self): raise e self.assertEqual(expected_body, response.headers.get("request-body")) self.assertEqual(response.body, b"Post method output") + + @gen_test + async def test_put(self): + expected_body = "da body" + expected_content = b"Put method output" + try: + response = await self.transport.put("/put", body="da body") + except Exception as e: + raise e + self.assertEqual(expected_body, response.headers.get("request-body")) + self.assertEqual(expected_content, response.body) From 8beb67c9815b5906b70206b3883872134c6de7f5 Mon Sep 17 00:00:00 2001 From: Flavio Garcia Date: Wed, 10 Apr 2024 20:24:51 -0400 Subject: [PATCH 12/13] build(release): deliver 0.7.0 Fixes: #9 --- peasant/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/peasant/__init__.py b/peasant/__init__.py index 9401a93..acdfaf1 100644 --- a/peasant/__init__.py +++ b/peasant/__init__.py @@ -13,7 +13,7 @@ # limitations under the License. __author__ = "Flavio Garcia " -__version__ = (0, 6, 1) +__version__ = (0, 7, 0) __licence__ = "Apache License V2.0" From 2f5264aa621a1174633d7f818196f0c919830804 Mon Sep 17 00:00:00 2001 From: Flavio Garcia Date: Thu, 11 Apr 2024 10:47:00 -0400 Subject: [PATCH 13/13] fix(build): change resolve_requires to be independent of pip This solution need only `os` pacakge, for some file handling. Fixes: #11 --- MANIFEST.in | 1 + requirements/tornado.txt | 2 +- setup.py | 39 ++++++++++++++++----------------------- 3 files changed, 18 insertions(+), 24 deletions(-) diff --git a/MANIFEST.in b/MANIFEST.in index 86fef0b..d11de22 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -3,3 +3,4 @@ include LICENSE include requirements/all.txt include requirements/basic.txt include requirements/tornado.txt +include requirements/requests.txt diff --git a/requirements/tornado.txt b/requirements/tornado.txt index 7941d78..0675fd4 100644 --- a/requirements/tornado.txt +++ b/requirements/tornado.txt @@ -1 +1 @@ -tornado >= 6.3 +tornado>=6.3 diff --git a/setup.py b/setup.py index 5033277..2ab8b9c 100644 --- a/setup.py +++ b/setup.py @@ -15,36 +15,29 @@ # limitations under the License. import peasant -from codecs import open +import os from setuptools import find_packages, setup -import sys - -try: - # for pip >= 10 - from pip._internal.req import parse_requirements -except ImportError: - # for pip <= 9.0.3 - print("error: Upgrade to a pip version newer than 10. Run \"pip install " - "--upgrade pip\".") - sys.exit(1) with open("README.md", "r") as fh: long_description = fh.read() -# Solution from http://bit.ly/29Yl8VN def resolve_requires(requirements_file): - try: - requirements = parse_requirements("./%s" % requirements_file, - session=False) - return [str(ir.req) for ir in requirements] - except AttributeError: - # for pip >= 20.1.x - # Need to run again as the first run was ruined by the exception - requirements = parse_requirements("./%s" % requirements_file, - session=False) - # pr stands for parsed_requirement - return [str(pr.requirement) for pr in requirements] + requires = [] + if os.path.isfile(f"./{requirements_file}"): + file_dir = os.path.dirname(f"./{requirements_file}") + with open(f"./{requirements_file}") as f: + for raw_line in f.readlines(): + line = raw_line.strip().replace("\n", "") + if len(line) > 0: + if line.startswith("-r "): + partial_file = os.path.join(file_dir, line.replace( + "-r ", "")) + partial_requires = resolve_requires(partial_file) + requires = requires + partial_requires + continue + requires.append(line) + return requires setup(