From 5f9833160959552231c72f7127cf6b7774f6746b Mon Sep 17 00:00:00 2001 From: Ben Darnell Date: Fri, 7 Jun 2024 14:28:07 -0400 Subject: [PATCH 1/5] Update mypy and typing packages to latest Adapt to newer upstream types for AbstractEventLoop --- requirements.txt | 8 ++++---- tornado/platform/asyncio.py | 15 +++++++++++++-- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/requirements.txt b/requirements.txt index 9118bdfde6..8dcc978d32 100644 --- a/requirements.txt +++ b/requirements.txt @@ -48,9 +48,9 @@ markupsafe==2.1.2 # via jinja2 mccabe==0.7.0 # via flake8 -mypy==1.0.1 +mypy==1.10.0 # via -r requirements.in -mypy-extensions==0.4.3 +mypy-extensions==1.0.0 # via # black # mypy @@ -111,9 +111,9 @@ sphinxcontrib-serializinghtml==1.1.5 # via sphinx tox==4.6.0 # via -r requirements.in -types-pycurl==7.45.2.0 +types-pycurl==7.45.3.20240421 # via -r requirements.in -typing-extensions==4.4.0 +typing-extensions==4.12.1 # via mypy urllib3==1.26.18 # via requests diff --git a/tornado/platform/asyncio.py b/tornado/platform/asyncio.py index 79e60848b4..2e9f424842 100644 --- a/tornado/platform/asyncio.py +++ b/tornado/platform/asyncio.py @@ -49,6 +49,9 @@ Union, ) +if typing.TYPE_CHECKING: + from typing_extensions import TypeVarTuple, Unpack + class _HasFileno(Protocol): def fileno(self) -> int: @@ -59,6 +62,8 @@ def fileno(self) -> int: _T = TypeVar("_T") +if typing.TYPE_CHECKING: + _Ts = TypeVarTuple("_Ts") # Collection of selector thread event loops to shut down on exit. _selector_loops: Set["SelectorThread"] = set() @@ -702,12 +707,18 @@ def close(self) -> None: self._real_loop.close() def add_reader( - self, fd: "_FileDescriptorLike", callback: Callable[..., None], *args: Any + self, + fd: "_FileDescriptorLike", + callback: Callable[..., None], + *args: "Unpack[_Ts]", ) -> None: return self._selector.add_reader(fd, callback, *args) def add_writer( - self, fd: "_FileDescriptorLike", callback: Callable[..., None], *args: Any + self, + fd: "_FileDescriptorLike", + callback: Callable[..., None], + *args: "Unpack[_Ts]", ) -> None: return self._selector.add_writer(fd, callback, *args) From e7dff512f8329dfc92590f144abbeaff55fce6ad Mon Sep 17 00:00:00 2001 From: Ben Darnell Date: Fri, 7 Jun 2024 14:42:28 -0400 Subject: [PATCH 2/5] httputil: Add types for elements of HTTPHeaders Revealed an issue in websocket.py in which bytes were used when it should have been str. This avoided being a bug because something down the line was converting it to str but it was still a logical type error. The change to httputil.py was taken from #3329 (thanks mslynch). Closes #3329 Fixes #3328 --- tornado/httputil.py | 8 +++++++- tornado/web.py | 6 +++--- tornado/websocket.py | 2 +- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/tornado/httputil.py b/tornado/httputil.py index 9ce992d82b..8155035330 100644 --- a/tornado/httputil.py +++ b/tornado/httputil.py @@ -62,6 +62,12 @@ from asyncio import Future # noqa: F401 import unittest # noqa: F401 + # This can be done unconditionally in the base class of HTTPHeaders + # after we drop support for Python 3.8. + StrMutableMapping = collections.abc.MutableMapping[str, str] +else: + StrMutableMapping = collections.abc.MutableMapping + # To be used with str.strip() and related methods. HTTP_WHITESPACE = " \t" @@ -76,7 +82,7 @@ def _normalize_header(name: str) -> str: return "-".join([w.capitalize() for w in name.split("-")]) -class HTTPHeaders(collections.abc.MutableMapping): +class HTTPHeaders(StrMutableMapping): """A dictionary that maintains ``Http-Header-Case`` for all keys. Supports multiple values per key via a pair of new methods, diff --git a/tornado/web.py b/tornado/web.py index 039396470f..21d0cad689 100644 --- a/tornado/web.py +++ b/tornado/web.py @@ -1596,14 +1596,14 @@ def check_xsrf_cookie(self) -> None: # information please see # http://www.djangoproject.com/weblog/2011/feb/08/security/ # http://weblog.rubyonrails.org/2011/2/8/csrf-protection-bypass-in-ruby-on-rails - token = ( + input_token = ( self.get_argument("_xsrf", None) or self.request.headers.get("X-Xsrftoken") or self.request.headers.get("X-Csrftoken") ) - if not token: + if not input_token: raise HTTPError(403, "'_xsrf' argument missing from POST") - _, token, _ = self._decode_xsrf_token(token) + _, token, _ = self._decode_xsrf_token(input_token) _, expected_token, _ = self._get_raw_xsrf_token() if not token: raise HTTPError(403, "'_xsrf' argument has invalid format") diff --git a/tornado/websocket.py b/tornado/websocket.py index 8f0e0aefe8..0127303076 100644 --- a/tornado/websocket.py +++ b/tornado/websocket.py @@ -1380,7 +1380,7 @@ def __init__( { "Upgrade": "websocket", "Connection": "Upgrade", - "Sec-WebSocket-Key": self.key, + "Sec-WebSocket-Key": to_unicode(self.key), "Sec-WebSocket-Version": "13", } ) From 70dc6f73fb9b130e9e548b62b3ff9ff2a8a9e870 Mon Sep 17 00:00:00 2001 From: Ben Darnell Date: Fri, 7 Jun 2024 14:51:34 -0400 Subject: [PATCH 3/5] web: Type SUPPORTED_METHODS so it can be overridden Its default type is `Tuple[str, str, str, str, str, str, str]`, which can only be overridden by a tuple of the exact same length. This change originated in #3354 (thanks alexmv and andersk). Closes #3354 --- tornado/web.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/tornado/web.py b/tornado/web.py index 21d0cad689..207b5ddd0b 100644 --- a/tornado/web.py +++ b/tornado/web.py @@ -192,7 +192,15 @@ class RequestHandler(object): """ - SUPPORTED_METHODS = ("GET", "HEAD", "POST", "DELETE", "PATCH", "PUT", "OPTIONS") + SUPPORTED_METHODS: Tuple[str, ...] = ( + "GET", + "HEAD", + "POST", + "DELETE", + "PATCH", + "PUT", + "OPTIONS", + ) _template_loaders = {} # type: Dict[str, template.BaseLoader] _template_loader_lock = threading.Lock() From 4337efe236168d0b8b73188abb6944976ce5394a Mon Sep 17 00:00:00 2001 From: Ben Darnell Date: Fri, 7 Jun 2024 15:23:45 -0400 Subject: [PATCH 4/5] web: Improve typing for UIModule.render In practice, UIModule.render often returns the result of self.render_string, which returns bytes. In fact, we have an example of that in this file which had a type ignore comment. UIModule.render may now return either str or bytes and downstream code is responsible for handling this. (Note that the new call to _unicode appears to be redundant since the Template module's bytes return was already working correctly, but this conversion is necessary to satisfy the type checker.) Fixes #3050 --- tornado/web.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tornado/web.py b/tornado/web.py index 207b5ddd0b..3a85bc41f1 100644 --- a/tornado/web.py +++ b/tornado/web.py @@ -1894,7 +1894,7 @@ def render(*args, **kwargs) -> str: # type: ignore if name not in self._active_modules: self._active_modules[name] = module(self) rendered = self._active_modules[name].render(*args, **kwargs) - return rendered + return _unicode(rendered) return render @@ -3331,7 +3331,7 @@ def __init__(self, handler: RequestHandler) -> None: def current_user(self) -> Any: return self.handler.current_user - def render(self, *args: Any, **kwargs: Any) -> str: + def render(self, *args: Any, **kwargs: Any) -> Union[str, bytes]: """Override in subclasses to return this module's output.""" raise NotImplementedError() @@ -3379,12 +3379,12 @@ def render_string(self, path: str, **kwargs: Any) -> bytes: class _linkify(UIModule): - def render(self, text: str, **kwargs: Any) -> str: # type: ignore + def render(self, text: str, **kwargs: Any) -> str: return escape.linkify(text, **kwargs) class _xsrf_form_html(UIModule): - def render(self) -> str: # type: ignore + def render(self) -> str: return self.handler.xsrf_form_html() @@ -3410,7 +3410,7 @@ def __init__(self, handler: RequestHandler) -> None: self._resource_list = [] # type: List[Dict[str, Any]] self._resource_dict = {} # type: Dict[str, Dict[str, Any]] - def render(self, path: str, **kwargs: Any) -> bytes: # type: ignore + def render(self, path: str, **kwargs: Any) -> bytes: def set_resources(**kwargs) -> str: # type: ignore if path not in self._resource_dict: self._resource_list.append(kwargs) From 385af837f278ec0a6489cb61db158960e648fa94 Mon Sep 17 00:00:00 2001 From: Ben Darnell Date: Fri, 7 Jun 2024 15:54:08 -0400 Subject: [PATCH 5/5] concurrent: Update type hint on chain_future to match implementation This method has always accepted both asyncio and concurrent futures, but the type hint incorrectly indicated that it only accepted asyncio futures. Fixes #3314 --- tornado/concurrent.py | 5 ++++- tornado/test/concurrent_test.py | 26 ++++++++++++++++++++++++++ 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/tornado/concurrent.py b/tornado/concurrent.py index 5047c5389f..e98093f215 100644 --- a/tornado/concurrent.py +++ b/tornado/concurrent.py @@ -145,7 +145,10 @@ def wrapper(self: Any, *args: Any, **kwargs: Any) -> Future: _NO_RESULT = object() -def chain_future(a: "Future[_T]", b: "Future[_T]") -> None: +def chain_future( + a: Union["Future[_T]", "futures.Future[_T]"], + b: Union["Future[_T]", "futures.Future[_T]"], +) -> None: """Chain two futures together so that when one completes, so does the other. The result (success or failure) of ``a`` will be copied to ``b``, unless diff --git a/tornado/test/concurrent_test.py b/tornado/test/concurrent_test.py index 33fcb6505e..009d6ed43f 100644 --- a/tornado/test/concurrent_test.py +++ b/tornado/test/concurrent_test.py @@ -21,6 +21,7 @@ from tornado.concurrent import ( Future, + chain_future, run_on_executor, future_set_result_unless_cancelled, ) @@ -47,6 +48,31 @@ def test_future_set_result_unless_cancelled(self): self.assertEqual(fut.result(), 42) +class ChainFutureTest(AsyncTestCase): + @gen_test + async def test_asyncio_futures(self): + fut: Future[int] = Future() + fut2: Future[int] = Future() + chain_future(fut, fut2) + fut.set_result(42) + result = await fut2 + self.assertEqual(result, 42) + + @gen_test + async def test_concurrent_futures(self): + # A three-step chain: two concurrent futures (showing that both arguments to chain_future + # can be concurrent futures), and then one from a concurrent future to an asyncio future so + # we can use it in await. + fut: futures.Future[int] = futures.Future() + fut2: futures.Future[int] = futures.Future() + fut3: Future[int] = Future() + chain_future(fut, fut2) + chain_future(fut2, fut3) + fut.set_result(42) + result = await fut3 + self.assertEqual(result, 42) + + # The following series of classes demonstrate and test various styles # of use, with and without generators and futures.