From f890efba68fa5affbdb13c845f34539c6c9350ca Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Thu, 21 Nov 2024 15:03:13 -0500 Subject: [PATCH] undo some --- chia/_tests/core/server/test_dos.py | 3 +- chia/_tests/util/misc.py | 3 +- chia/_tests/util/time_out_assert.py | 17 ++++++++- chia/util/introspection.py | 19 ---------- chia/util/task_referencer.py | 55 ++++++----------------------- pytest.ini | 2 +- 6 files changed, 31 insertions(+), 68 deletions(-) delete mode 100644 chia/util/introspection.py diff --git a/chia/_tests/core/server/test_dos.py b/chia/_tests/core/server/test_dos.py index 0ea11daed202..5d33c580f7c7 100644 --- a/chia/_tests/core/server/test_dos.py +++ b/chia/_tests/core/server/test_dos.py @@ -171,8 +171,9 @@ async def test_invalid_protocol_handshake(self, setup_two_nodes_fixture, self_ho await session.close() await asyncio.sleep(1) # give some time for cleanup to work + @pytest.mark.parametrize("n", range(10)) @pytest.mark.anyio - async def test_spam_tx(self, setup_two_nodes_fixture, self_hostname): + async def test_spam_tx(self, setup_two_nodes_fixture, self_hostname, n): nodes, _, _ = setup_two_nodes_fixture _full_node_1, full_node_2 = nodes server_1 = nodes[0].full_node.server diff --git a/chia/_tests/util/misc.py b/chia/_tests/util/misc.py index dfa999b5aba5..0671817b467e 100644 --- a/chia/_tests/util/misc.py +++ b/chia/_tests/util/misc.py @@ -34,7 +34,7 @@ import chia._tests from chia._tests import ether from chia._tests.core.data_layer.util import ChiaRoot -from chia._tests.util.time_out_assert import DataTypeProtocol +from chia._tests.util.time_out_assert import DataTypeProtocol, caller_file_and_line from chia.full_node.mempool import Mempool from chia.protocols.protocol_message_types import ProtocolMessageTypes from chia.server.api_protocol import ApiMetadata, ApiProtocol @@ -42,7 +42,6 @@ from chia.types.blockchain_format.sized_bytes import bytes32 from chia.types.condition_opcodes import ConditionOpcode from chia.util.hash import std_hash -from chia.util.introspection import caller_file_and_line from chia.util.ints import uint16, uint32, uint64 from chia.util.network import WebServer from chia.wallet.util.compute_hints import HintedCoin diff --git a/chia/_tests/util/time_out_assert.py b/chia/_tests/util/time_out_assert.py index 5bce383e202d..b1b99917077f 100644 --- a/chia/_tests/util/time_out_assert.py +++ b/chia/_tests/util/time_out_assert.py @@ -6,6 +6,8 @@ import logging import pathlib import time +from collections.abc import Iterable +from inspect import getframeinfo, stack from pathlib import Path from typing import TYPE_CHECKING, Any, Callable, ClassVar, Protocol, TypeVar, cast, final @@ -13,7 +15,6 @@ import chia._tests from chia._tests import ether from chia.protocols.protocol_message_types import ProtocolMessageTypes -from chia.util.introspection import caller_file_and_line from chia.util.timing import adjusted_timeout log = logging.getLogger(__name__) @@ -174,3 +175,17 @@ async def bool_f(): return True return bool_f + + +def caller_file_and_line(distance: int = 1, relative_to: Iterable[Path] = ()) -> tuple[str, int]: + caller = getframeinfo(stack()[distance + 1][0]) + + caller_path = Path(caller.filename) + options: list[str] = [caller_path.as_posix()] + for path in relative_to: + try: + options.append(caller_path.relative_to(path).as_posix()) + except ValueError: + pass + + return min(options, key=len), caller.lineno diff --git a/chia/util/introspection.py b/chia/util/introspection.py deleted file mode 100644 index 61821211e9ca..000000000000 --- a/chia/util/introspection.py +++ /dev/null @@ -1,19 +0,0 @@ -from __future__ import annotations - -from inspect import getframeinfo, stack -from pathlib import Path -from typing import Iterable - - -def caller_file_and_line(distance: int = 1, relative_to: Iterable[Path] = ()) -> tuple[str, int]: - caller = getframeinfo(stack()[distance + 1][0]) - - caller_path = Path(caller.filename) - options: list[str] = [caller_path.as_posix()] - for path in relative_to: - try: - options.append(caller_path.relative_to(path).as_posix()) - except ValueError: - pass - - return min(options, key=len), caller.lineno diff --git a/chia/util/task_referencer.py b/chia/util/task_referencer.py index bcc246445b64..be7bb82376bf 100644 --- a/chia/util/task_referencer.py +++ b/chia/util/task_referencer.py @@ -5,37 +5,28 @@ import asyncio import dataclasses -import gc import logging import math -import pathlib import time import typing -import warnings - -import chia -from chia.util.introspection import caller_file_and_line -from chia.util.log_exceptions import log_exceptions T = typing.TypeVar("T") logger = logging.getLogger(__name__) -@dataclasses.dataclass(frozen=True) +@dataclasses.dataclass(frozen=True, slots=True) class _TaskInfo: task: asyncio.Task[object] = dataclasses.field(hash=False) task_object_id: int name: str known_unreferenced: bool - creation_file: str - creation_line: int def __str__(self) -> str: - return f"{self.name} ({self.creation_file}:{self.creation_line})" + return self.name -@dataclasses.dataclass +@dataclasses.dataclass(slots=True) class _TaskReferencer: """Holds strong references to tasks until they are done. This compensates for asyncio holding only weak references. This should be replaced by patterns using @@ -43,29 +34,21 @@ class _TaskReferencer: """ tasks: list[_TaskInfo] = dataclasses.field(default_factory=list) - unreported_tasks: set[_TaskInfo] = dataclasses.field(default_factory=set) clock: typing.Callable[[], float] = time.monotonic - last_cull: float = -math.inf + last_cull_time: float = -math.inf + last_cull_length: int = 0 cull_period: float = 30 cull_count: int = 1000 - reporting_task: typing.Optional[asyncio.Task[None]] = None def create_task( self, coroutine: typing.Coroutine[object, object, T], + *, name: typing.Optional[str] = None, known_unreferenced: bool = False, ) -> asyncio.Task[T]: - if self.reporting_task is None: - self.reporting_task = asyncio.create_task(self.report_unreferenced_tasks()) - self.maybe_cull() - file, line = caller_file_and_line( - distance=1, - relative_to=(pathlib.Path(chia.__file__).parent.parent,), - ) - task = asyncio.create_task(coro=coroutine, name=name) self.tasks.append( _TaskInfo( @@ -73,8 +56,6 @@ def create_task( task_object_id=id(task), name=task.get_name(), known_unreferenced=known_unreferenced, - creation_file=file, - creation_line=line, ) ) @@ -82,29 +63,15 @@ def create_task( def maybe_cull(self) -> None: now = self.clock() - since_last = now - self.last_cull + since_last = now - self.last_cull_time - if len(self.tasks) <= self.cull_count and since_last <= self.cull_period: + if len(self.tasks) <= self.last_cull_length + self.cull_count and since_last <= self.cull_period: return # TODO: consider collecting results and logging errors - self.tasks[:] = (task_info for task_info in self.tasks if not task_info.task.done()) - self.last_cull = now - - async def report_unreferenced_tasks(self) -> None: - while True: - with log_exceptions(log=logger, consume=True, message="unreferenced task reporting"): - await asyncio.sleep(1) - - for task_info in self.tasks: - if task_info.known_unreferenced or task_info.task.done(): - continue - - if len(gc.get_referrers(task_info.task)) == 1: - # presently coded to repeat every time - message = f"unexpected incomplete unreferenced task found: {task_info}" - logger.error(message) - warnings.warn(message) + self.tasks = [task_info for task_info in self.tasks if not task_info.task.done()] + self.last_cull_time = now + self.last_cull_length = len(self.tasks) _global_task_referencer = _TaskReferencer() diff --git a/pytest.ini b/pytest.ini index 740b149761a8..f05e2f4fee2b 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,6 +1,6 @@ [pytest] ; logging options -log_cli = False +log_cli = True addopts = --verbose --tb=short -n auto -p no:monitor log_level = WARNING console_output_style = count