From ca38ea90d4cd26c1ad4de472721e72ba5d85e5ef Mon Sep 17 00:00:00 2001 From: fselmo Date: Thu, 10 Oct 2024 11:16:52 -0600 Subject: [PATCH 1/6] Add support for non-mainnet request cache validation thresholds: - If non-mainnet ethereum, allow the use of an integer value for the request cache validation threshold. Pre-configure "safe" default values for some common non-mainnet chain ids based on their varied finality mechanisms. - This integer value represents the number of seconds from time.now() that the request cache deems as a safe enough time window to allow the request to be cached. - Update the tests to reflect these changes. Bonus: - Add a note to the ``request_mocker`` to make sure that mocked results are of the correct expected types based on JSON-RPC spec (e.g. hex strings instead of ints for numbers, etc.) --- .../caching-utils/test_request_caching.py | 173 +++++++++------ web3/_utils/caching/caching_utils.py | 153 +++++++++++-- .../caching/request_caching_validation.py | 210 ++++++++++++++---- web3/_utils/module_testing/utils.py | 8 + web3/providers/async_base.py | 9 +- web3/providers/base.py | 9 +- 6 files changed, 431 insertions(+), 131 deletions(-) diff --git a/tests/core/caching-utils/test_request_caching.py b/tests/core/caching-utils/test_request_caching.py index c48c03f0a6..abbf917dfe 100644 --- a/tests/core/caching-utils/test_request_caching.py +++ b/tests/core/caching-utils/test_request_caching.py @@ -21,9 +21,11 @@ ) from web3._utils.caching.caching_utils import ( ASYNC_INTERNAL_VALIDATION_MAP, + BLOCK_IN_RESULT, BLOCKHASH_IN_PARAMS, BLOCKNUM_IN_PARAMS, - BLOCKNUM_IN_RESULT, + CHAIN_VALIDATION_THRESHOLD_DEFAULTS, + DEFAULT_VALIDATION_THRESHOLD, INTERNAL_VALIDATION_MAP, ) from web3.exceptions import ( @@ -64,6 +66,7 @@ def w3(request_mocker): mock_results={ "fake_endpoint": lambda *_: uuid.uuid4(), "not_on_allowlist": lambda *_: uuid.uuid4(), + "eth_chainId": "0x1", # mainnet }, ): yield _w3 @@ -134,10 +137,10 @@ def test_caching_requests_does_not_share_state_between_providers(request_mocker) # strap w3_a_shared_cache with w3_a's cache w3_a_shared_cache.provider._request_cache = w3_a.provider._request_cache - mock_results_a = {RPCEndpoint("eth_chainId"): 11111} - mock_results_a_shared_cache = {RPCEndpoint("eth_chainId"): 00000} - mock_results_b = {RPCEndpoint("eth_chainId"): 22222} - mock_results_c = {RPCEndpoint("eth_chainId"): 33333} + mock_results_a = {RPCEndpoint("eth_chainId"): hex(11111)} + mock_results_a_shared_cache = {RPCEndpoint("eth_chainId"): hex(00000)} + mock_results_b = {RPCEndpoint("eth_chainId"): hex(22222)} + mock_results_c = {RPCEndpoint("eth_chainId"): hex(33333)} with request_mocker(w3_a, mock_results=mock_results_a): with request_mocker(w3_b, mock_results=mock_results_b): @@ -154,10 +157,10 @@ def test_caching_requests_does_not_share_state_between_providers(request_mocker) "eth_chainId", [] ) - assert result_a == 11111 - assert result_b == 22222 - assert result_c == 33333 - assert result_a_shared_cache == 11111 + assert result_a == hex(11111) + assert result_b == hex(22222) + assert result_c == hex(33333) + assert result_a_shared_cache == hex(11111) @pytest.mark.parametrize( @@ -199,7 +202,7 @@ def test_all_providers_do_not_cache_by_default_and_can_set_caching_properties(pr "threshold", (RequestCacheValidationThreshold.FINALIZED, RequestCacheValidationThreshold.SAFE), ) -@pytest.mark.parametrize("endpoint", BLOCKNUM_IN_PARAMS | BLOCKNUM_IN_RESULT) +@pytest.mark.parametrize("endpoint", BLOCKNUM_IN_PARAMS | BLOCK_IN_RESULT) @pytest.mark.parametrize( "blocknum,should_cache", ( @@ -211,11 +214,11 @@ def test_all_providers_do_not_cache_by_default_and_can_set_caching_properties(pr ("0x5", False), ), ) -def test_blocknum_validation_against_validation_threshold_when_caching( +def test_blocknum_validation_against_validation_threshold_when_caching_mainnet( threshold, endpoint, blocknum, should_cache, request_mocker ): w3 = Web3( - HTTPProvider( + BaseProvider( cache_allowed_requests=True, request_cache_validation_threshold=threshold ) ) @@ -224,7 +227,7 @@ def test_blocknum_validation_against_validation_threshold_when_caching( mock_results={ endpoint: ( # mock the result to requests that return blocks - {"number": blocknum} + {"number": blocknum, "timestamp": "0x0"} if "getBlock" in endpoint # mock the result to requests that return transactions else {"blockNumber": blocknum} @@ -232,16 +235,17 @@ def test_blocknum_validation_against_validation_threshold_when_caching( "eth_getBlockByNumber": lambda _method, params: ( # mock the threshold block to be blocknum "0x2", return # blocknum otherwise - {"number": "0x2"} + {"number": "0x2", "timestamp": "0x0"} if params[0] == threshold.value - else {"number": params[0]} + else {"number": params[0], "timestamp": "0x0"} ), + "eth_chainId": "0x1", # mainnet }, ): assert len(w3.provider._request_cache.items()) == 0 w3.manager.request_blocking(endpoint, [blocknum, False]) cached_items = len(w3.provider._request_cache.items()) - assert cached_items == 1 if should_cache else cached_items == 0 + assert cached_items > 0 if should_cache else cached_items == 0 @pytest.mark.parametrize( @@ -260,30 +264,31 @@ def test_blocknum_validation_against_validation_threshold_when_caching( ("pending", None, False), ), ) -def test_block_id_param_caching( +def test_block_id_param_caching_mainnet( threshold, endpoint, block_id, blocknum, should_cache, request_mocker ): w3 = Web3( - HTTPProvider( + BaseProvider( cache_allowed_requests=True, request_cache_validation_threshold=threshold ) ) with request_mocker( w3, mock_results={ + "eth_chainId": "0x1", # mainnet endpoint: "0x0", "eth_getBlockByNumber": lambda _method, params: ( # mock the threshold block to be blocknum "0x2" for all test cases - {"number": "0x2"} + {"number": "0x2", "timestamp": "0x0"} if params[0] == threshold.value - else {"number": blocknum} + else {"number": blocknum, "timestamp": "0x0"} ), }, ): assert len(w3.provider._request_cache.items()) == 0 w3.manager.request_blocking(RPCEndpoint(endpoint), [block_id, False]) cached_items = len(w3.provider._request_cache.items()) - assert cached_items == 1 if should_cache else cached_items == 0 + assert cached_items > 0 if should_cache else cached_items == 0 @pytest.mark.parametrize( @@ -302,24 +307,25 @@ def test_block_id_param_caching( ("0x5", False), ), ) -def test_blockhash_validation_against_validation_threshold_when_caching( +def test_blockhash_validation_against_validation_threshold_when_caching_mainnet( threshold, endpoint, blocknum, should_cache, request_mocker ): w3 = Web3( - HTTPProvider( + BaseProvider( cache_allowed_requests=True, request_cache_validation_threshold=threshold ) ) with request_mocker( w3, mock_results={ + "eth_chainId": "0x1", # mainnet "eth_getBlockByNumber": lambda _method, params: ( # mock the threshold block to be blocknum "0x2" - {"number": "0x2"} + {"number": "0x2", "timestamp": "0x0"} if params[0] == threshold.value - else {"number": params[0]} + else {"number": params[0], "timestamp": "0x0"} ), - "eth_getBlockByHash": {"number": blocknum}, + "eth_getBlockByHash": {"number": blocknum, "timestamp": "0x0"}, endpoint: "0x0", }, ): @@ -329,23 +335,37 @@ def test_blockhash_validation_against_validation_threshold_when_caching( assert cached_items == 2 if should_cache else cached_items == 0 -def test_request_caching_validation_threshold_is_finalized_by_default(): - w3 = Web3(HTTPProvider(cache_allowed_requests=True)) - assert ( - w3.provider.request_cache_validation_threshold - == RequestCacheValidationThreshold.FINALIZED - ) +@pytest.mark.parametrize( + "chain_id,expected_threshold", + ( + *CHAIN_VALIDATION_THRESHOLD_DEFAULTS.items(), + (3456787654567654, DEFAULT_VALIDATION_THRESHOLD), + (11111111111444444444444444, DEFAULT_VALIDATION_THRESHOLD), + (-11111111111111111117, DEFAULT_VALIDATION_THRESHOLD), + ), +) +def test_request_caching_validation_threshold_defaults( + chain_id, expected_threshold, request_mocker +): + w3 = Web3(BaseProvider(cache_allowed_requests=True)) + with request_mocker(w3, mock_results={"eth_chainId": hex(chain_id)}): + w3.manager.request_blocking(RPCEndpoint("eth_chainId"), []) + assert w3.provider.request_cache_validation_threshold == expected_threshold + # assert chain_id is cached + cache_items = w3.provider._request_cache.items() + assert len(cache_items) == 1 + assert cache_items[0][1]["result"] == hex(chain_id) @pytest.mark.parametrize( - "endpoint", BLOCKNUM_IN_PARAMS | BLOCKNUM_IN_RESULT | BLOCKHASH_IN_PARAMS + "endpoint", BLOCKNUM_IN_PARAMS | BLOCK_IN_RESULT | BLOCKHASH_IN_PARAMS ) @pytest.mark.parametrize("blocknum", ("0x0", "0x1", "0x2", "0x3", "0x4", "0x5")) def test_request_caching_with_validation_threshold_set_to_none( endpoint, blocknum, request_mocker ): w3 = Web3( - HTTPProvider( + BaseProvider( cache_allowed_requests=True, request_cache_validation_threshold=None, ) @@ -376,6 +396,7 @@ async def async_w3(request_mocker): mock_results={ "fake_endpoint": lambda *_: uuid.uuid4(), "not_on_allowlist": lambda *_: uuid.uuid4(), + "eth_chainId": "0x1", # mainnet }, ): yield _async_w3 @@ -460,10 +481,10 @@ async def test_async_request_caching_does_not_share_state_between_providers( # strap async_w3_a_shared_cache with async_w3_a's cache async_w3_a_shared_cache.provider._request_cache = async_w3_a.provider._request_cache - mock_results_a = {RPCEndpoint("eth_chainId"): 11111} - mock_results_a_shared_cache = {RPCEndpoint("eth_chainId"): 00000} - mock_results_b = {RPCEndpoint("eth_chainId"): 22222} - mock_results_c = {RPCEndpoint("eth_chainId"): 33333} + mock_results_a = {RPCEndpoint("eth_chainId"): hex(11111)} + mock_results_a_shared_cache = {RPCEndpoint("eth_chainId"): hex(00000)} + mock_results_b = {RPCEndpoint("eth_chainId"): hex(22222)} + mock_results_c = {RPCEndpoint("eth_chainId"): hex(33333)} async with request_mocker(async_w3_a, mock_results=mock_results_a): async with request_mocker(async_w3_b, mock_results=mock_results_b): @@ -480,10 +501,10 @@ async def test_async_request_caching_does_not_share_state_between_providers( "eth_chainId", [] ) - assert result_a == 11111 - assert result_b == 22222 - assert result_c == 33333 - assert result_a_shared_cache == 11111 + assert result_a == hex(11111) + assert result_b == hex(22222) + assert result_c == hex(33333) + assert result_a_shared_cache == hex(11111) @pytest.mark.asyncio @@ -491,7 +512,7 @@ async def test_async_request_caching_does_not_share_state_between_providers( "threshold", (RequestCacheValidationThreshold.FINALIZED, RequestCacheValidationThreshold.SAFE), ) -@pytest.mark.parametrize("endpoint", BLOCKNUM_IN_PARAMS | BLOCKNUM_IN_RESULT) +@pytest.mark.parametrize("endpoint", BLOCKNUM_IN_PARAMS | BLOCK_IN_RESULT) @pytest.mark.parametrize( "blocknum,should_cache", ( @@ -503,11 +524,11 @@ async def test_async_request_caching_does_not_share_state_between_providers( ("0x5", False), ), ) -async def test_async_blocknum_validation_against_validation_threshold( +async def test_async_blocknum_validation_against_validation_threshold_mainnet( threshold, endpoint, blocknum, should_cache, request_mocker ): async_w3 = AsyncWeb3( - AsyncHTTPProvider( + AsyncBaseProvider( cache_allowed_requests=True, request_cache_validation_threshold=threshold ) ) @@ -516,7 +537,7 @@ async def test_async_blocknum_validation_against_validation_threshold( mock_results={ endpoint: ( # mock the result to requests that return blocks - {"number": blocknum} + {"number": blocknum, "timestamp": "0x0"} if "getBlock" in endpoint # mock the result to requests that return transactions else {"blockNumber": blocknum} @@ -524,16 +545,17 @@ async def test_async_blocknum_validation_against_validation_threshold( "eth_getBlockByNumber": lambda _method, params: ( # mock the threshold block to be blocknum "0x2", return # blocknum otherwise - {"number": "0x2"} + {"number": "0x2", "timestamp": "0x0"} if params[0] == threshold.value - else {"number": params[0]} + else {"number": params[0], "timestamp": "0x0"} ), + "eth_chainId": "0x1", # mainnet }, ): assert len(async_w3.provider._request_cache.items()) == 0 await async_w3.manager.coro_request(endpoint, [blocknum, False]) cached_items = len(async_w3.provider._request_cache.items()) - assert cached_items == 1 if should_cache else cached_items == 0 + assert cached_items > 0 if should_cache else cached_items == 0 @pytest.mark.asyncio @@ -553,30 +575,31 @@ async def test_async_blocknum_validation_against_validation_threshold( ("pending", None, False), ), ) -async def test_async_block_id_param_caching( +async def test_async_block_id_param_caching_mainnet( threshold, endpoint, block_id, blocknum, should_cache, request_mocker ): async_w3 = AsyncWeb3( - AsyncHTTPProvider( + AsyncBaseProvider( cache_allowed_requests=True, request_cache_validation_threshold=threshold ) ) async with request_mocker( async_w3, mock_results={ + "eth_chainId": "0x1", # mainnet endpoint: "0x0", "eth_getBlockByNumber": lambda _method, params: ( # mock the threshold block to be blocknum "0x2" for all test cases - {"number": "0x2"} + {"number": "0x2", "timestamp": "0x0"} if params[0] == threshold.value - else {"number": blocknum} + else {"number": blocknum, "timestamp": "0x0"} ), }, ): assert len(async_w3.provider._request_cache.items()) == 0 await async_w3.manager.coro_request(RPCEndpoint(endpoint), [block_id, False]) cached_items = len(async_w3.provider._request_cache.items()) - assert cached_items == 1 if should_cache else cached_items == 0 + assert cached_items > 0 if should_cache else cached_items == 0 @pytest.mark.asyncio @@ -596,24 +619,25 @@ async def test_async_block_id_param_caching( ("0x5", False), ), ) -async def test_async_blockhash_validation_against_validation_threshold( +async def test_async_blockhash_validation_against_validation_threshold_mainnet( threshold, endpoint, blocknum, should_cache, request_mocker ): async_w3 = AsyncWeb3( - AsyncHTTPProvider( + AsyncBaseProvider( cache_allowed_requests=True, request_cache_validation_threshold=threshold ) ) async with request_mocker( async_w3, mock_results={ + "eth_chainId": "0x1", # mainnet "eth_getBlockByNumber": lambda _method, params: ( # mock the threshold block to be blocknum "0x2" - {"number": "0x2"} + {"number": "0x2", "timestamp": "0x0"} if params[0] == threshold.value - else {"number": params[0]} + else {"number": params[0], "timestamp": "0x0"} ), - "eth_getBlockByHash": {"number": blocknum}, + "eth_getBlockByHash": {"number": blocknum, "timestamp": "0x0"}, endpoint: "0x0", }, ): @@ -624,24 +648,39 @@ async def test_async_blockhash_validation_against_validation_threshold( @pytest.mark.asyncio -async def test_async_request_caching_validation_threshold_is_finalized_by_default(): - async_w3 = AsyncWeb3(AsyncHTTPProvider(cache_allowed_requests=True)) - assert ( - async_w3.provider.request_cache_validation_threshold - == RequestCacheValidationThreshold.FINALIZED - ) +@pytest.mark.parametrize( + "chain_id,expected_threshold", + ( + *CHAIN_VALIDATION_THRESHOLD_DEFAULTS.items(), + (3456787654567654, DEFAULT_VALIDATION_THRESHOLD), + (11111111111444444444444444, DEFAULT_VALIDATION_THRESHOLD), + (-11111111111111111117, DEFAULT_VALIDATION_THRESHOLD), + ), +) +async def test_async_request_caching_validation_threshold_defaults( + chain_id, expected_threshold, request_mocker +): + async_w3 = AsyncWeb3(AsyncBaseProvider(cache_allowed_requests=True)) + async with request_mocker(async_w3, mock_results={"eth_chainId": hex(chain_id)}): + await async_w3.manager.coro_request(RPCEndpoint("eth_chainId"), []) + assert ( + async_w3.provider.request_cache_validation_threshold == expected_threshold + ) + cache_items = async_w3.provider._request_cache.items() + assert len(cache_items) == 1 + assert cache_items[0][1]["result"] == hex(chain_id) @pytest.mark.asyncio @pytest.mark.parametrize( - "endpoint", BLOCKNUM_IN_PARAMS | BLOCKNUM_IN_RESULT | BLOCKHASH_IN_PARAMS + "endpoint", BLOCKNUM_IN_PARAMS | BLOCK_IN_RESULT | BLOCKHASH_IN_PARAMS ) @pytest.mark.parametrize("blocknum", ("0x0", "0x1", "0x2", "0x3", "0x4", "0x5")) async def test_async_request_caching_with_validation_threshold_set_to_none( endpoint, blocknum, request_mocker ): async_w3 = AsyncWeb3( - AsyncHTTPProvider( + AsyncBaseProvider( cache_allowed_requests=True, request_cache_validation_threshold=None, ) diff --git a/web3/_utils/caching/caching_utils.py b/web3/_utils/caching/caching_utils.py index dc640cee07..1fb0bc5ec4 100644 --- a/web3/_utils/caching/caching_utils.py +++ b/web3/_utils/caching/caching_utils.py @@ -16,6 +16,9 @@ Union, ) +from eth_typing import ( + ChainId, +) from eth_utils import ( is_boolean, is_bytes, @@ -34,12 +37,15 @@ from web3._utils.caching.request_caching_validation import ( UNCACHEABLE_BLOCK_IDS, always_cache_request, - async_validate_blockhash_in_params, - async_validate_blocknum_in_params, - async_validate_blocknum_in_result, - validate_blockhash_in_params, - validate_blocknum_in_params, - validate_blocknum_in_result, + async_validate_from_block_id_in_params, + async_validate_from_blockhash_in_params, + async_validate_from_blocknum_in_result, + validate_from_block_id_in_params, + validate_from_blockhash_in_params, + validate_from_blocknum_in_result, +) +from web3._utils.empty import ( + empty, ) from web3._utils.rpc_abi import ( RPC, @@ -47,6 +53,9 @@ from web3.exceptions import ( Web3TypeError, ) +from web3.utils import ( + RequestCacheValidationThreshold, +) if TYPE_CHECKING: from web3.providers import ( # noqa: F401 @@ -100,6 +109,28 @@ def __init__( self.middleware_response_processors: List[Callable[..., Any]] = [] +DEFAULT_VALIDATION_THRESHOLD = 60 * 60 # 1 hour + +CHAIN_VALIDATION_THRESHOLD_DEFAULTS: Dict[ + int, Union[RequestCacheValidationThreshold, int] +] = { + # Suggested safe values as defaults for each chain. Users can configure a different + # value if desired. + ChainId.ETH.value: RequestCacheValidationThreshold.FINALIZED, + ChainId.ARB1.value: 7 * 24 * 60 * 60, # 7 days + ChainId.ZKSYNC.value: 60 * 60, # 1 hour + ChainId.OETH.value: 3 * 60, # 3 minutes + ChainId.MATIC.value: 30 * 60, # 30 minutes + ChainId.ZKEVM.value: 60 * 60, # 1 hour + ChainId.BASE.value: 7 * 24 * 60 * 60, # 7 days + ChainId.SCR.value: 60 * 60, # 1 hour + ChainId.GNO.value: 5 * 60, # 5 minutes + ChainId.AVAX.value: 2 * 60, # 2 minutes + ChainId.BNB.value: 2 * 60, # 2 minutes + ChainId.FTM.value: 60, # 1 minute +} + + def is_cacheable_request( provider: Union[ASYNC_PROVIDER_TYPE, SYNC_PROVIDER_TYPE], method: "RPCEndpoint", @@ -128,7 +159,7 @@ def is_cacheable_request( RPC.eth_getUncleByBlockNumberAndIndex, RPC.eth_getUncleCountByBlockNumber, } -BLOCKNUM_IN_RESULT = { +BLOCK_IN_RESULT = { RPC.eth_getBlockByHash, RPC.eth_getTransactionByHash, RPC.eth_getTransactionByBlockNumberAndIndex, @@ -142,16 +173,60 @@ def is_cacheable_request( } INTERNAL_VALIDATION_MAP: Dict[ - "RPCEndpoint", Callable[[SYNC_PROVIDER_TYPE, Sequence[Any], Dict[str, Any]], bool] + "RPCEndpoint", + Callable[ + [SYNC_PROVIDER_TYPE, Sequence[Any], Dict[str, Any]], + bool, + ], ] = { **{endpoint: always_cache_request for endpoint in ALWAYS_CACHE}, - **{endpoint: validate_blocknum_in_params for endpoint in BLOCKNUM_IN_PARAMS}, - **{endpoint: validate_blocknum_in_result for endpoint in BLOCKNUM_IN_RESULT}, - **{endpoint: validate_blockhash_in_params for endpoint in BLOCKHASH_IN_PARAMS}, + **{endpoint: validate_from_block_id_in_params for endpoint in BLOCKNUM_IN_PARAMS}, + **{endpoint: validate_from_blocknum_in_result for endpoint in BLOCK_IN_RESULT}, + **{endpoint: validate_from_blockhash_in_params for endpoint in BLOCKHASH_IN_PARAMS}, } CACHEABLE_REQUESTS = tuple(INTERNAL_VALIDATION_MAP.keys()) +def set_threshold_if_empty(provider: SYNC_PROVIDER_TYPE) -> None: + current_threshold = provider.request_cache_validation_threshold + + if current_threshold is empty or isinstance( + current_threshold, RequestCacheValidationThreshold + ): + try: + # turn off momentarily to avoid recursion + provider.cache_allowed_requests = False + chain_id_result = provider.make_request("eth_chainId", [])["result"] + chain_id = int(chain_id_result, 16) + + if ( + isinstance( + current_threshold, + RequestCacheValidationThreshold, + ) + and chain_id != 1 + ): + provider.logger.debug( + "Request cache validation threshold is set to " + f"{current_threshold.value} " + f"for chain with chain_id `{chain_id}` but this value only works " + "on chain_id `1`. Setting to default value for chain_id " + f"`{chain_id}`.", + ) + provider.request_cache_validation_threshold = empty + + if current_threshold is empty: + provider.request_cache_validation_threshold = ( + CHAIN_VALIDATION_THRESHOLD_DEFAULTS.get( + chain_id, DEFAULT_VALIDATION_THRESHOLD + ) + ) + except Exception: + provider.request_cache_validation_threshold = DEFAULT_VALIDATION_THRESHOLD + finally: + provider.cache_allowed_requests = True + + def _should_cache_response( provider: SYNC_PROVIDER_TYPE, method: "RPCEndpoint", @@ -161,6 +236,8 @@ def _should_cache_response( result = response.get("result", None) if "error" in response or is_null(result): return False + + set_threshold_if_empty(provider) if ( method in INTERNAL_VALIDATION_MAP and provider.request_cache_validation_threshold is not None @@ -206,14 +283,60 @@ def wrapper( ASYNC_INTERNAL_VALIDATION_MAP: Dict["RPCEndpoint", ASYNC_VALIDATOR_TYPE] = { **{endpoint: always_cache_request for endpoint in ALWAYS_CACHE}, - **{endpoint: async_validate_blocknum_in_params for endpoint in BLOCKNUM_IN_PARAMS}, - **{endpoint: async_validate_blocknum_in_result for endpoint in BLOCKNUM_IN_RESULT}, **{ - endpoint: async_validate_blockhash_in_params for endpoint in BLOCKHASH_IN_PARAMS + endpoint: async_validate_from_block_id_in_params + for endpoint in BLOCKNUM_IN_PARAMS + }, + **{ + endpoint: async_validate_from_blocknum_in_result for endpoint in BLOCK_IN_RESULT + }, + **{ + endpoint: async_validate_from_blockhash_in_params + for endpoint in BLOCKHASH_IN_PARAMS }, } +async def async_set_threshold_if_empty(provider: ASYNC_PROVIDER_TYPE) -> None: + current_threshold = provider.request_cache_validation_threshold + + if current_threshold is empty or isinstance( + current_threshold, RequestCacheValidationThreshold + ): + try: + # turn off momentarily to avoid recursion + provider.cache_allowed_requests = False + chain_id_result = await provider.make_request("eth_chainId", []) + chain_id = int(chain_id_result["result"], 16) + + if ( + isinstance( + current_threshold, + RequestCacheValidationThreshold, + ) + and chain_id != 1 + ): + provider.logger.debug( + "Request cache validation threshold is set to " + f"{current_threshold.value} " + f"for chain with chain_id `{chain_id}` but this value only works " + "on chain_id `1`. Setting to default value for chain_id " + f"`{chain_id}`.", + ) + provider.request_cache_validation_threshold = empty + + if current_threshold is empty: + provider.request_cache_validation_threshold = ( + CHAIN_VALIDATION_THRESHOLD_DEFAULTS.get( + chain_id, DEFAULT_VALIDATION_THRESHOLD + ) + ) + except Exception: + provider.request_cache_validation_threshold = DEFAULT_VALIDATION_THRESHOLD + finally: + provider.cache_allowed_requests = True + + async def _async_should_cache_response( provider: ASYNC_PROVIDER_TYPE, method: "RPCEndpoint", @@ -223,6 +346,8 @@ async def _async_should_cache_response( result = response.get("result", None) if "error" in response or is_null(result): return False + + await async_set_threshold_if_empty(provider) if ( method in ASYNC_INTERNAL_VALIDATION_MAP and provider.request_cache_validation_threshold is not None diff --git a/web3/_utils/caching/request_caching_validation.py b/web3/_utils/caching/request_caching_validation.py index c5f963a3ed..b82a0d15fe 100644 --- a/web3/_utils/caching/request_caching_validation.py +++ b/web3/_utils/caching/request_caching_validation.py @@ -1,3 +1,4 @@ +import time from typing import ( TYPE_CHECKING, Any, @@ -7,8 +8,8 @@ Union, ) -from eth_utils import ( - to_int, +from web3.utils import ( + RequestCacheValidationThreshold, ) if TYPE_CHECKING: @@ -31,99 +32,216 @@ def _error_log( ) -def is_beyond_validation_threshold(provider: SYNC_PROVIDER_TYPE, blocknum: int) -> bool: +def always_cache_request(*_args: Any, **_kwargs: Any) -> bool: + return True + + +def is_beyond_validation_threshold( + provider: SYNC_PROVIDER_TYPE, + blocknum: int = None, + block_timestamp: int = None, +) -> bool: try: - # `threshold` is either "finalized" or "safe" - threshold = provider.request_cache_validation_threshold.value - response = provider.make_request("eth_getBlockByNumber", [threshold, False]) - return blocknum <= to_int(hexstr=response["result"]["number"]) + threshold = provider.request_cache_validation_threshold + + if isinstance(threshold, RequestCacheValidationThreshold): + # if mainnet and threshold is "finalized" or "safe" + threshold_block = provider.make_request( + "eth_getBlockByNumber", [threshold.value, False] + )["result"] + # we should have a `blocknum` to compare against + return blocknum <= int(threshold_block["number"], 16) + elif isinstance(threshold, int): + if not block_timestamp: + # if validating via `blocknum` from params, we need to get the timestamp + # for the block with `blocknum`. + block = provider.make_request( + "eth_getBlockByNumber", [hex(blocknum), False] + )["result"] + block_timestamp = int(block["timestamp"], 16) + + # if validating via `block_timestamp` from result, we should have a + # `block_timestamp` to compare against + return block_timestamp <= time.time() - threshold + else: + provider.logger.error( + "Invalid request_cache_validation_threshold value. This should not " + f"have happened. Request not cached.\n threshold: {threshold}" + ) + return False except Exception as e: _error_log(provider, e) return False -def always_cache_request(*_args: Any, **_kwargs: Any) -> bool: - return True - - -def validate_blocknum_in_params( - provider: SYNC_PROVIDER_TYPE, params: Sequence[Any], _result: Dict[str, Any] +def validate_from_block_id_in_params( + provider: SYNC_PROVIDER_TYPE, + params: Sequence[Any], + _result: Dict[str, Any], ) -> bool: block_id = params[0] if block_id == "earliest": # `earliest` should always be cacheable return True - blocknum = to_int(hexstr=block_id) - return is_beyond_validation_threshold(provider, blocknum) + + blocknum = int(block_id, 16) + return is_beyond_validation_threshold(provider, blocknum=blocknum) -def validate_blocknum_in_result( - provider: SYNC_PROVIDER_TYPE, _params: Sequence[Any], result: Dict[str, Any] +def validate_from_blocknum_in_result( + provider: SYNC_PROVIDER_TYPE, + _params: Sequence[Any], + result: Dict[str, Any], ) -> bool: - # `number` if block result, `blockNumber` if transaction result - blocknum = to_int(hexstr=result.get("number", result.get("blockNumber"))) - return is_beyond_validation_threshold(provider, blocknum) + try: + # transaction results + if "blockNumber" in result: + blocknum = result.get("blockNumber") + # make an extra call to get the block values + block = provider.make_request("eth_getBlockByNumber", [blocknum, False])[ + "result" + ] + return is_beyond_validation_threshold( + provider, + blocknum=int(blocknum, 16), + block_timestamp=int(block["timestamp"], 16), + ) + elif "number" in result: + return is_beyond_validation_threshold( + provider, + blocknum=int(result["number"], 16), + block_timestamp=int(result["timestamp"], 16), + ) + else: + provider.logger.error( + "Could not find block number in result. This should not have happened. " + f"Request not cached.\n result: {result}", + ) + return False + except Exception as e: + _error_log(provider, e) + return False -def validate_blockhash_in_params( - provider: SYNC_PROVIDER_TYPE, params: Sequence[Any], _result: Dict[str, Any] +def validate_from_blockhash_in_params( + provider: SYNC_PROVIDER_TYPE, + params: Sequence[Any], + _result: Dict[str, Any], ) -> bool: try: # make an extra call to get the block number from the hash - response = provider.make_request("eth_getBlockByHash", [params[0], False]) + block = provider.make_request("eth_getBlockByHash", [params[0], False])[ + "result" + ] + return is_beyond_validation_threshold( + provider, + blocknum=int(block["number"], 16), + block_timestamp=int(block["timestamp"], 16), + ) except Exception as e: _error_log(provider, e) return False - blocknum = to_int(hexstr=response["result"]["number"]) - return is_beyond_validation_threshold(provider, blocknum) - # -- async -- # async def async_is_beyond_validation_threshold( - provider: ASYNC_PROVIDER_TYPE, blocknum: int + provider: ASYNC_PROVIDER_TYPE, + blocknum: int = None, + block_timestamp: int = None, ) -> bool: try: - # `threshold` is either "finalized" or "safe" - threshold = provider.request_cache_validation_threshold.value - response = await provider.make_request( - "eth_getBlockByNumber", [threshold, False] - ) - return blocknum <= to_int(hexstr=response["result"]["number"]) + threshold = provider.request_cache_validation_threshold + + if isinstance(threshold, RequestCacheValidationThreshold): + # if mainnet and threshold is "finalized" or "safe" + threshold_block = await provider.make_request( + "eth_getBlockByNumber", [threshold.value, False] + ) + # we should have a `blocknum` to compare against + return blocknum <= int(threshold_block["result"]["number"], 16) + elif isinstance(threshold, int): + if not block_timestamp: + block = await provider.make_request( + "eth_getBlockByNumber", [hex(blocknum), False] + ) + block_timestamp = int(block["result"]["timestamp"], 16) + + # if validating via `block_timestamp` from result, we should have a + # `block_timestamp` to compare against + return block_timestamp <= time.time() - threshold + else: + provider.logger.error( + "Invalid request_cache_validation_threshold value. This should not " + f"have happened. Request not cached.\n threshold: {threshold}" + ) + return False except Exception as e: _error_log(provider, e) return False -async def async_validate_blocknum_in_params( - provider: ASYNC_PROVIDER_TYPE, params: Sequence[Any], _result: Dict[str, Any] +async def async_validate_from_block_id_in_params( + provider: ASYNC_PROVIDER_TYPE, + params: Sequence[Any], + _result: Dict[str, Any], ) -> bool: block_id = params[0] if block_id == "earliest": # `earliest` should always be cacheable return True - blocknum = to_int(hexstr=params[0]) - return await async_is_beyond_validation_threshold(provider, blocknum) + + blocknum = int(block_id, 16) + return await async_is_beyond_validation_threshold(provider, blocknum=blocknum) -async def async_validate_blocknum_in_result( - provider: ASYNC_PROVIDER_TYPE, _params: Sequence[Any], result: Dict[str, Any] +async def async_validate_from_blocknum_in_result( + provider: ASYNC_PROVIDER_TYPE, + _params: Sequence[Any], + result: Dict[str, Any], ) -> bool: - # `number` if block result, `blockNumber` if transaction result - blocknum = to_int(hexstr=result.get("number", result.get("blockNumber"))) - return await async_is_beyond_validation_threshold(provider, blocknum) + try: + # transaction results + if "blockNumber" in result: + blocknum = result.get("blockNumber") + # make an extra call to get the block values + block = await provider.make_request( + "eth_getBlockByNumber", [blocknum, False] + ) + return await async_is_beyond_validation_threshold( + provider, + blocknum=int(blocknum, 16), + block_timestamp=int(block["result"]["timestamp"], 16), + ) + elif "number" in result: + return await async_is_beyond_validation_threshold( + provider, + blocknum=int(result["number"], 16), + block_timestamp=int(result["timestamp"], 16), + ) + else: + provider.logger.error( + "Could not find block number in result. This should not have happened. " + f"Request not cached.\n result: {result}", + ) + return False + except Exception as e: + _error_log(provider, e) + return False -async def async_validate_blockhash_in_params( +async def async_validate_from_blockhash_in_params( provider: ASYNC_PROVIDER_TYPE, params: Sequence[Any], _result: Dict[str, Any] ) -> bool: try: + # make an extra call to get the block number from the hash response = await provider.make_request("eth_getBlockByHash", [params[0], False]) + return await async_is_beyond_validation_threshold( + provider, + blocknum=int(response["result"]["number"], 16), + block_timestamp=int(response["result"]["timestamp"], 16), + ) except Exception as e: _error_log(provider, e) return False - - blocknum = to_int(hexstr=response["result"]["number"]) - return await async_is_beyond_validation_threshold(provider, blocknum) diff --git a/web3/_utils/module_testing/utils.py b/web3/_utils/module_testing/utils.py index b2a4fd5084..01260bb5f3 100644 --- a/web3/_utils/module_testing/utils.py +++ b/web3/_utils/module_testing/utils.py @@ -35,6 +35,14 @@ class RequestMocker: Context manager to mock requests made by a web3 instance. This is meant to be used via a ``request_mocker`` fixture defined within the appropriate context. + ************************************************************************************ + Important: When mocking results, it's important to keep in mind the types that + clients return. For example, what we commonly translate to integers are returned + as hex strings in the RPC response and should be mocked as such for more + accurate testing. + ************************************************************************************ + + Example: ------- diff --git a/web3/providers/async_base.py b/web3/providers/async_base.py index 5a4f3f415f..c09756c83a 100644 --- a/web3/providers/async_base.py +++ b/web3/providers/async_base.py @@ -10,6 +10,7 @@ Optional, Set, Tuple, + Union, cast, ) @@ -23,6 +24,10 @@ CACHEABLE_REQUESTS, async_handle_request_caching, ) +from web3._utils.empty import ( + Empty, + empty, +) from web3._utils.encoding import ( FriendlyJsonSerde, Web3JsonEncoder, @@ -88,8 +93,8 @@ def __init__( cache_allowed_requests: bool = False, cacheable_requests: Set[RPCEndpoint] = None, request_cache_validation_threshold: Optional[ - RequestCacheValidationThreshold - ] = RequestCacheValidationThreshold.FINALIZED, + Union[RequestCacheValidationThreshold, int, Empty] + ] = empty, ) -> None: self._request_cache = SimpleCache(1000) self.cache_allowed_requests = cache_allowed_requests diff --git a/web3/providers/base.py b/web3/providers/base.py index b653f267bc..d165c8a61a 100644 --- a/web3/providers/base.py +++ b/web3/providers/base.py @@ -9,6 +9,7 @@ Optional, Set, Tuple, + Union, cast, ) @@ -21,6 +22,10 @@ CACHEABLE_REQUESTS, handle_request_caching, ) +from web3._utils.empty import ( + Empty, + empty, +) from web3._utils.encoding import ( FriendlyJsonSerde, Web3JsonEncoder, @@ -71,8 +76,8 @@ def __init__( cache_allowed_requests: bool = False, cacheable_requests: Set[RPCEndpoint] = None, request_cache_validation_threshold: Optional[ - RequestCacheValidationThreshold - ] = RequestCacheValidationThreshold.FINALIZED, + Union[RequestCacheValidationThreshold, int, Empty] + ] = empty, ) -> None: self._request_cache = SimpleCache(1000) self.cache_allowed_requests = cache_allowed_requests From 12f59a9ec44dc630b531416ab647efc7324cf997 Mon Sep 17 00:00:00 2001 From: fselmo Date: Thu, 10 Oct 2024 16:31:29 -0600 Subject: [PATCH 2/6] Prevent recursion internally when caching requests; add tests: - In order to prevent recursion, we need to turn off caching while we make internal calls to get the block information we need. Turn it back on via a ``finally`` so we are sure to turn it back on. - Add some comprehensive tests for the now complex request caching functionality due to multi-chain support and internal-based caching threshold configuration. --- .../caching-utils/test_request_caching.py | 262 +++++++++++++++++- .../caching/request_caching_validation.py | 28 ++ 2 files changed, 275 insertions(+), 15 deletions(-) diff --git a/tests/core/caching-utils/test_request_caching.py b/tests/core/caching-utils/test_request_caching.py index abbf917dfe..b86c476752 100644 --- a/tests/core/caching-utils/test_request_caching.py +++ b/tests/core/caching-utils/test_request_caching.py @@ -1,6 +1,7 @@ import itertools import pytest import threading +import time import uuid import pytest_asyncio @@ -245,7 +246,7 @@ def test_blocknum_validation_against_validation_threshold_when_caching_mainnet( assert len(w3.provider._request_cache.items()) == 0 w3.manager.request_blocking(endpoint, [blocknum, False]) cached_items = len(w3.provider._request_cache.items()) - assert cached_items > 0 if should_cache else cached_items == 0 + assert cached_items == 1 if should_cache else cached_items == 0 @pytest.mark.parametrize( @@ -332,7 +333,7 @@ def test_blockhash_validation_against_validation_threshold_when_caching_mainnet( assert len(w3.provider._request_cache.items()) == 0 w3.manager.request_blocking(endpoint, [b"\x00" * 32, False]) cached_items = len(w3.provider._request_cache.items()) - assert cached_items == 2 if should_cache else cached_items == 0 + assert cached_items == 1 if should_cache else cached_items == 0 @pytest.mark.parametrize( @@ -360,20 +361,119 @@ def test_request_caching_validation_threshold_defaults( @pytest.mark.parametrize( "endpoint", BLOCKNUM_IN_PARAMS | BLOCK_IN_RESULT | BLOCKHASH_IN_PARAMS ) -@pytest.mark.parametrize("blocknum", ("0x0", "0x1", "0x2", "0x3", "0x4", "0x5")) -def test_request_caching_with_validation_threshold_set_to_none( - endpoint, blocknum, request_mocker +@pytest.mark.parametrize( + "time_from_threshold,should_cache", + # -0.2 to give some time for the test to run + ((-2, True), (-1, True), (-0.2, True), (1, False), (2, False)), +) +@pytest.mark.parametrize( + "chain_id,expected_threshold_in_seconds", + [ + (chain_id, threshold) + for chain_id, threshold in CHAIN_VALIDATION_THRESHOLD_DEFAULTS.items() + if isinstance(threshold, int) + ] + + [ + (3456787654567654, DEFAULT_VALIDATION_THRESHOLD), + (11111111111444444444444444, DEFAULT_VALIDATION_THRESHOLD), + (-11111111111111111117, DEFAULT_VALIDATION_THRESHOLD), + ], +) +def test_sync_validation_against_validation_threshold_time_based( + endpoint, + time_from_threshold, + should_cache, + chain_id, + expected_threshold_in_seconds, + request_mocker, +): + w3 = Web3(BaseProvider(cache_allowed_requests=True)) + blocknum = "0x2" + # mock the timestamp so that we are at the threshold +/- the time_from_threshold + mocked_time = hex( + int(round(time.time() - expected_threshold_in_seconds) + time_from_threshold) + ) + + with request_mocker( + w3, + mock_results={ + "eth_chainId": hex(chain_id), + endpoint: lambda *_: ( + # mock the result to requests that return blocks + {"number": blocknum, "timestamp": mocked_time} + if "getBlock" in endpoint + # mock the result to requests that return transactions with the blocknum + # for our block under test + else {"blockNumber": blocknum} + ), + "eth_getBlockByNumber": lambda _method, params: ( + {"number": blocknum, "timestamp": mocked_time} + ), + "eth_getBlockByHash": {"number": blocknum, "timestamp": mocked_time}, + }, + ): + assert len(w3.provider._request_cache.items()) == 0 + w3.manager.request_blocking(endpoint, [blocknum, False]) + cached_items = w3.provider._request_cache.items() + assert len(cached_items) == 1 if should_cache else len(cached_items) == 0 + + +@pytest.mark.parametrize( + "time_from_threshold,should_cache", + # -0.2 to give some time for the test to run + ((-2, True), (-1, True), (-0.2, True), (1, False), (2, False)), +) +@pytest.mark.parametrize( + "chain_id", + ( + # test that defaults are also overridden by the configured validation threshold + *CHAIN_VALIDATION_THRESHOLD_DEFAULTS.keys(), + 3456787654567654, + 11111111111444444444444444, + -11111111111111111117, + ), +) +@pytest.mark.parametrize( + "endpoint", BLOCKNUM_IN_PARAMS | BLOCK_IN_RESULT | BLOCKHASH_IN_PARAMS +) +def test_validation_against_validation_threshold_time_based_configured( + time_from_threshold, should_cache, chain_id, endpoint, request_mocker ): + configured_time_threshold = 60 * 60 * 24 * 7 # 1 week w3 = Web3( BaseProvider( cache_allowed_requests=True, - request_cache_validation_threshold=None, + request_cache_validation_threshold=configured_time_threshold, ) ) - with request_mocker(w3, mock_results={endpoint: {"number": blocknum}}): + blocknum = "0x2" + # mock the timestamp so that we are at the threshold +/- the time_from_threshold + mocked_time = hex( + int(round(time.time()) - configured_time_threshold + time_from_threshold) + ) + + with request_mocker( + w3, + mock_results={ + "eth_chainId": chain_id, + endpoint: lambda *_: ( + # mock the result to requests that return blocks + {"number": blocknum, "timestamp": mocked_time} + if "getBlock" in endpoint + # mock the result to requests that return transactions with the blocknum + # for our block under test + else {"blockNumber": blocknum} + ), + "eth_getBlockByNumber": lambda _method, params: ( + {"number": blocknum, "timestamp": mocked_time} + ), + "eth_getBlockByHash": {"number": blocknum, "timestamp": mocked_time}, + }, + ): assert len(w3.provider._request_cache.items()) == 0 w3.manager.request_blocking(endpoint, [blocknum, False]) - assert len(w3.provider._request_cache.items()) == 1 + cached_items = w3.provider._request_cache.items() + assert len(cached_items) == 1 if should_cache else len(cached_items) == 0 # -- async -- # @@ -555,7 +655,7 @@ async def test_async_blocknum_validation_against_validation_threshold_mainnet( assert len(async_w3.provider._request_cache.items()) == 0 await async_w3.manager.coro_request(endpoint, [blocknum, False]) cached_items = len(async_w3.provider._request_cache.items()) - assert cached_items > 0 if should_cache else cached_items == 0 + assert cached_items == 1 if should_cache else cached_items == 0 @pytest.mark.asyncio @@ -599,7 +699,7 @@ async def test_async_block_id_param_caching_mainnet( assert len(async_w3.provider._request_cache.items()) == 0 await async_w3.manager.coro_request(RPCEndpoint(endpoint), [block_id, False]) cached_items = len(async_w3.provider._request_cache.items()) - assert cached_items > 0 if should_cache else cached_items == 0 + assert cached_items == 1 if should_cache else cached_items == 0 @pytest.mark.asyncio @@ -644,7 +744,7 @@ async def test_async_blockhash_validation_against_validation_threshold_mainnet( assert len(async_w3.provider._request_cache.items()) == 0 await async_w3.manager.coro_request(endpoint, [b"\x00" * 32, False]) cached_items = len(async_w3.provider._request_cache.items()) - assert cached_items == 2 if should_cache else cached_items == 0 + assert cached_items == 1 if should_cache else cached_items == 0 @pytest.mark.asyncio @@ -666,9 +766,67 @@ async def test_async_request_caching_validation_threshold_defaults( assert ( async_w3.provider.request_cache_validation_threshold == expected_threshold ) - cache_items = async_w3.provider._request_cache.items() - assert len(cache_items) == 1 - assert cache_items[0][1]["result"] == hex(chain_id) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "endpoint", BLOCKNUM_IN_PARAMS | BLOCK_IN_RESULT | BLOCKHASH_IN_PARAMS +) +@pytest.mark.parametrize( + "time_from_threshold,should_cache", + # -0.2 to give some time for the test to run + ((-2, True), (-1, True), (-0.2, True), (1, False), (2, False)), +) +@pytest.mark.parametrize( + "chain_id,expected_threshold_in_seconds", + [ + (chain_id, threshold) + for chain_id, threshold in CHAIN_VALIDATION_THRESHOLD_DEFAULTS.items() + if isinstance(threshold, int) + ] + + [ + (3456787654567654, DEFAULT_VALIDATION_THRESHOLD), + (11111111111444444444444444, DEFAULT_VALIDATION_THRESHOLD), + (-11111111111111111117, DEFAULT_VALIDATION_THRESHOLD), + ], +) +async def test_async_validation_against_validation_threshold_time_based( + endpoint, + time_from_threshold, + should_cache, + chain_id, + expected_threshold_in_seconds, + request_mocker, +): + async_w3 = AsyncWeb3(AsyncBaseProvider(cache_allowed_requests=True)) + blocknum = "0x2" + # mock the timestamp so that we are at the threshold +/- the time_from_threshold + mocked_time = hex( + int(round(time.time() - expected_threshold_in_seconds) + time_from_threshold) + ) + + async with request_mocker( + async_w3, + mock_results={ + "eth_chainId": hex(chain_id), + endpoint: lambda *_: ( + # mock the result to requests that return blocks + {"number": blocknum, "timestamp": mocked_time} + if "getBlock" in endpoint + # mock the result to requests that return transactions with the blocknum + # for our block under test + else {"blockNumber": blocknum} + ), + "eth_getBlockByNumber": lambda _method, params: ( + {"number": blocknum, "timestamp": mocked_time} + ), + "eth_getBlockByHash": {"number": blocknum, "timestamp": mocked_time}, + }, + ): + assert len(async_w3.provider._request_cache.items()) == 0 + await async_w3.manager.coro_request(endpoint, [blocknum, False]) + cached_items = async_w3.provider._request_cache.items() + assert len(cached_items) == 1 if should_cache else len(cached_items) == 0 @pytest.mark.asyncio @@ -685,7 +843,81 @@ async def test_async_request_caching_with_validation_threshold_set_to_none( request_cache_validation_threshold=None, ) ) - async with request_mocker(async_w3, mock_results={endpoint: {"number": blocknum}}): + async with request_mocker( + async_w3, + mock_results={ + # simulate the default settings for mainnet and show that we cache + # everything before and after the `finalized` block + "eth_chainId": "0x1", + endpoint: {"number": blocknum}, + "eth_getBlockByNumber": lambda _method, params: ( + # mock the threshold block to be blocknum "0x2", return + # blocknum otherwise + {"number": "0x2", "timestamp": "0x0"} + if params[0] == "finalized" + else {"number": blocknum, "timestamp": "0x0"} + ), + }, + ): assert len(async_w3.provider._request_cache.items()) == 0 await async_w3.manager.coro_request(endpoint, [blocknum, False]) assert len(async_w3.provider._request_cache.items()) == 1 + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "time_from_threshold,should_cache", + # -0.2 to give some time for the test to run + ((-2, True), (-1, True), (-0.2, True), (1, False), (2, False)), +) +@pytest.mark.parametrize( + "chain_id", + ( + # test that defaults are also overridden by the configured validation threshold + *CHAIN_VALIDATION_THRESHOLD_DEFAULTS.keys(), + 3456787654567654, + 11111111111444444444444444, + -11111111111111111117, + ), +) +@pytest.mark.parametrize( + "endpoint", BLOCKNUM_IN_PARAMS | BLOCK_IN_RESULT | BLOCKHASH_IN_PARAMS +) +async def test_async_validation_against_validation_threshold_time_based_configured( + time_from_threshold, should_cache, chain_id, endpoint, request_mocker +): + configured_time_threshold = 60 * 60 * 24 * 7 # 1 week + async_w3 = AsyncWeb3( + AsyncBaseProvider( + cache_allowed_requests=True, + request_cache_validation_threshold=configured_time_threshold, + ) + ) + blocknum = "0x2" + # mock the timestamp so that we are at the threshold +/- the time_from_threshold + mocked_time = hex( + int(round(time.time()) - configured_time_threshold + time_from_threshold) + ) + + async with request_mocker( + async_w3, + mock_results={ + "eth_chainId": chain_id, + endpoint: lambda *_: ( + # mock the result to requests that return blocks + {"number": blocknum, "timestamp": mocked_time} + if "getBlock" in endpoint + # mock the result to requests that return transactions with the blocknum + # for our block under test + else {"blockNumber": blocknum} + ), + "eth_getBlockByNumber": lambda _method, params: ( + {"number": blocknum, "timestamp": mocked_time} + ), + "eth_getBlockByHash": {"number": blocknum, "timestamp": mocked_time}, + }, + ): + assert len(async_w3.provider._request_cache.items()) == 0 + await async_w3.manager.coro_request(endpoint, [blocknum, False]) + cached_items = async_w3.provider._request_cache.items() + assert len(cached_items) == 1 if should_cache else len(cached_items) == 0 diff --git a/web3/_utils/caching/request_caching_validation.py b/web3/_utils/caching/request_caching_validation.py index b82a0d15fe..aaad0150f8 100644 --- a/web3/_utils/caching/request_caching_validation.py +++ b/web3/_utils/caching/request_caching_validation.py @@ -44,6 +44,8 @@ def is_beyond_validation_threshold( try: threshold = provider.request_cache_validation_threshold + # turn off caching to prevent recursion + provider.cache_allowed_requests = False if isinstance(threshold, RequestCacheValidationThreshold): # if mainnet and threshold is "finalized" or "safe" threshold_block = provider.make_request( @@ -72,6 +74,8 @@ def is_beyond_validation_threshold( except Exception as e: _error_log(provider, e) return False + finally: + provider.cache_allowed_requests = True def validate_from_block_id_in_params( @@ -94,6 +98,9 @@ def validate_from_blocknum_in_result( result: Dict[str, Any], ) -> bool: try: + # turn off caching to prevent recursion + provider.cache_allowed_requests = False + # transaction results if "blockNumber" in result: blocknum = result.get("blockNumber") @@ -121,6 +128,8 @@ def validate_from_blocknum_in_result( except Exception as e: _error_log(provider, e) return False + finally: + provider.cache_allowed_requests = True def validate_from_blockhash_in_params( @@ -129,6 +138,9 @@ def validate_from_blockhash_in_params( _result: Dict[str, Any], ) -> bool: try: + # turn off caching to prevent recursion + provider.cache_allowed_requests = False + # make an extra call to get the block number from the hash block = provider.make_request("eth_getBlockByHash", [params[0], False])[ "result" @@ -141,6 +153,8 @@ def validate_from_blockhash_in_params( except Exception as e: _error_log(provider, e) return False + finally: + provider.cache_allowed_requests = True # -- async -- # @@ -154,6 +168,8 @@ async def async_is_beyond_validation_threshold( try: threshold = provider.request_cache_validation_threshold + # turn off caching to prevent recursion + provider.cache_allowed_requests = False if isinstance(threshold, RequestCacheValidationThreshold): # if mainnet and threshold is "finalized" or "safe" threshold_block = await provider.make_request( @@ -180,6 +196,8 @@ async def async_is_beyond_validation_threshold( except Exception as e: _error_log(provider, e) return False + finally: + provider.cache_allowed_requests = True async def async_validate_from_block_id_in_params( @@ -202,6 +220,9 @@ async def async_validate_from_blocknum_in_result( result: Dict[str, Any], ) -> bool: try: + # turn off caching to prevent recursion + provider.cache_allowed_requests = False + # transaction results if "blockNumber" in result: blocknum = result.get("blockNumber") @@ -229,12 +250,17 @@ async def async_validate_from_blocknum_in_result( except Exception as e: _error_log(provider, e) return False + finally: + provider.cache_allowed_requests = True async def async_validate_from_blockhash_in_params( provider: ASYNC_PROVIDER_TYPE, params: Sequence[Any], _result: Dict[str, Any] ) -> bool: try: + # turn off caching to prevent recursion + provider.cache_allowed_requests = False + # make an extra call to get the block number from the hash response = await provider.make_request("eth_getBlockByHash", [params[0], False]) return await async_is_beyond_validation_threshold( @@ -245,3 +271,5 @@ async def async_validate_from_blockhash_in_params( except Exception as e: _error_log(provider, e) return False + finally: + provider.cache_allowed_requests = True From a4c659e44175e17c3cc35fe9a379562ba7b3b618 Mon Sep 17 00:00:00 2001 From: fselmo Date: Fri, 11 Oct 2024 10:33:07 -0600 Subject: [PATCH 3/6] Update request caching documentation --- docs/internals.rst | 117 +++++++++++++++++++++++---------- newsfragments/3508.bugfix.rst | 1 + newsfragments/3508.docs.rst | 1 + newsfragments/3508.feature.rst | 1 + 4 files changed, 87 insertions(+), 33 deletions(-) create mode 100644 newsfragments/3508.bugfix.rst create mode 100644 newsfragments/3508.docs.rst create mode 100644 newsfragments/3508.feature.rst diff --git a/docs/internals.rst b/docs/internals.rst index b5bae593c9..17e23922b5 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -119,12 +119,21 @@ Provider Configurations Request Caching ``````````````` +.. important:: + Familiarize yourself with the validation logic for request caching before + enabling it. Since this feature often requires making additional requests under the + hood to try to guarantee the validity of the data, it may create unnecessary + overhead for your use case. Validation can be turned off by setting the + ``request_cache_validation_threshold`` option to ``None``, caching all allowed + requests, or configured for adjusting performance to your needs. + + Request caching can be configured at the provider level via the following configuration options on the provider instance: - ``cache_allowed_requests: bool = False`` -- ``cacheable_requests: Set[RPCEndpoint] = CACHEABLE_REQUESTS`` -- ``request_cache_validation_threshold: RequestCacheValidationThreshold = RequestCacheValidationThreshold.FINALIZED`` +- ``cacheable_requests: Optional[Set[RPCEndpoint]]`` +- ``request_cache_validation_threshold: Optional[Union[RequestCacheValidationThreshold, int]]`` For requests that don't rely on block data (e.g., ``eth_chainId``), enabling request caching by setting the ``cache_allowed_requests`` option to ``True`` will cache all @@ -132,35 +141,78 @@ responses. This is safe to do. However, for requests that rely on block data (e.g., ``eth_getBlockByNumber``), it is not safe to always cache their responses because block data can change - during a -chain reorganization, for example. The ``request_cache_validation_threshold`` option -allows configuring a safe threshold for caching responses that depend on block data. By -default, the ``finalized`` block number is used as the validation threshold, meaning -that a request's response will be cached if the block number it relies on is less than -or equal to the ``finalized`` block number. If the block number exceeds the -``finalized`` block number, the response won't be cached. +chain reorganization or while finality has not been reached, for example. The +``request_cache_validation_threshold`` option allows configuring a safe threshold for +caching responses that depend on block data. By default, this option is configured +to internal values deemed "safe" for the chain id you are connected to. If you are +connected to mainnet Ethereum, this value is set to the ``finalized`` block number. +If you are connected to another chain, this value is set to a time internal in seconds, +from the current time, that is deemed "safe" for that chain's finality mechanism. + +**It's important to understand that, in order to perform these validations, extra +requests are sometimes made to the node to get the appropriate information. For a +transaction request, for example, it is necessary to get the block information to +validate the transaction is beyond the safe threshold. This can create overhead, +especially for high-frequency requests. For this reason, it is important to understand +when to turn on caching and how to configure the validation appropriately for your +use case in order to avoid unnecessary overhead.** + +We keep a list of some reasonable values for bigger chains and +use the time interval of 1 hour for everything else. Below is a list of the default +values for internally configured chains: + + - ETH: RequestCacheValidationThreshold.FINALIZED ("finalized" block) + - ARB1: 7 days + - ZKSYNC: 1 hour + - OETH: 3 minutes + - MATIC: 30 minutes + - ZKEVM: 1 hour + - BASE: 7 days + - SCR: 1 hour + - GNO: 5 minutes + - AVAX: 2 minutes + - BNB: 2 minutes + - FTM: 1 minute + +For Ethereum mainnet, for example, this means that a request's response will be cached +if the block number the request relies on is less than or equal to the ``finalized`` +block number. If the block number exceeds the ``finalized`` block number, the response +won't be cached. For all others, the response will be cached if the block timestamp +related to the data that is being requested is older than or equal to the time interval +configured for that chain. For any chain not on this list, the default value is set to +1 hour (this includes all testnets). This behavior can be modified by setting the ``request_cache_validation_threshold`` option to ``RequestCacheValidationThreshold.SAFE``, which uses the ``safe`` block as -the threshold, or to ``None``, which disables cache validation and caches all -requests (this is not recommended). The ``RequestCacheValidationThreshold`` enum is -imported from the ``web3.utils`` module. - -The current list of requests that are validated by this configuration before being -cached is: - - - RPC.eth_getBlockByNumber - - RPC.eth_getRawTransactionByBlockNumberAndIndex - - RPC.eth_getBlockTransactionCountByNumber - - RPC.eth_getUncleByBlockNumberAndIndex - - RPC.eth_getUncleCountByBlockNumber - - RPC.eth_getBlockByHash - - RPC.eth_getTransactionByHash - - RPC.eth_getTransactionByBlockNumberAndIndex - - RPC.eth_getTransactionByBlockHashAndIndex - - RPC.eth_getBlockTransactionCountByHash - - RPC.eth_getRawTransactionByBlockHashAndIndex - - RPC.eth_getUncleByBlockHashAndIndex - - RPC.eth_getUncleCountByBlockHash +the threshold (Ethereum mainnet only), to your own time interval in seconds (for any +chain, including mainnet Ethereum), or to ``None``, which disables any validation and +caches all requests (this is not recommended for non testnet chains). The +``RequestCacheValidationThreshold`` enum, for mainnet ``finalized`` and ``safe`` values, +is imported from the ``web3.utils`` module. + +Note that the ``cacheable_requests`` option can be used to specify a set of RPC +endpoints that are allowed to be cached. By default, this option is set to an internal +list of deemed-safe-to-cache endpoints, excluding endpoints such as ``eth_call``, whose +responses can vary and are not safe to cache. The default list of cacheable requests is +below, with requests validated by the ``request_cache_validation_threshold`` option in +bold: + + - eth_chainId + - web3_clientVersion + - net_version + - **eth_getBlockByNumber** + - **eth_getRawTransactionByBlockNumberAndIndex** + - **eth_getBlockTransactionCountByNumber** + - **eth_getUncleByBlockNumberAndIndex** + - **eth_getUncleCountByBlockNumber** + - **eth_getBlockByHash** + - **eth_getTransactionByHash** + - **eth_getTransactionByBlockNumberAndIndex** + - **eth_getTransactionByBlockHashAndIndex** + - **eth_getBlockTransactionCountByHash** + - **eth_getRawTransactionByBlockHashAndIndex** + - **eth_getUncleByBlockHashAndIndex** + - **eth_getUncleCountByBlockHash** .. code-block:: python @@ -173,13 +225,12 @@ cached is: # optional flag to turn on cached requests, defaults to ``False`` cache_allowed_requests=True, - # optional, defaults to an internal list of deemed-safe-to-cache endpoints + # optional, defaults to an internal list of deemed-safe-to-cache endpoints (see above) cacheable_requests={"eth_chainId", "eth_getBlockByNumber"}, - # optional, defaults to ``RequestCacheValidationThreshold.FINALIZED`` - # can be set to ``RequestCacheValidationThreshold.SAFE`` or turned off - # by setting to ``None``. - request_cache_validation_threshold=RequestCacheValidationThreshold.SAFE, + # optional, defaults to a value that is based on the chain id (see above) + request_cache_validation_threshold=60 * 60, # 1 hour + # request_cache_validation_threshold=RequestCacheValidationThreshold.SAFE, # Ethereum mainnet only )) .. _http_retry_requests: diff --git a/newsfragments/3508.bugfix.rst b/newsfragments/3508.bugfix.rst new file mode 100644 index 0000000000..d1a1a8631c --- /dev/null +++ b/newsfragments/3508.bugfix.rst @@ -0,0 +1 @@ +Fix a bug where non-mainnet chains could not cache requests based on missing ``finalized`` block number. diff --git a/newsfragments/3508.docs.rst b/newsfragments/3508.docs.rst new file mode 100644 index 0000000000..4817e0b0c4 --- /dev/null +++ b/newsfragments/3508.docs.rst @@ -0,0 +1 @@ +Update the request caching documentation to clarify on when to reach for request caching and how to configure the request validation threshold for certain endpoints. diff --git a/newsfragments/3508.feature.rst b/newsfragments/3508.feature.rst new file mode 100644 index 0000000000..3099025177 --- /dev/null +++ b/newsfragments/3508.feature.rst @@ -0,0 +1 @@ +Allow a time interval, in seconds, to be used as the ``request_cache_validation_threshold`` for request caching. Keep a list of internal default values based on the chain id for some of the bigger chains. From b5f5985e454174aa2e1b7963f66f0a3ecccf1beb Mon Sep 17 00:00:00 2001 From: fselmo Date: Mon, 14 Oct 2024 13:50:04 -0600 Subject: [PATCH 4/6] Changes from comments on PR #3508: - Fix typo in docs. - Fix test check, be more precise and check cache == 1. - Remove validation for threshold with chain_id == 1 since more chains than mainnet Ethereum can make use of ``finalized`` and ``safe`` thresholds. - Store and reset the current value of ``cache_allowed_requests`` rather than assuming ``True`` / ``False`` --- docs/internals.rst | 2 +- .../caching-utils/test_request_caching.py | 2 +- web3/_utils/caching/caching_utils.py | 38 ++----------------- .../caching/request_caching_validation.py | 18 ++++++--- 4 files changed, 18 insertions(+), 42 deletions(-) diff --git a/docs/internals.rst b/docs/internals.rst index 17e23922b5..2f6182c67a 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -146,7 +146,7 @@ chain reorganization or while finality has not been reached, for example. The caching responses that depend on block data. By default, this option is configured to internal values deemed "safe" for the chain id you are connected to. If you are connected to mainnet Ethereum, this value is set to the ``finalized`` block number. -If you are connected to another chain, this value is set to a time internal in seconds, +If you are connected to another chain, this value is set to a time interval in seconds, from the current time, that is deemed "safe" for that chain's finality mechanism. **It's important to understand that, in order to perform these validations, extra diff --git a/tests/core/caching-utils/test_request_caching.py b/tests/core/caching-utils/test_request_caching.py index b86c476752..4233e15f2a 100644 --- a/tests/core/caching-utils/test_request_caching.py +++ b/tests/core/caching-utils/test_request_caching.py @@ -289,7 +289,7 @@ def test_block_id_param_caching_mainnet( assert len(w3.provider._request_cache.items()) == 0 w3.manager.request_blocking(RPCEndpoint(endpoint), [block_id, False]) cached_items = len(w3.provider._request_cache.items()) - assert cached_items > 0 if should_cache else cached_items == 0 + assert cached_items == 1 if should_cache else cached_items == 0 @pytest.mark.parametrize( diff --git a/web3/_utils/caching/caching_utils.py b/web3/_utils/caching/caching_utils.py index 1fb0bc5ec4..790d91c417 100644 --- a/web3/_utils/caching/caching_utils.py +++ b/web3/_utils/caching/caching_utils.py @@ -193,28 +193,13 @@ def set_threshold_if_empty(provider: SYNC_PROVIDER_TYPE) -> None: if current_threshold is empty or isinstance( current_threshold, RequestCacheValidationThreshold ): + cache_allowed_requests = provider.cache_allowed_requests try: # turn off momentarily to avoid recursion provider.cache_allowed_requests = False chain_id_result = provider.make_request("eth_chainId", [])["result"] chain_id = int(chain_id_result, 16) - if ( - isinstance( - current_threshold, - RequestCacheValidationThreshold, - ) - and chain_id != 1 - ): - provider.logger.debug( - "Request cache validation threshold is set to " - f"{current_threshold.value} " - f"for chain with chain_id `{chain_id}` but this value only works " - "on chain_id `1`. Setting to default value for chain_id " - f"`{chain_id}`.", - ) - provider.request_cache_validation_threshold = empty - if current_threshold is empty: provider.request_cache_validation_threshold = ( CHAIN_VALIDATION_THRESHOLD_DEFAULTS.get( @@ -224,7 +209,7 @@ def set_threshold_if_empty(provider: SYNC_PROVIDER_TYPE) -> None: except Exception: provider.request_cache_validation_threshold = DEFAULT_VALIDATION_THRESHOLD finally: - provider.cache_allowed_requests = True + provider.cache_allowed_requests = cache_allowed_requests def _should_cache_response( @@ -303,28 +288,13 @@ async def async_set_threshold_if_empty(provider: ASYNC_PROVIDER_TYPE) -> None: if current_threshold is empty or isinstance( current_threshold, RequestCacheValidationThreshold ): + cache_allowed_requests = provider.cache_allowed_requests try: # turn off momentarily to avoid recursion provider.cache_allowed_requests = False chain_id_result = await provider.make_request("eth_chainId", []) chain_id = int(chain_id_result["result"], 16) - if ( - isinstance( - current_threshold, - RequestCacheValidationThreshold, - ) - and chain_id != 1 - ): - provider.logger.debug( - "Request cache validation threshold is set to " - f"{current_threshold.value} " - f"for chain with chain_id `{chain_id}` but this value only works " - "on chain_id `1`. Setting to default value for chain_id " - f"`{chain_id}`.", - ) - provider.request_cache_validation_threshold = empty - if current_threshold is empty: provider.request_cache_validation_threshold = ( CHAIN_VALIDATION_THRESHOLD_DEFAULTS.get( @@ -334,7 +304,7 @@ async def async_set_threshold_if_empty(provider: ASYNC_PROVIDER_TYPE) -> None: except Exception: provider.request_cache_validation_threshold = DEFAULT_VALIDATION_THRESHOLD finally: - provider.cache_allowed_requests = True + provider.cache_allowed_requests = cache_allowed_requests async def _async_should_cache_response( diff --git a/web3/_utils/caching/request_caching_validation.py b/web3/_utils/caching/request_caching_validation.py index aaad0150f8..fd51c686aa 100644 --- a/web3/_utils/caching/request_caching_validation.py +++ b/web3/_utils/caching/request_caching_validation.py @@ -41,6 +41,7 @@ def is_beyond_validation_threshold( blocknum: int = None, block_timestamp: int = None, ) -> bool: + cache_allowed_requests = provider.cache_allowed_requests try: threshold = provider.request_cache_validation_threshold @@ -75,7 +76,7 @@ def is_beyond_validation_threshold( _error_log(provider, e) return False finally: - provider.cache_allowed_requests = True + provider.cache_allowed_requests = cache_allowed_requests def validate_from_block_id_in_params( @@ -97,6 +98,7 @@ def validate_from_blocknum_in_result( _params: Sequence[Any], result: Dict[str, Any], ) -> bool: + cache_allowed_requests = provider.cache_allowed_requests try: # turn off caching to prevent recursion provider.cache_allowed_requests = False @@ -129,7 +131,7 @@ def validate_from_blocknum_in_result( _error_log(provider, e) return False finally: - provider.cache_allowed_requests = True + provider.cache_allowed_requests = cache_allowed_requests def validate_from_blockhash_in_params( @@ -137,6 +139,7 @@ def validate_from_blockhash_in_params( params: Sequence[Any], _result: Dict[str, Any], ) -> bool: + cache_allowed_requests = provider.cache_allowed_requests try: # turn off caching to prevent recursion provider.cache_allowed_requests = False @@ -154,7 +157,7 @@ def validate_from_blockhash_in_params( _error_log(provider, e) return False finally: - provider.cache_allowed_requests = True + provider.cache_allowed_requests = cache_allowed_requests # -- async -- # @@ -165,6 +168,7 @@ async def async_is_beyond_validation_threshold( blocknum: int = None, block_timestamp: int = None, ) -> bool: + cache_allowed_requests = provider.cache_allowed_requests try: threshold = provider.request_cache_validation_threshold @@ -197,7 +201,7 @@ async def async_is_beyond_validation_threshold( _error_log(provider, e) return False finally: - provider.cache_allowed_requests = True + provider.cache_allowed_requests = cache_allowed_requests async def async_validate_from_block_id_in_params( @@ -219,6 +223,7 @@ async def async_validate_from_blocknum_in_result( _params: Sequence[Any], result: Dict[str, Any], ) -> bool: + cache_allowed_requests = provider.cache_allowed_requests try: # turn off caching to prevent recursion provider.cache_allowed_requests = False @@ -251,12 +256,13 @@ async def async_validate_from_blocknum_in_result( _error_log(provider, e) return False finally: - provider.cache_allowed_requests = True + provider.cache_allowed_requests = cache_allowed_requests async def async_validate_from_blockhash_in_params( provider: ASYNC_PROVIDER_TYPE, params: Sequence[Any], _result: Dict[str, Any] ) -> bool: + cache_allowed_requests = provider.cache_allowed_requests try: # turn off caching to prevent recursion provider.cache_allowed_requests = False @@ -272,4 +278,4 @@ async def async_validate_from_blockhash_in_params( _error_log(provider, e) return False finally: - provider.cache_allowed_requests = True + provider.cache_allowed_requests = cache_allowed_requests From e240cc672080e62c64771ceb8c1745ee4bf1e353 Mon Sep 17 00:00:00 2001 From: fselmo Date: Mon, 14 Oct 2024 16:56:08 -0600 Subject: [PATCH 5/6] Test all providers for request caching; add caching to IPCProvider --- .../caching-utils/test_request_caching.py | 200 ++++++++++-------- web3/providers/ipc.py | 4 + 2 files changed, 114 insertions(+), 90 deletions(-) diff --git a/tests/core/caching-utils/test_request_caching.py b/tests/core/caching-utils/test_request_caching.py index 4233e15f2a..f61a94b6a0 100644 --- a/tests/core/caching-utils/test_request_caching.py +++ b/tests/core/caching-utils/test_request_caching.py @@ -2,6 +2,10 @@ import pytest import threading import time +from typing import ( + Optional, + Union, +) import uuid import pytest_asyncio @@ -13,6 +17,7 @@ HTTPProvider, IPCProvider, LegacyWebSocketProvider, + PersistentConnectionProvider, Web3, WebSocketProvider, ) @@ -32,14 +37,6 @@ from web3.exceptions import ( Web3RPCError, ) -from web3.providers import ( - AsyncBaseProvider, - BaseProvider, - JSONBaseProvider, -) -from web3.providers.async_base import ( - AsyncJSONBaseProvider, -) from web3.types import ( RPCEndpoint, ) @@ -48,6 +45,17 @@ SimpleCache, ) +SYNC_PROVIDERS = [ + HTTPProvider, + IPCProvider, + LegacyWebSocketProvider, # deprecated +] +ASYNC_PROVIDERS = [ + AsyncHTTPProvider, + AsyncIPCProvider, + WebSocketProvider, +] + def simple_cache_return_value_a(): _cache = SimpleCache() @@ -58,9 +66,14 @@ def simple_cache_return_value_a(): return _cache +@pytest.fixture(params=SYNC_PROVIDERS) +def sync_provider(request): + return request.param + + @pytest.fixture -def w3(request_mocker): - _w3 = Web3(provider=BaseProvider(cache_allowed_requests=True)) +def w3(sync_provider, request_mocker): + _w3 = Web3(provider=sync_provider(cache_allowed_requests=True)) _w3.provider.cacheable_requests += (RPCEndpoint("fake_endpoint"),) with request_mocker( _w3, @@ -88,8 +101,8 @@ def test_request_caching_populates_cache(w3): assert len(w3.provider._request_cache.items()) == 2 -def test_request_caching_does_not_cache_none_responses(request_mocker): - w3 = Web3(BaseProvider(cache_allowed_requests=True)) +def test_request_caching_does_not_cache_none_responses(sync_provider, request_mocker): + w3 = Web3(sync_provider(cache_allowed_requests=True)) w3.provider.cacheable_requests += (RPCEndpoint("fake_endpoint"),) counter = itertools.count() @@ -105,8 +118,8 @@ def result_cb(_method, _params): assert next(counter) == 2 -def test_request_caching_does_not_cache_error_responses(request_mocker): - w3 = Web3(BaseProvider(cache_allowed_requests=True)) +def test_request_caching_does_not_cache_error_responses(sync_provider, request_mocker): + w3 = Web3(sync_provider(cache_allowed_requests=True)) w3.provider.cacheable_requests += (RPCEndpoint("fake_endpoint"),) with request_mocker( @@ -127,12 +140,14 @@ def test_request_caching_does_not_cache_endpoints_not_in_allowlist(w3): assert result_a != result_b -def test_caching_requests_does_not_share_state_between_providers(request_mocker): +def test_caching_requests_does_not_share_state_between_providers( + sync_provider, request_mocker +): w3_a, w3_b, w3_c, w3_a_shared_cache = ( - Web3(provider=BaseProvider(cache_allowed_requests=True)), - Web3(provider=BaseProvider(cache_allowed_requests=True)), - Web3(provider=BaseProvider(cache_allowed_requests=True)), - Web3(provider=BaseProvider(cache_allowed_requests=True)), + Web3(provider=sync_provider(cache_allowed_requests=True)), + Web3(provider=sync_provider(cache_allowed_requests=True)), + Web3(provider=sync_provider(cache_allowed_requests=True)), + Web3(provider=sync_provider(cache_allowed_requests=True)), ) # strap w3_a_shared_cache with w3_a's cache @@ -164,21 +179,7 @@ def test_caching_requests_does_not_share_state_between_providers(request_mocker) assert result_a_shared_cache == hex(11111) -@pytest.mark.parametrize( - "provider", - [ - BaseProvider, - JSONBaseProvider, - HTTPProvider, - IPCProvider, - AsyncBaseProvider, - AsyncJSONBaseProvider, - AsyncHTTPProvider, - AsyncIPCProvider, - WebSocketProvider, - LegacyWebSocketProvider, # deprecated - ], -) +@pytest.mark.parametrize("provider", [*SYNC_PROVIDERS, *ASYNC_PROVIDERS]) def test_all_providers_do_not_cache_by_default_and_can_set_caching_properties(provider): _provider_default_init = provider() assert _provider_default_init.cache_allowed_requests is False @@ -216,10 +217,10 @@ def test_all_providers_do_not_cache_by_default_and_can_set_caching_properties(pr ), ) def test_blocknum_validation_against_validation_threshold_when_caching_mainnet( - threshold, endpoint, blocknum, should_cache, request_mocker + threshold, endpoint, blocknum, should_cache, sync_provider, request_mocker ): w3 = Web3( - BaseProvider( + sync_provider( cache_allowed_requests=True, request_cache_validation_threshold=threshold ) ) @@ -266,10 +267,10 @@ def test_blocknum_validation_against_validation_threshold_when_caching_mainnet( ), ) def test_block_id_param_caching_mainnet( - threshold, endpoint, block_id, blocknum, should_cache, request_mocker + threshold, endpoint, block_id, blocknum, should_cache, sync_provider, request_mocker ): w3 = Web3( - BaseProvider( + sync_provider( cache_allowed_requests=True, request_cache_validation_threshold=threshold ) ) @@ -309,10 +310,10 @@ def test_block_id_param_caching_mainnet( ), ) def test_blockhash_validation_against_validation_threshold_when_caching_mainnet( - threshold, endpoint, blocknum, should_cache, request_mocker + threshold, endpoint, blocknum, should_cache, sync_provider, request_mocker ): w3 = Web3( - BaseProvider( + sync_provider( cache_allowed_requests=True, request_cache_validation_threshold=threshold ) ) @@ -346,9 +347,9 @@ def test_blockhash_validation_against_validation_threshold_when_caching_mainnet( ), ) def test_request_caching_validation_threshold_defaults( - chain_id, expected_threshold, request_mocker + chain_id, expected_threshold, sync_provider, request_mocker ): - w3 = Web3(BaseProvider(cache_allowed_requests=True)) + w3 = Web3(sync_provider(cache_allowed_requests=True)) with request_mocker(w3, mock_results={"eth_chainId": hex(chain_id)}): w3.manager.request_blocking(RPCEndpoint("eth_chainId"), []) assert w3.provider.request_cache_validation_threshold == expected_threshold @@ -385,9 +386,10 @@ def test_sync_validation_against_validation_threshold_time_based( should_cache, chain_id, expected_threshold_in_seconds, + sync_provider, request_mocker, ): - w3 = Web3(BaseProvider(cache_allowed_requests=True)) + w3 = Web3(sync_provider(cache_allowed_requests=True)) blocknum = "0x2" # mock the timestamp so that we are at the threshold +/- the time_from_threshold mocked_time = hex( @@ -437,11 +439,11 @@ def test_sync_validation_against_validation_threshold_time_based( "endpoint", BLOCKNUM_IN_PARAMS | BLOCK_IN_RESULT | BLOCKHASH_IN_PARAMS ) def test_validation_against_validation_threshold_time_based_configured( - time_from_threshold, should_cache, chain_id, endpoint, request_mocker + time_from_threshold, should_cache, chain_id, endpoint, sync_provider, request_mocker ): configured_time_threshold = 60 * 60 * 24 * 7 # 1 week w3 = Web3( - BaseProvider( + sync_provider( cache_allowed_requests=True, request_cache_validation_threshold=configured_time_threshold, ) @@ -487,9 +489,32 @@ def test_async_cacheable_requests_are_the_same_as_sync(): ), "make sure the async and sync cacheable requests are the same" +@pytest_asyncio.fixture(params=ASYNC_PROVIDERS) +async def async_provider(request): + return request.param + + +async def _async_w3_init( + async_provider, + threshold: Optional[Union[RequestCacheValidationThreshold, int]] = "empty", +): + if isinstance(async_provider, PersistentConnectionProvider): + _async_w3 = await AsyncWeb3( + provider=async_provider( + cache_allowed_requests=True, + ) + ) + else: + _async_w3 = AsyncWeb3(provider=async_provider(cache_allowed_requests=True)) + + if threshold != "empty": + _async_w3.provider.request_cache_validation_threshold = threshold + return _async_w3 + + @pytest_asyncio.fixture -async def async_w3(request_mocker): - _async_w3 = AsyncWeb3(AsyncBaseProvider(cache_allowed_requests=True)) +async def async_w3(async_provider, request_mocker): + _async_w3 = await _async_w3_init(async_provider) _async_w3.provider.cacheable_requests += (RPCEndpoint("fake_endpoint"),) async with request_mocker( _async_w3, @@ -524,8 +549,10 @@ async def test_async_request_caching_populates_cache(async_w3): @pytest.mark.asyncio -async def test_async_request_caching_does_not_cache_none_responses(request_mocker): - async_w3 = AsyncWeb3(AsyncBaseProvider(cache_allowed_requests=True)) +async def test_async_request_caching_does_not_cache_none_responses( + async_provider, request_mocker +): + async_w3 = await _async_w3_init(async_provider) async_w3.provider.cacheable_requests += (RPCEndpoint("fake_endpoint"),) counter = itertools.count() @@ -542,8 +569,10 @@ def result_cb(_method, _params): @pytest.mark.asyncio -async def test_async_request_caching_does_not_cache_error_responses(request_mocker): - async_w3 = AsyncWeb3(AsyncBaseProvider(cache_allowed_requests=True)) +async def test_async_request_caching_does_not_cache_error_responses( + async_provider, request_mocker +): + async_w3 = await _async_w3_init(async_provider) async_w3.provider.cacheable_requests += (RPCEndpoint("fake_endpoint"),) async with request_mocker( @@ -569,13 +598,14 @@ async def test_async_request_caching_does_not_cache_non_allowlist_endpoints( @pytest.mark.asyncio async def test_async_request_caching_does_not_share_state_between_providers( + async_provider, request_mocker, ): async_w3_a, async_w3_b, async_w3_c, async_w3_a_shared_cache = ( - AsyncWeb3(AsyncBaseProvider(cache_allowed_requests=True)), - AsyncWeb3(AsyncBaseProvider(cache_allowed_requests=True)), - AsyncWeb3(AsyncBaseProvider(cache_allowed_requests=True)), - AsyncWeb3(AsyncBaseProvider(cache_allowed_requests=True)), + await _async_w3_init(async_provider), + await _async_w3_init(async_provider), + await _async_w3_init(async_provider), + await _async_w3_init(async_provider), ) # strap async_w3_a_shared_cache with async_w3_a's cache @@ -625,13 +655,9 @@ async def test_async_request_caching_does_not_share_state_between_providers( ), ) async def test_async_blocknum_validation_against_validation_threshold_mainnet( - threshold, endpoint, blocknum, should_cache, request_mocker + threshold, endpoint, blocknum, should_cache, async_provider, request_mocker ): - async_w3 = AsyncWeb3( - AsyncBaseProvider( - cache_allowed_requests=True, request_cache_validation_threshold=threshold - ) - ) + async_w3 = await _async_w3_init(async_provider, threshold=threshold) async with request_mocker( async_w3, mock_results={ @@ -676,13 +702,15 @@ async def test_async_blocknum_validation_against_validation_threshold_mainnet( ), ) async def test_async_block_id_param_caching_mainnet( - threshold, endpoint, block_id, blocknum, should_cache, request_mocker + threshold, + endpoint, + block_id, + blocknum, + should_cache, + async_provider, + request_mocker, ): - async_w3 = AsyncWeb3( - AsyncBaseProvider( - cache_allowed_requests=True, request_cache_validation_threshold=threshold - ) - ) + async_w3 = await _async_w3_init(async_provider, threshold=threshold) async with request_mocker( async_w3, mock_results={ @@ -720,13 +748,9 @@ async def test_async_block_id_param_caching_mainnet( ), ) async def test_async_blockhash_validation_against_validation_threshold_mainnet( - threshold, endpoint, blocknum, should_cache, request_mocker + threshold, endpoint, blocknum, should_cache, async_provider, request_mocker ): - async_w3 = AsyncWeb3( - AsyncBaseProvider( - cache_allowed_requests=True, request_cache_validation_threshold=threshold - ) - ) + async_w3 = await _async_w3_init(async_provider, threshold=threshold) async with request_mocker( async_w3, mock_results={ @@ -758,9 +782,9 @@ async def test_async_blockhash_validation_against_validation_threshold_mainnet( ), ) async def test_async_request_caching_validation_threshold_defaults( - chain_id, expected_threshold, request_mocker + chain_id, expected_threshold, async_provider, request_mocker ): - async_w3 = AsyncWeb3(AsyncBaseProvider(cache_allowed_requests=True)) + async_w3 = await _async_w3_init(async_provider) async with request_mocker(async_w3, mock_results={"eth_chainId": hex(chain_id)}): await async_w3.manager.coro_request(RPCEndpoint("eth_chainId"), []) assert ( @@ -796,9 +820,10 @@ async def test_async_validation_against_validation_threshold_time_based( should_cache, chain_id, expected_threshold_in_seconds, + async_provider, request_mocker, ): - async_w3 = AsyncWeb3(AsyncBaseProvider(cache_allowed_requests=True)) + async_w3 = await _async_w3_init(async_provider) blocknum = "0x2" # mock the timestamp so that we are at the threshold +/- the time_from_threshold mocked_time = hex( @@ -835,14 +860,9 @@ async def test_async_validation_against_validation_threshold_time_based( ) @pytest.mark.parametrize("blocknum", ("0x0", "0x1", "0x2", "0x3", "0x4", "0x5")) async def test_async_request_caching_with_validation_threshold_set_to_none( - endpoint, blocknum, request_mocker + endpoint, blocknum, async_provider, request_mocker ): - async_w3 = AsyncWeb3( - AsyncBaseProvider( - cache_allowed_requests=True, - request_cache_validation_threshold=None, - ) - ) + async_w3 = await _async_w3_init(async_provider, threshold=None) async with request_mocker( async_w3, mock_results={ @@ -884,15 +904,15 @@ async def test_async_request_caching_with_validation_threshold_set_to_none( "endpoint", BLOCKNUM_IN_PARAMS | BLOCK_IN_RESULT | BLOCKHASH_IN_PARAMS ) async def test_async_validation_against_validation_threshold_time_based_configured( - time_from_threshold, should_cache, chain_id, endpoint, request_mocker + time_from_threshold, + should_cache, + chain_id, + endpoint, + async_provider, + request_mocker, ): configured_time_threshold = 60 * 60 * 24 * 7 # 1 week - async_w3 = AsyncWeb3( - AsyncBaseProvider( - cache_allowed_requests=True, - request_cache_validation_threshold=configured_time_threshold, - ) - ) + async_w3 = await _async_w3_init(async_provider, threshold=configured_time_threshold) blocknum = "0x2" # mock the timestamp so that we are at the threshold +/- the time_from_threshold mocked_time = hex( diff --git a/web3/providers/ipc.py b/web3/providers/ipc.py index 9a4e815aa5..9ea72157d6 100644 --- a/web3/providers/ipc.py +++ b/web3/providers/ipc.py @@ -32,6 +32,9 @@ from .._utils.batching import ( sort_batch_response_by_response_ids, ) +from .._utils.caching import ( + handle_request_caching, +) from ..exceptions import ( Web3TypeError, Web3ValueError, @@ -189,6 +192,7 @@ def _make_request(self, request: bytes) -> RPCResponse: timeout.sleep(0) continue + @handle_request_caching def make_request(self, method: RPCEndpoint, params: Any) -> RPCResponse: self.logger.debug( f"Making request IPC. Path: {self.ipc_path}, Method: {method}" From aa695d41bd7b9f2befbdf83d187fae1921f122c1 Mon Sep 17 00:00:00 2001 From: fselmo Date: Tue, 15 Oct 2024 09:51:24 -0600 Subject: [PATCH 6/6] Remove caching decorator from base providers; fix typing: - Base providers don't implement ``make_request`` and so can't cache requests. Remove the request caching decorator from the base provider classes. - Fix typing in the request caching utils, which should now be tighter, expecting an ``RPCEndpoint`` instead of a generic ``str``. --- web3/_utils/caching/caching_utils.py | 32 +++++++++++-------- .../caching/request_caching_validation.py | 29 ++++++++++------- web3/_utils/module_testing/utils.py | 6 ++-- web3/providers/async_base.py | 2 -- web3/providers/base.py | 2 -- 5 files changed, 39 insertions(+), 32 deletions(-) diff --git a/web3/_utils/caching/caching_utils.py b/web3/_utils/caching/caching_utils.py index 790d91c417..fe0317424d 100644 --- a/web3/_utils/caching/caching_utils.py +++ b/web3/_utils/caching/caching_utils.py @@ -53,6 +53,9 @@ from web3.exceptions import ( Web3TypeError, ) +from web3.types import ( + RPCEndpoint, +) from web3.utils import ( RequestCacheValidationThreshold, ) @@ -65,7 +68,6 @@ from web3.types import ( # noqa: F401 AsyncMakeRequestFn, MakeRequestFn, - RPCEndpoint, RPCResponse, ) @@ -93,7 +95,7 @@ def generate_cache_key(value: Any) -> str: class RequestInformation: def __init__( self, - method: "RPCEndpoint", + method: RPCEndpoint, params: Any, response_formatters: Tuple[ Union[Dict[str, Callable[..., Any]], Callable[..., Any]], @@ -133,7 +135,7 @@ def __init__( def is_cacheable_request( provider: Union[ASYNC_PROVIDER_TYPE, SYNC_PROVIDER_TYPE], - method: "RPCEndpoint", + method: RPCEndpoint, params: Any, ) -> bool: if not (provider.cache_allowed_requests and method in provider.cacheable_requests): @@ -173,7 +175,7 @@ def is_cacheable_request( } INTERNAL_VALIDATION_MAP: Dict[ - "RPCEndpoint", + RPCEndpoint, Callable[ [SYNC_PROVIDER_TYPE, Sequence[Any], Dict[str, Any]], bool, @@ -197,7 +199,9 @@ def set_threshold_if_empty(provider: SYNC_PROVIDER_TYPE) -> None: try: # turn off momentarily to avoid recursion provider.cache_allowed_requests = False - chain_id_result = provider.make_request("eth_chainId", [])["result"] + chain_id_result = provider.make_request(RPCEndpoint("eth_chainId"), [])[ + "result" + ] chain_id = int(chain_id_result, 16) if current_threshold is empty: @@ -214,7 +218,7 @@ def set_threshold_if_empty(provider: SYNC_PROVIDER_TYPE) -> None: def _should_cache_response( provider: SYNC_PROVIDER_TYPE, - method: "RPCEndpoint", + method: RPCEndpoint, params: Sequence[Any], response: "RPCResponse", ) -> bool: @@ -232,10 +236,10 @@ def _should_cache_response( def handle_request_caching( - func: Callable[[SYNC_PROVIDER_TYPE, "RPCEndpoint", Any], "RPCResponse"] + func: Callable[[SYNC_PROVIDER_TYPE, RPCEndpoint, Any], "RPCResponse"] ) -> Callable[..., "RPCResponse"]: def wrapper( - provider: SYNC_PROVIDER_TYPE, method: "RPCEndpoint", params: Any + provider: SYNC_PROVIDER_TYPE, method: RPCEndpoint, params: Any ) -> "RPCResponse": if is_cacheable_request(provider, method, params): request_cache = provider._request_cache @@ -266,7 +270,7 @@ def wrapper( Union[bool, Coroutine[Any, Any, bool]], ] -ASYNC_INTERNAL_VALIDATION_MAP: Dict["RPCEndpoint", ASYNC_VALIDATOR_TYPE] = { +ASYNC_INTERNAL_VALIDATION_MAP: Dict[RPCEndpoint, ASYNC_VALIDATOR_TYPE] = { **{endpoint: always_cache_request for endpoint in ALWAYS_CACHE}, **{ endpoint: async_validate_from_block_id_in_params @@ -292,7 +296,9 @@ async def async_set_threshold_if_empty(provider: ASYNC_PROVIDER_TYPE) -> None: try: # turn off momentarily to avoid recursion provider.cache_allowed_requests = False - chain_id_result = await provider.make_request("eth_chainId", []) + chain_id_result = await provider.make_request( + RPCEndpoint("eth_chainId"), [] + ) chain_id = int(chain_id_result["result"], 16) if current_threshold is empty: @@ -309,7 +315,7 @@ async def async_set_threshold_if_empty(provider: ASYNC_PROVIDER_TYPE) -> None: async def _async_should_cache_response( provider: ASYNC_PROVIDER_TYPE, - method: "RPCEndpoint", + method: RPCEndpoint, params: Sequence[Any], response: "RPCResponse", ) -> bool: @@ -333,11 +339,11 @@ async def _async_should_cache_response( def async_handle_request_caching( func: Callable[ - [ASYNC_PROVIDER_TYPE, "RPCEndpoint", Any], Coroutine[Any, Any, "RPCResponse"] + [ASYNC_PROVIDER_TYPE, RPCEndpoint, Any], Coroutine[Any, Any, "RPCResponse"] ], ) -> Callable[..., Coroutine[Any, Any, "RPCResponse"]]: async def wrapper( - provider: ASYNC_PROVIDER_TYPE, method: "RPCEndpoint", params: Any + provider: ASYNC_PROVIDER_TYPE, method: RPCEndpoint, params: Any ) -> "RPCResponse": if is_cacheable_request(provider, method, params): request_cache = provider._request_cache diff --git a/web3/_utils/caching/request_caching_validation.py b/web3/_utils/caching/request_caching_validation.py index fd51c686aa..4f904c72ac 100644 --- a/web3/_utils/caching/request_caching_validation.py +++ b/web3/_utils/caching/request_caching_validation.py @@ -8,6 +8,9 @@ Union, ) +from web3.types import ( + RPCEndpoint, +) from web3.utils import ( RequestCacheValidationThreshold, ) @@ -50,7 +53,7 @@ def is_beyond_validation_threshold( if isinstance(threshold, RequestCacheValidationThreshold): # if mainnet and threshold is "finalized" or "safe" threshold_block = provider.make_request( - "eth_getBlockByNumber", [threshold.value, False] + RPCEndpoint("eth_getBlockByNumber"), [threshold.value, False] )["result"] # we should have a `blocknum` to compare against return blocknum <= int(threshold_block["number"], 16) @@ -59,7 +62,7 @@ def is_beyond_validation_threshold( # if validating via `blocknum` from params, we need to get the timestamp # for the block with `blocknum`. block = provider.make_request( - "eth_getBlockByNumber", [hex(blocknum), False] + RPCEndpoint("eth_getBlockByNumber"), [hex(blocknum), False] )["result"] block_timestamp = int(block["timestamp"], 16) @@ -107,9 +110,9 @@ def validate_from_blocknum_in_result( if "blockNumber" in result: blocknum = result.get("blockNumber") # make an extra call to get the block values - block = provider.make_request("eth_getBlockByNumber", [blocknum, False])[ - "result" - ] + block = provider.make_request( + RPCEndpoint("eth_getBlockByNumber"), [blocknum, False] + )["result"] return is_beyond_validation_threshold( provider, blocknum=int(blocknum, 16), @@ -145,9 +148,9 @@ def validate_from_blockhash_in_params( provider.cache_allowed_requests = False # make an extra call to get the block number from the hash - block = provider.make_request("eth_getBlockByHash", [params[0], False])[ - "result" - ] + block = provider.make_request( + RPCEndpoint("eth_getBlockByHash"), [params[0], False] + )["result"] return is_beyond_validation_threshold( provider, blocknum=int(block["number"], 16), @@ -177,14 +180,14 @@ async def async_is_beyond_validation_threshold( if isinstance(threshold, RequestCacheValidationThreshold): # if mainnet and threshold is "finalized" or "safe" threshold_block = await provider.make_request( - "eth_getBlockByNumber", [threshold.value, False] + RPCEndpoint("eth_getBlockByNumber"), [threshold.value, False] ) # we should have a `blocknum` to compare against return blocknum <= int(threshold_block["result"]["number"], 16) elif isinstance(threshold, int): if not block_timestamp: block = await provider.make_request( - "eth_getBlockByNumber", [hex(blocknum), False] + RPCEndpoint("eth_getBlockByNumber"), [hex(blocknum), False] ) block_timestamp = int(block["result"]["timestamp"], 16) @@ -233,7 +236,7 @@ async def async_validate_from_blocknum_in_result( blocknum = result.get("blockNumber") # make an extra call to get the block values block = await provider.make_request( - "eth_getBlockByNumber", [blocknum, False] + RPCEndpoint("eth_getBlockByNumber"), [blocknum, False] ) return await async_is_beyond_validation_threshold( provider, @@ -268,7 +271,9 @@ async def async_validate_from_blockhash_in_params( provider.cache_allowed_requests = False # make an extra call to get the block number from the hash - response = await provider.make_request("eth_getBlockByHash", [params[0], False]) + response = await provider.make_request( + RPCEndpoint("eth_getBlockByHash"), [params[0], False] + ) return await async_is_beyond_validation_threshold( provider, blocknum=int(response["result"]["number"], 16), diff --git a/web3/_utils/module_testing/utils.py b/web3/_utils/module_testing/utils.py index 01260bb5f3..4bce464e43 100644 --- a/web3/_utils/module_testing/utils.py +++ b/web3/_utils/module_testing/utils.py @@ -113,10 +113,9 @@ def __enter__(self) -> "Self": return self - # define __exit__ with typing information def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None: # mypy error: Cannot assign to a method - self.w3.provider.make_request = self._make_request # type: ignore[method-assign] # noqa: E501 + self.w3.provider.make_request = self._make_request # type: ignore[assignment] # reset request func cache to re-build request_func with original make_request self.w3.provider._request_func_cache = (None, None) @@ -175,6 +174,7 @@ def _mock_request_handler( return mocked_response # -- async -- # + async def __aenter__(self) -> "Self": # mypy error: Cannot assign to a method self.w3.provider.make_request = self._async_mock_request_handler # type: ignore[method-assign] # noqa: E501 @@ -184,7 +184,7 @@ async def __aenter__(self) -> "Self": async def __aexit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None: # mypy error: Cannot assign to a method - self.w3.provider.make_request = self._make_request # type: ignore[method-assign] # noqa: E501 + self.w3.provider.make_request = self._make_request # type: ignore[assignment] # reset request func cache to re-build request_func with original make_request self.w3.provider._request_func_cache = (None, None) diff --git a/web3/providers/async_base.py b/web3/providers/async_base.py index c09756c83a..eb8a7426e5 100644 --- a/web3/providers/async_base.py +++ b/web3/providers/async_base.py @@ -22,7 +22,6 @@ from web3._utils.caching import ( CACHEABLE_REQUESTS, - async_handle_request_caching, ) from web3._utils.empty import ( Empty, @@ -137,7 +136,6 @@ async def batch_request_func( self._batch_request_func_cache = (middleware, accumulator_fn) return self._batch_request_func_cache[-1] - @async_handle_request_caching async def make_request(self, method: RPCEndpoint, params: Any) -> RPCResponse: raise NotImplementedError("Providers must implement this method") diff --git a/web3/providers/base.py b/web3/providers/base.py index d165c8a61a..3d9249a875 100644 --- a/web3/providers/base.py +++ b/web3/providers/base.py @@ -20,7 +20,6 @@ from web3._utils.caching import ( CACHEABLE_REQUESTS, - handle_request_caching, ) from web3._utils.empty import ( Empty, @@ -109,7 +108,6 @@ def request_func( return self._request_func_cache[-1] - @handle_request_caching def make_request(self, method: RPCEndpoint, params: Any) -> RPCResponse: raise NotImplementedError("Providers must implement this method")