diff --git a/docs/internals.rst b/docs/internals.rst index b5bae593c9..2f6182c67a 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -119,12 +119,21 @@ Provider Configurations Request Caching ``````````````` +.. important:: + Familiarize yourself with the validation logic for request caching before + enabling it. Since this feature often requires making additional requests under the + hood to try to guarantee the validity of the data, it may create unnecessary + overhead for your use case. Validation can be turned off by setting the + ``request_cache_validation_threshold`` option to ``None``, caching all allowed + requests, or configured for adjusting performance to your needs. + + Request caching can be configured at the provider level via the following configuration options on the provider instance: - ``cache_allowed_requests: bool = False`` -- ``cacheable_requests: Set[RPCEndpoint] = CACHEABLE_REQUESTS`` -- ``request_cache_validation_threshold: RequestCacheValidationThreshold = RequestCacheValidationThreshold.FINALIZED`` +- ``cacheable_requests: Optional[Set[RPCEndpoint]]`` +- ``request_cache_validation_threshold: Optional[Union[RequestCacheValidationThreshold, int]]`` For requests that don't rely on block data (e.g., ``eth_chainId``), enabling request caching by setting the ``cache_allowed_requests`` option to ``True`` will cache all @@ -132,35 +141,78 @@ responses. This is safe to do. However, for requests that rely on block data (e.g., ``eth_getBlockByNumber``), it is not safe to always cache their responses because block data can change - during a -chain reorganization, for example. The ``request_cache_validation_threshold`` option -allows configuring a safe threshold for caching responses that depend on block data. By -default, the ``finalized`` block number is used as the validation threshold, meaning -that a request's response will be cached if the block number it relies on is less than -or equal to the ``finalized`` block number. If the block number exceeds the -``finalized`` block number, the response won't be cached. +chain reorganization or while finality has not been reached, for example. The +``request_cache_validation_threshold`` option allows configuring a safe threshold for +caching responses that depend on block data. By default, this option is configured +to internal values deemed "safe" for the chain id you are connected to. If you are +connected to mainnet Ethereum, this value is set to the ``finalized`` block number. +If you are connected to another chain, this value is set to a time interval in seconds, +from the current time, that is deemed "safe" for that chain's finality mechanism. + +**It's important to understand that, in order to perform these validations, extra +requests are sometimes made to the node to get the appropriate information. For a +transaction request, for example, it is necessary to get the block information to +validate the transaction is beyond the safe threshold. This can create overhead, +especially for high-frequency requests. For this reason, it is important to understand +when to turn on caching and how to configure the validation appropriately for your +use case in order to avoid unnecessary overhead.** + +We keep a list of some reasonable values for bigger chains and +use the time interval of 1 hour for everything else. Below is a list of the default +values for internally configured chains: + + - ETH: RequestCacheValidationThreshold.FINALIZED ("finalized" block) + - ARB1: 7 days + - ZKSYNC: 1 hour + - OETH: 3 minutes + - MATIC: 30 minutes + - ZKEVM: 1 hour + - BASE: 7 days + - SCR: 1 hour + - GNO: 5 minutes + - AVAX: 2 minutes + - BNB: 2 minutes + - FTM: 1 minute + +For Ethereum mainnet, for example, this means that a request's response will be cached +if the block number the request relies on is less than or equal to the ``finalized`` +block number. If the block number exceeds the ``finalized`` block number, the response +won't be cached. For all others, the response will be cached if the block timestamp +related to the data that is being requested is older than or equal to the time interval +configured for that chain. For any chain not on this list, the default value is set to +1 hour (this includes all testnets). This behavior can be modified by setting the ``request_cache_validation_threshold`` option to ``RequestCacheValidationThreshold.SAFE``, which uses the ``safe`` block as -the threshold, or to ``None``, which disables cache validation and caches all -requests (this is not recommended). The ``RequestCacheValidationThreshold`` enum is -imported from the ``web3.utils`` module. - -The current list of requests that are validated by this configuration before being -cached is: - - - RPC.eth_getBlockByNumber - - RPC.eth_getRawTransactionByBlockNumberAndIndex - - RPC.eth_getBlockTransactionCountByNumber - - RPC.eth_getUncleByBlockNumberAndIndex - - RPC.eth_getUncleCountByBlockNumber - - RPC.eth_getBlockByHash - - RPC.eth_getTransactionByHash - - RPC.eth_getTransactionByBlockNumberAndIndex - - RPC.eth_getTransactionByBlockHashAndIndex - - RPC.eth_getBlockTransactionCountByHash - - RPC.eth_getRawTransactionByBlockHashAndIndex - - RPC.eth_getUncleByBlockHashAndIndex - - RPC.eth_getUncleCountByBlockHash +the threshold (Ethereum mainnet only), to your own time interval in seconds (for any +chain, including mainnet Ethereum), or to ``None``, which disables any validation and +caches all requests (this is not recommended for non testnet chains). The +``RequestCacheValidationThreshold`` enum, for mainnet ``finalized`` and ``safe`` values, +is imported from the ``web3.utils`` module. + +Note that the ``cacheable_requests`` option can be used to specify a set of RPC +endpoints that are allowed to be cached. By default, this option is set to an internal +list of deemed-safe-to-cache endpoints, excluding endpoints such as ``eth_call``, whose +responses can vary and are not safe to cache. The default list of cacheable requests is +below, with requests validated by the ``request_cache_validation_threshold`` option in +bold: + + - eth_chainId + - web3_clientVersion + - net_version + - **eth_getBlockByNumber** + - **eth_getRawTransactionByBlockNumberAndIndex** + - **eth_getBlockTransactionCountByNumber** + - **eth_getUncleByBlockNumberAndIndex** + - **eth_getUncleCountByBlockNumber** + - **eth_getBlockByHash** + - **eth_getTransactionByHash** + - **eth_getTransactionByBlockNumberAndIndex** + - **eth_getTransactionByBlockHashAndIndex** + - **eth_getBlockTransactionCountByHash** + - **eth_getRawTransactionByBlockHashAndIndex** + - **eth_getUncleByBlockHashAndIndex** + - **eth_getUncleCountByBlockHash** .. code-block:: python @@ -173,13 +225,12 @@ cached is: # optional flag to turn on cached requests, defaults to ``False`` cache_allowed_requests=True, - # optional, defaults to an internal list of deemed-safe-to-cache endpoints + # optional, defaults to an internal list of deemed-safe-to-cache endpoints (see above) cacheable_requests={"eth_chainId", "eth_getBlockByNumber"}, - # optional, defaults to ``RequestCacheValidationThreshold.FINALIZED`` - # can be set to ``RequestCacheValidationThreshold.SAFE`` or turned off - # by setting to ``None``. - request_cache_validation_threshold=RequestCacheValidationThreshold.SAFE, + # optional, defaults to a value that is based on the chain id (see above) + request_cache_validation_threshold=60 * 60, # 1 hour + # request_cache_validation_threshold=RequestCacheValidationThreshold.SAFE, # Ethereum mainnet only )) .. _http_retry_requests: diff --git a/newsfragments/3508.bugfix.rst b/newsfragments/3508.bugfix.rst new file mode 100644 index 0000000000..d1a1a8631c --- /dev/null +++ b/newsfragments/3508.bugfix.rst @@ -0,0 +1 @@ +Fix a bug where non-mainnet chains could not cache requests based on missing ``finalized`` block number. diff --git a/newsfragments/3508.docs.rst b/newsfragments/3508.docs.rst new file mode 100644 index 0000000000..4817e0b0c4 --- /dev/null +++ b/newsfragments/3508.docs.rst @@ -0,0 +1 @@ +Update the request caching documentation to clarify on when to reach for request caching and how to configure the request validation threshold for certain endpoints. diff --git a/newsfragments/3508.feature.rst b/newsfragments/3508.feature.rst new file mode 100644 index 0000000000..3099025177 --- /dev/null +++ b/newsfragments/3508.feature.rst @@ -0,0 +1 @@ +Allow a time interval, in seconds, to be used as the ``request_cache_validation_threshold`` for request caching. Keep a list of internal default values based on the chain id for some of the bigger chains. diff --git a/tests/core/caching-utils/test_request_caching.py b/tests/core/caching-utils/test_request_caching.py index c48c03f0a6..f61a94b6a0 100644 --- a/tests/core/caching-utils/test_request_caching.py +++ b/tests/core/caching-utils/test_request_caching.py @@ -1,6 +1,11 @@ import itertools import pytest import threading +import time +from typing import ( + Optional, + Union, +) import uuid import pytest_asyncio @@ -12,6 +17,7 @@ HTTPProvider, IPCProvider, LegacyWebSocketProvider, + PersistentConnectionProvider, Web3, WebSocketProvider, ) @@ -21,22 +27,16 @@ ) from web3._utils.caching.caching_utils import ( ASYNC_INTERNAL_VALIDATION_MAP, + BLOCK_IN_RESULT, BLOCKHASH_IN_PARAMS, BLOCKNUM_IN_PARAMS, - BLOCKNUM_IN_RESULT, + CHAIN_VALIDATION_THRESHOLD_DEFAULTS, + DEFAULT_VALIDATION_THRESHOLD, INTERNAL_VALIDATION_MAP, ) from web3.exceptions import ( Web3RPCError, ) -from web3.providers import ( - AsyncBaseProvider, - BaseProvider, - JSONBaseProvider, -) -from web3.providers.async_base import ( - AsyncJSONBaseProvider, -) from web3.types import ( RPCEndpoint, ) @@ -45,6 +45,17 @@ SimpleCache, ) +SYNC_PROVIDERS = [ + HTTPProvider, + IPCProvider, + LegacyWebSocketProvider, # deprecated +] +ASYNC_PROVIDERS = [ + AsyncHTTPProvider, + AsyncIPCProvider, + WebSocketProvider, +] + def simple_cache_return_value_a(): _cache = SimpleCache() @@ -55,15 +66,21 @@ def simple_cache_return_value_a(): return _cache +@pytest.fixture(params=SYNC_PROVIDERS) +def sync_provider(request): + return request.param + + @pytest.fixture -def w3(request_mocker): - _w3 = Web3(provider=BaseProvider(cache_allowed_requests=True)) +def w3(sync_provider, request_mocker): + _w3 = Web3(provider=sync_provider(cache_allowed_requests=True)) _w3.provider.cacheable_requests += (RPCEndpoint("fake_endpoint"),) with request_mocker( _w3, mock_results={ "fake_endpoint": lambda *_: uuid.uuid4(), "not_on_allowlist": lambda *_: uuid.uuid4(), + "eth_chainId": "0x1", # mainnet }, ): yield _w3 @@ -84,8 +101,8 @@ def test_request_caching_populates_cache(w3): assert len(w3.provider._request_cache.items()) == 2 -def test_request_caching_does_not_cache_none_responses(request_mocker): - w3 = Web3(BaseProvider(cache_allowed_requests=True)) +def test_request_caching_does_not_cache_none_responses(sync_provider, request_mocker): + w3 = Web3(sync_provider(cache_allowed_requests=True)) w3.provider.cacheable_requests += (RPCEndpoint("fake_endpoint"),) counter = itertools.count() @@ -101,8 +118,8 @@ def result_cb(_method, _params): assert next(counter) == 2 -def test_request_caching_does_not_cache_error_responses(request_mocker): - w3 = Web3(BaseProvider(cache_allowed_requests=True)) +def test_request_caching_does_not_cache_error_responses(sync_provider, request_mocker): + w3 = Web3(sync_provider(cache_allowed_requests=True)) w3.provider.cacheable_requests += (RPCEndpoint("fake_endpoint"),) with request_mocker( @@ -123,21 +140,23 @@ def test_request_caching_does_not_cache_endpoints_not_in_allowlist(w3): assert result_a != result_b -def test_caching_requests_does_not_share_state_between_providers(request_mocker): +def test_caching_requests_does_not_share_state_between_providers( + sync_provider, request_mocker +): w3_a, w3_b, w3_c, w3_a_shared_cache = ( - Web3(provider=BaseProvider(cache_allowed_requests=True)), - Web3(provider=BaseProvider(cache_allowed_requests=True)), - Web3(provider=BaseProvider(cache_allowed_requests=True)), - Web3(provider=BaseProvider(cache_allowed_requests=True)), + Web3(provider=sync_provider(cache_allowed_requests=True)), + Web3(provider=sync_provider(cache_allowed_requests=True)), + Web3(provider=sync_provider(cache_allowed_requests=True)), + Web3(provider=sync_provider(cache_allowed_requests=True)), ) # strap w3_a_shared_cache with w3_a's cache w3_a_shared_cache.provider._request_cache = w3_a.provider._request_cache - mock_results_a = {RPCEndpoint("eth_chainId"): 11111} - mock_results_a_shared_cache = {RPCEndpoint("eth_chainId"): 00000} - mock_results_b = {RPCEndpoint("eth_chainId"): 22222} - mock_results_c = {RPCEndpoint("eth_chainId"): 33333} + mock_results_a = {RPCEndpoint("eth_chainId"): hex(11111)} + mock_results_a_shared_cache = {RPCEndpoint("eth_chainId"): hex(00000)} + mock_results_b = {RPCEndpoint("eth_chainId"): hex(22222)} + mock_results_c = {RPCEndpoint("eth_chainId"): hex(33333)} with request_mocker(w3_a, mock_results=mock_results_a): with request_mocker(w3_b, mock_results=mock_results_b): @@ -154,27 +173,13 @@ def test_caching_requests_does_not_share_state_between_providers(request_mocker) "eth_chainId", [] ) - assert result_a == 11111 - assert result_b == 22222 - assert result_c == 33333 - assert result_a_shared_cache == 11111 + assert result_a == hex(11111) + assert result_b == hex(22222) + assert result_c == hex(33333) + assert result_a_shared_cache == hex(11111) -@pytest.mark.parametrize( - "provider", - [ - BaseProvider, - JSONBaseProvider, - HTTPProvider, - IPCProvider, - AsyncBaseProvider, - AsyncJSONBaseProvider, - AsyncHTTPProvider, - AsyncIPCProvider, - WebSocketProvider, - LegacyWebSocketProvider, # deprecated - ], -) +@pytest.mark.parametrize("provider", [*SYNC_PROVIDERS, *ASYNC_PROVIDERS]) def test_all_providers_do_not_cache_by_default_and_can_set_caching_properties(provider): _provider_default_init = provider() assert _provider_default_init.cache_allowed_requests is False @@ -199,7 +204,7 @@ def test_all_providers_do_not_cache_by_default_and_can_set_caching_properties(pr "threshold", (RequestCacheValidationThreshold.FINALIZED, RequestCacheValidationThreshold.SAFE), ) -@pytest.mark.parametrize("endpoint", BLOCKNUM_IN_PARAMS | BLOCKNUM_IN_RESULT) +@pytest.mark.parametrize("endpoint", BLOCKNUM_IN_PARAMS | BLOCK_IN_RESULT) @pytest.mark.parametrize( "blocknum,should_cache", ( @@ -211,11 +216,11 @@ def test_all_providers_do_not_cache_by_default_and_can_set_caching_properties(pr ("0x5", False), ), ) -def test_blocknum_validation_against_validation_threshold_when_caching( - threshold, endpoint, blocknum, should_cache, request_mocker +def test_blocknum_validation_against_validation_threshold_when_caching_mainnet( + threshold, endpoint, blocknum, should_cache, sync_provider, request_mocker ): w3 = Web3( - HTTPProvider( + sync_provider( cache_allowed_requests=True, request_cache_validation_threshold=threshold ) ) @@ -224,7 +229,7 @@ def test_blocknum_validation_against_validation_threshold_when_caching( mock_results={ endpoint: ( # mock the result to requests that return blocks - {"number": blocknum} + {"number": blocknum, "timestamp": "0x0"} if "getBlock" in endpoint # mock the result to requests that return transactions else {"blockNumber": blocknum} @@ -232,10 +237,11 @@ def test_blocknum_validation_against_validation_threshold_when_caching( "eth_getBlockByNumber": lambda _method, params: ( # mock the threshold block to be blocknum "0x2", return # blocknum otherwise - {"number": "0x2"} + {"number": "0x2", "timestamp": "0x0"} if params[0] == threshold.value - else {"number": params[0]} + else {"number": params[0], "timestamp": "0x0"} ), + "eth_chainId": "0x1", # mainnet }, ): assert len(w3.provider._request_cache.items()) == 0 @@ -260,23 +266,24 @@ def test_blocknum_validation_against_validation_threshold_when_caching( ("pending", None, False), ), ) -def test_block_id_param_caching( - threshold, endpoint, block_id, blocknum, should_cache, request_mocker +def test_block_id_param_caching_mainnet( + threshold, endpoint, block_id, blocknum, should_cache, sync_provider, request_mocker ): w3 = Web3( - HTTPProvider( + sync_provider( cache_allowed_requests=True, request_cache_validation_threshold=threshold ) ) with request_mocker( w3, mock_results={ + "eth_chainId": "0x1", # mainnet endpoint: "0x0", "eth_getBlockByNumber": lambda _method, params: ( # mock the threshold block to be blocknum "0x2" for all test cases - {"number": "0x2"} + {"number": "0x2", "timestamp": "0x0"} if params[0] == threshold.value - else {"number": blocknum} + else {"number": blocknum, "timestamp": "0x0"} ), }, ): @@ -302,58 +309,173 @@ def test_block_id_param_caching( ("0x5", False), ), ) -def test_blockhash_validation_against_validation_threshold_when_caching( - threshold, endpoint, blocknum, should_cache, request_mocker +def test_blockhash_validation_against_validation_threshold_when_caching_mainnet( + threshold, endpoint, blocknum, should_cache, sync_provider, request_mocker ): w3 = Web3( - HTTPProvider( + sync_provider( cache_allowed_requests=True, request_cache_validation_threshold=threshold ) ) with request_mocker( w3, mock_results={ + "eth_chainId": "0x1", # mainnet "eth_getBlockByNumber": lambda _method, params: ( # mock the threshold block to be blocknum "0x2" - {"number": "0x2"} + {"number": "0x2", "timestamp": "0x0"} if params[0] == threshold.value - else {"number": params[0]} + else {"number": params[0], "timestamp": "0x0"} ), - "eth_getBlockByHash": {"number": blocknum}, + "eth_getBlockByHash": {"number": blocknum, "timestamp": "0x0"}, endpoint: "0x0", }, ): assert len(w3.provider._request_cache.items()) == 0 w3.manager.request_blocking(endpoint, [b"\x00" * 32, False]) cached_items = len(w3.provider._request_cache.items()) - assert cached_items == 2 if should_cache else cached_items == 0 + assert cached_items == 1 if should_cache else cached_items == 0 -def test_request_caching_validation_threshold_is_finalized_by_default(): - w3 = Web3(HTTPProvider(cache_allowed_requests=True)) - assert ( - w3.provider.request_cache_validation_threshold - == RequestCacheValidationThreshold.FINALIZED +@pytest.mark.parametrize( + "chain_id,expected_threshold", + ( + *CHAIN_VALIDATION_THRESHOLD_DEFAULTS.items(), + (3456787654567654, DEFAULT_VALIDATION_THRESHOLD), + (11111111111444444444444444, DEFAULT_VALIDATION_THRESHOLD), + (-11111111111111111117, DEFAULT_VALIDATION_THRESHOLD), + ), +) +def test_request_caching_validation_threshold_defaults( + chain_id, expected_threshold, sync_provider, request_mocker +): + w3 = Web3(sync_provider(cache_allowed_requests=True)) + with request_mocker(w3, mock_results={"eth_chainId": hex(chain_id)}): + w3.manager.request_blocking(RPCEndpoint("eth_chainId"), []) + assert w3.provider.request_cache_validation_threshold == expected_threshold + # assert chain_id is cached + cache_items = w3.provider._request_cache.items() + assert len(cache_items) == 1 + assert cache_items[0][1]["result"] == hex(chain_id) + + +@pytest.mark.parametrize( + "endpoint", BLOCKNUM_IN_PARAMS | BLOCK_IN_RESULT | BLOCKHASH_IN_PARAMS +) +@pytest.mark.parametrize( + "time_from_threshold,should_cache", + # -0.2 to give some time for the test to run + ((-2, True), (-1, True), (-0.2, True), (1, False), (2, False)), +) +@pytest.mark.parametrize( + "chain_id,expected_threshold_in_seconds", + [ + (chain_id, threshold) + for chain_id, threshold in CHAIN_VALIDATION_THRESHOLD_DEFAULTS.items() + if isinstance(threshold, int) + ] + + [ + (3456787654567654, DEFAULT_VALIDATION_THRESHOLD), + (11111111111444444444444444, DEFAULT_VALIDATION_THRESHOLD), + (-11111111111111111117, DEFAULT_VALIDATION_THRESHOLD), + ], +) +def test_sync_validation_against_validation_threshold_time_based( + endpoint, + time_from_threshold, + should_cache, + chain_id, + expected_threshold_in_seconds, + sync_provider, + request_mocker, +): + w3 = Web3(sync_provider(cache_allowed_requests=True)) + blocknum = "0x2" + # mock the timestamp so that we are at the threshold +/- the time_from_threshold + mocked_time = hex( + int(round(time.time() - expected_threshold_in_seconds) + time_from_threshold) ) + with request_mocker( + w3, + mock_results={ + "eth_chainId": hex(chain_id), + endpoint: lambda *_: ( + # mock the result to requests that return blocks + {"number": blocknum, "timestamp": mocked_time} + if "getBlock" in endpoint + # mock the result to requests that return transactions with the blocknum + # for our block under test + else {"blockNumber": blocknum} + ), + "eth_getBlockByNumber": lambda _method, params: ( + {"number": blocknum, "timestamp": mocked_time} + ), + "eth_getBlockByHash": {"number": blocknum, "timestamp": mocked_time}, + }, + ): + assert len(w3.provider._request_cache.items()) == 0 + w3.manager.request_blocking(endpoint, [blocknum, False]) + cached_items = w3.provider._request_cache.items() + assert len(cached_items) == 1 if should_cache else len(cached_items) == 0 + @pytest.mark.parametrize( - "endpoint", BLOCKNUM_IN_PARAMS | BLOCKNUM_IN_RESULT | BLOCKHASH_IN_PARAMS + "time_from_threshold,should_cache", + # -0.2 to give some time for the test to run + ((-2, True), (-1, True), (-0.2, True), (1, False), (2, False)), ) -@pytest.mark.parametrize("blocknum", ("0x0", "0x1", "0x2", "0x3", "0x4", "0x5")) -def test_request_caching_with_validation_threshold_set_to_none( - endpoint, blocknum, request_mocker +@pytest.mark.parametrize( + "chain_id", + ( + # test that defaults are also overridden by the configured validation threshold + *CHAIN_VALIDATION_THRESHOLD_DEFAULTS.keys(), + 3456787654567654, + 11111111111444444444444444, + -11111111111111111117, + ), +) +@pytest.mark.parametrize( + "endpoint", BLOCKNUM_IN_PARAMS | BLOCK_IN_RESULT | BLOCKHASH_IN_PARAMS +) +def test_validation_against_validation_threshold_time_based_configured( + time_from_threshold, should_cache, chain_id, endpoint, sync_provider, request_mocker ): + configured_time_threshold = 60 * 60 * 24 * 7 # 1 week w3 = Web3( - HTTPProvider( + sync_provider( cache_allowed_requests=True, - request_cache_validation_threshold=None, + request_cache_validation_threshold=configured_time_threshold, ) ) - with request_mocker(w3, mock_results={endpoint: {"number": blocknum}}): + blocknum = "0x2" + # mock the timestamp so that we are at the threshold +/- the time_from_threshold + mocked_time = hex( + int(round(time.time()) - configured_time_threshold + time_from_threshold) + ) + + with request_mocker( + w3, + mock_results={ + "eth_chainId": chain_id, + endpoint: lambda *_: ( + # mock the result to requests that return blocks + {"number": blocknum, "timestamp": mocked_time} + if "getBlock" in endpoint + # mock the result to requests that return transactions with the blocknum + # for our block under test + else {"blockNumber": blocknum} + ), + "eth_getBlockByNumber": lambda _method, params: ( + {"number": blocknum, "timestamp": mocked_time} + ), + "eth_getBlockByHash": {"number": blocknum, "timestamp": mocked_time}, + }, + ): assert len(w3.provider._request_cache.items()) == 0 w3.manager.request_blocking(endpoint, [blocknum, False]) - assert len(w3.provider._request_cache.items()) == 1 + cached_items = w3.provider._request_cache.items() + assert len(cached_items) == 1 if should_cache else len(cached_items) == 0 # -- async -- # @@ -367,15 +489,39 @@ def test_async_cacheable_requests_are_the_same_as_sync(): ), "make sure the async and sync cacheable requests are the same" +@pytest_asyncio.fixture(params=ASYNC_PROVIDERS) +async def async_provider(request): + return request.param + + +async def _async_w3_init( + async_provider, + threshold: Optional[Union[RequestCacheValidationThreshold, int]] = "empty", +): + if isinstance(async_provider, PersistentConnectionProvider): + _async_w3 = await AsyncWeb3( + provider=async_provider( + cache_allowed_requests=True, + ) + ) + else: + _async_w3 = AsyncWeb3(provider=async_provider(cache_allowed_requests=True)) + + if threshold != "empty": + _async_w3.provider.request_cache_validation_threshold = threshold + return _async_w3 + + @pytest_asyncio.fixture -async def async_w3(request_mocker): - _async_w3 = AsyncWeb3(AsyncBaseProvider(cache_allowed_requests=True)) +async def async_w3(async_provider, request_mocker): + _async_w3 = await _async_w3_init(async_provider) _async_w3.provider.cacheable_requests += (RPCEndpoint("fake_endpoint"),) async with request_mocker( _async_w3, mock_results={ "fake_endpoint": lambda *_: uuid.uuid4(), "not_on_allowlist": lambda *_: uuid.uuid4(), + "eth_chainId": "0x1", # mainnet }, ): yield _async_w3 @@ -403,8 +549,10 @@ async def test_async_request_caching_populates_cache(async_w3): @pytest.mark.asyncio -async def test_async_request_caching_does_not_cache_none_responses(request_mocker): - async_w3 = AsyncWeb3(AsyncBaseProvider(cache_allowed_requests=True)) +async def test_async_request_caching_does_not_cache_none_responses( + async_provider, request_mocker +): + async_w3 = await _async_w3_init(async_provider) async_w3.provider.cacheable_requests += (RPCEndpoint("fake_endpoint"),) counter = itertools.count() @@ -421,8 +569,10 @@ def result_cb(_method, _params): @pytest.mark.asyncio -async def test_async_request_caching_does_not_cache_error_responses(request_mocker): - async_w3 = AsyncWeb3(AsyncBaseProvider(cache_allowed_requests=True)) +async def test_async_request_caching_does_not_cache_error_responses( + async_provider, request_mocker +): + async_w3 = await _async_w3_init(async_provider) async_w3.provider.cacheable_requests += (RPCEndpoint("fake_endpoint"),) async with request_mocker( @@ -448,22 +598,23 @@ async def test_async_request_caching_does_not_cache_non_allowlist_endpoints( @pytest.mark.asyncio async def test_async_request_caching_does_not_share_state_between_providers( + async_provider, request_mocker, ): async_w3_a, async_w3_b, async_w3_c, async_w3_a_shared_cache = ( - AsyncWeb3(AsyncBaseProvider(cache_allowed_requests=True)), - AsyncWeb3(AsyncBaseProvider(cache_allowed_requests=True)), - AsyncWeb3(AsyncBaseProvider(cache_allowed_requests=True)), - AsyncWeb3(AsyncBaseProvider(cache_allowed_requests=True)), + await _async_w3_init(async_provider), + await _async_w3_init(async_provider), + await _async_w3_init(async_provider), + await _async_w3_init(async_provider), ) # strap async_w3_a_shared_cache with async_w3_a's cache async_w3_a_shared_cache.provider._request_cache = async_w3_a.provider._request_cache - mock_results_a = {RPCEndpoint("eth_chainId"): 11111} - mock_results_a_shared_cache = {RPCEndpoint("eth_chainId"): 00000} - mock_results_b = {RPCEndpoint("eth_chainId"): 22222} - mock_results_c = {RPCEndpoint("eth_chainId"): 33333} + mock_results_a = {RPCEndpoint("eth_chainId"): hex(11111)} + mock_results_a_shared_cache = {RPCEndpoint("eth_chainId"): hex(00000)} + mock_results_b = {RPCEndpoint("eth_chainId"): hex(22222)} + mock_results_c = {RPCEndpoint("eth_chainId"): hex(33333)} async with request_mocker(async_w3_a, mock_results=mock_results_a): async with request_mocker(async_w3_b, mock_results=mock_results_b): @@ -480,10 +631,10 @@ async def test_async_request_caching_does_not_share_state_between_providers( "eth_chainId", [] ) - assert result_a == 11111 - assert result_b == 22222 - assert result_c == 33333 - assert result_a_shared_cache == 11111 + assert result_a == hex(11111) + assert result_b == hex(22222) + assert result_c == hex(33333) + assert result_a_shared_cache == hex(11111) @pytest.mark.asyncio @@ -491,7 +642,7 @@ async def test_async_request_caching_does_not_share_state_between_providers( "threshold", (RequestCacheValidationThreshold.FINALIZED, RequestCacheValidationThreshold.SAFE), ) -@pytest.mark.parametrize("endpoint", BLOCKNUM_IN_PARAMS | BLOCKNUM_IN_RESULT) +@pytest.mark.parametrize("endpoint", BLOCKNUM_IN_PARAMS | BLOCK_IN_RESULT) @pytest.mark.parametrize( "blocknum,should_cache", ( @@ -503,20 +654,16 @@ async def test_async_request_caching_does_not_share_state_between_providers( ("0x5", False), ), ) -async def test_async_blocknum_validation_against_validation_threshold( - threshold, endpoint, blocknum, should_cache, request_mocker +async def test_async_blocknum_validation_against_validation_threshold_mainnet( + threshold, endpoint, blocknum, should_cache, async_provider, request_mocker ): - async_w3 = AsyncWeb3( - AsyncHTTPProvider( - cache_allowed_requests=True, request_cache_validation_threshold=threshold - ) - ) + async_w3 = await _async_w3_init(async_provider, threshold=threshold) async with request_mocker( async_w3, mock_results={ endpoint: ( # mock the result to requests that return blocks - {"number": blocknum} + {"number": blocknum, "timestamp": "0x0"} if "getBlock" in endpoint # mock the result to requests that return transactions else {"blockNumber": blocknum} @@ -524,10 +671,11 @@ async def test_async_blocknum_validation_against_validation_threshold( "eth_getBlockByNumber": lambda _method, params: ( # mock the threshold block to be blocknum "0x2", return # blocknum otherwise - {"number": "0x2"} + {"number": "0x2", "timestamp": "0x0"} if params[0] == threshold.value - else {"number": params[0]} + else {"number": params[0], "timestamp": "0x0"} ), + "eth_chainId": "0x1", # mainnet }, ): assert len(async_w3.provider._request_cache.items()) == 0 @@ -553,23 +701,26 @@ async def test_async_blocknum_validation_against_validation_threshold( ("pending", None, False), ), ) -async def test_async_block_id_param_caching( - threshold, endpoint, block_id, blocknum, should_cache, request_mocker +async def test_async_block_id_param_caching_mainnet( + threshold, + endpoint, + block_id, + blocknum, + should_cache, + async_provider, + request_mocker, ): - async_w3 = AsyncWeb3( - AsyncHTTPProvider( - cache_allowed_requests=True, request_cache_validation_threshold=threshold - ) - ) + async_w3 = await _async_w3_init(async_provider, threshold=threshold) async with request_mocker( async_w3, mock_results={ + "eth_chainId": "0x1", # mainnet endpoint: "0x0", "eth_getBlockByNumber": lambda _method, params: ( # mock the threshold block to be blocknum "0x2" for all test cases - {"number": "0x2"} + {"number": "0x2", "timestamp": "0x0"} if params[0] == threshold.value - else {"number": blocknum} + else {"number": blocknum, "timestamp": "0x0"} ), }, ): @@ -596,57 +747,197 @@ async def test_async_block_id_param_caching( ("0x5", False), ), ) -async def test_async_blockhash_validation_against_validation_threshold( - threshold, endpoint, blocknum, should_cache, request_mocker +async def test_async_blockhash_validation_against_validation_threshold_mainnet( + threshold, endpoint, blocknum, should_cache, async_provider, request_mocker ): - async_w3 = AsyncWeb3( - AsyncHTTPProvider( - cache_allowed_requests=True, request_cache_validation_threshold=threshold - ) - ) + async_w3 = await _async_w3_init(async_provider, threshold=threshold) async with request_mocker( async_w3, mock_results={ + "eth_chainId": "0x1", # mainnet "eth_getBlockByNumber": lambda _method, params: ( # mock the threshold block to be blocknum "0x2" - {"number": "0x2"} + {"number": "0x2", "timestamp": "0x0"} if params[0] == threshold.value - else {"number": params[0]} + else {"number": params[0], "timestamp": "0x0"} ), - "eth_getBlockByHash": {"number": blocknum}, + "eth_getBlockByHash": {"number": blocknum, "timestamp": "0x0"}, endpoint: "0x0", }, ): assert len(async_w3.provider._request_cache.items()) == 0 await async_w3.manager.coro_request(endpoint, [b"\x00" * 32, False]) cached_items = len(async_w3.provider._request_cache.items()) - assert cached_items == 2 if should_cache else cached_items == 0 + assert cached_items == 1 if should_cache else cached_items == 0 @pytest.mark.asyncio -async def test_async_request_caching_validation_threshold_is_finalized_by_default(): - async_w3 = AsyncWeb3(AsyncHTTPProvider(cache_allowed_requests=True)) - assert ( - async_w3.provider.request_cache_validation_threshold - == RequestCacheValidationThreshold.FINALIZED +@pytest.mark.parametrize( + "chain_id,expected_threshold", + ( + *CHAIN_VALIDATION_THRESHOLD_DEFAULTS.items(), + (3456787654567654, DEFAULT_VALIDATION_THRESHOLD), + (11111111111444444444444444, DEFAULT_VALIDATION_THRESHOLD), + (-11111111111111111117, DEFAULT_VALIDATION_THRESHOLD), + ), +) +async def test_async_request_caching_validation_threshold_defaults( + chain_id, expected_threshold, async_provider, request_mocker +): + async_w3 = await _async_w3_init(async_provider) + async with request_mocker(async_w3, mock_results={"eth_chainId": hex(chain_id)}): + await async_w3.manager.coro_request(RPCEndpoint("eth_chainId"), []) + assert ( + async_w3.provider.request_cache_validation_threshold == expected_threshold + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "endpoint", BLOCKNUM_IN_PARAMS | BLOCK_IN_RESULT | BLOCKHASH_IN_PARAMS +) +@pytest.mark.parametrize( + "time_from_threshold,should_cache", + # -0.2 to give some time for the test to run + ((-2, True), (-1, True), (-0.2, True), (1, False), (2, False)), +) +@pytest.mark.parametrize( + "chain_id,expected_threshold_in_seconds", + [ + (chain_id, threshold) + for chain_id, threshold in CHAIN_VALIDATION_THRESHOLD_DEFAULTS.items() + if isinstance(threshold, int) + ] + + [ + (3456787654567654, DEFAULT_VALIDATION_THRESHOLD), + (11111111111444444444444444, DEFAULT_VALIDATION_THRESHOLD), + (-11111111111111111117, DEFAULT_VALIDATION_THRESHOLD), + ], +) +async def test_async_validation_against_validation_threshold_time_based( + endpoint, + time_from_threshold, + should_cache, + chain_id, + expected_threshold_in_seconds, + async_provider, + request_mocker, +): + async_w3 = await _async_w3_init(async_provider) + blocknum = "0x2" + # mock the timestamp so that we are at the threshold +/- the time_from_threshold + mocked_time = hex( + int(round(time.time() - expected_threshold_in_seconds) + time_from_threshold) ) + async with request_mocker( + async_w3, + mock_results={ + "eth_chainId": hex(chain_id), + endpoint: lambda *_: ( + # mock the result to requests that return blocks + {"number": blocknum, "timestamp": mocked_time} + if "getBlock" in endpoint + # mock the result to requests that return transactions with the blocknum + # for our block under test + else {"blockNumber": blocknum} + ), + "eth_getBlockByNumber": lambda _method, params: ( + {"number": blocknum, "timestamp": mocked_time} + ), + "eth_getBlockByHash": {"number": blocknum, "timestamp": mocked_time}, + }, + ): + assert len(async_w3.provider._request_cache.items()) == 0 + await async_w3.manager.coro_request(endpoint, [blocknum, False]) + cached_items = async_w3.provider._request_cache.items() + assert len(cached_items) == 1 if should_cache else len(cached_items) == 0 + @pytest.mark.asyncio @pytest.mark.parametrize( - "endpoint", BLOCKNUM_IN_PARAMS | BLOCKNUM_IN_RESULT | BLOCKHASH_IN_PARAMS + "endpoint", BLOCKNUM_IN_PARAMS | BLOCK_IN_RESULT | BLOCKHASH_IN_PARAMS ) @pytest.mark.parametrize("blocknum", ("0x0", "0x1", "0x2", "0x3", "0x4", "0x5")) async def test_async_request_caching_with_validation_threshold_set_to_none( - endpoint, blocknum, request_mocker + endpoint, blocknum, async_provider, request_mocker ): - async_w3 = AsyncWeb3( - AsyncHTTPProvider( - cache_allowed_requests=True, - request_cache_validation_threshold=None, - ) - ) - async with request_mocker(async_w3, mock_results={endpoint: {"number": blocknum}}): + async_w3 = await _async_w3_init(async_provider, threshold=None) + async with request_mocker( + async_w3, + mock_results={ + # simulate the default settings for mainnet and show that we cache + # everything before and after the `finalized` block + "eth_chainId": "0x1", + endpoint: {"number": blocknum}, + "eth_getBlockByNumber": lambda _method, params: ( + # mock the threshold block to be blocknum "0x2", return + # blocknum otherwise + {"number": "0x2", "timestamp": "0x0"} + if params[0] == "finalized" + else {"number": blocknum, "timestamp": "0x0"} + ), + }, + ): assert len(async_w3.provider._request_cache.items()) == 0 await async_w3.manager.coro_request(endpoint, [blocknum, False]) assert len(async_w3.provider._request_cache.items()) == 1 + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "time_from_threshold,should_cache", + # -0.2 to give some time for the test to run + ((-2, True), (-1, True), (-0.2, True), (1, False), (2, False)), +) +@pytest.mark.parametrize( + "chain_id", + ( + # test that defaults are also overridden by the configured validation threshold + *CHAIN_VALIDATION_THRESHOLD_DEFAULTS.keys(), + 3456787654567654, + 11111111111444444444444444, + -11111111111111111117, + ), +) +@pytest.mark.parametrize( + "endpoint", BLOCKNUM_IN_PARAMS | BLOCK_IN_RESULT | BLOCKHASH_IN_PARAMS +) +async def test_async_validation_against_validation_threshold_time_based_configured( + time_from_threshold, + should_cache, + chain_id, + endpoint, + async_provider, + request_mocker, +): + configured_time_threshold = 60 * 60 * 24 * 7 # 1 week + async_w3 = await _async_w3_init(async_provider, threshold=configured_time_threshold) + blocknum = "0x2" + # mock the timestamp so that we are at the threshold +/- the time_from_threshold + mocked_time = hex( + int(round(time.time()) - configured_time_threshold + time_from_threshold) + ) + + async with request_mocker( + async_w3, + mock_results={ + "eth_chainId": chain_id, + endpoint: lambda *_: ( + # mock the result to requests that return blocks + {"number": blocknum, "timestamp": mocked_time} + if "getBlock" in endpoint + # mock the result to requests that return transactions with the blocknum + # for our block under test + else {"blockNumber": blocknum} + ), + "eth_getBlockByNumber": lambda _method, params: ( + {"number": blocknum, "timestamp": mocked_time} + ), + "eth_getBlockByHash": {"number": blocknum, "timestamp": mocked_time}, + }, + ): + assert len(async_w3.provider._request_cache.items()) == 0 + await async_w3.manager.coro_request(endpoint, [blocknum, False]) + cached_items = async_w3.provider._request_cache.items() + assert len(cached_items) == 1 if should_cache else len(cached_items) == 0 diff --git a/web3/_utils/caching/caching_utils.py b/web3/_utils/caching/caching_utils.py index dc640cee07..fe0317424d 100644 --- a/web3/_utils/caching/caching_utils.py +++ b/web3/_utils/caching/caching_utils.py @@ -16,6 +16,9 @@ Union, ) +from eth_typing import ( + ChainId, +) from eth_utils import ( is_boolean, is_bytes, @@ -34,12 +37,15 @@ from web3._utils.caching.request_caching_validation import ( UNCACHEABLE_BLOCK_IDS, always_cache_request, - async_validate_blockhash_in_params, - async_validate_blocknum_in_params, - async_validate_blocknum_in_result, - validate_blockhash_in_params, - validate_blocknum_in_params, - validate_blocknum_in_result, + async_validate_from_block_id_in_params, + async_validate_from_blockhash_in_params, + async_validate_from_blocknum_in_result, + validate_from_block_id_in_params, + validate_from_blockhash_in_params, + validate_from_blocknum_in_result, +) +from web3._utils.empty import ( + empty, ) from web3._utils.rpc_abi import ( RPC, @@ -47,6 +53,12 @@ from web3.exceptions import ( Web3TypeError, ) +from web3.types import ( + RPCEndpoint, +) +from web3.utils import ( + RequestCacheValidationThreshold, +) if TYPE_CHECKING: from web3.providers import ( # noqa: F401 @@ -56,7 +68,6 @@ from web3.types import ( # noqa: F401 AsyncMakeRequestFn, MakeRequestFn, - RPCEndpoint, RPCResponse, ) @@ -84,7 +95,7 @@ def generate_cache_key(value: Any) -> str: class RequestInformation: def __init__( self, - method: "RPCEndpoint", + method: RPCEndpoint, params: Any, response_formatters: Tuple[ Union[Dict[str, Callable[..., Any]], Callable[..., Any]], @@ -100,9 +111,31 @@ def __init__( self.middleware_response_processors: List[Callable[..., Any]] = [] +DEFAULT_VALIDATION_THRESHOLD = 60 * 60 # 1 hour + +CHAIN_VALIDATION_THRESHOLD_DEFAULTS: Dict[ + int, Union[RequestCacheValidationThreshold, int] +] = { + # Suggested safe values as defaults for each chain. Users can configure a different + # value if desired. + ChainId.ETH.value: RequestCacheValidationThreshold.FINALIZED, + ChainId.ARB1.value: 7 * 24 * 60 * 60, # 7 days + ChainId.ZKSYNC.value: 60 * 60, # 1 hour + ChainId.OETH.value: 3 * 60, # 3 minutes + ChainId.MATIC.value: 30 * 60, # 30 minutes + ChainId.ZKEVM.value: 60 * 60, # 1 hour + ChainId.BASE.value: 7 * 24 * 60 * 60, # 7 days + ChainId.SCR.value: 60 * 60, # 1 hour + ChainId.GNO.value: 5 * 60, # 5 minutes + ChainId.AVAX.value: 2 * 60, # 2 minutes + ChainId.BNB.value: 2 * 60, # 2 minutes + ChainId.FTM.value: 60, # 1 minute +} + + def is_cacheable_request( provider: Union[ASYNC_PROVIDER_TYPE, SYNC_PROVIDER_TYPE], - method: "RPCEndpoint", + method: RPCEndpoint, params: Any, ) -> bool: if not (provider.cache_allowed_requests and method in provider.cacheable_requests): @@ -128,7 +161,7 @@ def is_cacheable_request( RPC.eth_getUncleByBlockNumberAndIndex, RPC.eth_getUncleCountByBlockNumber, } -BLOCKNUM_IN_RESULT = { +BLOCK_IN_RESULT = { RPC.eth_getBlockByHash, RPC.eth_getTransactionByHash, RPC.eth_getTransactionByBlockNumberAndIndex, @@ -142,25 +175,58 @@ def is_cacheable_request( } INTERNAL_VALIDATION_MAP: Dict[ - "RPCEndpoint", Callable[[SYNC_PROVIDER_TYPE, Sequence[Any], Dict[str, Any]], bool] + RPCEndpoint, + Callable[ + [SYNC_PROVIDER_TYPE, Sequence[Any], Dict[str, Any]], + bool, + ], ] = { **{endpoint: always_cache_request for endpoint in ALWAYS_CACHE}, - **{endpoint: validate_blocknum_in_params for endpoint in BLOCKNUM_IN_PARAMS}, - **{endpoint: validate_blocknum_in_result for endpoint in BLOCKNUM_IN_RESULT}, - **{endpoint: validate_blockhash_in_params for endpoint in BLOCKHASH_IN_PARAMS}, + **{endpoint: validate_from_block_id_in_params for endpoint in BLOCKNUM_IN_PARAMS}, + **{endpoint: validate_from_blocknum_in_result for endpoint in BLOCK_IN_RESULT}, + **{endpoint: validate_from_blockhash_in_params for endpoint in BLOCKHASH_IN_PARAMS}, } CACHEABLE_REQUESTS = tuple(INTERNAL_VALIDATION_MAP.keys()) +def set_threshold_if_empty(provider: SYNC_PROVIDER_TYPE) -> None: + current_threshold = provider.request_cache_validation_threshold + + if current_threshold is empty or isinstance( + current_threshold, RequestCacheValidationThreshold + ): + cache_allowed_requests = provider.cache_allowed_requests + try: + # turn off momentarily to avoid recursion + provider.cache_allowed_requests = False + chain_id_result = provider.make_request(RPCEndpoint("eth_chainId"), [])[ + "result" + ] + chain_id = int(chain_id_result, 16) + + if current_threshold is empty: + provider.request_cache_validation_threshold = ( + CHAIN_VALIDATION_THRESHOLD_DEFAULTS.get( + chain_id, DEFAULT_VALIDATION_THRESHOLD + ) + ) + except Exception: + provider.request_cache_validation_threshold = DEFAULT_VALIDATION_THRESHOLD + finally: + provider.cache_allowed_requests = cache_allowed_requests + + def _should_cache_response( provider: SYNC_PROVIDER_TYPE, - method: "RPCEndpoint", + method: RPCEndpoint, params: Sequence[Any], response: "RPCResponse", ) -> bool: result = response.get("result", None) if "error" in response or is_null(result): return False + + set_threshold_if_empty(provider) if ( method in INTERNAL_VALIDATION_MAP and provider.request_cache_validation_threshold is not None @@ -170,10 +236,10 @@ def _should_cache_response( def handle_request_caching( - func: Callable[[SYNC_PROVIDER_TYPE, "RPCEndpoint", Any], "RPCResponse"] + func: Callable[[SYNC_PROVIDER_TYPE, RPCEndpoint, Any], "RPCResponse"] ) -> Callable[..., "RPCResponse"]: def wrapper( - provider: SYNC_PROVIDER_TYPE, method: "RPCEndpoint", params: Any + provider: SYNC_PROVIDER_TYPE, method: RPCEndpoint, params: Any ) -> "RPCResponse": if is_cacheable_request(provider, method, params): request_cache = provider._request_cache @@ -204,25 +270,60 @@ def wrapper( Union[bool, Coroutine[Any, Any, bool]], ] -ASYNC_INTERNAL_VALIDATION_MAP: Dict["RPCEndpoint", ASYNC_VALIDATOR_TYPE] = { +ASYNC_INTERNAL_VALIDATION_MAP: Dict[RPCEndpoint, ASYNC_VALIDATOR_TYPE] = { **{endpoint: always_cache_request for endpoint in ALWAYS_CACHE}, - **{endpoint: async_validate_blocknum_in_params for endpoint in BLOCKNUM_IN_PARAMS}, - **{endpoint: async_validate_blocknum_in_result for endpoint in BLOCKNUM_IN_RESULT}, **{ - endpoint: async_validate_blockhash_in_params for endpoint in BLOCKHASH_IN_PARAMS + endpoint: async_validate_from_block_id_in_params + for endpoint in BLOCKNUM_IN_PARAMS + }, + **{ + endpoint: async_validate_from_blocknum_in_result for endpoint in BLOCK_IN_RESULT + }, + **{ + endpoint: async_validate_from_blockhash_in_params + for endpoint in BLOCKHASH_IN_PARAMS }, } +async def async_set_threshold_if_empty(provider: ASYNC_PROVIDER_TYPE) -> None: + current_threshold = provider.request_cache_validation_threshold + + if current_threshold is empty or isinstance( + current_threshold, RequestCacheValidationThreshold + ): + cache_allowed_requests = provider.cache_allowed_requests + try: + # turn off momentarily to avoid recursion + provider.cache_allowed_requests = False + chain_id_result = await provider.make_request( + RPCEndpoint("eth_chainId"), [] + ) + chain_id = int(chain_id_result["result"], 16) + + if current_threshold is empty: + provider.request_cache_validation_threshold = ( + CHAIN_VALIDATION_THRESHOLD_DEFAULTS.get( + chain_id, DEFAULT_VALIDATION_THRESHOLD + ) + ) + except Exception: + provider.request_cache_validation_threshold = DEFAULT_VALIDATION_THRESHOLD + finally: + provider.cache_allowed_requests = cache_allowed_requests + + async def _async_should_cache_response( provider: ASYNC_PROVIDER_TYPE, - method: "RPCEndpoint", + method: RPCEndpoint, params: Sequence[Any], response: "RPCResponse", ) -> bool: result = response.get("result", None) if "error" in response or is_null(result): return False + + await async_set_threshold_if_empty(provider) if ( method in ASYNC_INTERNAL_VALIDATION_MAP and provider.request_cache_validation_threshold is not None @@ -238,11 +339,11 @@ async def _async_should_cache_response( def async_handle_request_caching( func: Callable[ - [ASYNC_PROVIDER_TYPE, "RPCEndpoint", Any], Coroutine[Any, Any, "RPCResponse"] + [ASYNC_PROVIDER_TYPE, RPCEndpoint, Any], Coroutine[Any, Any, "RPCResponse"] ], ) -> Callable[..., Coroutine[Any, Any, "RPCResponse"]]: async def wrapper( - provider: ASYNC_PROVIDER_TYPE, method: "RPCEndpoint", params: Any + provider: ASYNC_PROVIDER_TYPE, method: RPCEndpoint, params: Any ) -> "RPCResponse": if is_cacheable_request(provider, method, params): request_cache = provider._request_cache diff --git a/web3/_utils/caching/request_caching_validation.py b/web3/_utils/caching/request_caching_validation.py index c5f963a3ed..4f904c72ac 100644 --- a/web3/_utils/caching/request_caching_validation.py +++ b/web3/_utils/caching/request_caching_validation.py @@ -1,3 +1,4 @@ +import time from typing import ( TYPE_CHECKING, Any, @@ -7,8 +8,11 @@ Union, ) -from eth_utils import ( - to_int, +from web3.types import ( + RPCEndpoint, +) +from web3.utils import ( + RequestCacheValidationThreshold, ) if TYPE_CHECKING: @@ -31,99 +35,252 @@ def _error_log( ) -def is_beyond_validation_threshold(provider: SYNC_PROVIDER_TYPE, blocknum: int) -> bool: +def always_cache_request(*_args: Any, **_kwargs: Any) -> bool: + return True + + +def is_beyond_validation_threshold( + provider: SYNC_PROVIDER_TYPE, + blocknum: int = None, + block_timestamp: int = None, +) -> bool: + cache_allowed_requests = provider.cache_allowed_requests try: - # `threshold` is either "finalized" or "safe" - threshold = provider.request_cache_validation_threshold.value - response = provider.make_request("eth_getBlockByNumber", [threshold, False]) - return blocknum <= to_int(hexstr=response["result"]["number"]) + threshold = provider.request_cache_validation_threshold + + # turn off caching to prevent recursion + provider.cache_allowed_requests = False + if isinstance(threshold, RequestCacheValidationThreshold): + # if mainnet and threshold is "finalized" or "safe" + threshold_block = provider.make_request( + RPCEndpoint("eth_getBlockByNumber"), [threshold.value, False] + )["result"] + # we should have a `blocknum` to compare against + return blocknum <= int(threshold_block["number"], 16) + elif isinstance(threshold, int): + if not block_timestamp: + # if validating via `blocknum` from params, we need to get the timestamp + # for the block with `blocknum`. + block = provider.make_request( + RPCEndpoint("eth_getBlockByNumber"), [hex(blocknum), False] + )["result"] + block_timestamp = int(block["timestamp"], 16) + + # if validating via `block_timestamp` from result, we should have a + # `block_timestamp` to compare against + return block_timestamp <= time.time() - threshold + else: + provider.logger.error( + "Invalid request_cache_validation_threshold value. This should not " + f"have happened. Request not cached.\n threshold: {threshold}" + ) + return False except Exception as e: _error_log(provider, e) return False + finally: + provider.cache_allowed_requests = cache_allowed_requests -def always_cache_request(*_args: Any, **_kwargs: Any) -> bool: - return True - - -def validate_blocknum_in_params( - provider: SYNC_PROVIDER_TYPE, params: Sequence[Any], _result: Dict[str, Any] +def validate_from_block_id_in_params( + provider: SYNC_PROVIDER_TYPE, + params: Sequence[Any], + _result: Dict[str, Any], ) -> bool: block_id = params[0] if block_id == "earliest": # `earliest` should always be cacheable return True - blocknum = to_int(hexstr=block_id) - return is_beyond_validation_threshold(provider, blocknum) + + blocknum = int(block_id, 16) + return is_beyond_validation_threshold(provider, blocknum=blocknum) -def validate_blocknum_in_result( - provider: SYNC_PROVIDER_TYPE, _params: Sequence[Any], result: Dict[str, Any] +def validate_from_blocknum_in_result( + provider: SYNC_PROVIDER_TYPE, + _params: Sequence[Any], + result: Dict[str, Any], ) -> bool: - # `number` if block result, `blockNumber` if transaction result - blocknum = to_int(hexstr=result.get("number", result.get("blockNumber"))) - return is_beyond_validation_threshold(provider, blocknum) + cache_allowed_requests = provider.cache_allowed_requests + try: + # turn off caching to prevent recursion + provider.cache_allowed_requests = False + # transaction results + if "blockNumber" in result: + blocknum = result.get("blockNumber") + # make an extra call to get the block values + block = provider.make_request( + RPCEndpoint("eth_getBlockByNumber"), [blocknum, False] + )["result"] + return is_beyond_validation_threshold( + provider, + blocknum=int(blocknum, 16), + block_timestamp=int(block["timestamp"], 16), + ) + elif "number" in result: + return is_beyond_validation_threshold( + provider, + blocknum=int(result["number"], 16), + block_timestamp=int(result["timestamp"], 16), + ) + else: + provider.logger.error( + "Could not find block number in result. This should not have happened. " + f"Request not cached.\n result: {result}", + ) + return False + except Exception as e: + _error_log(provider, e) + return False + finally: + provider.cache_allowed_requests = cache_allowed_requests -def validate_blockhash_in_params( - provider: SYNC_PROVIDER_TYPE, params: Sequence[Any], _result: Dict[str, Any] + +def validate_from_blockhash_in_params( + provider: SYNC_PROVIDER_TYPE, + params: Sequence[Any], + _result: Dict[str, Any], ) -> bool: + cache_allowed_requests = provider.cache_allowed_requests try: + # turn off caching to prevent recursion + provider.cache_allowed_requests = False + # make an extra call to get the block number from the hash - response = provider.make_request("eth_getBlockByHash", [params[0], False]) + block = provider.make_request( + RPCEndpoint("eth_getBlockByHash"), [params[0], False] + )["result"] + return is_beyond_validation_threshold( + provider, + blocknum=int(block["number"], 16), + block_timestamp=int(block["timestamp"], 16), + ) except Exception as e: _error_log(provider, e) return False - - blocknum = to_int(hexstr=response["result"]["number"]) - return is_beyond_validation_threshold(provider, blocknum) + finally: + provider.cache_allowed_requests = cache_allowed_requests # -- async -- # async def async_is_beyond_validation_threshold( - provider: ASYNC_PROVIDER_TYPE, blocknum: int + provider: ASYNC_PROVIDER_TYPE, + blocknum: int = None, + block_timestamp: int = None, ) -> bool: + cache_allowed_requests = provider.cache_allowed_requests try: - # `threshold` is either "finalized" or "safe" - threshold = provider.request_cache_validation_threshold.value - response = await provider.make_request( - "eth_getBlockByNumber", [threshold, False] - ) - return blocknum <= to_int(hexstr=response["result"]["number"]) + threshold = provider.request_cache_validation_threshold + + # turn off caching to prevent recursion + provider.cache_allowed_requests = False + if isinstance(threshold, RequestCacheValidationThreshold): + # if mainnet and threshold is "finalized" or "safe" + threshold_block = await provider.make_request( + RPCEndpoint("eth_getBlockByNumber"), [threshold.value, False] + ) + # we should have a `blocknum` to compare against + return blocknum <= int(threshold_block["result"]["number"], 16) + elif isinstance(threshold, int): + if not block_timestamp: + block = await provider.make_request( + RPCEndpoint("eth_getBlockByNumber"), [hex(blocknum), False] + ) + block_timestamp = int(block["result"]["timestamp"], 16) + + # if validating via `block_timestamp` from result, we should have a + # `block_timestamp` to compare against + return block_timestamp <= time.time() - threshold + else: + provider.logger.error( + "Invalid request_cache_validation_threshold value. This should not " + f"have happened. Request not cached.\n threshold: {threshold}" + ) + return False except Exception as e: _error_log(provider, e) return False + finally: + provider.cache_allowed_requests = cache_allowed_requests -async def async_validate_blocknum_in_params( - provider: ASYNC_PROVIDER_TYPE, params: Sequence[Any], _result: Dict[str, Any] +async def async_validate_from_block_id_in_params( + provider: ASYNC_PROVIDER_TYPE, + params: Sequence[Any], + _result: Dict[str, Any], ) -> bool: block_id = params[0] if block_id == "earliest": # `earliest` should always be cacheable return True - blocknum = to_int(hexstr=params[0]) - return await async_is_beyond_validation_threshold(provider, blocknum) + + blocknum = int(block_id, 16) + return await async_is_beyond_validation_threshold(provider, blocknum=blocknum) -async def async_validate_blocknum_in_result( - provider: ASYNC_PROVIDER_TYPE, _params: Sequence[Any], result: Dict[str, Any] +async def async_validate_from_blocknum_in_result( + provider: ASYNC_PROVIDER_TYPE, + _params: Sequence[Any], + result: Dict[str, Any], ) -> bool: - # `number` if block result, `blockNumber` if transaction result - blocknum = to_int(hexstr=result.get("number", result.get("blockNumber"))) - return await async_is_beyond_validation_threshold(provider, blocknum) + cache_allowed_requests = provider.cache_allowed_requests + try: + # turn off caching to prevent recursion + provider.cache_allowed_requests = False + + # transaction results + if "blockNumber" in result: + blocknum = result.get("blockNumber") + # make an extra call to get the block values + block = await provider.make_request( + RPCEndpoint("eth_getBlockByNumber"), [blocknum, False] + ) + return await async_is_beyond_validation_threshold( + provider, + blocknum=int(blocknum, 16), + block_timestamp=int(block["result"]["timestamp"], 16), + ) + elif "number" in result: + return await async_is_beyond_validation_threshold( + provider, + blocknum=int(result["number"], 16), + block_timestamp=int(result["timestamp"], 16), + ) + else: + provider.logger.error( + "Could not find block number in result. This should not have happened. " + f"Request not cached.\n result: {result}", + ) + return False + except Exception as e: + _error_log(provider, e) + return False + finally: + provider.cache_allowed_requests = cache_allowed_requests -async def async_validate_blockhash_in_params( +async def async_validate_from_blockhash_in_params( provider: ASYNC_PROVIDER_TYPE, params: Sequence[Any], _result: Dict[str, Any] ) -> bool: + cache_allowed_requests = provider.cache_allowed_requests try: - response = await provider.make_request("eth_getBlockByHash", [params[0], False]) + # turn off caching to prevent recursion + provider.cache_allowed_requests = False + + # make an extra call to get the block number from the hash + response = await provider.make_request( + RPCEndpoint("eth_getBlockByHash"), [params[0], False] + ) + return await async_is_beyond_validation_threshold( + provider, + blocknum=int(response["result"]["number"], 16), + block_timestamp=int(response["result"]["timestamp"], 16), + ) except Exception as e: _error_log(provider, e) return False - - blocknum = to_int(hexstr=response["result"]["number"]) - return await async_is_beyond_validation_threshold(provider, blocknum) + finally: + provider.cache_allowed_requests = cache_allowed_requests diff --git a/web3/_utils/module_testing/utils.py b/web3/_utils/module_testing/utils.py index b2a4fd5084..4bce464e43 100644 --- a/web3/_utils/module_testing/utils.py +++ b/web3/_utils/module_testing/utils.py @@ -35,6 +35,14 @@ class RequestMocker: Context manager to mock requests made by a web3 instance. This is meant to be used via a ``request_mocker`` fixture defined within the appropriate context. + ************************************************************************************ + Important: When mocking results, it's important to keep in mind the types that + clients return. For example, what we commonly translate to integers are returned + as hex strings in the RPC response and should be mocked as such for more + accurate testing. + ************************************************************************************ + + Example: ------- @@ -105,10 +113,9 @@ def __enter__(self) -> "Self": return self - # define __exit__ with typing information def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None: # mypy error: Cannot assign to a method - self.w3.provider.make_request = self._make_request # type: ignore[method-assign] # noqa: E501 + self.w3.provider.make_request = self._make_request # type: ignore[assignment] # reset request func cache to re-build request_func with original make_request self.w3.provider._request_func_cache = (None, None) @@ -167,6 +174,7 @@ def _mock_request_handler( return mocked_response # -- async -- # + async def __aenter__(self) -> "Self": # mypy error: Cannot assign to a method self.w3.provider.make_request = self._async_mock_request_handler # type: ignore[method-assign] # noqa: E501 @@ -176,7 +184,7 @@ async def __aenter__(self) -> "Self": async def __aexit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None: # mypy error: Cannot assign to a method - self.w3.provider.make_request = self._make_request # type: ignore[method-assign] # noqa: E501 + self.w3.provider.make_request = self._make_request # type: ignore[assignment] # reset request func cache to re-build request_func with original make_request self.w3.provider._request_func_cache = (None, None) diff --git a/web3/providers/async_base.py b/web3/providers/async_base.py index 5a4f3f415f..eb8a7426e5 100644 --- a/web3/providers/async_base.py +++ b/web3/providers/async_base.py @@ -10,6 +10,7 @@ Optional, Set, Tuple, + Union, cast, ) @@ -21,7 +22,10 @@ from web3._utils.caching import ( CACHEABLE_REQUESTS, - async_handle_request_caching, +) +from web3._utils.empty import ( + Empty, + empty, ) from web3._utils.encoding import ( FriendlyJsonSerde, @@ -88,8 +92,8 @@ def __init__( cache_allowed_requests: bool = False, cacheable_requests: Set[RPCEndpoint] = None, request_cache_validation_threshold: Optional[ - RequestCacheValidationThreshold - ] = RequestCacheValidationThreshold.FINALIZED, + Union[RequestCacheValidationThreshold, int, Empty] + ] = empty, ) -> None: self._request_cache = SimpleCache(1000) self.cache_allowed_requests = cache_allowed_requests @@ -132,7 +136,6 @@ async def batch_request_func( self._batch_request_func_cache = (middleware, accumulator_fn) return self._batch_request_func_cache[-1] - @async_handle_request_caching async def make_request(self, method: RPCEndpoint, params: Any) -> RPCResponse: raise NotImplementedError("Providers must implement this method") diff --git a/web3/providers/base.py b/web3/providers/base.py index b653f267bc..3d9249a875 100644 --- a/web3/providers/base.py +++ b/web3/providers/base.py @@ -9,6 +9,7 @@ Optional, Set, Tuple, + Union, cast, ) @@ -19,7 +20,10 @@ from web3._utils.caching import ( CACHEABLE_REQUESTS, - handle_request_caching, +) +from web3._utils.empty import ( + Empty, + empty, ) from web3._utils.encoding import ( FriendlyJsonSerde, @@ -71,8 +75,8 @@ def __init__( cache_allowed_requests: bool = False, cacheable_requests: Set[RPCEndpoint] = None, request_cache_validation_threshold: Optional[ - RequestCacheValidationThreshold - ] = RequestCacheValidationThreshold.FINALIZED, + Union[RequestCacheValidationThreshold, int, Empty] + ] = empty, ) -> None: self._request_cache = SimpleCache(1000) self.cache_allowed_requests = cache_allowed_requests @@ -104,7 +108,6 @@ def request_func( return self._request_func_cache[-1] - @handle_request_caching def make_request(self, method: RPCEndpoint, params: Any) -> RPCResponse: raise NotImplementedError("Providers must implement this method") diff --git a/web3/providers/ipc.py b/web3/providers/ipc.py index 9a4e815aa5..9ea72157d6 100644 --- a/web3/providers/ipc.py +++ b/web3/providers/ipc.py @@ -32,6 +32,9 @@ from .._utils.batching import ( sort_batch_response_by_response_ids, ) +from .._utils.caching import ( + handle_request_caching, +) from ..exceptions import ( Web3TypeError, Web3ValueError, @@ -189,6 +192,7 @@ def _make_request(self, request: bytes) -> RPCResponse: timeout.sleep(0) continue + @handle_request_caching def make_request(self, method: RPCEndpoint, params: Any) -> RPCResponse: self.logger.debug( f"Making request IPC. Path: {self.ipc_path}, Method: {method}"