Skip to content

Commit

Permalink
Add support for non-mainnet request cache validation thresholds:
Browse files Browse the repository at this point in the history 
- If non-mainnet ethereum, allow the use of an integer value for the
  request cache validation threshold. Pre-configure "safe" default
  values for some common non-mainnet chain ids based on their
  varied finality mechanisms.

- This integer value represents the number of seconds from time.now()
  that the request cache deems as a safe enough time window to allow
  the request to be cached.

- Update the tests to reflect these changes.
  • Loading branch information
@fselmo
fselmo committed Oct 10, 2024
1 parent eb235c2 commit 0722a1e
Show file tree
Hide file tree
Showing 5 changed files with 413 additions and 115 deletions.
141 changes: 90 additions & 51 deletions tests/core/caching-utils/test_request_caching.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,9 +21,11 @@
)
from web3._utils.caching.caching_utils import (
ASYNC_INTERNAL_VALIDATION_MAP,
BLOCK_IN_RESULT,
BLOCKHASH_IN_PARAMS,
BLOCKNUM_IN_PARAMS,
BLOCKNUM_IN_RESULT,
CHAIN_VALIDATION_THRESHOLD_DEFAULTS,
DEFAULT_VALIDATION_THRESHOLD,
INTERNAL_VALIDATION_MAP,
)
from web3.exceptions import (
Expand Down Expand Up @@ -64,6 +66,7 @@ def w3(request_mocker):
mock_results={
"fake_endpoint": lambda *_: uuid.uuid4(),
"not_on_allowlist": lambda *_: uuid.uuid4(),
"eth_chainId": "0x1", # mainnet
},
):
yield _w3
Expand Down Expand Up @@ -199,7 +202,7 @@ def test_all_providers_do_not_cache_by_default_and_can_set_caching_properties(pr
"threshold",
(RequestCacheValidationThreshold.FINALIZED, RequestCacheValidationThreshold.SAFE),
)
@pytest.mark.parametrize("endpoint", BLOCKNUM_IN_PARAMS | BLOCKNUM_IN_RESULT)
@pytest.mark.parametrize("endpoint", BLOCKNUM_IN_PARAMS | BLOCK_IN_RESULT)
@pytest.mark.parametrize(
"blocknum,should_cache",
(
Expand All @@ -211,11 +214,11 @@ def test_all_providers_do_not_cache_by_default_and_can_set_caching_properties(pr
("0x5", False),
),
)
def test_blocknum_validation_against_validation_threshold_when_caching(
def test_blocknum_validation_against_validation_threshold_when_caching_mainnet(
threshold, endpoint, blocknum, should_cache, request_mocker
):
w3 = Web3(
HTTPProvider(
BaseProvider(
cache_allowed_requests=True, request_cache_validation_threshold=threshold
)
)
Expand All @@ -224,24 +227,25 @@ def test_blocknum_validation_against_validation_threshold_when_caching(
mock_results={
endpoint: (
# mock the result to requests that return blocks
{"number": blocknum}
{"number": blocknum, "timestamp": 0}
if "getBlock" in endpoint
# mock the result to requests that return transactions
else {"blockNumber": blocknum}
),
"eth_getBlockByNumber": lambda _method, params: (
# mock the threshold block to be blocknum "0x2", return
# blocknum otherwise
{"number": "0x2"}
{"number": "0x2", "timestamp": 0}
if params[0] == threshold.value
else {"number": params[0]}
else {"number": params[0], "timestamp": 0}
),
"eth_chainId": "0x1", # mainnet
},
):
assert len(w3.provider._request_cache.items()) == 0
w3.manager.request_blocking(endpoint, [blocknum, False])
cached_items = len(w3.provider._request_cache.items())
assert cached_items == 1 if should_cache else cached_items == 0
assert cached_items > 0 if should_cache else cached_items == 0


@pytest.mark.parametrize(
Expand All @@ -260,30 +264,31 @@ def test_blocknum_validation_against_validation_threshold_when_caching(
("pending", None, False),
),
)
def test_block_id_param_caching(
def test_block_id_param_caching_mainnet(
threshold, endpoint, block_id, blocknum, should_cache, request_mocker
):
w3 = Web3(
HTTPProvider(
BaseProvider(
cache_allowed_requests=True, request_cache_validation_threshold=threshold
)
)
with request_mocker(
w3,
mock_results={
"eth_chainId": "0x1", # mainnet
endpoint: "0x0",
"eth_getBlockByNumber": lambda _method, params: (
# mock the threshold block to be blocknum "0x2" for all test cases
{"number": "0x2"}
{"number": "0x2", "timestamp": 0}
if params[0] == threshold.value
else {"number": blocknum}
else {"number": blocknum, "timestamp": 0}
),
},
):
assert len(w3.provider._request_cache.items()) == 0
w3.manager.request_blocking(RPCEndpoint(endpoint), [block_id, False])
cached_items = len(w3.provider._request_cache.items())
assert cached_items == 1 if should_cache else cached_items == 0
assert cached_items > 0 if should_cache else cached_items == 0


@pytest.mark.parametrize(
Expand All @@ -302,24 +307,25 @@ def test_block_id_param_caching(
("0x5", False),
),
)
def test_blockhash_validation_against_validation_threshold_when_caching(
def test_blockhash_validation_against_validation_threshold_when_caching_mainnet(
threshold, endpoint, blocknum, should_cache, request_mocker
):
w3 = Web3(
HTTPProvider(
BaseProvider(
cache_allowed_requests=True, request_cache_validation_threshold=threshold
)
)
with request_mocker(
w3,
mock_results={
"eth_chainId": "0x1", # mainnet
"eth_getBlockByNumber": lambda _method, params: (
# mock the threshold block to be blocknum "0x2"
{"number": "0x2"}
{"number": "0x2", "timestamp": 0}
if params[0] == threshold.value
else {"number": params[0]}
else {"number": params[0], "timestamp": 0}
),
"eth_getBlockByHash": {"number": blocknum},
"eth_getBlockByHash": {"number": blocknum, "timestamp": 0},
endpoint: "0x0",
},
):
Expand All @@ -329,23 +335,37 @@ def test_blockhash_validation_against_validation_threshold_when_caching(
assert cached_items == 2 if should_cache else cached_items == 0


def test_request_caching_validation_threshold_is_finalized_by_default():
w3 = Web3(HTTPProvider(cache_allowed_requests=True))
assert (
w3.provider.request_cache_validation_threshold
== RequestCacheValidationThreshold.FINALIZED
)
@pytest.mark.parametrize(
"chain_id,expected_threshold",
(
*CHAIN_VALIDATION_THRESHOLD_DEFAULTS.items(),
(3456787654567654, DEFAULT_VALIDATION_THRESHOLD),
(11111111111444444444444444, DEFAULT_VALIDATION_THRESHOLD),
(-11111111111111111117, DEFAULT_VALIDATION_THRESHOLD),
),
)
def test_request_caching_validation_threshold_defaults(
chain_id, expected_threshold, request_mocker
):
w3 = Web3(BaseProvider(cache_allowed_requests=True))
with request_mocker(w3, mock_results={"eth_chainId": chain_id}):
w3.manager.request_blocking(RPCEndpoint("eth_chainId"), [])
assert w3.provider.request_cache_validation_threshold == expected_threshold
# assert chain_id is cached
cache_items = w3.provider._request_cache.items()
assert len(cache_items) == 1
assert cache_items[0][1]["result"] == chain_id


@pytest.mark.parametrize(
"endpoint", BLOCKNUM_IN_PARAMS | BLOCKNUM_IN_RESULT | BLOCKHASH_IN_PARAMS
"endpoint", BLOCKNUM_IN_PARAMS | BLOCK_IN_RESULT | BLOCKHASH_IN_PARAMS
)
@pytest.mark.parametrize("blocknum", ("0x0", "0x1", "0x2", "0x3", "0x4", "0x5"))
def test_request_caching_with_validation_threshold_set_to_none(
endpoint, blocknum, request_mocker
):
w3 = Web3(
HTTPProvider(
BaseProvider(
cache_allowed_requests=True,
request_cache_validation_threshold=None,
)
Expand Down Expand Up @@ -376,6 +396,7 @@ async def async_w3(request_mocker):
mock_results={
"fake_endpoint": lambda *_: uuid.uuid4(),
"not_on_allowlist": lambda *_: uuid.uuid4(),
"eth_chainId": "0x1", # mainnet
},
):
yield _async_w3
Expand Down Expand Up @@ -491,7 +512,7 @@ async def test_async_request_caching_does_not_share_state_between_providers(
"threshold",
(RequestCacheValidationThreshold.FINALIZED, RequestCacheValidationThreshold.SAFE),
)
@pytest.mark.parametrize("endpoint", BLOCKNUM_IN_PARAMS | BLOCKNUM_IN_RESULT)
@pytest.mark.parametrize("endpoint", BLOCKNUM_IN_PARAMS | BLOCK_IN_RESULT)
@pytest.mark.parametrize(
"blocknum,should_cache",
(
Expand All @@ -503,11 +524,11 @@ async def test_async_request_caching_does_not_share_state_between_providers(
("0x5", False),
),
)
async def test_async_blocknum_validation_against_validation_threshold(
async def test_async_blocknum_validation_against_validation_threshold_mainnet(
threshold, endpoint, blocknum, should_cache, request_mocker
):
async_w3 = AsyncWeb3(
AsyncHTTPProvider(
AsyncBaseProvider(
cache_allowed_requests=True, request_cache_validation_threshold=threshold
)
)
Expand All @@ -516,24 +537,25 @@ async def test_async_blocknum_validation_against_validation_threshold(
mock_results={
endpoint: (
# mock the result to requests that return blocks
{"number": blocknum}
{"number": blocknum, "timestamp": 0}
if "getBlock" in endpoint
# mock the result to requests that return transactions
else {"blockNumber": blocknum}
),
"eth_getBlockByNumber": lambda _method, params: (
# mock the threshold block to be blocknum "0x2", return
# blocknum otherwise
{"number": "0x2"}
{"number": "0x2", "timestamp": 0}
if params[0] == threshold.value
else {"number": params[0]}
else {"number": params[0], "timestamp": 0}
),
"eth_chainId": "0x1", # mainnet
},
):
assert len(async_w3.provider._request_cache.items()) == 0
await async_w3.manager.coro_request(endpoint, [blocknum, False])
cached_items = len(async_w3.provider._request_cache.items())
assert cached_items == 1 if should_cache else cached_items == 0
assert cached_items > 0 if should_cache else cached_items == 0


@pytest.mark.asyncio
Expand All @@ -553,30 +575,31 @@ async def test_async_blocknum_validation_against_validation_threshold(
("pending", None, False),
),
)
async def test_async_block_id_param_caching(
async def test_async_block_id_param_caching_mainnet(
threshold, endpoint, block_id, blocknum, should_cache, request_mocker
):
async_w3 = AsyncWeb3(
AsyncHTTPProvider(
AsyncBaseProvider(
cache_allowed_requests=True, request_cache_validation_threshold=threshold
)
)
async with request_mocker(
async_w3,
mock_results={
"eth_chainId": "0x1", # mainnet
endpoint: "0x0",
"eth_getBlockByNumber": lambda _method, params: (
# mock the threshold block to be blocknum "0x2" for all test cases
{"number": "0x2"}
{"number": "0x2", "timestamp": 0}
if params[0] == threshold.value
else {"number": blocknum}
else {"number": blocknum, "timestamp": 0}
),
},
):
assert len(async_w3.provider._request_cache.items()) == 0
await async_w3.manager.coro_request(RPCEndpoint(endpoint), [block_id, False])
cached_items = len(async_w3.provider._request_cache.items())
assert cached_items == 1 if should_cache else cached_items == 0
assert cached_items > 0 if should_cache else cached_items == 0


@pytest.mark.asyncio
Expand All @@ -596,24 +619,25 @@ async def test_async_block_id_param_caching(
("0x5", False),
),
)
async def test_async_blockhash_validation_against_validation_threshold(
async def test_async_blockhash_validation_against_validation_threshold_mainnet(
threshold, endpoint, blocknum, should_cache, request_mocker
):
async_w3 = AsyncWeb3(
AsyncHTTPProvider(
AsyncBaseProvider(
cache_allowed_requests=True, request_cache_validation_threshold=threshold
)
)
async with request_mocker(
async_w3,
mock_results={
"eth_chainId": "0x1", # mainnet
"eth_getBlockByNumber": lambda _method, params: (
# mock the threshold block to be blocknum "0x2"
{"number": "0x2"}
{"number": "0x2", "timestamp": 0}
if params[0] == threshold.value
else {"number": params[0]}
else {"number": params[0], "timestamp": 0}
),
"eth_getBlockByHash": {"number": blocknum},
"eth_getBlockByHash": {"number": blocknum, "timestamp": 0},
endpoint: "0x0",
},
):
Expand All @@ -624,24 +648,39 @@ async def test_async_blockhash_validation_against_validation_threshold(


@pytest.mark.asyncio
async def test_async_request_caching_validation_threshold_is_finalized_by_default():
async_w3 = AsyncWeb3(AsyncHTTPProvider(cache_allowed_requests=True))
assert (
async_w3.provider.request_cache_validation_threshold
== RequestCacheValidationThreshold.FINALIZED
)
@pytest.mark.parametrize(
"chain_id,expected_threshold",
(
*CHAIN_VALIDATION_THRESHOLD_DEFAULTS.items(),
(3456787654567654, DEFAULT_VALIDATION_THRESHOLD),
(11111111111444444444444444, DEFAULT_VALIDATION_THRESHOLD),
(-11111111111111111117, DEFAULT_VALIDATION_THRESHOLD),
),
)
async def test_async_request_caching_validation_threshold_defaults(
chain_id, expected_threshold, request_mocker
):
async_w3 = AsyncWeb3(AsyncBaseProvider(cache_allowed_requests=True))
async with request_mocker(async_w3, mock_results={"eth_chainId": chain_id}):
await async_w3.manager.coro_request(RPCEndpoint("eth_chainId"), [])
assert (
async_w3.provider.request_cache_validation_threshold == expected_threshold
)
cache_items = async_w3.provider._request_cache.items()
assert len(cache_items) == 1
assert cache_items[0][1]["result"] == chain_id


@pytest.mark.asyncio
@pytest.mark.parametrize(
"endpoint", BLOCKNUM_IN_PARAMS | BLOCKNUM_IN_RESULT | BLOCKHASH_IN_PARAMS
"endpoint", BLOCKNUM_IN_PARAMS | BLOCK_IN_RESULT | BLOCKHASH_IN_PARAMS
)
@pytest.mark.parametrize("blocknum", ("0x0", "0x1", "0x2", "0x3", "0x4", "0x5"))
async def test_async_request_caching_with_validation_threshold_set_to_none(
endpoint, blocknum, request_mocker
):
async_w3 = AsyncWeb3(
AsyncHTTPProvider(
AsyncBaseProvider(
cache_allowed_requests=True,
request_cache_validation_threshold=None,
)
Expand Down
Loading

0 comments on commit 0722a1e

Please sign in to comment.