add rate limit and faster default speed

ape_alchemy/__init__.py

@@ -1,6 +1,7 @@
 from ape import plugins
 
-from .provider import Alchemy
+from .provider import Alchemy, AlchemyConfig
+
 
 NETWORKS = {
     "ethereum": [
@@ -29,6 +30,11 @@
 
 @plugins.register(plugins.ProviderPlugin)
 def providers():
-    for ecosystem_name in NETWORKS:
-        for network_name in NETWORKS[ecosystem_name]:
+    for ecosystem_name, networks in NETWORKS.items():
+        for network_name in networks:
             yield ecosystem_name, network_name, Alchemy
+
+
+@plugins.register(plugins.Config)
+def config_class():
+    yield AlchemyConfig

ape_alchemy/provider.py

@@ -1,5 +1,5 @@
-import os
-from typing import Any, Dict, List, Optional
+import os, random, time
+from typing import Any, Dict, cast, List, Optional
 
 from ape.api import ReceiptAPI, TransactionAPI, UpstreamProvider
 from ape.exceptions import (
@@ -8,6 +8,7 @@
     ProviderError,
     VirtualMachineError,
 )
+from ape.api import PluginConfig
 from ape.logging import logger
 from ape.types import CallTreeNode
 from ape_ethereum.provider import Web3Provider
@@ -34,15 +35,71 @@
 PRIVATE_TX_BLOCK_WAIT = 25
 
 
+class AlchemyConfig(PluginConfig):
+    """Configuration for Alchemy.
+
+    Fields
+    ------
+        concurrency: int
+            The maximum number of concurrent requests to make.
+            Defaults to 1.
+        block_page_size: int
+            The maximum number of blocks to fetch in a single request.
+            Defaults to 250_000.
+        min_retry_delay: int
+            The amount of milliseconds to wait before retrying the request.
+            Defaults to one second.
+        retry_backoff_factor: int
+            The multiplier applied to the retry delay after each failed attempt.
+            Defaults to two.
+        max_retry_delay: int
+            The maximum length of the retry delay.
+            Defaults to 30 seconds.
+        max_retries: int
+            The maximum number of retries.
+            Defaults to 3.
+        retry_jitter: int
+            A random number of milliseconds up to this limit is added to each retry delay.
+            Defaults to 250 milliseconds.
+    """
+
+    concurrency: int = 1  # can't do exponential backoff with multiple threads
+    block_page_size: int = 250_000  # this acts as an upper limit, safe to set very high
+    min_retry_delay: int = 1_000  # 1 second
+    retry_backoff_factor: int = 2  # exponential backoff
+    max_retry_delay: int = 30_000  # 30 seconds
+    max_retries: int = 3
+    retry_jitter: int = 250  # 250 milliseconds
+
+
 class Alchemy(Web3Provider, UpstreamProvider):
     """
     A web3 provider using an HTTP connection to Alchemy.
 
     Docs: https://docs.alchemy.com/alchemy/
+
+    Fields
+    ------
+        concurrency: int
+            The maximum number of concurrent requests to make.
+            Defaults to 1.
+        block_page_size: int
+            The maximum number of blocks to fetch in a single request.
+            Defaults to 250_000.
+        network_uris: Dict[tuple, str]
+            A mapping of (ecosystem_name, network_name) -> URI
     """
 
+    concurrency: int = 1  # can't do exponential backoff with multiple threads
+    block_page_size: int = 250_000  # this acts as an upper limit, safe to set very high
     network_uris: Dict[tuple, str] = {}
 
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        alchemy_config = cast(AlchemyConfig, self.config_manager.get_config("alchemy"))
+        self.concurrency = alchemy_config.concurrency
+        self.block_page_size = alchemy_config.block_page_size
+
     @property
     def uri(self):
         """
@@ -179,26 +236,60 @@ def get_virtual_machine_error(self, exception: Exception, **kwargs) -> VirtualMa
 
         return VirtualMachineError(message=message, txn=txn)
 
-    def _make_request(self, endpoint: str, parameters: Optional[List] = None) -> Any:
-        try:
-            return super()._make_request(endpoint, parameters)
-        except HTTPError as err:
-            response_data = err.response.json() if err.response else {}
-            if "error" not in response_data:
-                raise AlchemyProviderError(str(err)) from err
-
-            error_data = response_data["error"]
-            message = (
-                error_data.get("message", str(error_data))
-                if isinstance(error_data, dict)
-                else error_data
-            )
-            cls = (
-                AlchemyFeatureNotAvailable
-                if "is not available" in message
-                else AlchemyProviderError
-            )
-            raise cls(message) from err
+    def _make_request(
+        self,
+        endpoint: str,
+        parameters: Optional[List] = None,
+        min_retry_delay: Optional[int] = None,
+        retry_backoff_factor: Optional[int] = None,
+        max_retry_delay: Optional[int] = None,
+        max_retries: Optional[int] = None,
+        retry_jitter: Optional[int] = None,
+    ) -> Any:
+        alchemy_config = cast(AlchemyConfig, self.config_manager.get_config("alchemy"))
+        min_retry_delay = (
+            min_retry_delay if min_retry_delay is not None else alchemy_config.min_retry_delay
+        )
+        retry_backoff_factor = (
+            retry_backoff_factor
+            if retry_backoff_factor is not None
+            else alchemy_config.retry_backoff_factor
+        )
+        max_retry_delay = (
+            max_retry_delay if max_retry_delay is not None else alchemy_config.max_retry_delay
+        )
+        max_retries = max_retries if max_retries is not None else alchemy_config.max_retries
+        retry_jitter = retry_jitter if retry_jitter is not None else alchemy_config.retry_jitter
+        for attempt in range(max_retries):
+            try:
+                return super()._make_request(endpoint, parameters)
+            except HTTPError as err:
+                message = str(err)
+                if any(
+                    error in message
+                    for error in ["exceeded its compute units", "Too Many Requests for url"]
+                ):
+                    retry_interval = min(
+                        max_retry_delay, min_retry_delay * retry_backoff_factor**attempt
+                    )
+                    logger.info(
+                        "Alchemy compute units exceeded, retrying, attempt %s/%s in %s ms",
+                        attempt + 1,
+                        max_retries,
+                        retry_interval,
+                    )
+                    delay = retry_interval + random.randint(0, retry_jitter)
+                    time.sleep(delay / 1000)
+                    continue
+                elif "error" not in message:
+                    raise AlchemyProviderError(str(err)) from err
+                cls = (
+                    AlchemyFeatureNotAvailable
+                    if "is not available" in message
+                    else AlchemyProviderError
+                )
+                raise cls(message) from err
+        raise AlchemyProviderError(f"Rate limit exceeded after {max_retries} attempts.")
 
     def send_private_transaction(self, txn: TransactionAPI, **kwargs) -> ReceiptAPI:
         """