feat: add support for Etherscan's rate limit [APE-774]

ape_etherscan/__init__.py

@@ -1,5 +1,6 @@
 from ape import plugins
 
+from .config import EtherscanConfig
 from .explorer import Etherscan
 from .query import EtherscanQueryEngine
 
@@ -35,6 +36,11 @@
 }
 
 
+@plugins.register(plugins.Config)
+def config_class():
+    return EtherscanConfig
+
+
 @plugins.register(plugins.ExplorerPlugin)
 def explorers():
     for ecosystem_name in NETWORKS:

ape_etherscan/client.py

@@ -1,11 +1,12 @@
 import json
 import os
 import random
+import time
 from io import StringIO
 from typing import Dict, Iterator, List, Optional
 
 from ape.logging import logger
-from ape.utils import USER_AGENT
+from ape.utils import USER_AGENT, ManagerAccessMixin
 from requests import Session
 
 from ape_etherscan.exceptions import UnhandledResultError, UnsupportedEcosystemError
@@ -109,14 +110,15 @@ def get_etherscan_api_uri(ecosystem_name: str, network_name: str):
     raise UnsupportedEcosystemError(ecosystem_name)
 
 
-class _APIClient:
+class _APIClient(ManagerAccessMixin):
     DEFAULT_HEADERS = {"User-Agent": USER_AGENT}
     session = Session()
 
     def __init__(self, ecosystem_name: str, network_name: str, module_name: str):
         self._ecosystem_name = ecosystem_name
         self._network_name = network_name
         self._module_name = module_name
+        self._last_call = 0.0
 
     @property
     def base_uri(self) -> str:
@@ -126,13 +128,32 @@ def base_uri(self) -> str:
     def base_params(self) -> Dict:
         return {"module": self._module_name}
 
+    @property
+    def _rate_limit(self) -> int:
+        config = self.config_manager.get_config("etherscan")
+        return getattr(config, self.network_manager.ecosystem.name).rate_limit
+
+    @property
+    def _min_time_between_calls(self) -> float:
+        return 1 / self._rate_limit  # seconds / calls per second
+
     def _get(
         self,
         params: Optional[Dict] = None,
         headers: Optional[Dict[str, str]] = None,
         raise_on_exceptions: bool = True,
     ) -> EtherscanResponse:
         params = self.__authorize(params)
+
+        # Rate limit
+        if time.time() - self._last_call < self._min_time_between_calls:
+            time_to_sleep = self._min_time_between_calls - (time.time() - self._last_call)
+            logger.debug(f"Sleeping {time_to_sleep} seconds to avoid rate limit")
+            # NOTE: Sleep time is in seconds (float for subseconds)
+            time.sleep(time_to_sleep)
+
+        self._last_call = time.time()
+
         return self._request(
             "GET",
             params=params,

ape_etherscan/config.py

@@ -0,0 +1,15 @@
+from ape.api.config import PluginConfig
+
+
+class EcosystemConfig(PluginConfig):
+    rate_limit: int = 5  # Requests per second
+
+
+class EtherscanConfig(PluginConfig):
+    ethereum: EcosystemConfig = EcosystemConfig()
+    arbitrum: EcosystemConfig = EcosystemConfig()
+    fantom: EcosystemConfig = EcosystemConfig()
+    optimism: EcosystemConfig = EcosystemConfig()
+    polygon: EcosystemConfig = EcosystemConfig()
+    avalanche: EcosystemConfig = EcosystemConfig()
+    bsc: EcosystemConfig = EcosystemConfig()

ape_etherscan/query.py

@@ -20,10 +20,18 @@ def _client_factory(self) -> ClientFactory:
     def estimate_query(self, query: QueryType) -> Optional[int]:  # type: ignore[override]
         return None
 
+    @property
+    def rate_limit(self) -> int:
+        config = self.config_manager.get_config("etherscan")
+        return getattr(config, self.network_manager.ecosystem.name).rate_limit
+
     @estimate_query.register
     def estimate_account_transaction_query(self, query: AccountTransactionQuery) -> int:
-        # About 15 ms per page of 100 transactions
-        return 1500 * (1 + query.stop_nonce - query.start_nonce) // 100
+        # About 15 ms per page of 100 transactions, with rate limit applied
+        if query.stop_nonce - query.stop_nonce <= 100:
+            return 15
+
+        return (10000 // self.rate_limit) * (1 + query.stop_nonce - query.start_nonce) // 100
 
     @singledispatchmethod
     def perform_query(self, query: QueryType) -> Iterator:  # type: ignore[override]