feat: add Gateway Provider and fix Fork Provider [APE-795]

.pre-commit-config.yaml

@@ -26,5 +26,11 @@ repos:
     -   id: mypy
         additional_dependencies: [types-requests, types-setuptools, pydantic]
 
+-   repo: https://github.com/executablebooks/mdformat
+    rev: 0.7.14
+    hooks:
+    -   id: mdformat
+        additional_dependencies: [mdformat-gfm, mdformat-frontmatter]
+
 default_language_version:
     python: python3

CONTRIBUTING.md

@@ -46,4 +46,4 @@ A pull request represents the start of a discussion, and doesn't necessarily nee
 If you are opening a work-in-progress pull request to verify that it passes CI tests, please consider
 [marking it as a draft](https://help.github.com/en/github/collaborating-with-issues-and-pull-requests/about-pull-requests#draft-pull-requests).
 
-Join the Ethereum Python [Discord](https://discord.gg/PcEJ54yX) if you have any questions.
+Join the ApeWorX [Discord](https://discord.gg/apeworx) if you have any questions.

ape_tenderly/__init__.py

@@ -1,9 +1,54 @@
 from ape import plugins
 
-from .provider import TenderlyProvider
+from .provider import TenderlyConfig, TenderlyForkProvider, TenderlyGatewayProvider
+
+NETWORKS = {
+    "ethereum": [
+        ("mainnet", TenderlyGatewayProvider),
+        ("mainnet-fork", TenderlyForkProvider),
+        ("goerli", TenderlyGatewayProvider),
+        ("goerli-fork", TenderlyForkProvider),
+        ("sepolia", TenderlyGatewayProvider),
+        ("sepolia-fork", TenderlyForkProvider),
+    ],
+    "polygon": [
+        ("mainnet", TenderlyGatewayProvider),
+        ("mainnet-fork", TenderlyForkProvider),
+        ("mumbai", TenderlyGatewayProvider),
+        ("mumbai-fork", TenderlyForkProvider),
+    ],
+    "arbitrum": [
+        ("mainnet-fork", TenderlyForkProvider),
+        ("goerli-fork", TenderlyForkProvider),
+    ],
+    "optimism": [
+        ("mainnet", TenderlyGatewayProvider),
+        ("mainnet-fork", TenderlyForkProvider),
+        ("goerli", TenderlyGatewayProvider),
+        ("goerli-fork", TenderlyForkProvider),
+    ],
+    "base": [
+        ("mainnet", TenderlyGatewayProvider),
+        ("mainnet-fork", TenderlyForkProvider),
+        ("goerli", TenderlyGatewayProvider),
+        ("goerli-fork", TenderlyForkProvider),
+    ],
+    "avalance": [
+        ("mainnet-fork", TenderlyForkProvider),
+    ],
+    "fantom": [
+        ("opera-fork", TenderlyForkProvider),
+    ],
+}
+
+
+@plugins.register(plugins.Config)
+def config_class():
+    return TenderlyConfig
 
 
 @plugins.register(plugins.ProviderPlugin)
 def providers():
-    yield "ethereum", "mainnet-fork", TenderlyProvider
-    yield "fantom", "opera-fork", TenderlyProvider
+    for ecosystem_name in NETWORKS:
+        for network_name, provider in NETWORKS[ecosystem_name]:
+            yield ecosystem_name, network_name, provider

ape_tenderly/client.py

@@ -0,0 +1,96 @@
+import os
+from typing import Any, Dict, List
+
+import requests
+from ape.exceptions import ConfigError
+from ape.utils import cached_property
+from pydantic import BaseModel, parse_obj_as
+
+TENDERLY_FORK_ID = "TENDERLY_FORK_ID"
+TENDERLY_PROJECT = "TENDERLY_PROJECT"
+TENDERLY_ACCESS_KEY = "TENDERLY_ACCESS_KEY"
+TENDERLY_GATEWAY_ACCESS_KEY = "TENDERLY_GATEWAY_ACCESS_KEY"
+
+
+class ForkDetails(BaseModel):
+    chain_config: Dict[str, Any] = {}
+
+
+class Fork(BaseModel):
+    id: str
+    network_id: int
+    block_number: int
+    details: ForkDetails
+    json_rpc_url: str
+    config: Dict[str, Any] = {}
+
+
+class TenderlyClientError(Exception):
+    pass
+
+
+class TenderlyClient:
+    @cached_property
+    def __access_key_header(self) -> Dict:
+        if not (access_key := os.environ.get(TENDERLY_ACCESS_KEY)):
+            raise ConfigError("No valid tenderly access key found.")
+
+        return {"X-Access-Key": access_key}
+
+    @cached_property
+    def _api_uri(self) -> str:
+        if not (project_name := os.environ.get(TENDERLY_PROJECT)):
+            raise ConfigError("No valid tenderly project name found.")
+
+        return f"https://api.tenderly.co/api/v2/project/{project_name}"
+
+    def get_forks(self) -> List[Fork]:
+        response = requests.get(
+            f"{self._api_uri}/forks",
+            headers=self.__access_key_header,
+        )
+
+        if not response.ok:
+            # NOTE: This will raise on any HTTP errors
+            response.raise_for_status()
+            # ...and this will raise for anything else
+            raise TenderlyClientError(f"Error processing request: {response.text}")
+
+        return parse_obj_as(List[Fork], response.json())
+
+    def create_fork(self, chain_id: int) -> Fork:
+        response = requests.post(
+            f"{self._api_uri}/forks",
+            json={
+                "name": f"ape-fork-{chain_id}",
+                "description": "Automatically created by Ape",
+                "network_id": str(chain_id),
+            },
+            headers=self.__access_key_header,
+        )
+
+        if not response.ok:
+            # NOTE: This will raise on any HTTP errors
+            response.raise_for_status()
+            # ...and this will raise for anything else
+            raise TenderlyClientError(f"Error processing request: {response.text}")
+
+        return parse_obj_as(Fork, response.json().get("fork"))
+
+    def remove_fork(self, fork_id: str):
+        response = requests.delete(
+            f"{self._api_uri}/forks/{fork_id}",
+            headers=self.__access_key_header,
+        )
+
+        if not response.ok:
+            # NOTE: This will raise on any HTTP errors
+            response.raise_for_status()
+            # ...and this will raise for anything else
+            raise TenderlyClientError(f"Error processing request: {response.text}")
+
+    def get_gateway_rpc_uri(self, network_name: str) -> str:
+        if not (project_id := os.environ.get(TENDERLY_GATEWAY_ACCESS_KEY)):
+            raise TenderlyClientError("No valid Tenderly Gateway Access Key found.")
+
+        return f"https://{network_name}.gateway.tenderly.co/{project_id}"

ape_tenderly/provider.py

@@ -1,39 +1,111 @@
-import os
+import atexit
 
-import requests
-from ape.api import ProviderAPI, Web3Provider
-from ape.exceptions import ConfigError
+from ape.api import PluginConfig, UpstreamProvider, Web3Provider
+from ape.exceptions import ProviderError
+from ape.logging import logger
 from ape.utils import cached_property
 from web3 import HTTPProvider, Web3
+from web3.gas_strategies.rpc import rpc_gas_price_strategy
+from web3.middleware import geth_poa_middleware
 
-TENDERLY_FORK_ID = "TENDERLY_FORK_ID"
-TENDERLY_FORK_SERVICE_URI = "TENDERLY_FORK_SERVICE_URI"
+from .client import Fork, TenderlyClient
 
 
-class TenderlyProvider(Web3Provider, ProviderAPI):
+class TenderlyConfig(PluginConfig):
+    auto_remove_forks: bool = True
+
+
+class TenderlyForkProvider(Web3Provider):
+    @cached_property
+    def _client(self) -> TenderlyClient:
+        return TenderlyClient()
+
+    def _create_fork(self) -> Fork:
+        ecosystem_name = self.network.ecosystem.name
+        network_name = self.network.name.replace("-fork", "")
+        chain_id = self.network.ecosystem.get_network(network_name).chain_id
+
+        logger.debug(f"Creating tenderly fork for '{ecosystem_name}:{network_name}'...")
+        fork = self._client.create_fork(chain_id)
+        logger.success(f"Created tenderly fork '{fork.id}'.")
+        return fork
+
     @cached_property
-    def fork_id(self) -> str:
-        if TENDERLY_FORK_ID in os.environ:
-            return os.environ[TENDERLY_FORK_ID]
-
-        elif TENDERLY_FORK_SERVICE_URI in os.environ:
-            fork_network_name = self.network.name.replace("-fork", "")
-            chain_id = self.network.ecosystem.get_network(fork_network_name).chain_id
-            response = requests.post(
-                os.environ[TENDERLY_FORK_SERVICE_URI],
-                json={"network_id": str(chain_id)},
-            )
-            return response.json()["simulation_fork"]["id"]
+    def fork(self) -> Fork:
+        # NOTE: Always create a new fork, because the fork will get cached here
+        #       per-instance of this class, and "released" when the fork is closed
+        return self._create_fork()
+
+    @property
+    def uri(self) -> str:
+        return f"https://rpc.tenderly.co/fork/{self.fork.id}"
+
+    def connect(self):
+        self._web3 = Web3(HTTPProvider(self.uri))
+        atexit.register(self.disconnect)  # NOTE: Make sure we de-provision forks
+
+    def disconnect(self):
+        if self.config.auto_remove_forks:
+            try:
+                fork_id = self.fork.id
+                logger.debug(f"Removing tenderly fork '{fork_id}'...")
+                self._client.remove_fork(fork_id)
+                logger.success(f"Removed tenderly fork '{fork_id}'.")
+            except Exception as e:
+                logger.error(f"Couldn't remove tenderly fork '{fork_id}': {e}.")
 
         else:
-            raise ConfigError("No valid tenderly fork ID found.")
+            logger.info(f"Not removing tenderly fork '{self.fork.id}.'")
+
+        self._web3 = None
+
+
+class TenderlyGatewayProvider(Web3Provider, UpstreamProvider):
+    """
+    A web3 provider using an HTTP connection to Tenderly's RPC nodes.
+
+    Docs: https://docs.tenderly.co/web3-gateway/web3-gateway
+    """
+
+    @cached_property
+    def _client(self) -> TenderlyClient:
+        return TenderlyClient()
 
     @property
     def uri(self) -> str:
-        return f"https://rpc.tenderly.co/fork/{self.fork_id}"
+        ecosystem_name = self.network.ecosystem.name
+        network_name = self.network.name
+
+        if ecosystem_name == "ethereum":
+            # e.g. Sepolia, Goerli, etc.
+            network_subdomain = network_name
+        elif network_name == "mainnet":
+            # e.g. Polygon mainnet, Optimism, etc.
+            network_subdomain = ecosystem_name
+        else:
+            network_subdomain = f"{ecosystem_name}-{network_name}"
+
+        return self._client.get_gateway_rpc_uri(network_subdomain)
+
+    @property
+    def connection_str(self) -> str:
+        return self.uri
 
     def connect(self):
         self._web3 = Web3(HTTPProvider(self.uri))
 
+        try:
+            # Any chain that *began* as PoA needs the middleware for pre-merge blocks
+            ethereum_goerli = 5
+            optimism = (10, 420)
+            polygon = (137, 80001)
+
+            if self._web3.eth.chain_id in (ethereum_goerli, *optimism, *polygon):
+                self._web3.middleware_onion.inject(geth_poa_middleware, layer=0)
+
+            self._web3.eth.set_gas_price_strategy(rpc_gas_price_strategy)
+        except Exception as err:
+            raise ProviderError(f"Failed to connect to Tenderly Gateway.\n{repr(err)}") from err
+
     def disconnect(self):
         self._web3 = None

setup.py

@@ -10,10 +10,10 @@
         "hypothesis>=6.2.0,<7.0",  # Strategy-based fuzzer
     ],
     "lint": [
-        "black>=23.3.0,<24",  # auto-formatter and linter
+        "black>=23.3.0,<24",  # Auto-formatter and linter
         "mypy>=0.991,<1",  # Static type analyzer
-        "types-setuptools",  # Needed due to mypy typeshed
-        "types-requests",  # Needed due to mypy typeshed
+        "types-setuptools",  # Needed for mypy type shed
+        "types-requests",  # Needed for mypy typeshed
         "flake8>=6.0.0,<7",  # Style linter
         "isort>=5.10.1,<6",  # Import sorting linter
         "mdformat>=0.7.16",  # Auto-formatter for markdown