From 410fdb54fd36b4eb77989d79ad168b68a38a1570 Mon Sep 17 00:00:00 2001 From: Mike Shultz Date: Mon, 19 Feb 2024 09:59:36 -0700 Subject: [PATCH] feat: adds compiler_cache_folder config var (#1897) * feat: adds compiler_cache_folder config var * docs: document compiler_cache_folder config var * feat: adds compiler_cache_folder to project manager * test: tests compiler_cache_folder * test: resolve in fixt * refactor: move compiler_cache_folder to ape-compile plugin config * test(fix): config structure change * docs: improved inline commentary * feat: adds compile config prop base_dir for compilation base directory, tighten up cache_folder relative path handling, and prevent escape of project root * fix: resolve() path when checking for escape so symlinks don't bust things up * fix: compare resolved paths, do not mix. instrument error a bit better. * docs: update for config structure changes. add warning for cache_folder config * docs: how to spell Co-authored-by: antazoey * fix: unnecessary f in chat Co-authored-by: antazoey * refactor: base_dir -> base_path --------- Co-authored-by: Juliya Smith Co-authored-by: antazoey --- docs/userguides/compile.md | 18 ++++++++- src/ape/api/config.py | 13 ++++++- src/ape/managers/compilers.py | 4 +- src/ape/managers/config.py | 10 ++++- src/ape/managers/project/manager.py | 19 +++++++--- src/ape_compile/__init__.py | 57 ++++++++++++++++++++++++++++- tests/conftest.py | 2 +- tests/functional/test_config.py | 9 +++++ 8 files changed, 117 insertions(+), 15 deletions(-) diff --git a/docs/userguides/compile.md b/docs/userguides/compile.md index 6524b17771..83122e013d 100644 --- a/docs/userguides/compile.md +++ b/docs/userguides/compile.md @@ -85,7 +85,23 @@ compile: ## Settings Generally, configure compiler plugins using your `ape-config.yaml` file. -For example, when using the `vyper` plugin, you can configure settings under the `vyper` key: +One setting that applies to many compiler plugins is `cache_folder`, which holds dependency source files the compiler uses when compiling your contracts. +By default, the folder is in your `contracts/.cache` folder but there are times you may want to move this to another location. +Paths are relative to the project directory. +For instance, to move the dependency cahce to the root project directory: + +```yaml +compile: + cache_folder: .cache +``` + +```{caution} +Changing the location of the dependency cache folder may alter the the output bytecode of your contracts from some compilers. +Specifically, the [solc compiler will apend a hash of the input metadata to the contract bytecode](https://docs.soliditylang.org/en/latest/metadata.html#encoding-of-the-metadata-hash-in-the-bytecode) which will change with contract path or compiler settings changes. +This may impact things like contract verification on existing projects. +``` + +As another example, when using the `vyper` plugin, you can configure settings under the `vyper` key: ```yaml vyper: diff --git a/src/ape/api/config.py b/src/ape/api/config.py index 80e5d3c86c..39ac1887f8 100644 --- a/src/ape/api/config.py +++ b/src/ape/api/config.py @@ -1,11 +1,14 @@ from enum import Enum -from typing import Any, Dict, Optional, TypeVar +from typing import TYPE_CHECKING, Any, Dict, Optional, TypeVar from pydantic import ConfigDict from pydantic_settings import BaseSettings from ape.utils.basemodel import _assert_not_ipython_check +if TYPE_CHECKING: + from ape.managers.config import ConfigManager + ConfigItemType = TypeVar("ConfigItemType") @@ -35,8 +38,14 @@ class PluginConfig(BaseSettings): a config API must register a subclass of this class. """ + # NOTE: This is probably partially initialized at the time of assignment + _config_manager: Optional["ConfigManager"] + @classmethod - def from_overrides(cls, overrides: Dict) -> "PluginConfig": + def from_overrides( + cls, overrides: Dict, config_manager: Optional["ConfigManager"] = None + ) -> "PluginConfig": + cls._config_manager = config_manager default_values = cls().model_dump() def update(root: Dict, value_map: Dict): diff --git a/src/ape/managers/compilers.py b/src/ape/managers/compilers.py index 20a7cd2581..3994cfa4c4 100644 --- a/src/ape/managers/compilers.py +++ b/src/ape/managers/compilers.py @@ -164,7 +164,9 @@ def compile( # For mypy - should not be possible. raise ValueError("Compiler should not be None") - compiled_contracts = compiler.compile(paths_to_compile, base_path=contracts_folder) + compiled_contracts = compiler.compile( + paths_to_compile, base_path=self.config_manager.get_config("compile").base_path + ) # Validate some things about the compile contracts. for contract_type in compiled_contracts: diff --git a/src/ape/managers/config.py b/src/ape/managers/config.py index 1fd0c04c20..21b28ffe24 100644 --- a/src/ape/managers/config.py +++ b/src/ape/managers/config.py @@ -200,6 +200,7 @@ def _plugin_configs(self) -> Dict[str, PluginConfig]: ) self.contracts_folder = configs["contracts_folder"] = contracts_folder + deployments = user_config.pop("deployments", {}) valid_ecosystems = dict(self.plugin_manager.ecosystems) valid_network_names = [n[1] for n in [e[1] for e in self.plugin_manager.networks]] @@ -221,8 +222,13 @@ def _plugin_configs(self) -> Dict[str, PluginConfig]: ethereum_config_cls = config_class if config_class != ConfigDict: - # NOTE: Will raise if improperly provided keys - config = config_class.from_overrides(user_override) # type: ignore + config = config_class.from_overrides( # type: ignore + # NOTE: Will raise if improperly provided keys + user_override, + # NOTE: Sending ourselves in case the PluginConfig needs access to the root + # config vars. + config_manager=self, + ) else: # NOTE: Just use it directly as a dict if `ConfigDict` is passed config = user_override diff --git a/src/ape/managers/project/manager.py b/src/ape/managers/project/manager.py index aa4f854189..1184820652 100644 --- a/src/ape/managers/project/manager.py +++ b/src/ape/managers/project/manager.py @@ -105,6 +105,16 @@ def contracts_folder(self) -> Path: return folder + @property + def compiler_cache_folder(self) -> Path: + """ + The path to the project's compiler source cache folder. + """ + # NOTE: as long as config has come up properly, this should not be None + compile_conf = self.config_manager.get_config("compile") + assert compile_conf.cache_folder is not None + return compile_conf.cache_folder + @property def source_paths(self) -> List[Path]: """ @@ -122,14 +132,12 @@ def source_paths(self) -> List[Path]: # Dependency sources should be ignored, as they are pulled in # independently to compiler via import. - in_contracts_cache_dir = contracts_folder / ".cache" - for extension in self.compiler_manager.registered_compilers: files.extend( ( x for x in contracts_folder.rglob(f"*{extension}") - if x.is_file() and in_contracts_cache_dir not in x.parents + if x.is_file() and self.compiler_cache_folder not in x.parents ) ) @@ -714,9 +722,8 @@ def load_contracts( types for each compiled contract. """ - in_source_cache = self.contracts_folder / ".cache" - if not use_cache and in_source_cache.is_dir(): - shutil.rmtree(str(in_source_cache)) + if not use_cache and self.compiler_cache_folder.is_dir(): + shutil.rmtree(str(self.compiler_cache_folder)) if isinstance(file_paths, Path): file_path_list = [file_paths] diff --git a/src/ape_compile/__init__.py b/src/ape_compile/__init__.py index c363651d4b..ed9c64653f 100644 --- a/src/ape_compile/__init__.py +++ b/src/ape_compile/__init__.py @@ -1,10 +1,13 @@ -from typing import List +from pathlib import Path +from typing import List, Optional -from pydantic import field_validator +from pydantic import field_validator, model_validator from ape import plugins from ape.api import PluginConfig +DEFAULT_CACHE_FOLDER_NAME = ".cache" # default relative to contracts/ + class Config(PluginConfig): include_dependencies: bool = False @@ -23,6 +26,56 @@ class Config(PluginConfig): Source exclusion globs across all file types. """ + cache_folder: Optional[Path] = None + """ + Path to contract dependency cache directory (e.g. `contracts/.cache`) + """ + + @property + def base_path(self) -> Path: + """The base directory for compilation file path references""" + + # These should be initialized by plugin config loading and pydantic validators before the + # time this prop is accessed. + assert self._config_manager is not None + assert self.cache_folder is not None + + # If the dependency cache folder is configured, to be outside of the contracts dir, we want + # to use the projects folder to be the base dir for copmilation. + if self._config_manager.contracts_folder not in self.cache_folder.parents: + return self._config_manager.PROJECT_FOLDER + + # Otherwise, we're defaulting to contracts folder for backwards compatibility. Changing this + # will cause existing projects to compile to different bytecode. + return self._config_manager.contracts_folder + + @model_validator(mode="after") + def validate_cache_folder(self): + if self._config_manager is None: + return # Not enough information to continue at this time + + contracts_folder = self._config_manager.contracts_folder + project_folder = self._config_manager.PROJECT_FOLDER + + # Set unconfigured default + if self.cache_folder is None: + self.cache_folder = contracts_folder / DEFAULT_CACHE_FOLDER_NAME + + # If we get a relative path, assume it's relative to project root (where the config file + # lives) + elif not self.cache_folder.is_absolute(): + self.cache_folder = project_folder / self.cache_folder + + # Do not allow escape of the project folder for security and functionality reasons. Paths + # outside the relative compilation root are not portable and will cause bytecode changes. + project_resolved = project_folder.resolve() + cache_resolved = self.cache_folder.resolve() + if project_resolved not in cache_resolved.parents: + raise ValueError( + "cache_folder must be a child of the project directory. " + f"{project_resolved} not in {cache_resolved}" + ) + @field_validator("exclude", mode="before") @classmethod def validate_exclude(cls, value): diff --git a/tests/conftest.py b/tests/conftest.py index c6dfb9129a..8c2dc735cc 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -298,7 +298,7 @@ def temp_config(config): def func(data: Optional[Dict] = None): data = data or {} with tempfile.TemporaryDirectory() as temp_dir_str: - temp_dir = Path(temp_dir_str) + temp_dir = Path(temp_dir_str).resolve() config._cached_configs = {} config_file = temp_dir / CONFIG_FILE_NAME config_file.touch() diff --git a/tests/functional/test_config.py b/tests/functional/test_config.py index 09fd4daac4..e5227d09b1 100644 --- a/tests/functional/test_config.py +++ b/tests/functional/test_config.py @@ -324,6 +324,15 @@ def test_contracts_folder_with_hyphen(temp_config): assert project.contracts_folder.name == "src" +def test_compiler_cache_folder(temp_config): + with temp_config( + {"contracts_folder": "smarts", "compile": {"cache_folder": ".cash"}} + ) as project: + assert project.contracts_folder.name == "smarts" + assert project.compiler_cache_folder.name == ".cash" + assert str(project.contracts_folder) not in str(project.compiler_cache_folder) + + def test_custom_network(): chain_id = 11191919191991918223773 data = {