From 0248a95ee22d97155c1c100ed4bb2ec39e1e2abc Mon Sep 17 00:00:00 2001 From: Mike Shultz Date: Wed, 31 Jan 2024 15:42:37 -0700 Subject: [PATCH 1/8] feat: use compiler cache dir from ape-compile config --- ape_solidity/_utils.py | 6 +++--- ape_solidity/compiler.py | 11 +++++------ 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/ape_solidity/_utils.py b/ape_solidity/_utils.py index ba2e5d5..7a7237f 100644 --- a/ape_solidity/_utils.py +++ b/ape_solidity/_utils.py @@ -95,15 +95,15 @@ def package_id(self) -> Path: class ImportRemappingBuilder: def __init__(self, contracts_cache: Path): # import_map maps import keys like `@openzeppelin/contracts` - # to str paths in the contracts' .cache folder. + # to str paths in the compiler cache folder. self.import_map: Dict[str, str] = {} self.dependencies_added: Set[Path] = set() self.contracts_cache = contracts_cache def add_entry(self, remapping: ImportRemapping): path = remapping.package_id - if not str(path).startswith(f".cache{os.path.sep}"): - path = Path(".cache") / path + if str(self.contracts_cache) not in str(path): + path = self.contracts_cache / path self.import_map[remapping.key] = str(path) diff --git a/ape_solidity/compiler.py b/ape_solidity/compiler.py index df50f45..9922008 100644 --- a/ape_solidity/compiler.py +++ b/ape_solidity/compiler.py @@ -221,9 +221,8 @@ def get_import_remapping(self, base_path: Optional[Path] = None) -> Dict[str, st raise IncorrectMappingFormatError() # We use these helpers to transform the values configured - # to values matching files in the `contracts/.cache` folder. - contracts_cache = base_path / ".cache" - builder = ImportRemappingBuilder(contracts_cache) + # to values matching files in the compiler cache folder. + builder = ImportRemappingBuilder(self.project_manager.compiler_cache_folder) packages_cache = self.config_manager.packages_folder # Here we hash and validate if there were changes to remappings. @@ -233,7 +232,7 @@ def get_import_remapping(self, base_path: Optional[Path] = None) -> Dict[str, st if ( self._import_remapping_hash and self._import_remapping_hash == hash(remappings_tuple) - and contracts_cache.is_dir() + and self.project_manager.compiler_cache_folder.is_dir() ): return self._cached_import_map @@ -265,7 +264,7 @@ def get_import_remapping(self, base_path: Optional[Path] = None) -> Dict[str, st data_folder_cache = packages_cache / package_id # Re-build a downloaded dependency manifest into the .cache directory for imports. - sub_contracts_cache = contracts_cache / package_id + sub_contracts_cache = self.project_manager.compiler_cache_folder / package_id if not sub_contracts_cache.is_dir() or not list(sub_contracts_cache.iterdir()): cached_manifest_file = data_folder_cache / f"{remapping_obj.name}.json" if not cached_manifest_file.is_file(): @@ -435,7 +434,7 @@ def _get_used_remappings( x for sources in self.get_imports(list(sources), base_path=base_path).values() for x in sources - if x.startswith(".cache") + if str(self.project_manager.compiler_cache_folder) in x ) for parent_key in ( os.path.sep.join(source.split(os.path.sep)[:3]) for source in [source] From cdaf8412f452e9aa59cfb72a2cf05d084b55d194 Mon Sep 17 00:00:00 2001 From: Mike Shultz Date: Tue, 6 Feb 2024 13:58:30 -0700 Subject: [PATCH 2/8] refactor: idk my bff Jill --- ape_solidity/compiler.py | 35 ++++++++++++++++++++++++++++------- 1 file changed, 28 insertions(+), 7 deletions(-) diff --git a/ape_solidity/compiler.py b/ape_solidity/compiler.py index 9922008..e540c45 100644 --- a/ape_solidity/compiler.py +++ b/ape_solidity/compiler.py @@ -391,10 +391,7 @@ def get_compiler_settings( for solc_version, sources in files_by_solc_version.items(): version_settings: Dict[str, Union[Any, List[Any]]] = { "optimizer": {"enabled": self.settings.optimize, "runs": DEFAULT_OPTIMIZATION_RUNS}, - "outputSelection": { - str(get_relative_path(p, base_path)): {"*": OUTPUT_SELECTION, "": ["ast"]} - for p in sources - }, + "outputSelection": {str(p): {"*": OUTPUT_SELECTION, "": ["ast"]} for p in sources}, } if remappings_used := self._get_used_remappings( sources, remappings=import_remappings, base_path=base_path @@ -432,8 +429,8 @@ def _get_used_remappings( k: v for source in ( x - for sources in self.get_imports(list(sources), base_path=base_path).values() - for x in sources + for sourceset in self.get_imports(list(sources), base_path=base_path).values() + for x in sourceset if str(self.project_manager.compiler_cache_folder) in x ) for parent_key in ( @@ -470,6 +467,7 @@ def get_standard_input_json( x: {"content": (base_path / x).read_text()} for x in vers_settings["outputSelection"] } + input_jsons[solc_version] = { "sources": sources, "settings": vers_settings, @@ -495,6 +493,11 @@ def compile( if solc_version >= Version("0.6.9"): arguments["base_path"] = base_path + if self.project_manager.compiler_cache_folder.is_dir(): + # TODO: equivalent of --include-path? + # arguments["include_path"] = self.project_manager.compiler_cache_folder + arguments["allow_paths"] = self.project_manager.compiler_cache_folder + # Allow empty contracts, like Vyper does. arguments["allow_empty"] = True @@ -835,7 +838,9 @@ def _get_imported_source_paths( return set() source_ids_checked.append(source_identifier) - import_file_paths = [base_path / i for i in imports.get(source_identifier, []) if i] + import_file_paths = [ + (base_path / i).resolve() for i in imports.get(source_identifier, []) if i + ] return_set = {i for i in import_file_paths} for import_path in import_file_paths: indirect_imports = self._get_imported_source_paths( @@ -1108,3 +1113,19 @@ def _import_str_to_source_id( def _try_max(ls: List[Any]): return max(ls) if ls else None + + +def _get_best_relative_path(path: Path, base_paths: List[Path]) -> Path: + """Return the first matching relative path to the given base paths""" + relatives = [] + + for base in base_paths: + try: + relatives.append(path.relative_to(base)) + except ValueError: + continue + + if not relatives: + raise ValueError(f"{path} is not a subpath of any given base paths") + + return min(relatives, key=lambda x: len(x.parts)) From 64801ceea634e38586b3b99325850bad0934b353 Mon Sep 17 00:00:00 2001 From: Mike Shultz Date: Thu, 8 Feb 2024 02:48:26 -0700 Subject: [PATCH 3/8] fix: all path references should be relative to base_dir --- ape_solidity/compiler.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/ape_solidity/compiler.py b/ape_solidity/compiler.py index e540c45..d0b3052 100644 --- a/ape_solidity/compiler.py +++ b/ape_solidity/compiler.py @@ -222,7 +222,9 @@ def get_import_remapping(self, base_path: Optional[Path] = None) -> Dict[str, st # We use these helpers to transform the values configured # to values matching files in the compiler cache folder. - builder = ImportRemappingBuilder(self.project_manager.compiler_cache_folder) + builder = ImportRemappingBuilder( + get_relative_path(self.project_manager.compiler_cache_folder, base_path) + ) packages_cache = self.config_manager.packages_folder # Here we hash and validate if there were changes to remappings. @@ -391,7 +393,10 @@ def get_compiler_settings( for solc_version, sources in files_by_solc_version.items(): version_settings: Dict[str, Union[Any, List[Any]]] = { "optimizer": {"enabled": self.settings.optimize, "runs": DEFAULT_OPTIMIZATION_RUNS}, - "outputSelection": {str(p): {"*": OUTPUT_SELECTION, "": ["ast"]} for p in sources}, + "outputSelection": { + str(get_relative_path(p, base_path)): {"*": OUTPUT_SELECTION, "": ["ast"]} + for p in sources + }, } if remappings_used := self._get_used_remappings( sources, remappings=import_remappings, base_path=base_path @@ -424,6 +429,8 @@ def _get_used_remappings( # No remappings used at all. return {} + relative_cache = get_relative_path(self.project_manager.compiler_cache_folder, base_path) + # Filter out unused import remapping. return { k: v @@ -431,7 +438,7 @@ def _get_used_remappings( x for sourceset in self.get_imports(list(sources), base_path=base_path).values() for x in sourceset - if str(self.project_manager.compiler_cache_folder) in x + if str(relative_cache) in x ) for parent_key in ( os.path.sep.join(source.split(os.path.sep)[:3]) for source in [source] @@ -838,9 +845,7 @@ def _get_imported_source_paths( return set() source_ids_checked.append(source_identifier) - import_file_paths = [ - (base_path / i).resolve() for i in imports.get(source_identifier, []) if i - ] + import_file_paths = [base_path / i for i in imports.get(source_identifier, []) if i] return_set = {i for i in import_file_paths} for import_path in import_file_paths: indirect_imports = self._get_imported_source_paths( From f587be84275dfb2b76051d8221a5ca2b0b02b49f Mon Sep 17 00:00:00 2001 From: Mike Shultz Date: Thu, 8 Feb 2024 02:50:22 -0700 Subject: [PATCH 4/8] style: cleanup unused code and commentary --- ape_solidity/compiler.py | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/ape_solidity/compiler.py b/ape_solidity/compiler.py index d0b3052..65965c5 100644 --- a/ape_solidity/compiler.py +++ b/ape_solidity/compiler.py @@ -501,8 +501,6 @@ def compile( arguments["base_path"] = base_path if self.project_manager.compiler_cache_folder.is_dir(): - # TODO: equivalent of --include-path? - # arguments["include_path"] = self.project_manager.compiler_cache_folder arguments["allow_paths"] = self.project_manager.compiler_cache_folder # Allow empty contracts, like Vyper does. @@ -1118,19 +1116,3 @@ def _import_str_to_source_id( def _try_max(ls: List[Any]): return max(ls) if ls else None - - -def _get_best_relative_path(path: Path, base_paths: List[Path]) -> Path: - """Return the first matching relative path to the given base paths""" - relatives = [] - - for base in base_paths: - try: - relatives.append(path.relative_to(base)) - except ValueError: - continue - - if not relatives: - raise ValueError(f"{path} is not a subpath of any given base paths") - - return min(relatives, key=lambda x: len(x.parts)) From 97fe4196fd2daacc87f18390598d1793ec5555e4 Mon Sep 17 00:00:00 2001 From: Mike Shultz Date: Wed, 21 Feb 2024 23:33:49 -0700 Subject: [PATCH 5/8] chore: bump ape dep to >=0.7.10 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index db349d6..36247c6 100644 --- a/setup.py +++ b/setup.py @@ -69,7 +69,7 @@ include_package_data=True, install_requires=[ "py-solc-x>=2.0.2,<3", - "eth-ape>=0.7.0,<0.8", + "eth-ape>=0.7.10,<0.8", "ethpm-types", # Use the version ape requires "eth-pydantic-types", # Use the version ape requires "packaging", # Use the version ape requires From 62ccb0f6cb93b026add53c643e15a4e36a92a3ca Mon Sep 17 00:00:00 2001 From: Mike Shultz Date: Fri, 23 Feb 2024 18:13:55 -0700 Subject: [PATCH 6/8] test: adds test_get_imports_cache_folder and fixes test_get_imports --- ape_solidity/_utils.py | 3 +-- tests/test_compiler.py | 39 ++++++++++++++++++++++++++++++++++++++- 2 files changed, 39 insertions(+), 3 deletions(-) diff --git a/ape_solidity/_utils.py b/ape_solidity/_utils.py index 7a7237f..e34739a 100644 --- a/ape_solidity/_utils.py +++ b/ape_solidity/_utils.py @@ -102,7 +102,7 @@ def __init__(self, contracts_cache: Path): def add_entry(self, remapping: ImportRemapping): path = remapping.package_id - if str(self.contracts_cache) not in str(path): + if self.contracts_cache not in path.parents: path = self.contracts_cache / path self.import_map[remapping.key] = str(path) @@ -110,7 +110,6 @@ def add_entry(self, remapping: ImportRemapping): def get_import_lines(source_paths: Set[Path]) -> Dict[Path, List[str]]: imports_dict: Dict[Path, List[str]] = {} - for filepath in source_paths: import_set = set() if not filepath.is_file(): diff --git a/tests/test_compiler.py b/tests/test_compiler.py index f57c5ff..ff5e914 100644 --- a/tests/test_compiler.py +++ b/tests/test_compiler.py @@ -3,6 +3,7 @@ import shutil from pathlib import Path +import ape import pytest import solcx from ape import reverts @@ -141,7 +142,12 @@ def test_compile_just_a_struct(compiler, project): def test_get_imports(project, compiler): - import_dict = compiler.get_imports(TEST_CONTRACT_PATHS, BASE_PATH) + test_contract_paths = [ + p + for p in project.contracts_folder.iterdir() + if ".cache" not in str(p) and not p.is_dir() and p.suffix == Extension.SOL.value + ] + import_dict = compiler.get_imports(test_contract_paths, project.contracts_folder) contract_imports = import_dict["Imports.sol"] # NOTE: make sure there aren't duplicates assert len([x for x in contract_imports if contract_imports.count(x) > 1]) == 0 @@ -161,6 +167,37 @@ def test_get_imports(project, compiler): assert set(contract_imports) == expected +def test_get_imports_cache_folder(project, compiler): + """Test imports when cache folder is configured""" + config = project.config_manager.get_config("compile") + config.cache_folder = project.path / ".cash" + # assert False + test_contract_paths = [ + p + for p in project.contracts_folder.iterdir() + if ".cache" not in str(p) and not p.is_dir() and p.suffix == Extension.SOL.value + ] + # Using a different base path here because the cache folder is in the project root + import_dict = compiler.get_imports(test_contract_paths, project.path) + contract_imports = import_dict["contracts/Imports.sol"] + # NOTE: make sure there aren't duplicates + assert len([x for x in contract_imports if contract_imports.count(x) > 1]) == 0 + # NOTE: returning a list + assert isinstance(contract_imports, list) + # NOTE: in case order changes + expected = { + ".cash/BrownieDependency/local/BrownieContract.sol", + ".cash/BrownieStyleDependency/local/BrownieStyleDependency.sol", + ".cash/TestDependency/local/Dependency.sol", + ".cash/gnosis/v1.3.0/common/Enum.sol", + "contracts/CompilesOnce.sol", + "contracts/MissingPragma.sol", + "contracts/NumerousDefinitions.sol", + "contracts/subfolder/Relativecontract.sol", + } + assert set(contract_imports) == expected + + def test_get_imports_raises_when_non_solidity_files(compiler, vyper_source_path): with raises_because_not_sol: compiler.get_imports([vyper_source_path]) From 202f73250214a9ff80e20d8d21a8fb80ca3aebef Mon Sep 17 00:00:00 2001 From: Mike Shultz Date: Fri, 23 Feb 2024 18:20:26 -0700 Subject: [PATCH 7/8] style: remove unused import --- tests/test_compiler.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_compiler.py b/tests/test_compiler.py index ff5e914..2288818 100644 --- a/tests/test_compiler.py +++ b/tests/test_compiler.py @@ -3,7 +3,6 @@ import shutil from pathlib import Path -import ape import pytest import solcx from ape import reverts From 51ccf9bc290af9973894fa602ace287c89666385 Mon Sep 17 00:00:00 2001 From: Mike Shultz Date: Fri, 23 Feb 2024 23:55:26 -0700 Subject: [PATCH 8/8] test: fix cache config setting --- tests/test_compiler.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tests/test_compiler.py b/tests/test_compiler.py index 2288818..2dc14a3 100644 --- a/tests/test_compiler.py +++ b/tests/test_compiler.py @@ -168,8 +168,9 @@ def test_get_imports(project, compiler): def test_get_imports_cache_folder(project, compiler): """Test imports when cache folder is configured""" - config = project.config_manager.get_config("compile") - config.cache_folder = project.path / ".cash" + compile_config = project.config_manager.get_config("compile") + og_cache_colder = compile_config.cache_folder + compile_config.cache_folder = project.path / ".cash" # assert False test_contract_paths = [ p @@ -196,6 +197,10 @@ def test_get_imports_cache_folder(project, compiler): } assert set(contract_imports) == expected + # Reset because this config is stateful across tests + compile_config.cache_folder = og_cache_colder + shutil.rmtree(og_cache_colder) + def test_get_imports_raises_when_non_solidity_files(compiler, vyper_source_path): with raises_because_not_sol: