From 55946cb527041f0814158f961bd969bfa24c12cd Mon Sep 17 00:00:00 2001 From: niv vaknin Date: Thu, 26 Dec 2024 16:23:21 +0200 Subject: [PATCH 1/8] Fix address filter from comments --- Quorum/checks/price_feed.py | 41 +++++++++++++++++++++++++++++++++---- 1 file changed, 37 insertions(+), 4 deletions(-) diff --git a/Quorum/checks/price_feed.py b/Quorum/checks/price_feed.py index 0be7006..078a2a9 100644 --- a/Quorum/checks/price_feed.py +++ b/Quorum/checks/price_feed.py @@ -8,6 +8,31 @@ import Quorum.utils.pretty_printer as pp +def remove_solidity_comments(source_code: str) -> str: + """ + Removes single-line and multi-line comments from Solidity source code. + + Args: + source_code (str): The Solidity source code as a single string. + + Returns: + str: The source code with comments removed. + """ + # Regex pattern to match single-line comments (//...) + single_line_comment_pattern = r'//.*?$' + + # Regex pattern to match multi-line comments (/*...*/) + multi_line_comment_pattern = r'/\*.*?\*/' + + # First, remove multi-line comments + source_code = re.sub(multi_line_comment_pattern, '', source_code, flags=re.DOTALL) + + # Then, remove single-line comments + source_code = re.sub(single_line_comment_pattern, '', source_code, flags=re.MULTILINE) + + return source_code + + class PriceFeedCheck(Check): """ The PriceFeedCheck class is responsible for verifying the price feed addresses in the source code @@ -37,12 +62,13 @@ def __init__( self.address_pattern = r'0x[a-fA-F0-9]{40}' self.providers = providers - def __check_price_feed_address(self, address: str) -> dict | None: + def __check_price_feed_address(self, address: str, file_name: str) -> dict | None: """ Check if the given address is present in the price feed providers. Args: address (str): The address to be checked. + file_name (str): The name of the source code file where the address was found. Returns: dict | None: The price feed data if the address is found, otherwise None. @@ -57,7 +83,7 @@ def __check_price_feed_address(self, address: str) -> dict | None: return price_feed.model_dump() pp.pretty_print( - f"Address {address} not found in any price feed provider: {[p.get_name() for p in self.providers]}", + f"Address {address} from {file_name} not found in any price feed provider: {[p.get_name() for p in self.providers]}", pp.Colors.INFO ) return None @@ -75,10 +101,17 @@ def verify_price_feed(self) -> None: verified_sources_path = f"{Path(source_code.file_name).stem.removesuffix('.sol')}/verified_sources.json" verified_variables = [] + # Combine all lines into a single string contract_text = '\n'.join(source_code.file_content) - addresses = set(re.findall(self.address_pattern, contract_text)) + + # Remove comments from the source code + clean_text = remove_solidity_comments(contract_text) + + # Extract unique addresses using regex + addresses = set(re.findall(self.address_pattern, clean_text)) + for address in addresses: - if feed := self.__check_price_feed_address(address): + if feed := self.__check_price_feed_address(address.lower(), source_code.file_name): verified_variables.append(feed) if verified_variables: From 849ac5dcec2b40146609c690c4483e33801bf464 Mon Sep 17 00:00:00 2001 From: niv vaknin Date: Thu, 26 Dec 2024 17:29:14 +0200 Subject: [PATCH 2/8] Fix tests --- Quorum/checks/price_feed.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Quorum/checks/price_feed.py b/Quorum/checks/price_feed.py index 078a2a9..a672682 100644 --- a/Quorum/checks/price_feed.py +++ b/Quorum/checks/price_feed.py @@ -111,7 +111,7 @@ def verify_price_feed(self) -> None: addresses = set(re.findall(self.address_pattern, clean_text)) for address in addresses: - if feed := self.__check_price_feed_address(address.lower(), source_code.file_name): + if feed := self.__check_price_feed_address(address, source_code.file_name): verified_variables.append(feed) if verified_variables: From 9f3092c69d26071df08fdbd6c0074339377df4fb Mon Sep 17 00:00:00 2001 From: niv vaknin Date: Sun, 29 Dec 2024 10:50:05 +0200 Subject: [PATCH 3/8] Add test --- Quorum/tests/checks/__init__.py | 0 Quorum/tests/checks/test_checks.py | 52 ++++++++++++++++++ Quorum/tests/checks/test_price_feed.py | 76 ++++++++++++++++++++++++++ 3 files changed, 128 insertions(+) create mode 100644 Quorum/tests/checks/__init__.py create mode 100644 Quorum/tests/checks/test_checks.py create mode 100644 Quorum/tests/checks/test_price_feed.py diff --git a/Quorum/tests/checks/__init__.py b/Quorum/tests/checks/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/Quorum/tests/checks/test_checks.py b/Quorum/tests/checks/test_checks.py new file mode 100644 index 0000000..55ac18c --- /dev/null +++ b/Quorum/tests/checks/test_checks.py @@ -0,0 +1,52 @@ +import pytest + +import Quorum.tests.conftest as conftest + +from Quorum.apis.block_explorers.source_code import SourceCode +import Quorum.checks as Checks +from Quorum.utils.chain_enum import Chain +from Quorum.apis.price_feeds import ChainLinkAPI + +from pathlib import Path + + +@pytest.mark.parametrize('source_codes', ['ETH/0xAD6c03BF78A3Ee799b86De5aCE32Bb116eD24637'], indirect=True) +def test_diff(source_codes: list[SourceCode], tmp_output_path: Path): + diff_check = Checks.DiffCheck('Aave', Chain.ETH, '', source_codes) + diff_check.target_repo = conftest.RESOURCES_DIR / 'clones/Aave/modules' + + missing_files = diff_check.find_diffs() + + assert len(missing_files) == 1 + assert (missing_files[0].file_name == + 'src/20240711_Multi_ReserveFactorUpdatesMidJuly/AaveV2Ethereum_ReserveFactorUpdatesMidJuly_20240711.sol') + + diffs = [p.stem for p in diff_check.check_folder.rglob('*.patch')] + assert sorted(diffs) == sorted(['AggregatorInterface', 'AaveV2Ethereum', 'AaveV2']) + + +@pytest.mark.parametrize('source_codes', ['bad_global_variables'], indirect=True) +def test_global_variables(source_codes: list[SourceCode], tmp_output_path: Path): + global_variables_check = Checks.GlobalVariableCheck('Aave', Chain.ETH, '', source_codes) + global_variables_check.check_global_variables() + + bad_files = [p.stem for p in global_variables_check.check_folder.iterdir()] + assert len(bad_files) == 2 + assert sorted(bad_files) == sorted(['AaveV2Ethereum', 'AaveV2Ethereum_ReserveFactorUpdatesMidJuly_20240711']) + + +@pytest.mark.parametrize('source_codes', ['ETH/0xAD6c03BF78A3Ee799b86De5aCE32Bb116eD24637'], indirect=True) +def test_price_feed(source_codes: list[SourceCode], tmp_output_path: Path): + price_feed_check = Checks.PriceFeedCheck('Aave', Chain.ETH, '', source_codes, [ + ChainLinkAPI()]) + price_feed_check.verify_price_feed() + + assert sorted([p.name for p in price_feed_check.check_folder.iterdir()]) == ['AaveV2Ethereum'] + + +@pytest.mark.parametrize('source_codes', ['ETH/0xAD6c03BF78A3Ee799b86De5aCE32Bb116eD24637'], indirect=True) +def test_new_listing(source_codes: list[SourceCode], tmp_output_path: Path): + new_listing_check = Checks.NewListingCheck('Aave', Chain.ETH, '', source_codes) + new_listing_check.new_listing_check() + + assert next(new_listing_check.check_folder.iterdir(), None) is None diff --git a/Quorum/tests/checks/test_price_feed.py b/Quorum/tests/checks/test_price_feed.py new file mode 100644 index 0000000..adfb520 --- /dev/null +++ b/Quorum/tests/checks/test_price_feed.py @@ -0,0 +1,76 @@ +import pytest + + +from Quorum.apis.block_explorers.source_code import SourceCode +import Quorum.checks as Checks +from Quorum.utils.chain_enum import Chain +from Quorum.apis.price_feeds import ChainLinkAPI + +from pathlib import Path + + +@pytest.mark.parametrize('source_codes', ['ETH/0xAD6c03BF78A3Ee799b86De5aCE32Bb116eD24637'], indirect=True) +def test_price_feed(source_codes: list[SourceCode], tmp_output_path: Path): + price_feed_check = Checks.PriceFeedCheck('Aave', Chain.ETH, '', source_codes, [ + ChainLinkAPI()]) + price_feed_check.verify_price_feed() + + assert sorted([p.name for p in price_feed_check.check_folder.iterdir()]) == ['AaveV2Ethereum'] + +def test_source_code_clean(): + code = """ +// SPDX-License-Identifier: MIT +pragma solidity ^0.8.0; + +import {IProposalGenericExecutor} from 'aave-helpers/interfaces/IProposalGenericExecutor.sol'; +import {AaveV2Ethereum, AaveV2EthereumAssets, ILendingPoolConfigurator} from 'aave-address-book/AaveV2Ethereum.sol'; + +/** + * @title Reserve Factor Updates Mid July + * @author karpatkey_TokenLogic + * - Snapshot: https://snapshot.org/#/aave.eth/proposal/0x9cc7906f04f45cebeaa48a05ed281f49da00d89c4dd988a968272fa179f14d06 + * - Discussion: https://governance.aave.com/t/arfc-increase-bridged-usdc-reserve-factor-across-all-deployments/17787 + */ +contract AaveV2Ethereum_ReserveFactorUpdatesMidJuly_20240711 is IProposalGenericExecutor { + ILendingPoolConfigurator public constant POOL_CONFIGURATOR = + ILendingPoolConfigurator(AaveV2Ethereum.POOL_CONFIGURATOR); + + uint256 public constant DAI_RF = 65_00; + uint256 public constant LINK_RF = 70_00; + uint256 public constant USDC_RF = 65_00; + uint256 public constant USDT_RF = 65_00; + uint256 public constant WBTC_RF = 70_00; + uint256 public constant WETH_RF = 65_00; + + function execute() external { + POOL_CONFIGURATOR.setReserveFactor(AaveV2EthereumAssets.DAI_UNDERLYING, DAI_RF); + POOL_CONFIGURATOR.setReserveFactor(AaveV2EthereumAssets.LINK_UNDERLYING, LINK_RF); + POOL_CONFIGURATOR.setReserveFactor(AaveV2EthereumAssets.USDC_UNDERLYING, USDC_RF); + POOL_CONFIGURATOR.setReserveFactor(AaveV2EthereumAssets.USDT_UNDERLYING, USDT_RF); + POOL_CONFIGURATOR.setReserveFactor(AaveV2EthereumAssets.WBTC_UNDERLYING, WBTC_RF); + POOL_CONFIGURATOR.setReserveFactor(AaveV2EthereumAssets.WETH_UNDERLYING, WETH_RF); + } +} + """ + cleaned = Checks.clean_source_code(code) + expected = """contract AaveV2Ethereum_ReserveFactorUpdatesMidJuly_20240711 is IProposalGenericExecutor { + ILendingPoolConfigurator public constant POOL_CONFIGURATOR = + ILendingPoolConfigurator(AaveV2Ethereum.POOL_CONFIGURATOR); + + uint256 public constant DAI_RF = 65_00; + uint256 public constant LINK_RF = 70_00; + uint256 public constant USDC_RF = 65_00; + uint256 public constant USDT_RF = 65_00; + uint256 public constant WBTC_RF = 70_00; + uint256 public constant WETH_RF = 65_00; + + function execute() external { + POOL_CONFIGURATOR.setReserveFactor(AaveV2EthereumAssets.DAI_UNDERLYING, DAI_RF); + POOL_CONFIGURATOR.setReserveFactor(AaveV2EthereumAssets.LINK_UNDERLYING, LINK_RF); + POOL_CONFIGURATOR.setReserveFactor(AaveV2EthereumAssets.USDC_UNDERLYING, USDC_RF); + POOL_CONFIGURATOR.setReserveFactor(AaveV2EthereumAssets.USDT_UNDERLYING, USDT_RF); + POOL_CONFIGURATOR.setReserveFactor(AaveV2EthereumAssets.WBTC_UNDERLYING, WBTC_RF); + POOL_CONFIGURATOR.setReserveFactor(AaveV2EthereumAssets.WETH_UNDERLYING, WETH_RF); + } +}""" + assert cleaned.replace(" ", "") == expected.replace(" ", "") \ No newline at end of file From 000b3c2de9385ee1cdcfdc73828709dbabd351a0 Mon Sep 17 00:00:00 2001 From: niv vaknin Date: Sun, 29 Dec 2024 10:51:59 +0200 Subject: [PATCH 4/8] Add test --- Quorum/tests/test_checks.py | 51 ------------------------------------- 1 file changed, 51 deletions(-) delete mode 100644 Quorum/tests/test_checks.py diff --git a/Quorum/tests/test_checks.py b/Quorum/tests/test_checks.py deleted file mode 100644 index 9e0943f..0000000 --- a/Quorum/tests/test_checks.py +++ /dev/null @@ -1,51 +0,0 @@ -import pytest - -import Quorum.tests.conftest as conftest - -from Quorum.apis.block_explorers.source_code import SourceCode -import Quorum.checks as Checks -from Quorum.utils.chain_enum import Chain -from Quorum.apis.price_feeds import ChainLinkAPI - -from pathlib import Path - - -@pytest.mark.parametrize('source_codes', ['ETH/0xAD6c03BF78A3Ee799b86De5aCE32Bb116eD24637'], indirect=True) -def test_diff(source_codes: list[SourceCode], tmp_output_path: Path): - diff_check = Checks.DiffCheck('Aave', Chain.ETH, '', source_codes) - diff_check.target_repo = conftest.RESOURCES_DIR / 'clones/Aave/modules' - - missing_files = diff_check.find_diffs() - - assert len(missing_files) == 1 - assert (missing_files[0].file_name == - 'src/20240711_Multi_ReserveFactorUpdatesMidJuly/AaveV2Ethereum_ReserveFactorUpdatesMidJuly_20240711.sol') - - diffs = [p.stem for p in diff_check.check_folder.rglob('*.patch')] - assert sorted(diffs) == sorted(['AggregatorInterface', 'AaveV2Ethereum', 'AaveV2']) - - -@pytest.mark.parametrize('source_codes', ['bad_global_variables'], indirect=True) -def test_global_variables(source_codes: list[SourceCode], tmp_output_path: Path): - global_variables_check = Checks.GlobalVariableCheck('Aave', Chain.ETH, '', source_codes) - global_variables_check.check_global_variables() - - bad_files = [p.stem for p in global_variables_check.check_folder.iterdir()] - assert len(bad_files) == 2 - assert sorted(bad_files) == sorted(['AaveV2Ethereum', 'AaveV2Ethereum_ReserveFactorUpdatesMidJuly_20240711']) - - -@pytest.mark.parametrize('source_codes', ['ETH/0xAD6c03BF78A3Ee799b86De5aCE32Bb116eD24637'], indirect=True) -def test_price_feed(source_codes: list[SourceCode], tmp_output_path: Path): - price_feed_check = Checks.PriceFeedCheck('Aave', Chain.ETH, '', source_codes, [ChainLinkAPI()]) - price_feed_check.verify_price_feed() - - assert sorted([p.name for p in price_feed_check.check_folder.iterdir()]) == ['AaveV2Ethereum'] - - -@pytest.mark.parametrize('source_codes', ['ETH/0xAD6c03BF78A3Ee799b86De5aCE32Bb116eD24637'], indirect=True) -def test_new_listing(source_codes: list[SourceCode], tmp_output_path: Path): - new_listing_check = Checks.NewListingCheck('Aave', Chain.ETH, '', source_codes) - new_listing_check.new_listing_check() - - assert next(new_listing_check.check_folder.iterdir(), None) is None From b9a1d960ada1d014b8427f37c463651d0e645a5c Mon Sep 17 00:00:00 2001 From: niv vaknin Date: Sun, 29 Dec 2024 10:58:05 +0200 Subject: [PATCH 5/8] Fix test --- Quorum/tests/checks/test_price_feed.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Quorum/tests/checks/test_price_feed.py b/Quorum/tests/checks/test_price_feed.py index adfb520..9e98ea4 100644 --- a/Quorum/tests/checks/test_price_feed.py +++ b/Quorum/tests/checks/test_price_feed.py @@ -52,7 +52,7 @@ def test_source_code_clean(): } } """ - cleaned = Checks.clean_source_code(code) + cleaned = Checks.price_feed.remove_solidity_comments(code) expected = """contract AaveV2Ethereum_ReserveFactorUpdatesMidJuly_20240711 is IProposalGenericExecutor { ILendingPoolConfigurator public constant POOL_CONFIGURATOR = ILendingPoolConfigurator(AaveV2Ethereum.POOL_CONFIGURATOR); From 04d87a66b469ad4918198bb675561099a1d947fd Mon Sep 17 00:00:00 2001 From: niv vaknin Date: Sun, 29 Dec 2024 11:09:48 +0200 Subject: [PATCH 6/8] Fix print --- Quorum/apis/block_explorers/chains_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Quorum/apis/block_explorers/chains_api.py b/Quorum/apis/block_explorers/chains_api.py index f5fa40c..eb3f890 100644 --- a/Quorum/apis/block_explorers/chains_api.py +++ b/Quorum/apis/block_explorers/chains_api.py @@ -49,7 +49,7 @@ def __init__(self, chain: Chain) -> None: chain_id = self.CHAIN_ID_MAP[chain] api_key = os.getenv("ETHSCAN_API_KEY") if not api_key: - raise ValueError(f"{chain}SCAN_API_KEY environment variable is not set.") + raise ValueError("ETHSCAN_API_KEY environment variable is not set.") self.base_url = self.BASE_URL.format(chain_id=chain_id, api_key=api_key) From 1af90e116c88e1bdaa6143cf1eaaedb741359dfe Mon Sep 17 00:00:00 2001 From: niv vaknin Date: Sun, 29 Dec 2024 11:17:20 +0200 Subject: [PATCH 7/8] Fix testcase --- Quorum/tests/checks/test_price_feed.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/Quorum/tests/checks/test_price_feed.py b/Quorum/tests/checks/test_price_feed.py index 9e98ea4..e6dc266 100644 --- a/Quorum/tests/checks/test_price_feed.py +++ b/Quorum/tests/checks/test_price_feed.py @@ -53,7 +53,12 @@ def test_source_code_clean(): } """ cleaned = Checks.price_feed.remove_solidity_comments(code) - expected = """contract AaveV2Ethereum_ReserveFactorUpdatesMidJuly_20240711 is IProposalGenericExecutor { + expected = """pragma solidity ^0.8.0; + +import {IProposalGenericExecutor} from 'aave-helpers/interfaces/IProposalGenericExecutor.sol'; +import {AaveV2Ethereum, AaveV2EthereumAssets, ILendingPoolConfigurator} from 'aave-address-book/AaveV2Ethereum.sol'; + +contract AaveV2Ethereum_ReserveFactorUpdatesMidJuly_20240711 is IProposalGenericExecutor { ILendingPoolConfigurator public constant POOL_CONFIGURATOR = ILendingPoolConfigurator(AaveV2Ethereum.POOL_CONFIGURATOR); @@ -73,4 +78,4 @@ def test_source_code_clean(): POOL_CONFIGURATOR.setReserveFactor(AaveV2EthereumAssets.WETH_UNDERLYING, WETH_RF); } }""" - assert cleaned.replace(" ", "") == expected.replace(" ", "") \ No newline at end of file + assert cleaned.strip() == expected.strip() From 1efbbef07ebb9e3e18edbbd0a5fae01b3e0c7aef Mon Sep 17 00:00:00 2001 From: niv vaknin Date: Sun, 29 Dec 2024 12:11:31 +0200 Subject: [PATCH 8/8] Fix tests --- Quorum/tests/checks/__init__.py | 0 Quorum/tests/{checks => }/test_checks.py | 2 +- Quorum/tests/{checks => }/test_price_feed.py | 3 +++ 3 files changed, 4 insertions(+), 1 deletion(-) delete mode 100644 Quorum/tests/checks/__init__.py rename Quorum/tests/{checks => }/test_checks.py (96%) rename Quorum/tests/{checks => }/test_price_feed.py (98%) diff --git a/Quorum/tests/checks/__init__.py b/Quorum/tests/checks/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/Quorum/tests/checks/test_checks.py b/Quorum/tests/test_checks.py similarity index 96% rename from Quorum/tests/checks/test_checks.py rename to Quorum/tests/test_checks.py index 55ac18c..3db3976 100644 --- a/Quorum/tests/checks/test_checks.py +++ b/Quorum/tests/test_checks.py @@ -36,7 +36,7 @@ def test_global_variables(source_codes: list[SourceCode], tmp_output_path: Path) @pytest.mark.parametrize('source_codes', ['ETH/0xAD6c03BF78A3Ee799b86De5aCE32Bb116eD24637'], indirect=True) -def test_price_feed(source_codes: list[SourceCode], tmp_output_path: Path): +def test_price_feed_check(source_codes: list[SourceCode], tmp_output_path: Path): price_feed_check = Checks.PriceFeedCheck('Aave', Chain.ETH, '', source_codes, [ ChainLinkAPI()]) price_feed_check.verify_price_feed() diff --git a/Quorum/tests/checks/test_price_feed.py b/Quorum/tests/test_price_feed.py similarity index 98% rename from Quorum/tests/checks/test_price_feed.py rename to Quorum/tests/test_price_feed.py index e6dc266..7f2fe70 100644 --- a/Quorum/tests/checks/test_price_feed.py +++ b/Quorum/tests/test_price_feed.py @@ -17,6 +17,7 @@ def test_price_feed(source_codes: list[SourceCode], tmp_output_path: Path): assert sorted([p.name for p in price_feed_check.check_folder.iterdir()]) == ['AaveV2Ethereum'] + def test_source_code_clean(): code = """ // SPDX-License-Identifier: MIT @@ -58,6 +59,7 @@ def test_source_code_clean(): import {IProposalGenericExecutor} from 'aave-helpers/interfaces/IProposalGenericExecutor.sol'; import {AaveV2Ethereum, AaveV2EthereumAssets, ILendingPoolConfigurator} from 'aave-address-book/AaveV2Ethereum.sol'; + contract AaveV2Ethereum_ReserveFactorUpdatesMidJuly_20240711 is IProposalGenericExecutor { ILendingPoolConfigurator public constant POOL_CONFIGURATOR = ILendingPoolConfigurator(AaveV2Ethereum.POOL_CONFIGURATOR); @@ -78,4 +80,5 @@ def test_source_code_clean(): POOL_CONFIGURATOR.setReserveFactor(AaveV2EthereumAssets.WETH_UNDERLYING, WETH_RF); } }""" + # Strip leading/trailing whitespace for accurate comparison assert cleaned.strip() == expected.strip()