diff --git a/docs/installing-vyper.rst b/docs/installing-vyper.rst index 8eaa93590a..0c7d54903f 100644 --- a/docs/installing-vyper.rst +++ b/docs/installing-vyper.rst @@ -7,37 +7,37 @@ any errors. .. note:: - The easiest way to experiment with the language is to use the `Remix online compiler `_. - (Activate the vyper-remix plugin in the Plugin manager.) + The easiest way to experiment with the language is to use either `Try Vyper! `_ (maintained by the Vyper team) or the `Remix online compiler `_ (maintained by the Ethereum Foundation). + - To use Try Vyper, go to https://try.vyperlang.org and log in (requires Github login). + - To use remix, go to https://remix.ethereum.org and activate the vyper-remix plugin in the Plugin manager. -Docker -****** -Vyper can be downloaded as docker image from `dockerhub `_: -:: +Standalone +********** - docker pull vyperlang/vyper +The Vyper CLI can be installed with any ``pip`` compatible tool, for example, ``pipx`` or ``uv tool``. If you do not have ``pipx`` or ``uv`` installed, first, go to the respective tool's installation page: -To run the compiler use the ``docker run`` command: -:: +- https://github.com/pypa/pipx?tab=readme-ov-file +- https://github.com/astral-sh/uv?tab=readme-ov-file#uv - docker run -v $(pwd):/code vyperlang/vyper /code/ +Then, the command to install Vyper would be -Alternatively you can log into the docker image and execute vyper on the prompt. :: - docker run -v $(pwd):/code/ -it --entrypoint /bin/bash vyperlang/vyper - root@d35252d1fb1b:/code# vyper + pipx install vyper + +Or, -The normal parameters are also supported, for example: :: - docker run -v $(pwd):/code vyperlang/vyper -f abi /code/ - [{'name': 'test1', 'outputs': [], 'inputs': [{'type': 'uint256', 'name': 'a'}, {'type': 'bytes', 'name': 'b'}], 'constant': False, 'payable': False, 'type': 'function', 'gas': 441}, {'name': 'test2', 'outputs': [], 'inputs': [{'type': 'uint256', 'name': 'a'}], 'constant': False, 'payable': False, 'type': 'function', 'gas': 316}] + uv tool install vyper -.. note:: - If you would like to know how to install Docker, please follow their `documentation `_. +Binaries +******** + +Alternatively, prebuilt Vyper binaries for Windows, Mac and Linux are available for download from the GitHub releases page: https://github.com/vyperlang/vyper/releases. + PIP *** @@ -45,12 +45,17 @@ PIP Installing Python ================= -Vyper can only be built using Python 3.6 and higher. If you need to know how to install the correct version of python, +Vyper can only be built using Python 3.10 and higher. If you need to know how to install the correct version of python, follow the instructions from the official `Python website `_. Creating a virtual environment ============================== +Because pip installations are not isolated by default, this method of +installation is meant for more experienced Python developers who are using +Vyper as a library, or want to use it within a Python project with other +pip dependencies. + It is **strongly recommended** to install Vyper in **a virtual Python environment**, so that new packages installed and dependencies built are strictly contained in your Vyper project and will not alter or affect your @@ -76,13 +81,43 @@ Each tagged version of vyper is uploaded to `pypi `_: +:: + + docker pull vyperlang/vyper + +To run the compiler use the ``docker run`` command: +:: + + docker run -v $(pwd):/code vyperlang/vyper /code/ + +Alternatively you can log into the docker image and execute vyper on the prompt. +:: + + docker run -v $(pwd):/code/ -it --entrypoint /bin/bash vyperlang/vyper + root@d35252d1fb1b:/code# vyper + +The normal parameters are also supported, for example: +:: + + docker run -v $(pwd):/code vyperlang/vyper -f abi /code/ + [{'name': 'test1', 'outputs': [], 'inputs': [{'type': 'uint256', 'name': 'a'}, {'type': 'bytes', 'name': 'b'}], 'constant': False, 'payable': False, 'type': 'function', 'gas': 441}, {'name': 'test2', 'outputs': [], 'inputs': [{'type': 'uint256', 'name': 'a'}], 'constant': False, 'payable': False, 'type': 'function', 'gas': 316}] + +.. note:: + + If you would like to know how to install Docker, please follow their `documentation `_. + nix *** diff --git a/docs/types.rst b/docs/types.rst index 752e06b14f..807c83848f 100644 --- a/docs/types.rst +++ b/docs/types.rst @@ -359,11 +359,12 @@ A byte array with a max size. The syntax being ``Bytes[maxLen]``, where ``maxLen`` is an integer which denotes the maximum number of bytes. On the ABI level the Fixed-size bytes array is annotated as ``bytes``. -Bytes literals may be given as bytes strings. +Bytes literals may be given as bytes strings or as hex strings. .. code-block:: vyper bytes_string: Bytes[100] = b"\x01" + bytes_string: Bytes[100] = x"01" .. index:: !string diff --git a/tests/functional/builtins/codegen/test_empty.py b/tests/functional/builtins/codegen/test_empty.py index dd6c5c7cc1..3088162238 100644 --- a/tests/functional/builtins/codegen/test_empty.py +++ b/tests/functional/builtins/codegen/test_empty.py @@ -672,11 +672,11 @@ def test_empty_array_in_event_logging(get_contract, get_logs): @external def foo(): log MyLog( - b'hellohellohellohellohellohellohellohellohello', - empty(int128[2][3]), - 314159, - b'helphelphelphelphelphelphelphelphelphelphelp', - empty(uint256[3]) + arg1=b'hellohellohellohellohellohellohellohellohello', + arg2=empty(int128[2][3]), + arg3=314159, + arg4=b'helphelphelphelphelphelphelphelphelphelphelp', + arg5=empty(uint256[3]) ) """ diff --git a/tests/functional/builtins/codegen/test_raw_call.py b/tests/functional/builtins/codegen/test_raw_call.py index 4107f9a4d0..bf953ff018 100644 --- a/tests/functional/builtins/codegen/test_raw_call.py +++ b/tests/functional/builtins/codegen/test_raw_call.py @@ -261,6 +261,12 @@ def __default__(): assert env.message_call(caller.address, data=sig) == b"" +def _strip_initcode_suffix(bytecode): + bs = bytes.fromhex(bytecode.removeprefix("0x")) + to_strip = int.from_bytes(bs[-2:], "big") + return bs[:-to_strip].hex() + + # check max_outsize=0 does same thing as not setting max_outsize. # compile to bytecode and compare bytecode directly. def test_max_outsize_0(): @@ -276,7 +282,11 @@ def test_raw_call(_target: address): """ output1 = compile_code(code1, output_formats=["bytecode", "bytecode_runtime"]) output2 = compile_code(code2, output_formats=["bytecode", "bytecode_runtime"]) - assert output1 == output2 + assert output1["bytecode_runtime"] == output2["bytecode_runtime"] + + bytecode1 = output1["bytecode"] + bytecode2 = output2["bytecode"] + assert _strip_initcode_suffix(bytecode1) == _strip_initcode_suffix(bytecode2) # check max_outsize=0 does same thing as not setting max_outsize, @@ -298,7 +308,11 @@ def test_raw_call(_target: address) -> bool: """ output1 = compile_code(code1, output_formats=["bytecode", "bytecode_runtime"]) output2 = compile_code(code2, output_formats=["bytecode", "bytecode_runtime"]) - assert output1 == output2 + assert output1["bytecode_runtime"] == output2["bytecode_runtime"] + + bytecode1 = output1["bytecode"] + bytecode2 = output2["bytecode"] + assert _strip_initcode_suffix(bytecode1) == _strip_initcode_suffix(bytecode2) # test functionality of max_outsize=0 diff --git a/tests/functional/codegen/calling_convention/test_default_function.py b/tests/functional/codegen/calling_convention/test_default_function.py index 4d54e31f91..08d9c08678 100644 --- a/tests/functional/codegen/calling_convention/test_default_function.py +++ b/tests/functional/codegen/calling_convention/test_default_function.py @@ -28,7 +28,7 @@ def test_basic_default(env, get_logs, get_contract): @external @payable def __default__(): - log Sent(msg.sender) + log Sent(sender=msg.sender) """ c = get_contract(code) env.set_balance(env.deployer, 10**18) @@ -46,13 +46,13 @@ def test_basic_default_default_param_function(env, get_logs, get_contract): @external @payable def fooBar(a: int128 = 12345) -> int128: - log Sent(empty(address)) + log Sent(sender=empty(address)) return a @external @payable def __default__(): - log Sent(msg.sender) + log Sent(sender=msg.sender) """ c = get_contract(code) env.set_balance(env.deployer, 10**18) @@ -69,7 +69,7 @@ def test_basic_default_not_payable(env, tx_failed, get_contract): @external def __default__(): - log Sent(msg.sender) + log Sent(sender=msg.sender) """ c = get_contract(code) env.set_balance(env.deployer, 10**17) @@ -103,7 +103,7 @@ def test_always_public_2(assert_compile_failed, get_contract): sender: indexed(address) def __default__(): - log Sent(msg.sender) + log Sent(sender=msg.sender) """ assert_compile_failed(lambda: get_contract(code)) @@ -119,12 +119,12 @@ def test_zero_method_id(env, get_logs, get_contract, tx_failed): @payable # function selector: 0x00000000 def blockHashAskewLimitary(v: uint256) -> uint256: - log Sent(2) + log Sent(sig=2) return 7 @external def __default__(): - log Sent(1) + log Sent(sig=1) """ c = get_contract(code) @@ -165,12 +165,12 @@ def test_another_zero_method_id(env, get_logs, get_contract, tx_failed): @payable # function selector: 0x00000000 def wycpnbqcyf() -> uint256: - log Sent(2) + log Sent(sig=2) return 7 @external def __default__(): - log Sent(1) + log Sent(sig=1) """ c = get_contract(code) @@ -205,12 +205,12 @@ def test_partial_selector_match_trailing_zeroes(env, get_logs, get_contract): @payable # function selector: 0xd88e0b00 def fow() -> uint256: - log Sent(2) + log Sent(sig=2) return 7 @external def __default__(): - log Sent(1) + log Sent(sig=1) """ c = get_contract(code) diff --git a/tests/functional/codegen/features/decorators/test_private.py b/tests/functional/codegen/features/decorators/test_private.py index d313aa3bda..b9e34ea49b 100644 --- a/tests/functional/codegen/features/decorators/test_private.py +++ b/tests/functional/codegen/features/decorators/test_private.py @@ -436,7 +436,7 @@ def i_am_me() -> bool: @external @nonpayable def whoami() -> address: - log Addr(self._whoami()) + log Addr(addr=self._whoami()) return self._whoami() """ diff --git a/tests/functional/codegen/features/test_logging.py b/tests/functional/codegen/features/test_logging.py index cf77a30bd9..2bb646e6ef 100644 --- a/tests/functional/codegen/features/test_logging.py +++ b/tests/functional/codegen/features/test_logging.py @@ -5,13 +5,14 @@ from tests.utils import decimal_to_int from vyper import compile_code from vyper.exceptions import ( - ArgumentException, EventDeclarationException, + InstantiationException, InvalidType, NamespaceCollision, StructureException, TypeMismatch, UndeclaredDefinition, + UnknownAttribute, ) from vyper.utils import keccak256 @@ -50,7 +51,7 @@ def test_event_logging_with_topics(get_logs, keccak, get_contract): @external def foo(): self.a = b"bar" - log MyLog(self.a) + log MyLog(arg1=self.a) """ c = get_contract(loggy_code) @@ -78,7 +79,7 @@ def test_event_logging_with_multiple_topics(env, keccak, get_logs, get_contract) @external def foo(): - log MyLog(-2, True, self) + log MyLog(arg1=-2, arg2=True, arg3=self) """ c = get_contract(loggy_code) @@ -120,7 +121,7 @@ def test_event_logging_with_multiple_topics_var_and_store(get_contract, get_logs def foo(arg1: int128): a: bool = True self.b = self - log MyLog(arg1, a, self.b) + log MyLog(arg1=arg1, arg2=a, arg3=self.b) """ c = get_contract(code) @@ -141,13 +142,13 @@ def test_logging_the_same_event_multiple_times_with_topics(env, keccak, get_logs @external def foo(): - log MyLog(1, self) - log MyLog(1, self) + log MyLog(arg1=1, arg2=self) + log MyLog(arg1=1, arg2=self) @external def bar(): - log MyLog(1, self) - log MyLog(1, self) + log MyLog(arg1=1, arg2=self) + log MyLog(arg1=1, arg2=self) """ c = get_contract(loggy_code) @@ -198,7 +199,7 @@ def test_event_logging_with_data(get_logs, keccak, get_contract): @external def foo(): - log MyLog(123) + log MyLog(arg1=123) """ c = get_contract(loggy_code) @@ -231,8 +232,16 @@ def test_event_logging_with_fixed_array_data(env, keccak, get_logs, get_contract @external def foo(): - log MyLog([1,2], [block.timestamp, block.timestamp+1, block.timestamp+2], [[1,2],[1,2]]) - log MyLog([1,2], [block.timestamp, block.timestamp+1, block.timestamp+2], [[1,2],[1,2]]) + log MyLog( + arg1=[1,2], + arg2=[block.timestamp, block.timestamp+1, block.timestamp+2], + arg3=[[1,2],[1,2]] + ) + log MyLog( + arg1=[1,2], + arg2=[block.timestamp, block.timestamp+1, block.timestamp+2], + arg3=[[1,2],[1,2]] + ) """ c = get_contract(loggy_code) @@ -271,7 +280,7 @@ def test_logging_with_input_bytes_1(env, keccak, get_logs, get_contract): @external def foo(arg1: Bytes[29], arg2: Bytes[31]): - log MyLog(b'bar', arg1, arg2) + log MyLog(arg1=b'bar', arg2=arg1, arg3=arg2) """ c = get_contract(loggy_code) @@ -307,7 +316,7 @@ def test_event_logging_with_bytes_input_2(env, keccak, get_logs, get_contract): @external def foo(_arg1: Bytes[20]): - log MyLog(_arg1) + log MyLog(arg1=_arg1) """ c = get_contract(loggy_code) @@ -335,7 +344,7 @@ def test_event_logging_with_bytes_input_3(get_logs, keccak, get_contract): @external def foo(_arg1: Bytes[5]): - log MyLog(_arg1) + log MyLog(arg1=_arg1) """ c = get_contract(loggy_code) @@ -369,7 +378,7 @@ def test_event_logging_with_data_with_different_types(env, keccak, get_logs, get @external def foo(): - log MyLog(123, b'home', b'bar', 0xc305c901078781C232A2a521C2aF7980f8385ee9, self, block.timestamp) # noqa: E501 + log MyLog(arg1=123, arg2=b'home', arg3=b'bar', arg4=0xc305c901078781C232A2a521C2aF7980f8385ee9, arg5=self, arg6=block.timestamp) # noqa: E501 """ c = get_contract(loggy_code) @@ -412,7 +421,7 @@ def test_event_logging_with_topics_and_data_1(env, keccak, get_logs, get_contrac @external def foo(): - log MyLog(1, b'bar') + log MyLog(arg1=1, arg2=b'bar') """ c = get_contract(loggy_code) @@ -457,8 +466,8 @@ def test_event_logging_with_multiple_logs_topics_and_data(env, keccak, get_logs, @external def foo(): - log MyLog(1, b'bar') - log YourLog(self, MyStruct(x=1, y=b'abc', z=SmallStruct(t='house', w=13.5))) + log MyLog(arg1=1, arg2=b'bar') + log YourLog(arg1=self, arg2=MyStruct(x=1, y=b'abc', z=SmallStruct(t='house', w=13.5))) """ c = get_contract(loggy_code) @@ -524,7 +533,7 @@ def test_fails_when_input_is_the_wrong_type(tx_failed, get_contract): @external def foo_(): - log MyLog(b'yo') + log MyLog(arg1=b'yo') """ with tx_failed(TypeMismatch): @@ -539,7 +548,7 @@ def test_fails_when_topic_is_the_wrong_size(tx_failed, get_contract): @external def foo(): - log MyLog(b'bars') + log MyLog(arg1=b'bars') """ with tx_failed(TypeMismatch): @@ -553,7 +562,7 @@ def test_fails_when_input_topic_is_the_wrong_size(tx_failed, get_contract): @external def foo(arg1: Bytes[4]): - log MyLog(arg1) + log MyLog(arg1=arg1) """ with tx_failed(TypeMismatch): @@ -567,7 +576,7 @@ def test_fails_when_data_is_the_wrong_size(tx_failed, get_contract): @external def foo(): - log MyLog(b'bars') + log MyLog(arg1=b'bars') """ with tx_failed(TypeMismatch): @@ -581,7 +590,7 @@ def test_fails_when_input_data_is_the_wrong_size(tx_failed, get_contract): @external def foo(arg1: Bytes[4]): - log MyLog(arg1) + log MyLog(arg1=arg1) """ with tx_failed(TypeMismatch): @@ -610,7 +619,7 @@ def test_logging_fails_with_over_three_topics(tx_failed, get_contract): @deploy def __init__(): - log MyLog(1, 2, 3, 4) + log MyLog(arg1=1, arg2=2, arg3=3, arg4=4) """ with tx_failed(EventDeclarationException): @@ -650,7 +659,7 @@ def test_logging_fails_with_topic_type_mismatch(tx_failed, get_contract): @external def foo(): - log MyLog(self) + log MyLog(arg1=self) """ with tx_failed(TypeMismatch): @@ -664,7 +673,7 @@ def test_logging_fails_with_data_type_mismatch(tx_failed, get_contract): @external def foo(): - log MyLog(self) + log MyLog(arg1=self) """ with tx_failed(TypeMismatch): @@ -680,9 +689,9 @@ def test_logging_fails_when_number_of_arguments_is_greater_than_declaration( @external def foo(): - log MyLog(1, 2) + log MyLog(arg1=1, arg2=2) """ - with tx_failed(ArgumentException): + with tx_failed(UnknownAttribute): get_contract(loggy_code) @@ -694,9 +703,9 @@ def test_logging_fails_when_number_of_arguments_is_less_than_declaration(tx_fail @external def foo(): - log MyLog(1) + log MyLog(arg1=1) """ - with tx_failed(ArgumentException): + with tx_failed(InstantiationException): get_contract(loggy_code) @@ -852,7 +861,7 @@ def test_variable_list_packing(get_logs, get_contract): @external def foo(): a: int128[4] = [1, 2, 3, 4] - log Bar(a) + log Bar(_value=a) """ c = get_contract(code) @@ -868,7 +877,7 @@ def test_literal_list_packing(get_logs, get_contract): @external def foo(): - log Bar([1, 2, 3, 4]) + log Bar(_value=[1, 2, 3, 4]) """ c = get_contract(code) @@ -886,7 +895,7 @@ def test_storage_list_packing(get_logs, get_contract): @external def foo(): - log Bar(self.x) + log Bar(_value=self.x) @external def set_list(): @@ -910,7 +919,7 @@ def test_passed_list_packing(get_logs, get_contract): @external def foo(barbaric: int128[4]): - log Bar(barbaric) + log Bar(_value=barbaric) """ c = get_contract(code) @@ -926,7 +935,7 @@ def test_variable_decimal_list_packing(get_logs, get_contract): @external def foo(): - log Bar([1.11, 2.22, 3.33, 4.44]) + log Bar(_value=[1.11, 2.22, 3.33, 4.44]) """ c = get_contract(code) @@ -949,7 +958,7 @@ def test_storage_byte_packing(get_logs, get_contract): @external def foo(a: int128): - log MyLog(self.x) + log MyLog(arg1=self.x) @external def setbytez(): @@ -975,7 +984,7 @@ def test_storage_decimal_list_packing(get_logs, get_contract): @external def foo(): - log Bar(self.x) + log Bar(_value=self.x) @external def set_list(): @@ -1004,7 +1013,7 @@ def test_logging_fails_when_input_is_too_big(tx_failed, get_contract): @external def foo(inp: Bytes[33]): - log Bar(inp) + log Bar(_value=inp) """ with tx_failed(TypeMismatch): get_contract(code) @@ -1019,7 +1028,7 @@ def test_2nd_var_list_packing(get_logs, get_contract): @external def foo(): a: int128[4] = [1, 2, 3, 4] - log Bar(10, a) + log Bar(arg1=10, arg2=a) """ c = get_contract(code) @@ -1037,7 +1046,7 @@ def test_2nd_var_storage_list_packing(get_logs, get_contract): @external def foo(): - log Bar(10, self.x) + log Bar(arg1=10, arg2=self.x) @external def set_list(): @@ -1071,7 +1080,7 @@ def __init__(): @external def foo(): v: int128[3] = [7, 8, 9] - log Bar(10, self.x, b"test", v, self.y) + log Bar(arg1=10, arg2=self.x, arg3=b"test", arg4=v, arg5=self.y) @external def set_list(): @@ -1104,7 +1113,7 @@ def test_hashed_indexed_topics_calldata(get_logs, keccak, get_contract): @external def foo(a: Bytes[36], b: int128, c: String[7]): - log MyLog(a, b, c) + log MyLog(arg1=a, arg2=b, arg3=c) """ c = get_contract(loggy_code) @@ -1144,7 +1153,7 @@ def foo(): a: Bytes[10] = b"potato" b: int128 = -777 c: String[44] = "why hello, neighbor! how are you today?" - log MyLog(a, b, c) + log MyLog(arg1=a, arg2=b, arg3=c) """ c = get_contract(loggy_code) @@ -1191,7 +1200,7 @@ def setter(_a: Bytes[32], _b: int128, _c: String[6]): @external def foo(): - log MyLog(self.a, self.b, self.c) + log MyLog(arg1=self.a, arg2=self.b, arg3=self.c) """ c = get_contract(loggy_code) @@ -1229,7 +1238,7 @@ def test_hashed_indexed_topics_storxxage(get_logs, keccak, get_contract): @external def foo(): - log MyLog(b"wow", 666, "madness!") + log MyLog(arg1=b"wow", arg2=666, arg3="madness!") """ c = get_contract(loggy_code) diff --git a/tests/functional/codegen/features/test_logging_bytes_extended.py b/tests/functional/codegen/features/test_logging_bytes_extended.py index 6b84cdd23a..64c848bb8e 100644 --- a/tests/functional/codegen/features/test_logging_bytes_extended.py +++ b/tests/functional/codegen/features/test_logging_bytes_extended.py @@ -7,7 +7,7 @@ def test_bytes_logging_extended(get_contract, get_logs): @external def foo(): - log MyLog(667788, b'hellohellohellohellohellohellohellohellohello', 334455) + log MyLog(arg1=667788, arg2=b'hellohellohellohellohellohellohellohellohello', arg3=334455) """ c = get_contract(code) @@ -31,7 +31,7 @@ def foo(): a: Bytes[64] = b'hellohellohellohellohellohellohellohellohello' b: Bytes[64] = b'hellohellohellohellohellohellohellohello' # test literal much smaller than buffer - log MyLog(a, b, b'hello') + log MyLog(arg1=a, arg2=b, arg3=b'hello') """ c = get_contract(code) @@ -51,7 +51,7 @@ def test_bytes_logging_extended_passthrough(get_contract, get_logs): @external def foo(a: int128, b: Bytes[64], c: int128): - log MyLog(a, b, c) + log MyLog(arg1=a, arg2=b, arg3=c) """ c = get_contract(code) @@ -77,7 +77,7 @@ def test_bytes_logging_extended_storage(get_contract, get_logs): @external def foo(): - log MyLog(self.a, self.b, self.c) + log MyLog(arg1=self.a, arg2=self.b, arg3=self.c) @external def set(x: int128, y: Bytes[64], z: int128): @@ -114,10 +114,10 @@ def test_bytes_logging_extended_mixed_with_lists(get_contract, get_logs): @external def foo(): log MyLog( - [[24, 26], [12, 10]], - b'hellohellohellohellohellohellohellohellohello', - 314159, - b'helphelphelphelphelphelphelphelphelphelphelp' + arg1=[[24, 26], [12, 10]], + arg2=b'hellohellohellohellohellohellohellohellohello', + arg3=314159, + arg4=b'helphelphelphelphelphelphelphelphelphelphelp' ) """ diff --git a/tests/functional/codegen/features/test_logging_from_call.py b/tests/functional/codegen/features/test_logging_from_call.py index 190be7b4f4..2b14cd8398 100644 --- a/tests/functional/codegen/features/test_logging_from_call.py +++ b/tests/functional/codegen/features/test_logging_from_call.py @@ -21,11 +21,11 @@ def to_bytes32(_value: uint256) -> bytes32: @external def test_func(_value: uint256): data2: Bytes[60] = concat(self.to_bytes32(_value),self.to_bytes(_value),b"testing") - log TestLog(self.to_bytes32(_value), data2, self.to_bytes(_value)) + log TestLog(testData1=self.to_bytes32(_value), testData2=data2, testData3=self.to_bytes(_value)) loggedValue: bytes32 = self.to_bytes32(_value) loggedValue2: Bytes[8] = self.to_bytes(_value) - log TestLog(loggedValue, data2, loggedValue2) + log TestLog(testData1=loggedValue, testData2=data2, testData3=loggedValue2) """ c = get_contract(code) @@ -65,8 +65,8 @@ def test_func(_value: uint256,input: Bytes[133]): data2: Bytes[200] = b"hello world" - # log TestLog(self.to_bytes32(_value),input,self.to_bytes(_value)) - log TestLog(self.to_bytes32(_value),input,"bababa") + # log TestLog(testData1=self.to_bytes32(_value),testData2=input,testData3=self.to_bytes(_value)) + log TestLog(testData1=self.to_bytes32(_value),testData2=input,testData3="bababa") """ c = get_contract(code) @@ -99,8 +99,8 @@ def test_func(_value: uint256,input: Bytes[133]): data2: Bytes[200] = b"hello world" # log will be malformed - # log TestLog(self.to_bytes32(_value),input,self.to_bytes(_value)) - log TestLog(self.to_bytes32(_value), input) + # log TestLog(testData1=self.to_bytes32(_value),testData2=input,testData3=self.to_bytes(_value)) + log TestLog(testData1=self.to_bytes32(_value), testData2=input) """ c = get_contract(code) @@ -137,12 +137,12 @@ def test_func(_value: uint256,input: Bytes[2048]): data2: Bytes[2064] = concat(self.to_bytes(_value),self.to_bytes(_value),input) # log will be malformed - log TestLog(self.to_bytes32(_value), data2, self.to_bytes(_value)) + log TestLog(testData1=self.to_bytes32(_value), testData2=data2, testData3=self.to_bytes(_value)) loggedValue: Bytes[8] = self.to_bytes(_value) # log will be normal - log TestLog(self.to_bytes32(_value),data2,loggedValue) + log TestLog(testData1=self.to_bytes32(_value),testData2=data2,testData3=loggedValue) """ c = get_contract(code) diff --git a/tests/functional/codegen/features/test_memory_dealloc.py b/tests/functional/codegen/features/test_memory_dealloc.py index 3be57038ef..b733de736b 100644 --- a/tests/functional/codegen/features/test_memory_dealloc.py +++ b/tests/functional/codegen/features/test_memory_dealloc.py @@ -9,7 +9,7 @@ def sendit(): nonpayable @external def foo(target: address) -> uint256[2]: - log Shimmy(empty(address), 3) + log Shimmy(a=empty(address), b=3) amount: uint256 = 1 flargen: uint256 = 42 extcall Other(target).sendit() diff --git a/tests/functional/codegen/modules/test_events.py b/tests/functional/codegen/modules/test_events.py index ae5198cf90..c32a66caec 100644 --- a/tests/functional/codegen/modules/test_events.py +++ b/tests/functional/codegen/modules/test_events.py @@ -50,7 +50,7 @@ def test_module_event_indexed(get_contract, make_input_bundle, get_logs): @internal def foo(): - log MyEvent(5, 6) + log MyEvent(x=5, y=6) """ main = """ import lib1 diff --git a/tests/functional/codegen/test_interfaces.py b/tests/functional/codegen/test_interfaces.py index 9ea0b58d89..8887bf07cb 100644 --- a/tests/functional/codegen/test_interfaces.py +++ b/tests/functional/codegen/test_interfaces.py @@ -13,6 +13,7 @@ ) +# TODO CMC 2024-10-13: this should probably be in tests/unit/compiler/ def test_basic_extract_interface(): code = """ # Events @@ -22,6 +23,7 @@ def test_basic_extract_interface(): _to: address _value: uint256 + # Functions @view @@ -37,6 +39,7 @@ def allowance(_owner: address, _spender: address) -> (uint256, uint256): assert code_pass.strip() == out.strip() +# TODO CMC 2024-10-13: this should probably be in tests/unit/compiler/ def test_basic_extract_external_interface(): code = """ @view @@ -68,6 +71,7 @@ def test(_owner: address): nonpayable assert interface.strip() == out.strip() +# TODO CMC 2024-10-13: should probably be in syntax tests def test_basic_interface_implements(assert_compile_failed): code = """ from ethereum.ercs import IERC20 @@ -82,6 +86,7 @@ def test() -> bool: assert_compile_failed(lambda: compile_code(code), InterfaceViolation) +# TODO CMC 2024-10-13: should probably be in syntax tests def test_external_interface_parsing(make_input_bundle, assert_compile_failed): interface_code = """ @external @@ -126,6 +131,7 @@ def foo() -> uint256: compile_code(not_implemented_code, input_bundle=input_bundle) +# TODO CMC 2024-10-13: should probably be in syntax tests def test_log_interface_event(make_input_bundle, assert_compile_failed): interface_code = """ event Foo: @@ -160,6 +166,7 @@ def bar() -> uint256: ] +# TODO CMC 2024-10-13: should probably be in syntax tests @pytest.mark.parametrize("code,filename", VALID_IMPORT_CODE) def test_extract_file_interface_imports(code, filename, make_input_bundle): input_bundle = make_input_bundle({filename: ""}) @@ -177,6 +184,7 @@ def test_extract_file_interface_imports(code, filename, make_input_bundle): ] +# TODO CMC 2024-10-13: should probably be in syntax tests @pytest.mark.parametrize("code,exception_type", BAD_IMPORT_CODE) def test_extract_file_interface_imports_raises( code, exception_type, assert_compile_failed, make_input_bundle @@ -726,3 +734,43 @@ def bar() -> uint256: c = get_contract(code, input_bundle=input_bundle) assert c.foo() == c.bar() == 1 + + +def test_interface_with_structures(): + code = """ +struct MyStruct: + a: address + b: uint256 + +event Transfer: + sender: indexed(address) + receiver: indexed(address) + value: uint256 + +struct Voter: + weight: int128 + voted: bool + delegate: address + vote: int128 + +@external +def bar(): + pass + +event Buy: + buyer: indexed(address) + buy_order: uint256 + +@external +@view +def foo(s: MyStruct) -> MyStruct: + return s + """ + + out = compile_code(code, contract_path="code.vy", output_formats=["interface"])["interface"] + + assert "# Structs" in out + assert "struct MyStruct:" in out + assert "b: uint256" in out + assert "struct Voter:" in out + assert "voted: bool" in out diff --git a/tests/functional/codegen/types/test_bytes.py b/tests/functional/codegen/types/test_bytes.py index a5b119f143..6473be4348 100644 --- a/tests/functional/codegen/types/test_bytes.py +++ b/tests/functional/codegen/types/test_bytes.py @@ -259,6 +259,28 @@ def test2(l: bytes{m} = {vyper_literal}) -> bool: assert c.test2(vyper_literal) is True +@pytest.mark.parametrize("m,val", [(2, "ab"), (3, "ab"), (4, "abcd")]) +def test_native_hex_literals(get_contract, m, val): + vyper_literal = bytes.fromhex(val) + code = f""" +@external +def test() -> bool: + l: Bytes[{m}] = x"{val}" + return l == {vyper_literal} + +@external +def test2(l: Bytes[{m}] = x"{val}") -> bool: + return l == {vyper_literal} + """ + print(code) + + c = get_contract(code) + + assert c.test() is True + assert c.test2() is True + assert c.test2(vyper_literal) is True + + def test_zero_padding_with_private(get_contract): code = """ counter: uint256 diff --git a/tests/functional/codegen/types/test_string.py b/tests/functional/codegen/types/test_string.py index 51899b50f3..1c186eeb6e 100644 --- a/tests/functional/codegen/types/test_string.py +++ b/tests/functional/codegen/types/test_string.py @@ -116,7 +116,7 @@ def test_logging_extended_string(get_contract, get_logs): @external def foo(): - log MyLog(667788, 'hellohellohellohellohellohellohellohellohello', 334455) + log MyLog(arg1=667788, arg2='hellohellohellohellohellohellohellohellohello', arg3=334455) """ c = get_contract(code) diff --git a/tests/functional/grammar/test_grammar.py b/tests/functional/grammar/test_grammar.py index 2af5385b3d..0ff8c23477 100644 --- a/tests/functional/grammar/test_grammar.py +++ b/tests/functional/grammar/test_grammar.py @@ -102,6 +102,6 @@ def has_no_docstrings(c): max_examples=500, suppress_health_check=[HealthCheck.too_slow, HealthCheck.filter_too_much] ) def test_grammar_bruteforce(code): - _, _, _, reformatted_code = pre_parse(code + "\n") - tree = parse_to_ast(reformatted_code) + pre_parse_result = pre_parse(code + "\n") + tree = parse_to_ast(pre_parse_result.reformatted_code) assert isinstance(tree, Module) diff --git a/tests/functional/syntax/exceptions/test_type_mismatch_exception.py b/tests/functional/syntax/exceptions/test_type_mismatch_exception.py index 514f2df618..76c5c481f0 100644 --- a/tests/functional/syntax/exceptions/test_type_mismatch_exception.py +++ b/tests/functional/syntax/exceptions/test_type_mismatch_exception.py @@ -41,7 +41,7 @@ def foo(): message: String[1] @external def foo(): - log Foo("abcd") + log Foo(message="abcd") """, # Address literal must be checksummed """ diff --git a/tests/functional/syntax/names/test_event_names.py b/tests/functional/syntax/names/test_event_names.py index 367b646bfe..28cd6bdad0 100644 --- a/tests/functional/syntax/names/test_event_names.py +++ b/tests/functional/syntax/names/test_event_names.py @@ -26,7 +26,7 @@ def foo(i: int128) -> int128: @external def foo(i: int128) -> int128: temp_var : int128 = i - log int128(temp_var) + log int128(variable=temp_var) return temp_var """, NamespaceCollision, @@ -39,7 +39,7 @@ def foo(i: int128) -> int128: @external def foo(i: int128) -> int128: temp_var : int128 = i - log decimal(temp_var) + log decimal(variable=temp_var) return temp_var """, NamespaceCollision, @@ -52,7 +52,7 @@ def foo(i: int128) -> int128: @external def foo(i: int128) -> int128: temp_var : int128 = i - log wei(temp_var) + log wei(variable=temp_var) return temp_var """, StructureException, @@ -65,7 +65,7 @@ def foo(i: int128) -> int128: @external def foo(i: int128) -> int128: temp_var : int128 = i - log false(temp_var) + log false(variable=temp_var) return temp_var """, StructureException, @@ -102,7 +102,7 @@ def test_varname_validity_fail(bad_code, exc): @external def foo(i: int128) -> int128: variable : int128 = i - log Assigned(variable) + log Assigned(variable=variable) return variable """, """ @@ -122,7 +122,7 @@ def foo(i: int128) -> int128: @external def foo(i: int128) -> int128: variable : int128 = i - log Assigned1(variable) + log Assigned1(variable=variable) return variable """, ] diff --git a/tests/functional/syntax/test_ann_assign.py b/tests/functional/syntax/test_ann_assign.py index 23ebeb9560..fba9eff38d 100644 --- a/tests/functional/syntax/test_ann_assign.py +++ b/tests/functional/syntax/test_ann_assign.py @@ -3,11 +3,11 @@ from vyper import compiler from vyper.exceptions import ( + InstantiationException, InvalidAttribute, TypeMismatch, UndeclaredDefinition, UnknownAttribute, - VariableDeclarationException, ) fail_list = [ @@ -73,7 +73,7 @@ def foo() -> int128: def foo() -> int128: s: S = S(a=1) """, - VariableDeclarationException, + InstantiationException, ), ( """ diff --git a/tests/functional/syntax/test_bytes.py b/tests/functional/syntax/test_bytes.py index 0ca3b27fee..9df2962f2e 100644 --- a/tests/functional/syntax/test_bytes.py +++ b/tests/functional/syntax/test_bytes.py @@ -80,6 +80,15 @@ def test() -> Bytes[1]: ( """ @external +def test() -> Bytes[2]: + a: Bytes[2] = x"abc" + return a + """, + SyntaxException, + ), + ( + """ +@external def foo(): a: Bytes = b"abc" """, diff --git a/tests/functional/syntax/test_external_calls.py b/tests/functional/syntax/test_external_calls.py index a8fb5ae87b..fd6fa28cc9 100644 --- a/tests/functional/syntax/test_external_calls.py +++ b/tests/functional/syntax/test_external_calls.py @@ -61,7 +61,7 @@ def foo(f: Foo): s: uint256 = staticcall f.foo() """, # TODO: tokenizer currently has issue with log+staticcall/extcall, e.g. - # `log Bar(staticcall f.foo() + extcall f.bar())` + # `log Bar(_value=staticcall f.foo() + extcall f.bar())` ] diff --git a/tests/functional/syntax/test_interfaces.py b/tests/functional/syntax/test_interfaces.py index ea06e0ab2f..20813c48d1 100644 --- a/tests/functional/syntax/test_interfaces.py +++ b/tests/functional/syntax/test_interfaces.py @@ -158,12 +158,12 @@ def f(a: uint256): # visibility is nonpayable instead of view @external def transfer(_to : address, _value : uint256) -> bool: - log Transfer(msg.sender, _to, _value) + log Transfer(sender=msg.sender, receiver=_to, value=_value) return True @external def transferFrom(_from : address, _to : address, _value : uint256) -> bool: - log IERC20.Transfer(_from, _to, _value) + log IERC20.Transfer(sender=_from, receiver=_to, value=_value) return True @external diff --git a/tests/functional/syntax/test_logging.py b/tests/functional/syntax/test_logging.py index b96700a128..7f8f141b99 100644 --- a/tests/functional/syntax/test_logging.py +++ b/tests/functional/syntax/test_logging.py @@ -1,7 +1,13 @@ import pytest from vyper import compiler -from vyper.exceptions import StructureException, TypeMismatch +from vyper.exceptions import ( + InstantiationException, + InvalidAttribute, + StructureException, + TypeMismatch, + UnknownAttribute, +) fail_list = [ """ @@ -12,7 +18,7 @@ @external def foo(): - log Bar(self.x) + log Bar(_value=self.x) """, """ event Bar: @@ -21,7 +27,7 @@ def foo(): @external def foo(): x: decimal[4] = [0.0, 0.0, 0.0, 0.0] - log Bar(x) + log Bar(_value=x) """, """ struct Foo: @@ -37,7 +43,7 @@ def foo(): @external def test(): - log Test(-7) + log Test(n=-7) """, ] @@ -48,6 +54,61 @@ def test_logging_fail(bad_code): compiler.compile_code(bad_code) +def test_logging_fail_mixed_positional_kwargs(): + code = """ +event Test: + n: uint256 + o: uint256 + +@external +def test(): + log Test(7, o=12) + """ + with pytest.raises(InstantiationException): + compiler.compile_code(code) + + +def test_logging_fail_unknown_kwarg(): + code = """ +event Test: + n: uint256 + +@external +def test(): + log Test(n=7, o=12) + """ + with pytest.raises(UnknownAttribute): + compiler.compile_code(code) + + +def test_logging_fail_missing_kwarg(): + code = """ +event Test: + n: uint256 + o: uint256 + +@external +def test(): + log Test(n=7) + """ + with pytest.raises(InstantiationException): + compiler.compile_code(code) + + +def test_logging_fail_kwargs_out_of_order(): + code = """ +event Test: + n: uint256 + o: uint256 + +@external +def test(): + log Test(o=12, n=7) + """ + with pytest.raises(InvalidAttribute): + compiler.compile_code(code) + + @pytest.mark.parametrize("mutability", ["@pure", "@view"]) @pytest.mark.parametrize("visibility", ["@internal", "@external"]) def test_logging_from_non_mutable(mutability, visibility): @@ -58,7 +119,23 @@ def test_logging_from_non_mutable(mutability, visibility): {visibility} {mutability} def test(): - log Test(1) + log Test(n=1) """ with pytest.raises(StructureException): compiler.compile_code(code) + + +def test_logging_with_positional_args(get_contract, get_logs): + # TODO: Remove when positional arguments are fully deprecated + code = """ +event Test: + n: uint256 + +@external +def test(): + log Test(1) + """ + c = get_contract(code) + c.test() + (log,) = get_logs(c, "Test") + assert log.args.n == 1 diff --git a/tests/functional/syntax/test_structs.py b/tests/functional/syntax/test_structs.py index 9a9a397c48..c08859cd92 100644 --- a/tests/functional/syntax/test_structs.py +++ b/tests/functional/syntax/test_structs.py @@ -5,6 +5,7 @@ from vyper import compiler from vyper.exceptions import ( InstantiationException, + InvalidAttribute, StructureException, SyntaxException, TypeMismatch, @@ -32,7 +33,8 @@ def foo(): """, UnknownAttribute, ), - """ + ( + """ struct A: x: int128 y: int128 @@ -41,6 +43,8 @@ def foo(): def foo(): self.a = A(x=1) """, + InstantiationException, + ), """ struct A: x: int128 @@ -61,7 +65,8 @@ def foo(): def foo(): self.a = A(self.b) """, - """ + ( + """ struct A: x: int128 y: int128 @@ -70,6 +75,8 @@ def foo(): def foo(): self.a = A({x: 1}) """, + InstantiationException, + ), """ struct C: c: int128 @@ -386,7 +393,7 @@ def foo(): def foo(): self.b = B(foo=1, foo=2) """, - UnknownAttribute, + InvalidAttribute, ), ( """ diff --git a/tests/integration/test_pickle_ast.py b/tests/integration/test_pickle_ast.py new file mode 100644 index 0000000000..2c6144603a --- /dev/null +++ b/tests/integration/test_pickle_ast.py @@ -0,0 +1,19 @@ +import copy +import pickle + +from vyper.compiler.phases import CompilerData + + +def test_pickle_ast(): + code = """ +@external +def foo(): + self.bar() + y: uint256 = 5 + x: uint256 = 5 +def bar(): + pass + """ + f = CompilerData(code) + copy.deepcopy(f.annotated_vyper_module) + pickle.loads(pickle.dumps(f.annotated_vyper_module)) diff --git a/tests/unit/ast/test_annotate_and_optimize_ast.py b/tests/unit/ast/test_annotate_and_optimize_ast.py index 7e1641e49e..39ea899bd9 100644 --- a/tests/unit/ast/test_annotate_and_optimize_ast.py +++ b/tests/unit/ast/test_annotate_and_optimize_ast.py @@ -28,12 +28,12 @@ def foo() -> int128: def get_contract_info(source_code): - _, loop_var_annotations, class_types, reformatted_code = pre_parse(source_code) - py_ast = python_ast.parse(reformatted_code) + pre_parse_result = pre_parse(source_code) + py_ast = python_ast.parse(pre_parse_result.reformatted_code) - annotate_python_ast(py_ast, reformatted_code, loop_var_annotations, class_types) + annotate_python_ast(py_ast, pre_parse_result.reformatted_code, pre_parse_result) - return py_ast, reformatted_code + return py_ast, pre_parse_result.reformatted_code def test_it_annotates_ast_with_source_code(): diff --git a/tests/unit/ast/test_ast_dict.py b/tests/unit/ast/test_ast_dict.py index 07da3c0ace..c9d7248809 100644 --- a/tests/unit/ast/test_ast_dict.py +++ b/tests/unit/ast/test_ast_dict.py @@ -1,3 +1,4 @@ +import copy import json from vyper import compiler @@ -216,24 +217,27 @@ def foo(): input_bundle = make_input_bundle({"lib1.vy": lib1, "main.vy": main}) lib1_file = input_bundle.load_file("lib1.vy") - out = compiler.compile_from_file_input( + lib1_out = compiler.compile_from_file_input( lib1_file, input_bundle=input_bundle, output_formats=["annotated_ast_dict"] ) - lib1_ast = out["annotated_ast_dict"]["ast"] + + lib1_ast = copy.deepcopy(lib1_out["annotated_ast_dict"]["ast"]) lib1_sha256sum = lib1_ast.pop("source_sha256sum") assert lib1_sha256sum == lib1_file.sha256sum to_strip = NODE_SRC_ATTRIBUTES + ("resolved_path", "variable_reads", "variable_writes") _strip_source_annotations(lib1_ast, to_strip=to_strip) main_file = input_bundle.load_file("main.vy") - out = compiler.compile_from_file_input( + main_out = compiler.compile_from_file_input( main_file, input_bundle=input_bundle, output_formats=["annotated_ast_dict"] ) - main_ast = out["annotated_ast_dict"]["ast"] + main_ast = main_out["annotated_ast_dict"]["ast"] main_sha256sum = main_ast.pop("source_sha256sum") assert main_sha256sum == main_file.sha256sum _strip_source_annotations(main_ast, to_strip=to_strip) + assert main_out["annotated_ast_dict"]["imports"][0] == lib1_out["annotated_ast_dict"]["ast"] + # TODO: would be nice to refactor this into bunch of small test cases assert main_ast == { "ast_type": "Module", @@ -1776,3 +1780,49 @@ def qux2(): }, } ] + + +def test_annotated_ast_export_recursion(make_input_bundle): + sources = { + "main.vy": """ +import lib1 + +@external +def foo(): + lib1.foo() + """, + "lib1.vy": """ +import lib2 + +def foo(): + lib2.foo() + """, + "lib2.vy": """ +def foo(): + pass + """, + } + + input_bundle = make_input_bundle(sources) + + def compile_and_get_ast(file_name): + file = input_bundle.load_file(file_name) + output = compiler.compile_from_file_input( + file, input_bundle=input_bundle, output_formats=["annotated_ast_dict"] + ) + return output["annotated_ast_dict"] + + lib1_ast = compile_and_get_ast("lib1.vy")["ast"] + lib2_ast = compile_and_get_ast("lib2.vy")["ast"] + main_out = compile_and_get_ast("main.vy") + + lib1_import_ast = main_out["imports"][1] + lib2_import_ast = main_out["imports"][0] + + # path is once virtual, once libX.vy + # type contains name which is based on path + keys = [s for s in lib1_import_ast.keys() if s not in {"path", "type"}] + + for key in keys: + assert lib1_ast[key] == lib1_import_ast[key] + assert lib2_ast[key] == lib2_import_ast[key] diff --git a/tests/unit/ast/test_pre_parser.py b/tests/unit/ast/test_pre_parser.py index da7d72b8ec..4190725f7e 100644 --- a/tests/unit/ast/test_pre_parser.py +++ b/tests/unit/ast/test_pre_parser.py @@ -174,9 +174,9 @@ def test_prerelease_invalid_version_pragma(file_version, mock_version): @pytest.mark.parametrize("code, pre_parse_settings, compiler_data_settings", pragma_examples) def test_parse_pragmas(code, pre_parse_settings, compiler_data_settings, mock_version): mock_version("0.3.10") - settings, _, _, _ = pre_parse(code) + pre_parse_result = pre_parse(code) - assert settings == pre_parse_settings + assert pre_parse_result.settings == pre_parse_settings compiler_data = CompilerData(code) diff --git a/tests/unit/compiler/test_bytecode_runtime.py b/tests/unit/compiler/test_bytecode_runtime.py index 213adce017..1d38130c49 100644 --- a/tests/unit/compiler/test_bytecode_runtime.py +++ b/tests/unit/compiler/test_bytecode_runtime.py @@ -55,13 +55,17 @@ def test_bytecode_runtime(): def test_bytecode_signature(): - out = vyper.compile_code(simple_contract_code, output_formats=["bytecode_runtime", "bytecode"]) + out = vyper.compile_code( + simple_contract_code, output_formats=["bytecode_runtime", "bytecode", "integrity"] + ) runtime_code = bytes.fromhex(out["bytecode_runtime"].removeprefix("0x")) initcode = bytes.fromhex(out["bytecode"].removeprefix("0x")) metadata = _parse_cbor_metadata(initcode) - runtime_len, data_section_lengths, immutables_len, compiler = metadata + integrity_hash, runtime_len, data_section_lengths, immutables_len, compiler = metadata + + assert integrity_hash.hex() == out["integrity"] assert runtime_len == len(runtime_code) assert data_section_lengths == [] @@ -73,14 +77,18 @@ def test_bytecode_signature_dense_jumptable(): settings = Settings(optimize=OptimizationLevel.CODESIZE) out = vyper.compile_code( - many_functions, output_formats=["bytecode_runtime", "bytecode"], settings=settings + many_functions, + output_formats=["bytecode_runtime", "bytecode", "integrity"], + settings=settings, ) runtime_code = bytes.fromhex(out["bytecode_runtime"].removeprefix("0x")) initcode = bytes.fromhex(out["bytecode"].removeprefix("0x")) metadata = _parse_cbor_metadata(initcode) - runtime_len, data_section_lengths, immutables_len, compiler = metadata + integrity_hash, runtime_len, data_section_lengths, immutables_len, compiler = metadata + + assert integrity_hash.hex() == out["integrity"] assert runtime_len == len(runtime_code) assert data_section_lengths == [5, 35] @@ -92,14 +100,18 @@ def test_bytecode_signature_sparse_jumptable(): settings = Settings(optimize=OptimizationLevel.GAS) out = vyper.compile_code( - many_functions, output_formats=["bytecode_runtime", "bytecode"], settings=settings + many_functions, + output_formats=["bytecode_runtime", "bytecode", "integrity"], + settings=settings, ) runtime_code = bytes.fromhex(out["bytecode_runtime"].removeprefix("0x")) initcode = bytes.fromhex(out["bytecode"].removeprefix("0x")) metadata = _parse_cbor_metadata(initcode) - runtime_len, data_section_lengths, immutables_len, compiler = metadata + integrity_hash, runtime_len, data_section_lengths, immutables_len, compiler = metadata + + assert integrity_hash.hex() == out["integrity"] assert runtime_len == len(runtime_code) assert data_section_lengths == [8] @@ -108,13 +120,17 @@ def test_bytecode_signature_sparse_jumptable(): def test_bytecode_signature_immutables(): - out = vyper.compile_code(has_immutables, output_formats=["bytecode_runtime", "bytecode"]) + out = vyper.compile_code( + has_immutables, output_formats=["bytecode_runtime", "bytecode", "integrity"] + ) runtime_code = bytes.fromhex(out["bytecode_runtime"].removeprefix("0x")) initcode = bytes.fromhex(out["bytecode"].removeprefix("0x")) metadata = _parse_cbor_metadata(initcode) - runtime_len, data_section_lengths, immutables_len, compiler = metadata + integrity_hash, runtime_len, data_section_lengths, immutables_len, compiler = metadata + + assert integrity_hash.hex() == out["integrity"] assert runtime_len == len(runtime_code) assert data_section_lengths == [] @@ -129,7 +145,10 @@ def test_bytecode_signature_deployed(code, get_contract, env): deployed_code = env.get_code(c.address) metadata = _parse_cbor_metadata(c.bytecode) - runtime_len, data_section_lengths, immutables_len, compiler = metadata + integrity_hash, runtime_len, data_section_lengths, immutables_len, compiler = metadata + + out = vyper.compile_code(code, output_formats=["integrity"]) + assert integrity_hash.hex() == out["integrity"] assert compiler == {"vyper": list(vyper.version.version_tuple)} diff --git a/tests/unit/compiler/venom/test_algebraic_optimizer.py b/tests/unit/compiler/venom/test_algebraic_optimizer.py index b5d55efbdc..39008649ea 100644 --- a/tests/unit/compiler/venom/test_algebraic_optimizer.py +++ b/tests/unit/compiler/venom/test_algebraic_optimizer.py @@ -1,11 +1,9 @@ import pytest -from vyper.venom.analysis.analysis import IRAnalysesCache +from vyper.venom.analysis import IRAnalysesCache from vyper.venom.basicblock import IRBasicBlock, IRLabel from vyper.venom.context import IRContext -from vyper.venom.passes.algebraic_optimization import AlgebraicOptimizationPass -from vyper.venom.passes.make_ssa import MakeSSA -from vyper.venom.passes.remove_unused_variables import RemoveUnusedVariablesPass +from vyper.venom.passes import AlgebraicOptimizationPass, MakeSSA, RemoveUnusedVariablesPass @pytest.mark.parametrize("iszero_count", range(5)) diff --git a/tests/unit/compiler/venom/test_branch_optimizer.py b/tests/unit/compiler/venom/test_branch_optimizer.py index b6e806e217..82dff4777d 100644 --- a/tests/unit/compiler/venom/test_branch_optimizer.py +++ b/tests/unit/compiler/venom/test_branch_optimizer.py @@ -1,9 +1,7 @@ -from vyper.venom.analysis.analysis import IRAnalysesCache -from vyper.venom.analysis.dfg import DFGAnalysis +from vyper.venom.analysis import DFGAnalysis, IRAnalysesCache from vyper.venom.basicblock import IRBasicBlock, IRLabel from vyper.venom.context import IRContext -from vyper.venom.passes.branch_optimization import BranchOptimizationPass -from vyper.venom.passes.make_ssa import MakeSSA +from vyper.venom.passes import BranchOptimizationPass, MakeSSA def test_simple_jump_case(): diff --git a/tests/unit/compiler/venom/test_dominator_tree.py b/tests/unit/compiler/venom/test_dominator_tree.py index 29f86df221..30a2e4564e 100644 --- a/tests/unit/compiler/venom/test_dominator_tree.py +++ b/tests/unit/compiler/venom/test_dominator_tree.py @@ -2,12 +2,11 @@ from vyper.exceptions import CompilerPanic from vyper.utils import OrderedSet -from vyper.venom.analysis.analysis import IRAnalysesCache -from vyper.venom.analysis.dominators import DominatorTreeAnalysis +from vyper.venom.analysis import DominatorTreeAnalysis, IRAnalysesCache from vyper.venom.basicblock import IRBasicBlock, IRInstruction, IRLabel, IRLiteral, IRVariable from vyper.venom.context import IRContext from vyper.venom.function import IRFunction -from vyper.venom.passes.make_ssa import MakeSSA +from vyper.venom.passes import MakeSSA def _add_bb( diff --git a/tests/unit/compiler/venom/test_duplicate_operands.py b/tests/unit/compiler/venom/test_duplicate_operands.py index fbff0835d2..89b06796e3 100644 --- a/tests/unit/compiler/venom/test_duplicate_operands.py +++ b/tests/unit/compiler/venom/test_duplicate_operands.py @@ -1,6 +1,8 @@ from vyper.compiler.settings import OptimizationLevel from vyper.venom import generate_assembly_experimental +from vyper.venom.analysis import IRAnalysesCache from vyper.venom.context import IRContext +from vyper.venom.passes import StoreExpansionPass def test_duplicate_operands(): @@ -13,7 +15,7 @@ def test_duplicate_operands(): %3 = mul %1, %2 stop - Should compile to: [PUSH1, 10, DUP1, DUP1, DUP1, ADD, MUL, POP, STOP] + Should compile to: [PUSH1, 10, DUP1, DUP2, ADD, MUL, POP, STOP] """ ctx = IRContext() fn = ctx.create_function("test") @@ -23,5 +25,9 @@ def test_duplicate_operands(): bb.append_instruction("mul", sum_, op) bb.append_instruction("stop") - asm = generate_assembly_experimental(ctx, optimize=OptimizationLevel.GAS) - assert asm == ["PUSH1", 10, "DUP1", "DUP1", "ADD", "MUL", "POP", "STOP"] + ac = IRAnalysesCache(fn) + StoreExpansionPass(ac, fn).run_pass() + + optimize = OptimizationLevel.GAS + asm = generate_assembly_experimental(ctx, optimize=optimize) + assert asm == ["PUSH1", 10, "DUP1", "DUP2", "ADD", "MUL", "POP", "STOP"] diff --git a/tests/unit/compiler/venom/test_make_ssa.py b/tests/unit/compiler/venom/test_make_ssa.py index 9cea1a20a4..aa3fead6bf 100644 --- a/tests/unit/compiler/venom/test_make_ssa.py +++ b/tests/unit/compiler/venom/test_make_ssa.py @@ -1,7 +1,7 @@ -from vyper.venom.analysis.analysis import IRAnalysesCache +from vyper.venom.analysis import IRAnalysesCache from vyper.venom.basicblock import IRBasicBlock, IRLabel from vyper.venom.context import IRContext -from vyper.venom.passes.make_ssa import MakeSSA +from vyper.venom.passes import MakeSSA def test_phi_case(): diff --git a/tests/unit/compiler/venom/test_multi_entry_block.py b/tests/unit/compiler/venom/test_multi_entry_block.py index 313fbb3ebd..a38e4b4158 100644 --- a/tests/unit/compiler/venom/test_multi_entry_block.py +++ b/tests/unit/compiler/venom/test_multi_entry_block.py @@ -1,8 +1,7 @@ -from vyper.venom.analysis.analysis import IRAnalysesCache -from vyper.venom.analysis.cfg import CFGAnalysis +from vyper.venom.analysis import CFGAnalysis, IRAnalysesCache from vyper.venom.context import IRContext from vyper.venom.function import IRBasicBlock, IRLabel -from vyper.venom.passes.normalization import NormalizationPass +from vyper.venom.passes import NormalizationPass def test_multi_entry_block_1(): diff --git a/tests/unit/compiler/venom/test_sccp.py b/tests/unit/compiler/venom/test_sccp.py index e65839136e..375dfd5dac 100644 --- a/tests/unit/compiler/venom/test_sccp.py +++ b/tests/unit/compiler/venom/test_sccp.py @@ -1,11 +1,10 @@ import pytest from vyper.exceptions import StaticAssertionException -from vyper.venom.analysis.analysis import IRAnalysesCache +from vyper.venom.analysis import IRAnalysesCache from vyper.venom.basicblock import IRBasicBlock, IRLabel, IRLiteral, IRVariable from vyper.venom.context import IRContext -from vyper.venom.passes.make_ssa import MakeSSA -from vyper.venom.passes.sccp import SCCP +from vyper.venom.passes import SCCP, MakeSSA from vyper.venom.passes.sccp.sccp import LatticeEnum @@ -211,3 +210,34 @@ def test_cont_phi_const_case(): assert sccp.lattice[IRVariable("%5", version=1)].value == 106 assert sccp.lattice[IRVariable("%5", version=2)].value == 97 assert sccp.lattice[IRVariable("%5")].value == 2 + + +def test_phi_reduction_after_unreachable_block(): + ctx = IRContext() + fn = ctx.create_function("_global") + + bb = fn.get_basic_block() + + br1 = IRBasicBlock(IRLabel("then"), fn) + fn.append_basic_block(br1) + join = IRBasicBlock(IRLabel("join"), fn) + fn.append_basic_block(join) + + op = bb.append_instruction("store", 1) + true = IRLiteral(1) + bb.append_instruction("jnz", true, br1.label, join.label) + + op1 = br1.append_instruction("store", 2) + + br1.append_instruction("jmp", join.label) + + join.append_instruction("phi", bb.label, op, br1.label, op1) + join.append_instruction("stop") + + ac = IRAnalysesCache(fn) + SCCP(ac, fn).run_pass() + + assert join.instructions[0].opcode == "store", join.instructions[0] + assert join.instructions[0].operands == [op1] + + assert join.instructions[1].opcode == "stop" diff --git a/tests/unit/compiler/venom/test_simplify_cfg.py b/tests/unit/compiler/venom/test_simplify_cfg.py new file mode 100644 index 0000000000..3de6a77cc9 --- /dev/null +++ b/tests/unit/compiler/venom/test_simplify_cfg.py @@ -0,0 +1,48 @@ +from vyper.venom.analysis import IRAnalysesCache +from vyper.venom.basicblock import IRBasicBlock, IRLabel, IRLiteral +from vyper.venom.context import IRContext +from vyper.venom.passes import SCCP, SimplifyCFGPass + + +def test_phi_reduction_after_block_pruning(): + ctx = IRContext() + fn = ctx.create_function("_global") + + bb = fn.get_basic_block() + + br1 = IRBasicBlock(IRLabel("then"), fn) + fn.append_basic_block(br1) + br2 = IRBasicBlock(IRLabel("else"), fn) + fn.append_basic_block(br2) + + join = IRBasicBlock(IRLabel("join"), fn) + fn.append_basic_block(join) + + true = IRLiteral(1) + bb.append_instruction("jnz", true, br1.label, br2.label) + + op1 = br1.append_instruction("store", 1) + op2 = br2.append_instruction("store", 2) + + br1.append_instruction("jmp", join.label) + br2.append_instruction("jmp", join.label) + + join.append_instruction("phi", br1.label, op1, br2.label, op2) + join.append_instruction("stop") + + ac = IRAnalysesCache(fn) + SCCP(ac, fn).run_pass() + SimplifyCFGPass(ac, fn).run_pass() + + bbs = list(fn.get_basic_blocks()) + + assert len(bbs) == 1 + final_bb = bbs[0] + + inst0, inst1, inst2 = final_bb.instructions + + assert inst0.opcode == "store" + assert inst0.operands == [IRLiteral(1)] + assert inst1.opcode == "store" + assert inst1.operands == [inst0.output] + assert inst2.opcode == "stop" diff --git a/tests/unit/compiler/venom/test_stack_cleanup.py b/tests/unit/compiler/venom/test_stack_cleanup.py index 6015cf1c41..7198861771 100644 --- a/tests/unit/compiler/venom/test_stack_cleanup.py +++ b/tests/unit/compiler/venom/test_stack_cleanup.py @@ -9,7 +9,8 @@ def test_cleanup_stack(): bb = fn.get_basic_block() ret_val = bb.append_instruction("param") op = bb.append_instruction("store", 10) - bb.append_instruction("add", op, op) + op2 = bb.append_instruction("store", op) + bb.append_instruction("add", op, op2) bb.append_instruction("ret", ret_val) asm = generate_assembly_experimental(ctx, optimize=OptimizationLevel.GAS) diff --git a/tests/unit/compiler/venom/test_stack_reorder.py b/tests/unit/compiler/venom/test_stack_reorder.py new file mode 100644 index 0000000000..8f38e00cdb --- /dev/null +++ b/tests/unit/compiler/venom/test_stack_reorder.py @@ -0,0 +1,33 @@ +from vyper.venom import generate_assembly_experimental +from vyper.venom.analysis import IRAnalysesCache +from vyper.venom.context import IRContext +from vyper.venom.passes import StoreExpansionPass + + +def test_stack_reorder(): + """ + Test to was created from the example in the + issue https://github.com/vyperlang/vyper/issues/4215 + this example should fail with original stack reorder + algorithm but succeed with new one + """ + ctx = IRContext() + fn = ctx.create_function("_global") + + bb = fn.get_basic_block() + var0 = bb.append_instruction("store", 1) + var1 = bb.append_instruction("store", 2) + var2 = bb.append_instruction("store", 3) + var3 = bb.append_instruction("store", 4) + var4 = bb.append_instruction("store", 5) + + bb.append_instruction("staticcall", var0, var1, var2, var3, var4, var3) + + ret_val = bb.append_instruction("add", var4, var4) + + bb.append_instruction("ret", ret_val) + + ac = IRAnalysesCache(fn) + StoreExpansionPass(ac, fn).run_pass() + + generate_assembly_experimental(ctx) diff --git a/vyper/ast/grammar.lark b/vyper/ast/grammar.lark index 97f9f70e24..bc2f9ba77c 100644 --- a/vyper/ast/grammar.lark +++ b/vyper/ast/grammar.lark @@ -318,7 +318,7 @@ COMMENT: /#[^\n\r]*/ _NEWLINE: ( /\r?\n[\t ]*/ | COMMENT )+ -STRING: /b?("(?!"").*?(? Any: ... # context manager class FunctionDef(TopLevel): @@ -109,9 +110,7 @@ class ExprNode(VyperNode): class Constant(ExprNode): value: Any = ... -class Num(Constant): - @property - def n(self): ... +class Num(Constant): ... class Int(Num): value: int = ... @@ -122,14 +121,9 @@ class Hex(Num): @property def n_bytes(self): ... -class Str(Constant): - @property - def s(self): ... - -class Bytes(Constant): - @property - def s(self): ... - +class Str(Constant): ... +class Bytes(Constant): ... +class HexBytes(Constant): ... class NameConstant(Constant): ... class Ellipsis(Constant): ... diff --git a/vyper/ast/parse.py b/vyper/ast/parse.py index d4569dd644..1e88241186 100644 --- a/vyper/ast/parse.py +++ b/vyper/ast/parse.py @@ -6,10 +6,9 @@ import asttokens from vyper.ast import nodes as vy_ast -from vyper.ast.pre_parser import pre_parse +from vyper.ast.pre_parser import PreParseResult, pre_parse from vyper.compiler.settings import Settings from vyper.exceptions import CompilerPanic, ParserException, SyntaxException -from vyper.typing import ModificationOffsets from vyper.utils import sha256sum, vyper_warn @@ -55,9 +54,9 @@ def parse_to_ast_with_settings( """ if "\x00" in vyper_source: raise ParserException("No null bytes (\\x00) allowed in the source code.") - settings, class_types, for_loop_annotations, python_source = pre_parse(vyper_source) + pre_parse_result = pre_parse(vyper_source) try: - py_ast = python_ast.parse(python_source) + py_ast = python_ast.parse(pre_parse_result.reformatted_code) except SyntaxError as e: # TODO: Ensure 1-to-1 match of source_code:reformatted_code SyntaxErrors raise SyntaxException(str(e), vyper_source, e.lineno, e.offset) from None @@ -73,21 +72,20 @@ def parse_to_ast_with_settings( annotate_python_ast( py_ast, vyper_source, - class_types, - for_loop_annotations, + pre_parse_result, source_id=source_id, module_path=module_path, resolved_path=resolved_path, ) # postcondition: consumed all the for loop annotations - assert len(for_loop_annotations) == 0 + assert len(pre_parse_result.for_loop_annotations) == 0 # Convert to Vyper AST. module = vy_ast.get_node(py_ast) assert isinstance(module, vy_ast.Module) # mypy hint - return settings, module + return pre_parse_result.settings, module def ast_to_dict(ast_struct: Union[vy_ast.VyperNode, List]) -> Union[Dict, List]: @@ -118,8 +116,7 @@ def dict_to_ast(ast_struct: Union[Dict, List]) -> Union[vy_ast.VyperNode, List]: def annotate_python_ast( parsed_ast: python_ast.AST, vyper_source: str, - modification_offsets: ModificationOffsets, - for_loop_annotations: dict, + pre_parse_result: PreParseResult, source_id: int = 0, module_path: Optional[str] = None, resolved_path: Optional[str] = None, @@ -133,11 +130,8 @@ def annotate_python_ast( The AST to be annotated and optimized. vyper_source: str The original vyper source code - loop_var_annotations: dict - A mapping of line numbers of `For` nodes to the tokens of the type - annotation of the iterator extracted during pre-parsing. - modification_offsets : dict - A mapping of class names to their original class types. + pre_parse_result: PreParseResult + Outputs from pre-parsing. Returns ------- @@ -148,8 +142,7 @@ def annotate_python_ast( tokens.mark_tokens(parsed_ast) visitor = AnnotatingVisitor( vyper_source, - modification_offsets, - for_loop_annotations, + pre_parse_result, tokens, source_id, module_path=module_path, @@ -162,14 +155,12 @@ def annotate_python_ast( class AnnotatingVisitor(python_ast.NodeTransformer): _source_code: str - _modification_offsets: ModificationOffsets - _loop_var_annotations: dict[int, dict[str, Any]] + _pre_parse_result: PreParseResult def __init__( self, source_code: str, - modification_offsets: ModificationOffsets, - for_loop_annotations: dict, + pre_parse_result: PreParseResult, tokens: asttokens.ASTTokens, source_id: int, module_path: Optional[str] = None, @@ -180,8 +171,7 @@ def __init__( self._module_path = module_path self._resolved_path = resolved_path self._source_code = source_code - self._modification_offsets = modification_offsets - self._for_loop_annotations = for_loop_annotations + self._pre_parse_result = pre_parse_result self.counter: int = 0 @@ -275,7 +265,7 @@ def visit_ClassDef(self, node): """ self.generic_visit(node) - node.ast_type = self._modification_offsets[(node.lineno, node.col_offset)] + node.ast_type = self._pre_parse_result.modification_offsets[(node.lineno, node.col_offset)] return node def visit_For(self, node): @@ -283,7 +273,8 @@ def visit_For(self, node): Visit a For node, splicing in the loop variable annotation provided by the pre-parser """ - annotation_tokens = self._for_loop_annotations.pop((node.lineno, node.col_offset)) + key = (node.lineno, node.col_offset) + annotation_tokens = self._pre_parse_result.for_loop_annotations.pop(key) if not annotation_tokens: # a common case for people migrating to 0.4.0, provide a more @@ -350,14 +341,15 @@ def visit_Expr(self, node): if isinstance(node.value, python_ast.Yield): # CMC 2024-03-03 consider unremoving this from the enclosing Expr node = node.value - node.ast_type = self._modification_offsets[(node.lineno, node.col_offset)] + key = (node.lineno, node.col_offset) + node.ast_type = self._pre_parse_result.modification_offsets[key] return node def visit_Await(self, node): start_pos = node.lineno, node.col_offset # grab these before generic_visit modifies them self.generic_visit(node) - node.ast_type = self._modification_offsets[start_pos] + node.ast_type = self._pre_parse_result.modification_offsets[start_pos] return node def visit_Call(self, node): @@ -401,7 +393,18 @@ def visit_Constant(self, node): if node.value is None or isinstance(node.value, bool): node.ast_type = "NameConstant" elif isinstance(node.value, str): - node.ast_type = "Str" + key = (node.lineno, node.col_offset) + if key in self._pre_parse_result.native_hex_literal_locations: + if len(node.value) % 2 != 0: + raise SyntaxException( + "Native hex string must have an even number of characters", + self._source_code, + node.lineno, + node.col_offset, + ) + node.ast_type = "HexBytes" + else: + node.ast_type = "Str" elif isinstance(node.value, bytes): node.ast_type = "Bytes" elif isinstance(node.value, Ellipsis.__class__): diff --git a/vyper/ast/pre_parser.py b/vyper/ast/pre_parser.py index b12aecd0bf..07ba1d2d0d 100644 --- a/vyper/ast/pre_parser.py +++ b/vyper/ast/pre_parser.py @@ -2,7 +2,7 @@ import io import re from collections import defaultdict -from tokenize import COMMENT, NAME, OP, TokenError, TokenInfo, tokenize, untokenize +from tokenize import COMMENT, NAME, OP, STRING, TokenError, TokenInfo, tokenize, untokenize from packaging.specifiers import InvalidSpecifier, SpecifierSet @@ -12,7 +12,7 @@ # evm-version pragma from vyper.evm.opcodes import EVM_VERSIONS from vyper.exceptions import StructureException, SyntaxException, VersionException -from vyper.typing import ModificationOffsets, ParserPosition +from vyper.typing import ParserPosition def validate_version_pragma(version_str: str, full_source_code: str, start: ParserPosition) -> None: @@ -48,7 +48,7 @@ def validate_version_pragma(version_str: str, full_source_code: str, start: Pars ) -class ForParserState(enum.Enum): +class ParserState(enum.Enum): NOT_RUNNING = enum.auto() START_SOON = enum.auto() RUNNING = enum.auto() @@ -63,7 +63,7 @@ def __init__(self, code): self.annotations = {} self._current_annotation = None - self._state = ForParserState.NOT_RUNNING + self._state = ParserState.NOT_RUNNING self._current_for_loop = None def consume(self, token): @@ -71,15 +71,15 @@ def consume(self, token): if token.type == NAME and token.string == "for": # note: self._state should be NOT_RUNNING here, but we don't sanity # check here as that should be an error the parser will handle. - self._state = ForParserState.START_SOON + self._state = ParserState.START_SOON self._current_for_loop = token.start - if self._state == ForParserState.NOT_RUNNING: + if self._state == ParserState.NOT_RUNNING: return False # state machine: start slurping tokens if token.type == OP and token.string == ":": - self._state = ForParserState.RUNNING + self._state = ParserState.RUNNING # sanity check -- this should never really happen, but if it does, # try to raise an exception which pinpoints the source. @@ -93,12 +93,12 @@ def consume(self, token): # state machine: end slurping tokens if token.type == NAME and token.string == "in": - self._state = ForParserState.NOT_RUNNING + self._state = ParserState.NOT_RUNNING self.annotations[self._current_for_loop] = self._current_annotation or [] self._current_annotation = None return False - if self._state != ForParserState.RUNNING: + if self._state != ParserState.RUNNING: return False # slurp the token @@ -106,6 +106,42 @@ def consume(self, token): return True +class HexStringParser: + def __init__(self): + self.locations = [] + self._current_x = None + self._state = ParserState.NOT_RUNNING + + def consume(self, token, result): + # prepare to check if the next token is a STRING + if token.type == NAME and token.string == "x": + self._state = ParserState.RUNNING + self._current_x = token + return True + + if self._state == ParserState.NOT_RUNNING: + return False + + if self._state == ParserState.RUNNING: + current_x = self._current_x + self._current_x = None + self._state = ParserState.NOT_RUNNING + + toks = [current_x] + + # drop the leading x token if the next token is a STRING to avoid a python + # parser error + if token.type == STRING: + self.locations.append(current_x.start) + toks = [TokenInfo(STRING, token.string, current_x.start, token.end, token.line)] + result.extend(toks) + return True + + result.extend(toks) + + return False + + # compound statements that are replaced with `class` # TODO remove enum in favor of flag VYPER_CLASS_TYPES = { @@ -122,7 +158,34 @@ def consume(self, token): CUSTOM_EXPRESSION_TYPES = {"extcall": "ExtCall", "staticcall": "StaticCall"} -def pre_parse(code: str) -> tuple[Settings, ModificationOffsets, dict, str]: +class PreParseResult: + # Compilation settings based on the directives in the source code + settings: Settings + # A mapping of class names to their original class types. + modification_offsets: dict[tuple[int, int], str] + # A mapping of line/column offsets of `For` nodes to the annotation of the for loop target + for_loop_annotations: dict[tuple[int, int], list[TokenInfo]] + # A list of line/column offsets of native hex literals + native_hex_literal_locations: list[tuple[int, int]] + # Reformatted python source string. + reformatted_code: str + + def __init__( + self, + settings, + modification_offsets, + for_loop_annotations, + native_hex_literal_locations, + reformatted_code, + ): + self.settings = settings + self.modification_offsets = modification_offsets + self.for_loop_annotations = for_loop_annotations + self.native_hex_literal_locations = native_hex_literal_locations + self.reformatted_code = reformatted_code + + +def pre_parse(code: str) -> PreParseResult: """ Re-formats a vyper source string into a python source string and performs some validation. More specifically, @@ -144,19 +207,14 @@ def pre_parse(code: str) -> tuple[Settings, ModificationOffsets, dict, str]: Returns ------- - Settings - Compilation settings based on the directives in the source code - ModificationOffsets - A mapping of class names to their original class types. - dict[tuple[int, int], list[TokenInfo]] - A mapping of line/column offsets of `For` nodes to the annotation of the for loop target - str - Reformatted python source string. + PreParseResult + Outputs for transforming the python AST to vyper AST """ - result = [] - modification_offsets: ModificationOffsets = {} + result: list[TokenInfo] = [] + modification_offsets: dict[tuple[int, int], str] = {} settings = Settings() for_parser = ForParser(code) + native_hex_parser = HexStringParser() _col_adjustments: dict[int, int] = defaultdict(lambda: 0) @@ -264,7 +322,7 @@ def pre_parse(code: str) -> tuple[Settings, ModificationOffsets, dict, str]: if (typ, string) == (OP, ";"): raise SyntaxException("Semi-colon statements not allowed", code, start[0], start[1]) - if not for_parser.consume(token): + if not for_parser.consume(token) and not native_hex_parser.consume(token, result): result.extend(toks) except TokenError as e: @@ -274,4 +332,10 @@ def pre_parse(code: str) -> tuple[Settings, ModificationOffsets, dict, str]: for k, v in for_parser.annotations.items(): for_loop_annotations[k] = v.copy() - return settings, modification_offsets, for_loop_annotations, untokenize(result).decode("utf-8") + return PreParseResult( + settings, + modification_offsets, + for_loop_annotations, + native_hex_parser.locations, + untokenize(result).decode("utf-8"), + ) diff --git a/vyper/codegen/expr.py b/vyper/codegen/expr.py index 0b3b29b9d0..cd51966710 100644 --- a/vyper/codegen/expr.py +++ b/vyper/codegen/expr.py @@ -140,6 +140,12 @@ def parse_Str(self): # Byte literals def parse_Bytes(self): + return self._parse_bytes() + + def parse_HexBytes(self): + return self._parse_bytes() + + def _parse_bytes(self): bytez = self.expr.value bytez_length = len(self.expr.value) typ = BytesT(bytez_length) diff --git a/vyper/codegen/stmt.py b/vyper/codegen/stmt.py index 225cede747..b1e26d7d5f 100644 --- a/vyper/codegen/stmt.py +++ b/vyper/codegen/stmt.py @@ -93,7 +93,13 @@ def parse_If(self): def parse_Log(self): event = self.stmt._metadata["type"] - args = [Expr(arg, self.context).ir_node for arg in self.stmt.value.args] + if len(self.stmt.value.keywords) > 0: + # keyword arguments + to_compile = [arg.value for arg in self.stmt.value.keywords] + else: + # positional arguments + to_compile = self.stmt.value.args + args = [Expr(arg, self.context).ir_node for arg in to_compile] topic_ir = [] data_ir = [] diff --git a/vyper/compiler/output.py b/vyper/compiler/output.py index 577afd3822..f1be894e58 100644 --- a/vyper/compiler/output.py +++ b/vyper/compiler/output.py @@ -3,7 +3,8 @@ from collections import deque from pathlib import PurePath -from vyper.ast import ast_to_dict +import vyper.ast as vy_ast +from vyper.ast.utils import ast_to_dict from vyper.codegen.ir_node import IRnode from vyper.compiler.output_bundle import SolcJSONWriter, VyperArchiveWriter from vyper.compiler.phases import CompilerData @@ -11,7 +12,9 @@ from vyper.evm import opcodes from vyper.exceptions import VyperException from vyper.ir import compile_ir +from vyper.semantics.analysis.base import ModuleInfo from vyper.semantics.types.function import FunctionVisibility, StateMutability +from vyper.semantics.types.module import InterfaceT from vyper.typing import StorageLayout from vyper.utils import vyper_warn from vyper.warnings import ContractSizeLimitWarning @@ -26,9 +29,32 @@ def build_ast_dict(compiler_data: CompilerData) -> dict: def build_annotated_ast_dict(compiler_data: CompilerData) -> dict: + module_t = compiler_data.annotated_vyper_module._metadata["type"] + # get all reachable imports including recursion + imported_module_infos = module_t.reachable_imports + unique_modules: dict[str, vy_ast.Module] = {} + for info in imported_module_infos: + if isinstance(info.typ, InterfaceT): + ast = info.typ.decl_node + if ast is None: # json abi + continue + else: + assert isinstance(info.typ, ModuleInfo) + ast = info.typ.module_t._module + + assert isinstance(ast, vy_ast.Module) # help mypy + # use resolved_path for uniqueness, since Module objects can actually + # come from multiple InputBundles (particularly builtin interfaces), + # so source_id is not guaranteed to be unique. + if ast.resolved_path in unique_modules: + # sanity check -- objects must be identical + assert unique_modules[ast.resolved_path] is ast + unique_modules[ast.resolved_path] = ast + annotated_ast_dict = { "contract_name": str(compiler_data.contract_path), "ast": ast_to_dict(compiler_data.annotated_vyper_module), + "imports": [ast_to_dict(ast) for ast in unique_modules.values()], } return annotated_ast_dict @@ -102,22 +128,33 @@ def build_interface_output(compiler_data: CompilerData) -> str: interface = compiler_data.annotated_vyper_module._metadata["type"].interface out = "" - if interface.events: - out = "# Events\n\n" + if len(interface.structs) > 0: + out += "# Structs\n\n" + for struct in interface.structs.values(): + out += f"struct {struct.name}:\n" + for member_name, member_type in struct.members.items(): + out += f" {member_name}: {member_type}\n" + out += "\n\n" + + if len(interface.events) > 0: + out += "# Events\n\n" for event in interface.events.values(): encoded_args = "\n ".join(f"{name}: {typ}" for name, typ in event.arguments.items()) - out = f"{out}event {event.name}:\n {encoded_args if event.arguments else 'pass'}\n" + out += f"event {event.name}:\n {encoded_args if event.arguments else 'pass'}\n\n\n" - if interface.functions: - out = f"{out}\n# Functions\n\n" + if len(interface.functions) > 0: + out += "# Functions\n\n" for func in interface.functions.values(): if func.visibility == FunctionVisibility.INTERNAL or func.name == "__init__": continue if func.mutability != StateMutability.NONPAYABLE: - out = f"{out}@{func.mutability.value}\n" + out += f"@{func.mutability.value}\n" args = ", ".join([f"{arg.name}: {arg.typ}" for arg in func.arguments]) return_value = f" -> {func.return_type}" if func.return_type is not None else "" - out = f"{out}@external\ndef {func.name}({args}){return_value}:\n ...\n\n" + out += f"@external\ndef {func.name}({args}){return_value}:\n ...\n\n\n" + + out = out.rstrip("\n") + out += "\n" return out @@ -320,15 +357,13 @@ def _build_source_map_output(compiler_data, bytecode, pc_maps): def build_source_map_output(compiler_data: CompilerData) -> dict: - bytecode, pc_maps = compile_ir.assembly_to_evm( - compiler_data.assembly, insert_compiler_metadata=False - ) + bytecode, pc_maps = compile_ir.assembly_to_evm(compiler_data.assembly, compiler_metadata=None) return _build_source_map_output(compiler_data, bytecode, pc_maps) def build_source_map_runtime_output(compiler_data: CompilerData) -> dict: bytecode, pc_maps = compile_ir.assembly_to_evm( - compiler_data.assembly_runtime, insert_compiler_metadata=False + compiler_data.assembly_runtime, compiler_metadata=None ) return _build_source_map_output(compiler_data, bytecode, pc_maps) diff --git a/vyper/compiler/output_bundle.py b/vyper/compiler/output_bundle.py index 92494e3a70..06a84064a1 100644 --- a/vyper/compiler/output_bundle.py +++ b/vyper/compiler/output_bundle.py @@ -1,7 +1,6 @@ import importlib import io import json -import os import zipfile from dataclasses import dataclass from functools import cached_property @@ -13,7 +12,7 @@ from vyper.compiler.settings import Settings from vyper.exceptions import CompilerPanic from vyper.semantics.analysis.module import _is_builtin -from vyper.utils import get_long_version +from vyper.utils import get_long_version, safe_relpath # data structures and routines for constructing "output bundles", # basically reproducible builds of a vyper contract, with varying @@ -62,7 +61,7 @@ def compiler_inputs(self) -> dict[str, CompilerInput]: sources = {} for c in inputs: - path = os.path.relpath(c.resolved_path) + path = safe_relpath(c.resolved_path) # note: there should be a 1:1 correspondence between # resolved_path and source_id, but for clarity use resolved_path # since it corresponds more directly to search path semantics. @@ -73,7 +72,7 @@ def compiler_inputs(self) -> dict[str, CompilerInput]: @cached_property def compilation_target_path(self): p = PurePath(self.compiler_data.file_input.resolved_path) - p = os.path.relpath(p) + p = safe_relpath(p) return _anonymize(p) @cached_property @@ -121,7 +120,7 @@ def used_search_paths(self) -> list[str]: sps = [sp for sp, count in tmp.items() if count > 0] assert len(sps) > 0 - return [_anonymize(os.path.relpath(sp)) for sp in sps] + return [_anonymize(safe_relpath(sp)) for sp in sps] class OutputBundleWriter: diff --git a/vyper/compiler/phases.py b/vyper/compiler/phases.py index 147af24d67..97df73cdae 100644 --- a/vyper/compiler/phases.py +++ b/vyper/compiler/phases.py @@ -2,7 +2,7 @@ import warnings from functools import cached_property from pathlib import Path, PurePath -from typing import Optional +from typing import Any, Optional from vyper import ast as vy_ast from vyper.ast import natspec @@ -249,12 +249,15 @@ def assembly_runtime(self) -> list: @cached_property def bytecode(self) -> bytes: - insert_compiler_metadata = not self.no_bytecode_metadata - return generate_bytecode(self.assembly, insert_compiler_metadata=insert_compiler_metadata) + metadata = None + if not self.no_bytecode_metadata: + module_t = self.compilation_target._metadata["type"] + metadata = bytes.fromhex(module_t.integrity_sum) + return generate_bytecode(self.assembly, compiler_metadata=metadata) @cached_property def bytecode_runtime(self) -> bytes: - return generate_bytecode(self.assembly_runtime, insert_compiler_metadata=False) + return generate_bytecode(self.assembly_runtime, compiler_metadata=None) @cached_property def blueprint_bytecode(self) -> bytes: @@ -351,7 +354,7 @@ def _find_nested_opcode(assembly, key): return any(_find_nested_opcode(x, key) for x in sublists) -def generate_bytecode(assembly: list, insert_compiler_metadata: bool) -> bytes: +def generate_bytecode(assembly: list, compiler_metadata: Optional[Any]) -> bytes: """ Generate bytecode from assembly instructions. @@ -365,6 +368,4 @@ def generate_bytecode(assembly: list, insert_compiler_metadata: bool) -> bytes: bytes Final compiled bytecode. """ - return compile_ir.assembly_to_evm(assembly, insert_compiler_metadata=insert_compiler_metadata)[ - 0 - ] + return compile_ir.assembly_to_evm(assembly, compiler_metadata=compiler_metadata)[0] diff --git a/vyper/ir/compile_ir.py b/vyper/ir/compile_ir.py index 4c68aa2c8f..2cc951b188 100644 --- a/vyper/ir/compile_ir.py +++ b/vyper/ir/compile_ir.py @@ -1155,22 +1155,24 @@ def _relocate_segments(assembly): # TODO: change API to split assembly_to_evm and assembly_to_source/symbol_maps -def assembly_to_evm(assembly, pc_ofst=0, insert_compiler_metadata=False): +def assembly_to_evm(assembly, pc_ofst=0, compiler_metadata=None): bytecode, source_maps, _ = assembly_to_evm_with_symbol_map( - assembly, pc_ofst=pc_ofst, insert_compiler_metadata=insert_compiler_metadata + assembly, pc_ofst=pc_ofst, compiler_metadata=compiler_metadata ) return bytecode, source_maps -def assembly_to_evm_with_symbol_map(assembly, pc_ofst=0, insert_compiler_metadata=False): +def assembly_to_evm_with_symbol_map(assembly, pc_ofst=0, compiler_metadata=None): """ Assembles assembly into EVM assembly: list of asm instructions pc_ofst: when constructing the source map, the amount to offset all pcs by (no effect until we add deploy code source map) - insert_compiler_metadata: whether to append vyper metadata to output - (should be true for runtime code) + compiler_metadata: any compiler metadata to add. pass `None` to indicate + no metadata to be added (should always be `None` for + runtime code). the value is opaque, and will be passed + directly to `cbor2.dumps()`. """ line_number_map = { "breakpoints": set(), @@ -1278,10 +1280,11 @@ def assembly_to_evm_with_symbol_map(assembly, pc_ofst=0, insert_compiler_metadat pc += 1 bytecode_suffix = b"" - if insert_compiler_metadata: + if compiler_metadata is not None: # this will hold true when we are in initcode assert immutables_len is not None metadata = ( + compiler_metadata, len(runtime_code), data_section_lengths, immutables_len, diff --git a/vyper/semantics/analysis/base.py b/vyper/semantics/analysis/base.py index 65bc8df3ab..982b6eb01d 100644 --- a/vyper/semantics/analysis/base.py +++ b/vyper/semantics/analysis/base.py @@ -1,5 +1,5 @@ import enum -from dataclasses import dataclass +from dataclasses import dataclass, fields from functools import cached_property from typing import TYPE_CHECKING, Any, ClassVar, Dict, Optional, Union @@ -234,6 +234,17 @@ class VarAccess: # A sentinel indicating a subscript access SUBSCRIPT_ACCESS: ClassVar[Any] = object() + # custom __reduce__ and _produce implementations to work around + # a pickle bug. + # see https://github.com/python/cpython/issues/124937#issuecomment-2392227290 + def __reduce__(self): + dict_obj = {f.name: getattr(self, f.name) for f in fields(self)} + return self.__class__._produce, (dict_obj,) + + @classmethod + def _produce(cls, data): + return cls(**data) + @cached_property def attrs(self): ret = [] @@ -286,7 +297,6 @@ def __post_init__(self): for attr in should_match: if getattr(self.var_info, attr) != getattr(self, attr): raise CompilerPanic(f"Bad analysis: non-matching {attr}: {self}") - self._writes: OrderedSet[VarAccess] = OrderedSet() self._reads: OrderedSet[VarAccess] = OrderedSet() diff --git a/vyper/semantics/analysis/local.py b/vyper/semantics/analysis/local.py index b5292b1dad..809c6532c6 100644 --- a/vyper/semantics/analysis/local.py +++ b/vyper/semantics/analysis/local.py @@ -810,13 +810,17 @@ def visit_Call(self, node: vy_ast.Call, typ: VyperType) -> None: self.visit(kwarg.value, typ) elif is_type_t(func_type, EventT): - # events have no kwargs + # event ctors expected_types = func_type.typedef.arguments.values() # type: ignore - for arg, typ in zip(node.args, expected_types): - self.visit(arg, typ) + # Handle keyword args if present, otherwise use positional args + if len(node.keywords) > 0: + for kwarg, arg_type in zip(node.keywords, expected_types): + self.visit(kwarg.value, arg_type) + else: + for arg, typ in zip(node.args, expected_types): + self.visit(arg, typ) elif is_type_t(func_type, StructT): # struct ctors - # ctors have no kwargs expected_types = func_type.typedef.members.values() # type: ignore for kwarg, arg_type in zip(node.keywords, expected_types): self.visit(kwarg.value, arg_type) diff --git a/vyper/semantics/analysis/module.py b/vyper/semantics/analysis/module.py index d05e494b80..6816fbed98 100644 --- a/vyper/semantics/analysis/module.py +++ b/vyper/semantics/analysis/module.py @@ -1,4 +1,3 @@ -import os from pathlib import Path, PurePath from typing import Any, Optional @@ -58,7 +57,7 @@ from vyper.semantics.types.function import ContractFunctionT from vyper.semantics.types.module import ModuleT from vyper.semantics.types.utils import type_from_annotation -from vyper.utils import OrderedSet +from vyper.utils import OrderedSet, safe_relpath def analyze_module( @@ -898,7 +897,7 @@ def _import_to_path(level: int, module_str: str) -> PurePath: base_path = "../" * (level - 1) elif level == 1: base_path = "./" - return PurePath(f"{base_path}{module_str.replace('.','/')}/") + return PurePath(f"{base_path}{module_str.replace('.', '/')}/") # can add more, e.g. "vyper.builtins.interfaces", etc. @@ -921,7 +920,7 @@ def _load_builtin_import(level: int, module_str: str) -> tuple[CompilerInput, In # hygiene: convert to relpath to avoid leaking user directory info # (note Path.relative_to cannot handle absolute to relative path # conversion, so we must use the `os` module). - builtins_path = os.path.relpath(builtins_path) + builtins_path = safe_relpath(builtins_path) search_path = Path(builtins_path).parent.parent.parent # generate an input bundle just because it knows how to build paths. diff --git a/vyper/semantics/analysis/utils.py b/vyper/semantics/analysis/utils.py index d30eee79e0..9734087fc3 100644 --- a/vyper/semantics/analysis/utils.py +++ b/vyper/semantics/analysis/utils.py @@ -1,9 +1,11 @@ import itertools -from typing import Callable, Iterable, List +from typing import Any, Callable, Iterable, List from vyper import ast as vy_ast from vyper.exceptions import ( CompilerPanic, + InstantiationException, + InvalidAttribute, InvalidLiteral, InvalidOperation, InvalidReference, @@ -694,3 +696,43 @@ def get_expr_writes(node: vy_ast.VyperNode) -> OrderedSet[VarAccess]: ret |= get_expr_writes(c) node._metadata["writes_r"] = ret return ret + + +def validate_kwargs(node: vy_ast.Call, members: dict[str, VyperType], typeclass: str): + # manually validate kwargs for better error messages instead of + # relying on `validate_call_args` + + seen: dict[str, vy_ast.keyword] = {} + membernames = list(members.keys()) + + # check duplicate kwargs + for i, kwarg in enumerate(node.keywords): + # x=5 => kwarg(arg="x", value=Int(5)) + argname = kwarg.arg + if argname in seen: + prev = seen[argname] + raise InvalidAttribute(f"Duplicate {typeclass} argument", prev, kwarg) + seen[argname] = kwarg + + hint: Any # mypy kludge + if argname not in members: + hint = get_levenshtein_error_suggestions(argname, members, 1.0) + raise UnknownAttribute(f"Unknown {typeclass} argument.", kwarg, hint=hint) + + expect_name = membernames[i] + if argname != expect_name: + # out of order key + msg = f"{typeclass} keys are required to be in order, but got" + msg += f" `{argname}` instead of `{expect_name}`." + hint = "as a reminder, the order of the keys in this" + hint += f" {typeclass} are {list(members)}" + raise InvalidAttribute(msg, kwarg, hint=hint) + + expected_type = members[argname] + validate_expected_type(kwarg.value, expected_type) + + missing = OrderedSet(members.keys()) - OrderedSet(seen.keys()) + if len(missing) > 0: + msg = f"{typeclass} instantiation missing fields:" + msg += f" {', '.join(list(missing))}" + raise InstantiationException(msg, node) diff --git a/vyper/semantics/types/bytestrings.py b/vyper/semantics/types/bytestrings.py index cd330681cf..02e3bb213f 100644 --- a/vyper/semantics/types/bytestrings.py +++ b/vyper/semantics/types/bytestrings.py @@ -159,7 +159,7 @@ class BytesT(_BytestringT): typeclass = "bytes" _id = "Bytes" - _valid_literal = (vy_ast.Bytes,) + _valid_literal = (vy_ast.Bytes, vy_ast.HexBytes) @property def abi_type(self) -> ABIType: diff --git a/vyper/semantics/types/user.py b/vyper/semantics/types/user.py index ca8e99bc92..73fa4878c7 100644 --- a/vyper/semantics/types/user.py +++ b/vyper/semantics/types/user.py @@ -7,21 +7,23 @@ from vyper.exceptions import ( EventDeclarationException, FlagDeclarationException, - InvalidAttribute, + InstantiationException, NamespaceCollision, StructureException, UnfoldableNode, - UnknownAttribute, VariableDeclarationException, ) from vyper.semantics.analysis.base import Modifiability -from vyper.semantics.analysis.levenshtein_utils import get_levenshtein_error_suggestions -from vyper.semantics.analysis.utils import check_modifiability, validate_expected_type +from vyper.semantics.analysis.utils import ( + check_modifiability, + validate_expected_type, + validate_kwargs, +) from vyper.semantics.data_locations import DataLocation from vyper.semantics.types.base import VyperType from vyper.semantics.types.subscriptable import HashMapT from vyper.semantics.types.utils import type_from_abi, type_from_annotation -from vyper.utils import keccak256 +from vyper.utils import keccak256, vyper_warn # user defined type @@ -281,6 +283,25 @@ def from_EventDef(cls, base_node: vy_ast.EventDef) -> "EventT": return cls(base_node.name, members, indexed, base_node) def _ctor_call_return(self, node: vy_ast.Call) -> None: + # validate keyword arguments if provided + if len(node.keywords) > 0: + if len(node.args) > 0: + raise InstantiationException( + "Event instantiation requires either all keyword arguments " + "or all positional arguments", + node, + ) + + return validate_kwargs(node, self.arguments, self.typeclass) + + # warn about positional argument depreciation + msg = "Instantiating events with positional arguments is " + msg += "deprecated as of v0.4.1 and will be disallowed " + msg += "in a future release. Use kwargs instead eg. " + msg += "Foo(a=1, b=2)" + + vyper_warn(msg, node) + validate_call_args(node, len(self.arguments)) for arg, expected in zip(node.args, self.arguments.values()): validate_expected_type(arg, expected) @@ -415,31 +436,7 @@ def _ctor_call_return(self, node: vy_ast.Call) -> "StructT": "Struct contains a mapping and so cannot be declared as a literal", node ) - # manually validate kwargs for better error messages instead of - # relying on `validate_call_args` - members = self.member_types.copy() - keys = list(self.member_types.keys()) - for i, kwarg in enumerate(node.keywords): - # x=5 => kwarg(arg="x", value=Int(5)) - argname = kwarg.arg - if argname not in members: - hint = get_levenshtein_error_suggestions(argname, members, 1.0) - raise UnknownAttribute("Unknown or duplicate struct member.", kwarg, hint=hint) - expected = keys[i] - if argname != expected: - raise InvalidAttribute( - "Struct keys are required to be in order, but got " - f"`{argname}` instead of `{expected}`. (Reminder: the " - f"keys in this struct are {list(self.member_types.items())})", - kwarg, - ) - expected_type = members.pop(argname) - validate_expected_type(kwarg.value, expected_type) - - if members: - raise VariableDeclarationException( - f"Struct declaration does not define all fields: {', '.join(list(members))}", node - ) + validate_kwargs(node, self.member_types, self.typeclass) return self diff --git a/vyper/typing.py b/vyper/typing.py index ad3964dff9..108c0605bb 100644 --- a/vyper/typing.py +++ b/vyper/typing.py @@ -1,7 +1,6 @@ from typing import Dict, Optional, Sequence, Tuple, Union # Parser -ModificationOffsets = Dict[Tuple[int, int], str] ParserPosition = Tuple[int, int] # Compiler diff --git a/vyper/utils.py b/vyper/utils.py index 3f19a9d15c..d635c78383 100644 --- a/vyper/utils.py +++ b/vyper/utils.py @@ -4,6 +4,7 @@ import enum import functools import hashlib +import os import sys import time import traceback @@ -88,6 +89,7 @@ def update(self, other): def union(self, other): return self | other + # set dunders def __ior__(self, other): self.update(other) return self @@ -100,6 +102,15 @@ def __or__(self, other): def __eq__(self, other): return self._data == other._data + def __isub__(self, other): + self.dropmany(other) + return self + + def __sub__(self, other): + ret = self.copy() + ret.dropmany(other) + return ret + def copy(self): cls = self.__class__ ret = cls.__new__(cls) @@ -599,3 +610,12 @@ def annotate_source_code( cleanup_lines += [""] * (num_lines - len(cleanup_lines)) return "\n".join(cleanup_lines) + + +def safe_relpath(path): + try: + return os.path.relpath(path) + except ValueError: + # on Windows, if path and curdir are on different drives, an exception + # can be thrown + return path diff --git a/vyper/venom/__init__.py b/vyper/venom/__init__.py index afd79fc44f..310147baa7 100644 --- a/vyper/venom/__init__.py +++ b/vyper/venom/__init__.py @@ -9,16 +9,18 @@ from vyper.venom.context import IRContext from vyper.venom.function import IRFunction from vyper.venom.ir_node_to_venom import ir_node_to_venom -from vyper.venom.passes.algebraic_optimization import AlgebraicOptimizationPass -from vyper.venom.passes.branch_optimization import BranchOptimizationPass -from vyper.venom.passes.dft import DFTPass -from vyper.venom.passes.extract_literals import ExtractLiteralsPass -from vyper.venom.passes.make_ssa import MakeSSA -from vyper.venom.passes.mem2var import Mem2Var -from vyper.venom.passes.remove_unused_variables import RemoveUnusedVariablesPass -from vyper.venom.passes.sccp import SCCP -from vyper.venom.passes.simplify_cfg import SimplifyCFGPass -from vyper.venom.passes.store_elimination import StoreElimination +from vyper.venom.passes import ( + SCCP, + AlgebraicOptimizationPass, + BranchOptimizationPass, + DFTPass, + MakeSSA, + Mem2Var, + RemoveUnusedVariablesPass, + SimplifyCFGPass, + StoreElimination, + StoreExpansionPass, +) from vyper.venom.venom_to_assembly import VenomCompiler DEFAULT_OPT_LEVEL = OptimizationLevel.default() @@ -54,8 +56,9 @@ def _run_passes(fn: IRFunction, optimize: OptimizationLevel) -> None: SimplifyCFGPass(ac, fn).run_pass() AlgebraicOptimizationPass(ac, fn).run_pass() BranchOptimizationPass(ac, fn).run_pass() - ExtractLiteralsPass(ac, fn).run_pass() RemoveUnusedVariablesPass(ac, fn).run_pass() + + StoreExpansionPass(ac, fn).run_pass() DFTPass(ac, fn).run_pass() diff --git a/vyper/venom/analysis/__init__.py b/vyper/venom/analysis/__init__.py index e69de29bb2..4870de3fb7 100644 --- a/vyper/venom/analysis/__init__.py +++ b/vyper/venom/analysis/__init__.py @@ -0,0 +1,6 @@ +from .analysis import IRAnalysesCache, IRAnalysis +from .cfg import CFGAnalysis +from .dfg import DFGAnalysis +from .dominators import DominatorTreeAnalysis +from .equivalent_vars import VarEquivalenceAnalysis +from .liveness import LivenessAnalysis diff --git a/vyper/venom/analysis/cfg.py b/vyper/venom/analysis/cfg.py index bd2ae34b68..e4f130bc18 100644 --- a/vyper/venom/analysis/cfg.py +++ b/vyper/venom/analysis/cfg.py @@ -1,5 +1,5 @@ from vyper.utils import OrderedSet -from vyper.venom.analysis.analysis import IRAnalysis +from vyper.venom.analysis import IRAnalysis from vyper.venom.basicblock import CFG_ALTERING_INSTRUCTIONS @@ -32,8 +32,7 @@ def analyze(self) -> None: in_bb.add_cfg_out(bb) def invalidate(self): - from vyper.venom.analysis.dominators import DominatorTreeAnalysis - from vyper.venom.analysis.liveness import LivenessAnalysis + from vyper.venom.analysis import DominatorTreeAnalysis, LivenessAnalysis self.analyses_cache.invalidate_analysis(DominatorTreeAnalysis) self.analyses_cache.invalidate_analysis(LivenessAnalysis) diff --git a/vyper/venom/analysis/dominators.py b/vyper/venom/analysis/dominators.py index 129d1d0f22..e360df36b9 100644 --- a/vyper/venom/analysis/dominators.py +++ b/vyper/venom/analysis/dominators.py @@ -1,7 +1,6 @@ from vyper.exceptions import CompilerPanic from vyper.utils import OrderedSet -from vyper.venom.analysis.analysis import IRAnalysis -from vyper.venom.analysis.cfg import CFGAnalysis +from vyper.venom.analysis import CFGAnalysis, IRAnalysis from vyper.venom.basicblock import IRBasicBlock from vyper.venom.function import IRFunction diff --git a/vyper/venom/analysis/dup_requirements.py b/vyper/venom/analysis/dup_requirements.py deleted file mode 100644 index 7afb315035..0000000000 --- a/vyper/venom/analysis/dup_requirements.py +++ /dev/null @@ -1,15 +0,0 @@ -from vyper.utils import OrderedSet -from vyper.venom.analysis.analysis import IRAnalysis - - -class DupRequirementsAnalysis(IRAnalysis): - def analyze(self): - for bb in self.function.get_basic_blocks(): - last_liveness = bb.out_vars - for inst in reversed(bb.instructions): - inst.dup_requirements = OrderedSet() - ops = inst.get_input_variables() - for op in ops: - if op in last_liveness: - inst.dup_requirements.add(op) - last_liveness = inst.liveness diff --git a/vyper/venom/analysis/equivalent_vars.py b/vyper/venom/analysis/equivalent_vars.py new file mode 100644 index 0000000000..895895651a --- /dev/null +++ b/vyper/venom/analysis/equivalent_vars.py @@ -0,0 +1,40 @@ +from vyper.venom.analysis import DFGAnalysis, IRAnalysis +from vyper.venom.basicblock import IRVariable + + +class VarEquivalenceAnalysis(IRAnalysis): + """ + Generate equivalence sets of variables. This is used to avoid swapping + variables which are the same during venom_to_assembly. Theoretically, + the DFTPass should order variable declarations optimally, but, it is + not aware of the "pickaxe" heuristic in venom_to_assembly, so they can + interfere. + """ + + def analyze(self): + dfg = self.analyses_cache.request_analysis(DFGAnalysis) + + equivalence_set: dict[IRVariable, int] = {} + + for bag, (var, inst) in enumerate(dfg._dfg_outputs.items()): + if inst.opcode != "store": + continue + + source = inst.operands[0] + + assert var not in equivalence_set # invariant + if source in equivalence_set: + equivalence_set[var] = equivalence_set[source] + continue + else: + equivalence_set[var] = bag + equivalence_set[source] = bag + + self._equivalence_set = equivalence_set + + def equivalent(self, var1, var2): + if var1 not in self._equivalence_set: + return False + if var2 not in self._equivalence_set: + return False + return self._equivalence_set[var1] == self._equivalence_set[var2] diff --git a/vyper/venom/analysis/liveness.py b/vyper/venom/analysis/liveness.py index 2a471bc8be..b5d65961b7 100644 --- a/vyper/venom/analysis/liveness.py +++ b/vyper/venom/analysis/liveness.py @@ -2,8 +2,7 @@ from vyper.exceptions import CompilerPanic from vyper.utils import OrderedSet -from vyper.venom.analysis.analysis import IRAnalysis -from vyper.venom.analysis.cfg import CFGAnalysis +from vyper.venom.analysis import CFGAnalysis, IRAnalysis from vyper.venom.basicblock import IRBasicBlock, IRVariable diff --git a/vyper/venom/basicblock.py b/vyper/venom/basicblock.py index d6fb9560cd..799dcfb33b 100644 --- a/vyper/venom/basicblock.py +++ b/vyper/venom/basicblock.py @@ -1,5 +1,6 @@ from typing import TYPE_CHECKING, Any, Iterator, Optional, Union +import vyper.venom.effects as effects from vyper.codegen.ir_node import IRnode from vyper.utils import OrderedSet @@ -21,8 +22,6 @@ "istore", "tload", "tstore", - "assert", - "assert_unreachable", "mstore", "mload", "calldatacopy", @@ -209,7 +208,6 @@ class IRInstruction: output: Optional[IROperand] # set of live variables at this instruction liveness: OrderedSet[IRVariable] - dup_requirements: OrderedSet[IRVariable] parent: "IRBasicBlock" fence_id: int annotation: Optional[str] @@ -228,7 +226,6 @@ def __init__( self.operands = list(operands) # in case we get an iterator self.output = output self.liveness = OrderedSet() - self.dup_requirements = OrderedSet() self.fence_id = -1 self.annotation = None self.ast_source = None @@ -242,6 +239,12 @@ def is_volatile(self) -> bool: def is_bb_terminator(self) -> bool: return self.opcode in BB_TERMINATORS + def get_read_effects(self): + return effects.reads.get(self.opcode, effects.EMPTY) + + def get_write_effects(self): + return effects.writes.get(self.opcode, effects.EMPTY) + def get_label_operands(self) -> Iterator[IRLabel]: """ Get all labels in instruction. diff --git a/vyper/venom/effects.py b/vyper/venom/effects.py new file mode 100644 index 0000000000..20cc0e4b02 --- /dev/null +++ b/vyper/venom/effects.py @@ -0,0 +1,85 @@ +from enum import Flag, auto + + +class Effects(Flag): + STORAGE = auto() + TRANSIENT = auto() + MEMORY = auto() + MSIZE = auto() + IMMUTABLES = auto() + RETURNDATA = auto() + LOG = auto() + BALANCE = auto() + EXTCODE = auto() + + +EMPTY = Effects(0) +ALL = ~EMPTY +STORAGE = Effects.STORAGE +TRANSIENT = Effects.TRANSIENT +MEMORY = Effects.MEMORY +MSIZE = Effects.MSIZE +IMMUTABLES = Effects.IMMUTABLES +RETURNDATA = Effects.RETURNDATA +LOG = Effects.LOG +BALANCE = Effects.BALANCE +EXTCODE = Effects.EXTCODE + + +_writes = { + "sstore": STORAGE, + "tstore": TRANSIENT, + "mstore": MEMORY, + "istore": IMMUTABLES, + "call": ALL ^ IMMUTABLES, + "delegatecall": ALL ^ IMMUTABLES, + "staticcall": MEMORY | RETURNDATA, + "create": ALL ^ (MEMORY | IMMUTABLES), + "create2": ALL ^ (MEMORY | IMMUTABLES), + "invoke": ALL, # could be smarter, look up the effects of the invoked function + "log": LOG, + "dloadbytes": MEMORY, + "returndatacopy": MEMORY, + "calldatacopy": MEMORY, + "codecopy": MEMORY, + "extcodecopy": MEMORY, + "mcopy": MEMORY, +} + +_reads = { + "sload": STORAGE, + "tload": TRANSIENT, + "iload": IMMUTABLES, + "mload": MEMORY, + "mcopy": MEMORY, + "call": ALL, + "delegatecall": ALL, + "staticcall": ALL, + "create": ALL, + "create2": ALL, + "invoke": ALL, + "returndatasize": RETURNDATA, + "returndatacopy": RETURNDATA, + "balance": BALANCE, + "selfbalance": BALANCE, + "extcodecopy": EXTCODE, + "selfdestruct": BALANCE, # may modify code, but after the transaction + "log": MEMORY, + "revert": MEMORY, + "return": MEMORY, + "sha3": MEMORY, + "msize": MSIZE, +} + +reads = _reads.copy() +writes = _writes.copy() + +for k, v in reads.items(): + if MEMORY in v: + if k not in writes: + writes[k] = EMPTY + writes[k] |= MSIZE + +for k, v in writes.items(): + if MEMORY in v: + writes[k] |= MSIZE diff --git a/vyper/venom/ir_node_to_venom.py b/vyper/venom/ir_node_to_venom.py index 85172c70e1..e30f27f480 100644 --- a/vyper/venom/ir_node_to_venom.py +++ b/vyper/venom/ir_node_to_venom.py @@ -548,7 +548,7 @@ def emit_body_blocks(): _global_symbols[ir.value] = ptr elif ir.value.startswith("$palloca") and ir.value not in _global_symbols: alloca = ir.passthrough_metadata["alloca"] - ptr = fn.get_basic_block().append_instruction("store", alloca.offset) + ptr = fn.get_basic_block().append_instruction("palloca", alloca.offset, alloca.size) _global_symbols[ir.value] = ptr return _global_symbols.get(ir.value) or symbols.get(ir.value) diff --git a/vyper/venom/passes/__init__.py b/vyper/venom/passes/__init__.py new file mode 100644 index 0000000000..83098234c1 --- /dev/null +++ b/vyper/venom/passes/__init__.py @@ -0,0 +1,11 @@ +from .algebraic_optimization import AlgebraicOptimizationPass +from .branch_optimization import BranchOptimizationPass +from .dft import DFTPass +from .make_ssa import MakeSSA +from .mem2var import Mem2Var +from .normalization import NormalizationPass +from .remove_unused_variables import RemoveUnusedVariablesPass +from .sccp import SCCP +from .simplify_cfg import SimplifyCFGPass +from .store_elimination import StoreElimination +from .store_expansion import StoreExpansionPass diff --git a/vyper/venom/passes/algebraic_optimization.py b/vyper/venom/passes/algebraic_optimization.py index 1d375ea988..5d4291667e 100644 --- a/vyper/venom/passes/algebraic_optimization.py +++ b/vyper/venom/passes/algebraic_optimization.py @@ -1,5 +1,4 @@ -from vyper.venom.analysis.dfg import DFGAnalysis -from vyper.venom.analysis.liveness import LivenessAnalysis +from vyper.venom.analysis import DFGAnalysis, LivenessAnalysis from vyper.venom.basicblock import IRInstruction, IRLabel, IRLiteral, IROperand from vyper.venom.passes.base_pass import IRPass diff --git a/vyper/venom/passes/base_pass.py b/vyper/venom/passes/base_pass.py index 4d1bfe9647..3951ac4455 100644 --- a/vyper/venom/passes/base_pass.py +++ b/vyper/venom/passes/base_pass.py @@ -1,4 +1,4 @@ -from vyper.venom.analysis.analysis import IRAnalysesCache +from vyper.venom.analysis import IRAnalysesCache from vyper.venom.function import IRFunction diff --git a/vyper/venom/passes/branch_optimization.py b/vyper/venom/passes/branch_optimization.py index 354aab7900..d5b0ed9809 100644 --- a/vyper/venom/passes/branch_optimization.py +++ b/vyper/venom/passes/branch_optimization.py @@ -1,4 +1,4 @@ -from vyper.venom.analysis.dfg import DFGAnalysis +from vyper.venom.analysis import DFGAnalysis from vyper.venom.passes.base_pass import IRPass diff --git a/vyper/venom/passes/dft.py b/vyper/venom/passes/dft.py index f45a60079c..85f27867a7 100644 --- a/vyper/venom/passes/dft.py +++ b/vyper/venom/passes/dft.py @@ -1,5 +1,5 @@ from vyper.utils import OrderedSet -from vyper.venom.analysis.dfg import DFGAnalysis +from vyper.venom.analysis import DFGAnalysis from vyper.venom.basicblock import IRBasicBlock, IRInstruction, IRVariable from vyper.venom.function import IRFunction from vyper.venom.passes.base_pass import IRPass diff --git a/vyper/venom/passes/make_ssa.py b/vyper/venom/passes/make_ssa.py index a803514d8b..56d3e1b7d3 100644 --- a/vyper/venom/passes/make_ssa.py +++ b/vyper/venom/passes/make_ssa.py @@ -1,7 +1,5 @@ from vyper.utils import OrderedSet -from vyper.venom.analysis.cfg import CFGAnalysis -from vyper.venom.analysis.dominators import DominatorTreeAnalysis -from vyper.venom.analysis.liveness import LivenessAnalysis +from vyper.venom.analysis import CFGAnalysis, DominatorTreeAnalysis, LivenessAnalysis from vyper.venom.basicblock import IRBasicBlock, IRInstruction, IROperand, IRVariable from vyper.venom.passes.base_pass import IRPass diff --git a/vyper/venom/passes/mem2var.py b/vyper/venom/passes/mem2var.py index f4a37f5abb..f93924d449 100644 --- a/vyper/venom/passes/mem2var.py +++ b/vyper/venom/passes/mem2var.py @@ -1,8 +1,5 @@ -from vyper.utils import OrderedSet -from vyper.venom.analysis.cfg import CFGAnalysis -from vyper.venom.analysis.dfg import DFGAnalysis -from vyper.venom.analysis.liveness import LivenessAnalysis -from vyper.venom.basicblock import IRBasicBlock, IRInstruction, IRVariable +from vyper.venom.analysis import CFGAnalysis, DFGAnalysis, LivenessAnalysis +from vyper.venom.basicblock import IRInstruction, IRVariable from vyper.venom.function import IRFunction from vyper.venom.passes.base_pass import IRPass @@ -14,7 +11,6 @@ class Mem2Var(IRPass): """ function: IRFunction - defs: dict[IRVariable, OrderedSet[IRBasicBlock]] def run_pass(self): self.analyses_cache.request_analysis(CFGAnalysis) @@ -22,13 +18,20 @@ def run_pass(self): self.var_name_count = 0 for var, inst in dfg.outputs.items(): - if inst.opcode != "alloca": - continue - self._process_alloca_var(dfg, var) + if inst.opcode == "alloca": + self._process_alloca_var(dfg, var) + elif inst.opcode == "palloca": + self._process_palloca_var(dfg, inst, var) self.analyses_cache.invalidate_analysis(DFGAnalysis) self.analyses_cache.invalidate_analysis(LivenessAnalysis) + def _mk_varname(self, varname: str): + varname = varname.removeprefix("%") + varname = f"var{varname}_{self.var_name_count}" + self.var_name_count += 1 + return varname + def _process_alloca_var(self, dfg: DFGAnalysis, var: IRVariable): """ Process alloca allocated variable. If it is only used by mstore/mload/return @@ -40,8 +43,7 @@ def _process_alloca_var(self, dfg: DFGAnalysis, var: IRVariable): elif all([inst.opcode == "mstore" for inst in uses]): return elif all([inst.opcode in ["mstore", "mload", "return"] for inst in uses]): - var_name = f"addr{var.name}_{self.var_name_count}" - self.var_name_count += 1 + var_name = self._mk_varname(var.name) for inst in uses: if inst.opcode == "mstore": inst.opcode = "store" @@ -52,7 +54,32 @@ def _process_alloca_var(self, dfg: DFGAnalysis, var: IRVariable): inst.operands = [IRVariable(var_name)] elif inst.opcode == "return": bb = inst.parent - idx = bb.instructions.index(inst) + idx = len(bb.instructions) - 1 + assert inst == bb.instructions[idx] # sanity bb.insert_instruction( IRInstruction("mstore", [IRVariable(var_name), inst.operands[1]]), idx ) + + def _process_palloca_var(self, dfg: DFGAnalysis, palloca_inst: IRInstruction, var: IRVariable): + """ + Process alloca allocated variable. If it is only used by mstore/mload + instructions, it is promoted to a stack variable. Otherwise, it is left as is. + """ + uses = dfg.get_uses(var) + if not all(inst.opcode in ["mstore", "mload"] for inst in uses): + return + + var_name = self._mk_varname(var.name) + + palloca_inst.opcode = "mload" + palloca_inst.operands = [palloca_inst.operands[0]] + palloca_inst.output = IRVariable(var_name) + + for inst in uses: + if inst.opcode == "mstore": + inst.opcode = "store" + inst.output = IRVariable(var_name) + inst.operands = [inst.operands[0]] + elif inst.opcode == "mload": + inst.opcode = "store" + inst.operands = [IRVariable(var_name)] diff --git a/vyper/venom/passes/normalization.py b/vyper/venom/passes/normalization.py index cf44c3cf89..7ca242c74e 100644 --- a/vyper/venom/passes/normalization.py +++ b/vyper/venom/passes/normalization.py @@ -1,5 +1,5 @@ from vyper.exceptions import CompilerPanic -from vyper.venom.analysis.cfg import CFGAnalysis +from vyper.venom.analysis import CFGAnalysis from vyper.venom.basicblock import IRBasicBlock, IRLabel from vyper.venom.passes.base_pass import IRPass diff --git a/vyper/venom/passes/remove_unused_variables.py b/vyper/venom/passes/remove_unused_variables.py index e022b300cc..3ce5bdf2d3 100644 --- a/vyper/venom/passes/remove_unused_variables.py +++ b/vyper/venom/passes/remove_unused_variables.py @@ -1,6 +1,5 @@ from vyper.utils import OrderedSet -from vyper.venom.analysis.dfg import DFGAnalysis -from vyper.venom.analysis.liveness import LivenessAnalysis +from vyper.venom.analysis import DFGAnalysis, LivenessAnalysis from vyper.venom.basicblock import IRInstruction from vyper.venom.passes.base_pass import IRPass diff --git a/vyper/venom/passes/sccp/sccp.py b/vyper/venom/passes/sccp/sccp.py index 164d8e241d..7966863081 100644 --- a/vyper/venom/passes/sccp/sccp.py +++ b/vyper/venom/passes/sccp/sccp.py @@ -5,9 +5,7 @@ from vyper.exceptions import CompilerPanic, StaticAssertionException from vyper.utils import OrderedSet -from vyper.venom.analysis.analysis import IRAnalysesCache -from vyper.venom.analysis.cfg import CFGAnalysis -from vyper.venom.analysis.dominators import DominatorTreeAnalysis +from vyper.venom.analysis import CFGAnalysis, DominatorTreeAnalysis, IRAnalysesCache from vyper.venom.basicblock import ( IRBasicBlock, IRInstruction, @@ -56,9 +54,10 @@ class SCCP(IRPass): uses: dict[IRVariable, OrderedSet[IRInstruction]] lattice: Lattice work_list: list[WorkListItem] - cfg_dirty: bool cfg_in_exec: dict[IRBasicBlock, OrderedSet[IRBasicBlock]] + cfg_dirty: bool + def __init__(self, analyses_cache: IRAnalysesCache, function: IRFunction): super().__init__(analyses_cache, function) self.lattice = {} @@ -72,9 +71,9 @@ def run_pass(self): self._calculate_sccp(self.fn.entry) self._propagate_constants() - # self._propagate_variables() - - self.analyses_cache.invalidate_analysis(CFGAnalysis) + if self.cfg_dirty: + self.analyses_cache.force_analysis(CFGAnalysis) + self._fix_phi_nodes() def _calculate_sccp(self, entry: IRBasicBlock): """ @@ -178,7 +177,7 @@ def _visit_phi(self, inst: IRInstruction): def _visit_expr(self, inst: IRInstruction): opcode = inst.opcode - if opcode in ["store", "alloca"]: + if opcode in ["store", "alloca", "palloca"]: assert inst.output is not None, "Got store/alloca without output" out = self._eval_from_lattice(inst.operands[0]) self._set_lattice(inst.output, out) @@ -304,6 +303,7 @@ def _replace_constants(self, inst: IRInstruction): target = inst.operands[1] inst.opcode = "jmp" inst.operands = [target] + self.cfg_dirty = True elif inst.opcode in ("assert", "assert_unreachable"): @@ -329,6 +329,34 @@ def _replace_constants(self, inst: IRInstruction): if isinstance(lat, IRLiteral): inst.operands[i] = lat + def _fix_phi_nodes(self): + # fix basic blocks whose cfg in was changed + # maybe this should really be done in _visit_phi + needs_sort = False + + for bb in self.fn.get_basic_blocks(): + cfg_in_labels = OrderedSet(in_bb.label for in_bb in bb.cfg_in) + + for inst in bb.instructions: + if inst.opcode != "phi": + break + needs_sort |= self._fix_phi_inst(inst, cfg_in_labels) + + # move phi instructions to the top of the block + if needs_sort: + bb.instructions.sort(key=lambda inst: inst.opcode != "phi") + + def _fix_phi_inst(self, inst: IRInstruction, cfg_in_labels: OrderedSet): + operands = [op for label, op in inst.phi_operands if label in cfg_in_labels] + + if len(operands) != 1: + return False + + assert inst.output is not None + inst.opcode = "store" + inst.operands = operands + return True + def _meet(x: LatticeItem, y: LatticeItem) -> LatticeItem: if x == LatticeEnum.TOP: diff --git a/vyper/venom/passes/simplify_cfg.py b/vyper/venom/passes/simplify_cfg.py index 08582fee96..acf37376e0 100644 --- a/vyper/venom/passes/simplify_cfg.py +++ b/vyper/venom/passes/simplify_cfg.py @@ -1,6 +1,6 @@ from vyper.exceptions import CompilerPanic from vyper.utils import OrderedSet -from vyper.venom.analysis.cfg import CFGAnalysis +from vyper.venom.analysis import CFGAnalysis from vyper.venom.basicblock import IRBasicBlock, IRLabel from vyper.venom.passes.base_pass import IRPass @@ -9,23 +9,21 @@ class SimplifyCFGPass(IRPass): visited: OrderedSet def _merge_blocks(self, a: IRBasicBlock, b: IRBasicBlock): - a.instructions.pop() + a.instructions.pop() # pop terminating instruction for inst in b.instructions: - assert inst.opcode != "phi", "Not implemented yet" - if inst.opcode == "phi": - a.instructions.insert(0, inst) - else: - inst.parent = a - a.instructions.append(inst) + assert inst.opcode != "phi", f"Instruction should never be phi {b}" + inst.parent = a + a.instructions.append(inst) # Update CFG a.cfg_out = b.cfg_out - if len(b.cfg_out) > 0: - next_bb = b.cfg_out.first() + + for next_bb in a.cfg_out: next_bb.remove_cfg_in(b) next_bb.add_cfg_in(a) for inst in next_bb.instructions: + # assume phi instructions are at beginning of bb if inst.opcode != "phi": break inst.operands[inst.operands.index(b.label)] = a.label diff --git a/vyper/venom/passes/stack_reorder.py b/vyper/venom/passes/stack_reorder.py deleted file mode 100644 index a92fe0e626..0000000000 --- a/vyper/venom/passes/stack_reorder.py +++ /dev/null @@ -1,23 +0,0 @@ -from vyper.utils import OrderedSet -from vyper.venom.basicblock import IRBasicBlock -from vyper.venom.passes.base_pass import IRPass - - -class StackReorderPass(IRPass): - visited: OrderedSet - - def _reorder_stack(self): - pass - - def _visit(self, bb: IRBasicBlock): - if bb in self.visited: - return - self.visited.add(bb) - - for bb_out in bb.cfg_out: - self._visit(bb_out) - - def _run_pass(self): - entry = self.function.entry - self.visited = OrderedSet() - self._visit(entry) diff --git a/vyper/venom/passes/store_elimination.py b/vyper/venom/passes/store_elimination.py index 17b9ce995a..0ecd324e26 100644 --- a/vyper/venom/passes/store_elimination.py +++ b/vyper/venom/passes/store_elimination.py @@ -1,6 +1,4 @@ -from vyper.venom.analysis.cfg import CFGAnalysis -from vyper.venom.analysis.dfg import DFGAnalysis -from vyper.venom.analysis.liveness import LivenessAnalysis +from vyper.venom.analysis import CFGAnalysis, DFGAnalysis, LivenessAnalysis from vyper.venom.basicblock import IRVariable from vyper.venom.passes.base_pass import IRPass diff --git a/vyper/venom/passes/extract_literals.py b/vyper/venom/passes/store_expansion.py similarity index 68% rename from vyper/venom/passes/extract_literals.py rename to vyper/venom/passes/store_expansion.py index 91c0813e67..be5eb3d95d 100644 --- a/vyper/venom/passes/extract_literals.py +++ b/vyper/venom/passes/store_expansion.py @@ -1,12 +1,12 @@ -from vyper.venom.analysis.dfg import DFGAnalysis -from vyper.venom.analysis.liveness import LivenessAnalysis -from vyper.venom.basicblock import IRInstruction, IRLiteral +from vyper.venom.analysis import DFGAnalysis, LivenessAnalysis +from vyper.venom.basicblock import IRInstruction, IRLiteral, IRVariable from vyper.venom.passes.base_pass import IRPass -class ExtractLiteralsPass(IRPass): +class StoreExpansionPass(IRPass): """ - This pass extracts literals so that they can be reordered by the DFT pass + This pass extracts literals and variables so that they can be + reordered by the DFT pass """ def run_pass(self): @@ -20,7 +20,7 @@ def _process_bb(self, bb): i = 0 while i < len(bb.instructions): inst = bb.instructions[i] - if inst.opcode in ("store", "offset"): + if inst.opcode in ("store", "offset", "phi", "param"): i += 1 continue @@ -29,9 +29,11 @@ def _process_bb(self, bb): if inst.opcode == "log" and j == 0: continue - if isinstance(op, IRLiteral): + if isinstance(op, (IRVariable, IRLiteral)): var = self.function.get_next_variable() to_insert = IRInstruction("store", [op], var) bb.insert_instruction(to_insert, index=i) inst.operands[j] = var + i += 1 + i += 1 diff --git a/vyper/venom/stack_model.py b/vyper/venom/stack_model.py index a98e5bb25b..e284b41fb2 100644 --- a/vyper/venom/stack_model.py +++ b/vyper/venom/stack_model.py @@ -30,7 +30,7 @@ def push(self, op: IROperand) -> None: def pop(self, num: int = 1) -> None: del self._stack[len(self._stack) - num :] - def get_depth(self, op: IROperand, n: int = 1) -> int: + def get_depth(self, op: IROperand) -> int: """ Returns the depth of the n-th matching operand in the stack map. If the operand is not in the stack map, returns NOT_IN_STACK. @@ -39,10 +39,7 @@ def get_depth(self, op: IROperand, n: int = 1) -> int: for i, stack_op in enumerate(reversed(self._stack)): if stack_op.value == op.value: - if n <= 1: - return -i - else: - n -= 1 + return -i return StackModel.NOT_IN_STACK # type: ignore diff --git a/vyper/venom/venom_to_assembly.py b/vyper/venom/venom_to_assembly.py index 41a76319d7..ee4b83201d 100644 --- a/vyper/venom/venom_to_assembly.py +++ b/vyper/venom/venom_to_assembly.py @@ -1,4 +1,3 @@ -from collections import Counter from typing import Any from vyper.exceptions import CompilerPanic, StackTooDeep @@ -11,9 +10,7 @@ optimize_assembly, ) from vyper.utils import MemoryPositions, OrderedSet -from vyper.venom.analysis.analysis import IRAnalysesCache -from vyper.venom.analysis.dup_requirements import DupRequirementsAnalysis -from vyper.venom.analysis.liveness import LivenessAnalysis +from vyper.venom.analysis import IRAnalysesCache, LivenessAnalysis, VarEquivalenceAnalysis from vyper.venom.basicblock import ( IRBasicBlock, IRInstruction, @@ -23,9 +20,13 @@ IRVariable, ) from vyper.venom.context import IRContext -from vyper.venom.passes.normalization import NormalizationPass +from vyper.venom.passes import NormalizationPass from vyper.venom.stack_model import StackModel +DEBUG_SHOW_COST = False +if DEBUG_SHOW_COST: + import sys + # instructions which map one-to-one from venom to EVM _ONE_TO_ONE_INSTRUCTIONS = frozenset( [ @@ -153,7 +154,7 @@ def generate_evm(self, no_optimize: bool = False) -> list[str]: NormalizationPass(ac, fn).run_pass() self.liveness_analysis = ac.request_analysis(LivenessAnalysis) - ac.request_analysis(DupRequirementsAnalysis) + self.equivalence = ac.request_analysis(VarEquivalenceAnalysis) assert fn.normalized, "Non-normalized CFG!" @@ -200,21 +201,19 @@ def generate_evm(self, no_optimize: bool = False) -> list[str]: def _stack_reorder( self, assembly: list, stack: StackModel, stack_ops: list[IROperand], dry_run: bool = False ) -> int: - cost = 0 - if dry_run: assert len(assembly) == 0, "Dry run should not work on assembly" stack = stack.copy() - stack_ops_count = len(stack_ops) + if len(stack_ops) == 0: + return 0 - counts = Counter(stack_ops) + assert len(stack_ops) == len(set(stack_ops)) # precondition - for i in range(stack_ops_count): - op = stack_ops[i] - final_stack_depth = -(stack_ops_count - i - 1) - depth = stack.get_depth(op, counts[op]) # type: ignore - counts[op] -= 1 + cost = 0 + for i, op in enumerate(stack_ops): + final_stack_depth = -(len(stack_ops) - i - 1) + depth = stack.get_depth(op) if depth == StackModel.NOT_IN_STACK: raise CompilerPanic(f"Variable {op} not in stack") @@ -222,34 +221,40 @@ def _stack_reorder( if depth == final_stack_depth: continue - if op == stack.peek(final_stack_depth): + to_swap = stack.peek(final_stack_depth) + if self.equivalence.equivalent(op, to_swap): + # perform a "virtual" swap + stack.poke(final_stack_depth, op) + stack.poke(depth, to_swap) continue cost += self.swap(assembly, stack, depth) cost += self.swap(assembly, stack, final_stack_depth) + assert stack._stack[-len(stack_ops) :] == stack_ops, (stack, stack_ops) + return cost def _emit_input_operands( - self, assembly: list, inst: IRInstruction, ops: list[IROperand], stack: StackModel + self, + assembly: list, + inst: IRInstruction, + ops: list[IROperand], + stack: StackModel, + next_liveness: OrderedSet[IRVariable], ) -> None: # PRE: we already have all the items on the stack that have # been scheduled to be killed. now it's just a matter of emitting # SWAPs, DUPs and PUSHes until we match the `ops` argument - # dumb heuristic: if the top of stack is not wanted here, swap - # it with something that is wanted - if ops and stack.height > 0 and stack.peek(0) not in ops: - for op in ops: - if isinstance(op, IRVariable) and op not in inst.dup_requirements: - self.swap_op(assembly, stack, op) - break + # to validate store expansion invariant - + # each op is emitted at most once. + seen: set[IROperand] = set() - emitted_ops = OrderedSet[IROperand]() for op in ops: if isinstance(op, IRLabel): - # invoke emits the actual instruction itself so we don't need to emit it here - # but we need to add it to the stack map + # invoke emits the actual instruction itself so we don't need + # to emit it here but we need to add it to the stack map if inst.opcode != "invoke": assembly.append(f"_sym_{op.value}") stack.push(op) @@ -264,13 +269,12 @@ def _emit_input_operands( stack.push(op) continue - if op in inst.dup_requirements and op not in emitted_ops: + if op in next_liveness: self.dup_op(assembly, stack, op) - if op in emitted_ops: - self.dup_op(assembly, stack, op) - - emitted_ops.add(op) + # guaranteed by store expansion + assert op not in seen, (op, seen) + seen.add(op) def _generate_evm_for_basicblock_r( self, asm: list, basicblock: IRBasicBlock, stack: StackModel @@ -279,21 +283,36 @@ def _generate_evm_for_basicblock_r( return self.visited_basicblocks.add(basicblock) + if DEBUG_SHOW_COST: + print(basicblock, file=sys.stderr) + + ref = asm + asm = [] + # assembly entry point into the block asm.append(f"_sym_{basicblock.label}") asm.append("JUMPDEST") - self.clean_stack_from_cfg_in(asm, basicblock, stack) + if len(basicblock.cfg_in) == 1: + self.clean_stack_from_cfg_in(asm, basicblock, stack) all_insts = sorted(basicblock.instructions, key=lambda x: x.opcode != "param") for i, inst in enumerate(all_insts): - next_liveness = all_insts[i + 1].liveness if i + 1 < len(all_insts) else OrderedSet() + next_liveness = ( + all_insts[i + 1].liveness if i + 1 < len(all_insts) else basicblock.out_vars + ) asm.extend(self._generate_evm_for_instruction(inst, stack, next_liveness)) + if DEBUG_SHOW_COST: + print(" ".join(map(str, asm)), file=sys.stderr) + print("\n", file=sys.stderr) + + ref.extend(asm) + for bb in basicblock.reachable: - self._generate_evm_for_basicblock_r(asm, bb, stack.copy()) + self._generate_evm_for_basicblock_r(ref, bb, stack.copy()) # pop values from stack at entry to bb # note this produces the same result(!) no matter which basic block @@ -301,36 +320,37 @@ def _generate_evm_for_basicblock_r( def clean_stack_from_cfg_in( self, asm: list, basicblock: IRBasicBlock, stack: StackModel ) -> None: - if len(basicblock.cfg_in) == 0: - return - - to_pop = OrderedSet[IRVariable]() - for in_bb in basicblock.cfg_in: - # inputs is the input variables we need from in_bb - inputs = self.liveness_analysis.input_vars_from(in_bb, basicblock) - - # layout is the output stack layout for in_bb (which works - # for all possible cfg_outs from the in_bb). - layout = in_bb.out_vars - - # pop all the stack items which in_bb produced which we don't need. - to_pop |= layout.difference(inputs) - + # the input block is a splitter block, like jnz or djmp + assert len(basicblock.cfg_in) == 1 + in_bb = basicblock.cfg_in.first() + assert len(in_bb.cfg_out) > 1 + + # inputs is the input variables we need from in_bb + inputs = self.liveness_analysis.input_vars_from(in_bb, basicblock) + + # layout is the output stack layout for in_bb (which works + # for all possible cfg_outs from the in_bb, in_bb is responsible + # for making sure its output stack layout works no matter which + # bb it jumps into). + layout = in_bb.out_vars + to_pop = list(layout.difference(inputs)) + + # small heuristic: pop from shallowest first. + to_pop.sort(key=lambda var: -stack.get_depth(var)) + + # NOTE: we could get more fancy and try to optimize the swap + # operations here, there is probably some more room for optimization. for var in to_pop: depth = stack.get_depth(var) - # don't pop phantom phi inputs - if depth is StackModel.NOT_IN_STACK: - continue if depth != 0: self.swap(asm, stack, depth) self.pop(asm, stack) def _generate_evm_for_instruction( - self, inst: IRInstruction, stack: StackModel, next_liveness: OrderedSet = None + self, inst: IRInstruction, stack: StackModel, next_liveness: OrderedSet ) -> list[str]: assembly: list[str | int] = [] - next_liveness = next_liveness or OrderedSet() opcode = inst.opcode # @@ -341,7 +361,7 @@ def _generate_evm_for_instruction( if opcode in ["jmp", "djmp", "jnz", "invoke"]: operands = list(inst.get_non_label_operands()) - elif opcode == "alloca": + elif opcode in ("alloca", "palloca"): offset, _size = inst.operands operands = [offset] @@ -375,7 +395,8 @@ def _generate_evm_for_instruction( # example, for `%56 = %label1 %13 %label2 %14`, we will # find an instance of %13 *or* %14 in the stack and replace it with %56. to_be_replaced = stack.peek(depth) - if to_be_replaced in inst.dup_requirements: + if to_be_replaced in next_liveness: + # this branch seems unreachable (maybe due to make_ssa) # %13/%14 is still live(!), so we make a copy of it self.dup(assembly, stack, depth) stack.poke(0, ret) @@ -390,7 +411,7 @@ def _generate_evm_for_instruction( return apply_line_numbers(inst, assembly) # Step 2: Emit instruction's input operands - self._emit_input_operands(assembly, inst, operands, stack) + self._emit_input_operands(assembly, inst, operands, stack, next_liveness) # Step 3: Reorder stack before join points if opcode == "jmp": @@ -417,6 +438,13 @@ def _generate_evm_for_instruction( if cost_with_swap > cost_no_swap: operands[-1], operands[-2] = operands[-2], operands[-1] + cost = self._stack_reorder([], stack, operands, dry_run=True) + if DEBUG_SHOW_COST and cost: + print("ENTER", inst, file=sys.stderr) + print(" HAVE", stack, file=sys.stderr) + print(" WANT", operands, file=sys.stderr) + print(" COST", cost, file=sys.stderr) + # final step to get the inputs to this instruction ordered # correctly on the stack self._stack_reorder(assembly, stack, operands) @@ -433,7 +461,7 @@ def _generate_evm_for_instruction( # Step 5: Emit the EVM instruction(s) if opcode in _ONE_TO_ONE_INSTRUCTIONS: assembly.append(opcode.upper()) - elif opcode == "alloca": + elif opcode in ("alloca", "palloca"): pass elif opcode == "param": pass @@ -533,10 +561,21 @@ def _generate_evm_for_instruction( if inst.output not in next_liveness: self.pop(assembly, stack) else: - # peek at next_liveness to find the next scheduled item, - # and optimistically swap with it + # heuristic: peek at next_liveness to find the next scheduled + # item, and optimistically swap with it + if DEBUG_SHOW_COST: + stack0 = stack.copy() + next_scheduled = next_liveness.last() - self.swap_op(assembly, stack, next_scheduled) + cost = 0 + if not self.equivalence.equivalent(inst.output, next_scheduled): + cost = self.swap_op(assembly, stack, next_scheduled) + + if DEBUG_SHOW_COST and cost != 0: + print("ENTER", inst, file=sys.stderr) + print(" HAVE", stack0, file=sys.stderr) + print(" NEXT LIVENESS", next_liveness, file=sys.stderr) + print(" NEW_STACK", stack, file=sys.stderr) return apply_line_numbers(inst, assembly) @@ -558,7 +597,7 @@ def dup(self, assembly, stack, depth): assembly.append(_evm_dup_for(depth)) def swap_op(self, assembly, stack, op): - self.swap(assembly, stack, stack.get_depth(op)) + return self.swap(assembly, stack, stack.get_depth(op)) def dup_op(self, assembly, stack, op): self.dup(assembly, stack, stack.get_depth(op))