Merge branch 'master' into fix/pickle

tests/functional/builtins/codegen/test_raw_call.py

@@ -261,6 +261,12 @@ def __default__():
     assert env.message_call(caller.address, data=sig) == b""
 
 
+def _strip_initcode_suffix(bytecode):
+    bs = bytes.fromhex(bytecode.removeprefix("0x"))
+    to_strip = int.from_bytes(bs[-2:], "big")
+    return bs[:-to_strip].hex()
+
+
 # check max_outsize=0 does same thing as not setting max_outsize.
 # compile to bytecode and compare bytecode directly.
 def test_max_outsize_0():
@@ -276,7 +282,11 @@ def test_raw_call(_target: address):
     """
     output1 = compile_code(code1, output_formats=["bytecode", "bytecode_runtime"])
     output2 = compile_code(code2, output_formats=["bytecode", "bytecode_runtime"])
-    assert output1 == output2
+    assert output1["bytecode_runtime"] == output2["bytecode_runtime"]
+
+    bytecode1 = output1["bytecode"]
+    bytecode2 = output2["bytecode"]
+    assert _strip_initcode_suffix(bytecode1) == _strip_initcode_suffix(bytecode2)
 
 
 # check max_outsize=0 does same thing as not setting max_outsize,
@@ -298,7 +308,11 @@ def test_raw_call(_target: address) -> bool:
     """
     output1 = compile_code(code1, output_formats=["bytecode", "bytecode_runtime"])
     output2 = compile_code(code2, output_formats=["bytecode", "bytecode_runtime"])
-    assert output1 == output2
+    assert output1["bytecode_runtime"] == output2["bytecode_runtime"]
+
+    bytecode1 = output1["bytecode"]
+    bytecode2 = output2["bytecode"]
+    assert _strip_initcode_suffix(bytecode1) == _strip_initcode_suffix(bytecode2)
 
 
 # test functionality of max_outsize=0

tests/unit/compiler/test_bytecode_runtime.py

@@ -55,13 +55,17 @@ def test_bytecode_runtime():
 
 
 def test_bytecode_signature():
-    out = vyper.compile_code(simple_contract_code, output_formats=["bytecode_runtime", "bytecode"])
+    out = vyper.compile_code(
+        simple_contract_code, output_formats=["bytecode_runtime", "bytecode", "integrity"]
+    )
 
     runtime_code = bytes.fromhex(out["bytecode_runtime"].removeprefix("0x"))
     initcode = bytes.fromhex(out["bytecode"].removeprefix("0x"))
 
     metadata = _parse_cbor_metadata(initcode)
-    runtime_len, data_section_lengths, immutables_len, compiler = metadata
+    integrity_hash, runtime_len, data_section_lengths, immutables_len, compiler = metadata
+
+    assert integrity_hash.hex() == out["integrity"]
 
     assert runtime_len == len(runtime_code)
     assert data_section_lengths == []
@@ -73,14 +77,18 @@ def test_bytecode_signature_dense_jumptable():
     settings = Settings(optimize=OptimizationLevel.CODESIZE)
 
     out = vyper.compile_code(
-        many_functions, output_formats=["bytecode_runtime", "bytecode"], settings=settings
+        many_functions,
+        output_formats=["bytecode_runtime", "bytecode", "integrity"],
+        settings=settings,
     )
 
     runtime_code = bytes.fromhex(out["bytecode_runtime"].removeprefix("0x"))
     initcode = bytes.fromhex(out["bytecode"].removeprefix("0x"))
 
     metadata = _parse_cbor_metadata(initcode)
-    runtime_len, data_section_lengths, immutables_len, compiler = metadata
+    integrity_hash, runtime_len, data_section_lengths, immutables_len, compiler = metadata
+
+    assert integrity_hash.hex() == out["integrity"]
 
     assert runtime_len == len(runtime_code)
     assert data_section_lengths == [5, 35]
@@ -92,14 +100,18 @@ def test_bytecode_signature_sparse_jumptable():
     settings = Settings(optimize=OptimizationLevel.GAS)
 
     out = vyper.compile_code(
-        many_functions, output_formats=["bytecode_runtime", "bytecode"], settings=settings
+        many_functions,
+        output_formats=["bytecode_runtime", "bytecode", "integrity"],
+        settings=settings,
     )
 
     runtime_code = bytes.fromhex(out["bytecode_runtime"].removeprefix("0x"))
     initcode = bytes.fromhex(out["bytecode"].removeprefix("0x"))
 
     metadata = _parse_cbor_metadata(initcode)
-    runtime_len, data_section_lengths, immutables_len, compiler = metadata
+    integrity_hash, runtime_len, data_section_lengths, immutables_len, compiler = metadata
+
+    assert integrity_hash.hex() == out["integrity"]
 
     assert runtime_len == len(runtime_code)
     assert data_section_lengths == [8]
@@ -108,13 +120,17 @@ def test_bytecode_signature_sparse_jumptable():
 
 
 def test_bytecode_signature_immutables():
-    out = vyper.compile_code(has_immutables, output_formats=["bytecode_runtime", "bytecode"])
+    out = vyper.compile_code(
+        has_immutables, output_formats=["bytecode_runtime", "bytecode", "integrity"]
+    )
 
     runtime_code = bytes.fromhex(out["bytecode_runtime"].removeprefix("0x"))
     initcode = bytes.fromhex(out["bytecode"].removeprefix("0x"))
 
     metadata = _parse_cbor_metadata(initcode)
-    runtime_len, data_section_lengths, immutables_len, compiler = metadata
+    integrity_hash, runtime_len, data_section_lengths, immutables_len, compiler = metadata
+
+    assert integrity_hash.hex() == out["integrity"]
 
     assert runtime_len == len(runtime_code)
     assert data_section_lengths == []
@@ -129,7 +145,10 @@ def test_bytecode_signature_deployed(code, get_contract, env):
     deployed_code = env.get_code(c.address)
 
     metadata = _parse_cbor_metadata(c.bytecode)
-    runtime_len, data_section_lengths, immutables_len, compiler = metadata
+    integrity_hash, runtime_len, data_section_lengths, immutables_len, compiler = metadata
+
+    out = vyper.compile_code(code, output_formats=["integrity"])
+    assert integrity_hash.hex() == out["integrity"]
 
     assert compiler == {"vyper": list(vyper.version.version_tuple)}
 

tests/unit/compiler/venom/test_stack_reorder.py

@@ -0,0 +1,28 @@
+from vyper.venom import generate_assembly_experimental
+from vyper.venom.context import IRContext
+
+
+def test_stack_reorder():
+    """
+    Test to was created from the example in the
+    issue https://github.com/vyperlang/vyper/issues/4215
+    this example should fail with original stack reorder
+    algorithm but succeed with new one
+    """
+    ctx = IRContext()
+    fn = ctx.create_function("_global")
+
+    bb = fn.get_basic_block()
+    var0 = bb.append_instruction("store", 1)
+    var1 = bb.append_instruction("store", 2)
+    var2 = bb.append_instruction("store", 3)
+    var3 = bb.append_instruction("store", 4)
+    var4 = bb.append_instruction("store", 5)
+
+    bb.append_instruction("staticcall", var0, var1, var2, var3, var4, var3)
+
+    ret_val = bb.append_instruction("add", var4, var4)
+
+    bb.append_instruction("ret", ret_val)
+
+    generate_assembly_experimental(ctx)

vyper/compiler/output.py

@@ -320,15 +320,13 @@ def _build_source_map_output(compiler_data, bytecode, pc_maps):
 
 
 def build_source_map_output(compiler_data: CompilerData) -> dict:
-    bytecode, pc_maps = compile_ir.assembly_to_evm(
-        compiler_data.assembly, insert_compiler_metadata=False
-    )
+    bytecode, pc_maps = compile_ir.assembly_to_evm(compiler_data.assembly, compiler_metadata=None)
     return _build_source_map_output(compiler_data, bytecode, pc_maps)
 
 
 def build_source_map_runtime_output(compiler_data: CompilerData) -> dict:
     bytecode, pc_maps = compile_ir.assembly_to_evm(
-        compiler_data.assembly_runtime, insert_compiler_metadata=False
+        compiler_data.assembly_runtime, compiler_metadata=None
     )
     return _build_source_map_output(compiler_data, bytecode, pc_maps)
 

vyper/compiler/phases.py

@@ -2,7 +2,7 @@
 import warnings
 from functools import cached_property
 from pathlib import Path, PurePath
-from typing import Optional
+from typing import Any, Optional
 
 from vyper import ast as vy_ast
 from vyper.ast import natspec
@@ -249,12 +249,15 @@ def assembly_runtime(self) -> list:
 
     @cached_property
     def bytecode(self) -> bytes:
-        insert_compiler_metadata = not self.no_bytecode_metadata
-        return generate_bytecode(self.assembly, insert_compiler_metadata=insert_compiler_metadata)
+        metadata = None
+        if not self.no_bytecode_metadata:
+            module_t = self.compilation_target._metadata["type"]
+            metadata = bytes.fromhex(module_t.integrity_sum)
+        return generate_bytecode(self.assembly, compiler_metadata=metadata)
 
     @cached_property
     def bytecode_runtime(self) -> bytes:
-        return generate_bytecode(self.assembly_runtime, insert_compiler_metadata=False)
+        return generate_bytecode(self.assembly_runtime, compiler_metadata=None)
 
     @cached_property
     def blueprint_bytecode(self) -> bytes:
@@ -351,7 +354,7 @@ def _find_nested_opcode(assembly, key):
         return any(_find_nested_opcode(x, key) for x in sublists)
 
 
-def generate_bytecode(assembly: list, insert_compiler_metadata: bool) -> bytes:
+def generate_bytecode(assembly: list, compiler_metadata: Optional[Any]) -> bytes:
     """
     Generate bytecode from assembly instructions.
 
@@ -365,6 +368,4 @@ def generate_bytecode(assembly: list, insert_compiler_metadata: bool) -> bytes:
     bytes
         Final compiled bytecode.
     """
-    return compile_ir.assembly_to_evm(assembly, insert_compiler_metadata=insert_compiler_metadata)[
-        0
-    ]
+    return compile_ir.assembly_to_evm(assembly, compiler_metadata=compiler_metadata)[0]

vyper/ir/compile_ir.py

@@ -1155,22 +1155,24 @@ def _relocate_segments(assembly):
 
 
 # TODO: change API to split assembly_to_evm and assembly_to_source/symbol_maps
-def assembly_to_evm(assembly, pc_ofst=0, insert_compiler_metadata=False):
+def assembly_to_evm(assembly, pc_ofst=0, compiler_metadata=None):
     bytecode, source_maps, _ = assembly_to_evm_with_symbol_map(
-        assembly, pc_ofst=pc_ofst, insert_compiler_metadata=insert_compiler_metadata
+        assembly, pc_ofst=pc_ofst, compiler_metadata=compiler_metadata
     )
     return bytecode, source_maps
 
 
-def assembly_to_evm_with_symbol_map(assembly, pc_ofst=0, insert_compiler_metadata=False):
+def assembly_to_evm_with_symbol_map(assembly, pc_ofst=0, compiler_metadata=None):
     """
     Assembles assembly into EVM
 
     assembly: list of asm instructions
     pc_ofst: when constructing the source map, the amount to offset all
              pcs by (no effect until we add deploy code source map)
-    insert_compiler_metadata: whether to append vyper metadata to output
-                            (should be true for runtime code)
+    compiler_metadata: any compiler metadata to add. pass `None` to indicate
+                       no metadata to be added (should always be `None` for
+                       runtime code). the value is opaque, and will be passed
+                       directly to `cbor2.dumps()`.
     """
     line_number_map = {
         "breakpoints": set(),
@@ -1278,10 +1280,11 @@ def assembly_to_evm_with_symbol_map(assembly, pc_ofst=0, insert_compiler_metadat
             pc += 1
 
     bytecode_suffix = b""
-    if insert_compiler_metadata:
+    if compiler_metadata is not None:
         # this will hold true when we are in initcode
         assert immutables_len is not None
         metadata = (
+            compiler_metadata,
             len(runtime_code),
             data_section_lengths,
             immutables_len,

vyper/venom/basicblock.py

@@ -21,8 +21,6 @@
         "istore",
         "tload",
         "tstore",
-        "assert",
-        "assert_unreachable",
         "mstore",
         "mload",
         "calldatacopy",

vyper/venom/venom_to_assembly.py

@@ -1,4 +1,5 @@
-from collections import Counter
+from bisect import insort
+from collections import Counter, defaultdict
 from typing import Any
 
 from vyper.exceptions import CompilerPanic, StackTooDeep
@@ -205,14 +206,29 @@ def _stack_reorder(
             stack = stack.copy()
 
         stack_ops_count = len(stack_ops)
+        if stack_ops_count == 0:
+            return 0
 
         counts = Counter(stack_ops)
 
+        # positions stores the positions of relevant operands
+        # on stack for example operand %82 is on positions [0, 3]
+        # this operand could ocure even more deeper in the stack
+        # but only those that are needed/relevant in calculation
+        # are considered
+        positions: dict[IROperand, list[int]] = defaultdict(list)
+        for op in stack_ops:
+            positions[op] = []
+            for i in range(counts[op]):
+                positions[op].append(stack.get_depth(op, i + 1))
+
         for i in range(stack_ops_count):
             op = stack_ops[i]
             final_stack_depth = -(stack_ops_count - i - 1)
-            depth = stack.get_depth(op, counts[op])  # type: ignore
-            counts[op] -= 1
+            depth = positions[op].pop()  # type: ignore
+            assert depth not in range(
+                -stack_ops_count + 1, final_stack_depth
+            ), f"{depth} : ({-stack_ops_count - 1}, {final_stack_depth})"
 
             if depth == StackModel.NOT_IN_STACK:
                 raise CompilerPanic(f"Variable {op} not in stack")
@@ -223,9 +239,26 @@ def _stack_reorder(
             if op == stack.peek(final_stack_depth):
                 continue
 
+            # moves the top item to original position
+            top_item_positions = positions[stack.peek(0)]
+            if len(top_item_positions) != 0:
+                top_item_positions.remove(0)
+                insort(top_item_positions, depth)
+
             cost += self.swap(assembly, stack, depth)
+
+            # moves the item from final position to top
+            final_item_positions = positions[stack.peek(final_stack_depth)]
+            if final_stack_depth in final_item_positions:
+                final_item_positions.remove(final_stack_depth)
+                final_item_positions.insert(0, 0)
+            else:
+                final_item_positions.insert(0, 0)
+
             cost += self.swap(assembly, stack, final_stack_depth)
 
+        assert stack._stack[-len(stack_ops) :] == stack_ops, (stack, stack_ops)
+
         return cost
 
     def _emit_input_operands(