new(cli): Introduce eofwrap tool

pyproject.toml

@@ -86,6 +86,7 @@ checkfixtures = "cli.check_fixtures:check_fixtures"
 consume = "cli.pytest_commands.consume:consume"
 genindex = "cli.gen_index:generate_fixtures_index_cli"
 gentest = "cli.gentest:make_test"
+eofwrap = "cli.eofwrap:eof_wrap"
 pyspelling_soft_fail = "cli.tox_helpers:pyspelling"
 markdownlintcli2_soft_fail = "cli.tox_helpers:markdownlint"
 order_fixtures = "cli.order_fixtures:order_fixtures"

src/cli/eofwrap.py

@@ -0,0 +1,365 @@
+"""
+Generate a JSON blockchain test from an existing JSON blockchain test by wrapping its prestate code
+in EOF wherever possible.
+
+Example Usage:
+
+1. Wrap tests
+
+    ```console eofwrap TODO ```
+"""
+
+import json
+import os
+from pathlib import Path
+from typing import Any
+
+import click
+
+from ethereum_clis.clis.evmone import EvmOneTransitionTool
+from ethereum_test_base_types.base_types import Bytes
+from ethereum_test_base_types.conversions import to_hex
+from ethereum_test_fixtures.blockchain import FixtureBlock, InvalidFixtureBlock
+from ethereum_test_fixtures.file import BaseFixturesRootModel, BlockchainFixtures
+from ethereum_test_forks.forks.forks import CancunEIP7692
+from ethereum_test_specs.blockchain import Block, BlockchainFixture, BlockchainTest
+from ethereum_test_specs.eof import EOFParse
+from ethereum_test_tools import Opcodes as Op
+from ethereum_test_types import Transaction
+from ethereum_test_types.eof.v1 import OPCODE_MAP, Container, compute_code_stack_values
+from ethereum_test_types.types import Environment
+from ethereum_test_vm.bytecode import Bytecode
+
+GENERATION_ERROR_MESSAGE_THRESHOLD = 30
+
+
+@click.command()
+@click.argument("input", type=click.Path(exists=True, dir_okay=True, file_okay=True))
+@click.argument("output_dir", type=click.Path(dir_okay=True, file_okay=False))
+@click.option("--traces", is_flag=True, type=bool)
+def eof_wrap(input: str, output_dir: str, traces: bool):
+    """
+    Wraps JSON blockchain test file(s) found at `input` path and outputs them to the `output_dir`.
+    """
+    eof_wrapper = EofWrapper()
+
+    if os.path.isfile(input):
+        file = os.path.basename(input)
+        out_file = "eof_wrapped_" + file
+        out_path = os.path.join(output_dir, out_file)
+
+        eof_wrapper.wrap_file(input, out_path, traces)
+    else:
+        for subdir, dirs, files in os.walk(input):
+            for file in files:
+                rel_dir = Path(subdir).relative_to(input)
+                out_file = "eof_wrapped_" + file
+                out_path = os.path.join(output_dir, rel_dir, out_file)
+                in_path = os.path.join(subdir, file)
+
+                eof_wrapper.wrap_file(in_path, out_path, traces)
+
+    with open(os.path.join(output_dir, "metrics.json"), "w") as f:
+        json.dump(eof_wrapper.metrics, f, indent=4)
+
+
+class EofWrapper:
+    """
+    EOF wrapping of blockchain tests with some simple metrics tracking.
+    """
+
+    # JSON files had at least one fixture generated successfully with EOF
+    FILES_GENERATED = "files_generated"
+    # JSON files skipped explicitly or didn't have a fixture with EOF
+    FILES_SKIPPED = "files_skipped"
+    # Test fixtures with at least one EOF code and generated successfully
+    FIXTURES_GENERATED = "fixtures_generated"
+    # Test fixtures with no code able to be EOF-wrapped
+    FIXTURES_CANT_WRAP = "fixtures_cant_wrap"
+    # Test fixtures with EOF code but test doesn't pass and generation fails
+    FIXTURES_CANT_GENERATE = "fixtures_cant_generate"
+    # State accounts with code wrapped into valid EOF
+    ACCOUNTS_WRAPPED = "accounts_wrapped"
+    # State accounts with code wrapped into valid unique EOF
+    UNIQUE_ACCOUNTS_WRAPPED = "unique_accounts_wrapped"
+    # State accounts wrapped but the code is not valid EOF
+    ACCOUNTS_INVALID_EOF = "accounts_invalid_eof"
+    # State accounts wrapped into valid EOF but in a fixture of a failing test
+    ACCOUNTS_CANT_GENERATE = "accounts_cant_generate"
+    # Breakdown of EOF validation errors summing up to `accounts_invalid_eof`
+    VALIDATION_ERRORS = "validation_errors"
+    # Breakdown of runtime test failures summing up to `fixtures_cant_generate`
+    GENERATION_ERRORS = "generation_errors"
+
+    def __init__(self):
+        self.metrics = {
+            self.FILES_GENERATED: 0,
+            self.FILES_SKIPPED: 0,
+            self.FIXTURES_GENERATED: 0,
+            self.FIXTURES_CANT_WRAP: 0,
+            self.FIXTURES_CANT_GENERATE: 0,
+            self.ACCOUNTS_WRAPPED: 0,
+            self.UNIQUE_ACCOUNTS_WRAPPED: 0,
+            self.ACCOUNTS_INVALID_EOF: 0,
+            self.ACCOUNTS_CANT_GENERATE: 0,
+            self.VALIDATION_ERRORS: {},
+            self.GENERATION_ERRORS: {},
+        }
+        self.unique_eof = set()
+
+    file_skip_list = [
+        "Pyspecs",
+        # EXTCODE* opcodes return different results for EOF targets and that is tested elsewhere
+        "stExtCodeHash",
+        # bigint syntax
+        "ValueOverflowParis",
+        "bc4895-withdrawals",
+        # EOF opcodes at diff places - tests obsolete
+        "opcD0DiffPlaces",
+        "opcD1DiffPlaces",
+        "opcD2DiffPlaces",
+        "opcD3DiffPlaces",
+        "opcE0DiffPlaces",
+        "opcE1DiffPlaces",
+        "opcE2DiffPlaces",
+        "opcE3DiffPlaces",
+        "opcE4DiffPlaces",
+        "opcE5DiffPlaces",
+        "opcE6DiffPlaces",
+        "opcE7DiffPlaces",
+        "opcE8DiffPlaces",
+        "opcECDiffPlaces",
+        "opcEEDiffPlaces",
+        "opcF7DiffPlaces",
+        "opcF8DiffPlaces",
+        "opcF9DiffPlaces",
+        "opcFBDiffPlaces",
+        # stack overflow always (limit of `max_stack_height` is 1023!)
+        "push0_fill_stack",
+        "push0_stack_overflow",
+        "blobbasefee_stack_overflow",
+    ]
+
+    def wrap_file(self, in_path: str, out_path: str, traces: bool):
+        """
+        TODO
+        """
+        for skip in self.file_skip_list:
+            if skip in in_path:
+                self.metrics[self.FILES_SKIPPED] += 1
+                return
+
+        with open(in_path, "r") as input_file:
+            fixtures = BlockchainFixtures.from_json_data(json.load(input_file))
+
+        out_fixtures = BaseFixturesRootModel({})
+        fixture: BlockchainFixture
+        for id, fixture in fixtures.items():
+            fixture_eof_codes = []
+            wrapped_at_least_one_account = False
+
+            if fixture.pre:
+                for address, account in fixture.pre.root.items():
+                    if account is None or account.code is None or len(account.code) == 0:
+                        continue
+                    wrapped = wrap_code(account.code)
+                    if self._validate_eof(wrapped):
+                        account.code = Bytes(wrapped)
+                        wrapped_at_least_one_account = True
+                        self.metrics[self.ACCOUNTS_WRAPPED] += 1
+                        fixture_eof_codes.append(to_hex(account.code))
+
+                        # wrap the same account in post state the same way
+                        if fixture.post_state and fixture.post_state.root[address]:
+                            fixture.post_state.root[address].code = Bytes(wrapped)  # type: ignore
+                    else:
+                        self.metrics[self.ACCOUNTS_INVALID_EOF] += 1
+            if not wrapped_at_least_one_account:
+                self.metrics[self.FIXTURES_CANT_WRAP] += 1
+                continue
+
+            try:
+                out_fixture = self._wrap_fixture(fixture, traces)
+                out_fixtures[id] = out_fixture
+                self.metrics[self.FIXTURES_GENERATED] += 1
+                self.unique_eof.update(fixture_eof_codes)
+                self.metrics[self.UNIQUE_ACCOUNTS_WRAPPED] = len(self.unique_eof)
+            except Exception as e:
+                short = str(e)
+                if len(short) > GENERATION_ERROR_MESSAGE_THRESHOLD:
+                    short = short[:GENERATION_ERROR_MESSAGE_THRESHOLD] + "..."
+
+                _inc_counter(self.metrics[self.GENERATION_ERRORS], short)
+
+                self.metrics[self.FIXTURES_CANT_GENERATE] += 1
+                self.metrics[self.ACCOUNTS_CANT_GENERATE] += len(fixture_eof_codes)
+
+        if len(out_fixtures) == 0:
+            self.metrics[self.FILES_SKIPPED] += 1
+            return
+
+        os.makedirs(os.path.dirname(out_path), exist_ok=True)
+        out_fixtures.collect_into_file(Path(out_path))
+        self.metrics[self.FILES_GENERATED] += 1
+
+    def _wrap_fixture(self, fixture: BlockchainFixture, traces: bool):
+        env = Environment()
+
+        pre = fixture.pre
+
+        t8n = EvmOneTransitionTool(trace=traces)
+
+        test = BlockchainTest(
+            genesis_environment=env,
+            pre=pre.root,
+            post=fixture.post_state.root if fixture.post_state else {},
+            blocks=[],
+            tag="wrapped test",
+        )
+
+        for fixture_block in fixture.blocks:
+            if isinstance(fixture_block, FixtureBlock):
+                header = fixture_block.header
+                block = Block(
+                    parent_hash=header.parent_hash,
+                    ommers_hash=header.ommers_hash,
+                    fee_recipient=header.fee_recipient,
+                    state_root=header.state_root,
+                    transactions_trie=header.transactions_trie,
+                    receipts_root=header.receipts_root,
+                    logs_bloom=header.logs_bloom,
+                    difficulty=header.difficulty,
+                    number=header.number,
+                    gas_limit=header.gas_limit,
+                    gas_used=header.gas_used,
+                    timestamp=header.timestamp,
+                    extra_data=header.extra_data,
+                    prev_randao=header.prev_randao,
+                    nonce=header.nonce,
+                    base_fee_per_gas=header.base_fee_per_gas,
+                    withdrawals_root=header.withdrawals_root,
+                    blob_gas_used=header.blob_gas_used,
+                    excess_blob_gas=header.excess_blob_gas,
+                    parent_beacon_block_root=header.parent_beacon_block_root,
+                    requests_root=header.requests_root,
+                )
+                assert not fixture_block.ommers
+                assert not fixture_block.withdrawals
+                assert not fixture_block.deposit_requests
+                assert not fixture_block.withdrawal_requests
+                assert not fixture_block.consolidation_requests
+
+                for fixture_tx in fixture_block.txs:
+                    tx = Transaction(
+                        type=fixture_tx.ty,
+                        chain_id=fixture_tx.chain_id,
+                        nonce=fixture_tx.nonce,
+                        gas_price=fixture_tx.gas_price,
+                        max_priority_fee_per_gas=fixture_tx.max_priority_fee_per_gas,
+                        max_fee_per_gas=fixture_tx.max_fee_per_gas,
+                        gas_limit=fixture_tx.gas_limit,
+                        to=fixture_tx.to,
+                        value=fixture_tx.value,
+                        input=fixture_tx.data,
+                        access_list=fixture_tx.access_list,
+                        max_fee_per_blob_gas=fixture_tx.max_fee_per_blob_gas,
+                        blob_versioned_hashes=fixture_tx.blob_versioned_hashes,
+                        v=fixture_tx.v,
+                        r=fixture_tx.r,
+                        s=fixture_tx.s,
+                        sender=fixture_tx.sender,
+                        authorization_list=fixture_tx.authorization_list,
+                    )
+                    block.txs.append(tx)
+            elif isinstance(fixture_block, InvalidFixtureBlock):
+                block = Block(
+                    rlp=fixture_block.rlp,
+                    exception=fixture_block.expect_exception,
+                )
+            else:
+                raise TypeError("not a FixtureBlock")
+
+            test.blocks.append(block)
+
+        return test.generate(
+            request=None,  # type: ignore
+            t8n=t8n,
+            fork=CancunEIP7692,
+            fixture_format=BlockchainFixture,
+        )
+
+    def _validate_eof(self, container: Container, metrics: bool = True) -> bool:
+        eof_parse = EOFParse()
+
+        result = eof_parse.run(input=to_hex(container))
+        actual_message = result.stdout.strip()
+        if "OK" not in actual_message:
+            if metrics:
+                _inc_counter(self.metrics[self.VALIDATION_ERRORS], actual_message)
+            return False
+
+        return True
+
+
+def _has_opcode_at(code: Bytecode, i: int, op: Op) -> bool:
+    opcode_at = OPCODE_MAP.get(
+        bytes(code)[i],
+    )
+    if not opcode_at:
+        return False
+
+    return opcode_at == op
+
+
+def wrap_code(account_code: Bytes) -> Container:
+    """
+    Wraps `account_code` into a simplest EOF container, applying some simple heuristics in
+    order to obtain a valid code section termination.
+    """
+    assert len(account_code) > 0
+
+    (
+        auto_code_inputs,
+        auto_code_outputs,
+        auto_max_height,
+    ) = compute_code_stack_values(account_code)
+    code = Bytecode(
+        account_code,
+        popped_stack_items=auto_code_inputs,
+        pushed_stack_items=auto_code_outputs,
+    )
+
+    if len(code) >= 2 and _has_opcode_at(code, -2, Op.PUSH1):
+        # Non-truncated PUSH1 at the end, a common case causing "no termination" but looks like
+        # `00` at the end
+        code += Op.STOP
+
+    # Something other than STOP at the end - let's add it and potentially cleanup below
+    if not _has_opcode_at(code, -1, Op.STOP):
+        code += Op.STOP
+
+    while len(code) >= 2:
+        if len(code) >= 3 and bytes(code)[-3:] == bytes(Op.PUSH1(0) + Op.STOP):
+            break
+        elif (
+            bytes(code)[-2:] == bytes(Op.STOP + Op.STOP)
+            or bytes(code)[-2:] == bytes(Op.RETURN + Op.STOP)
+            or bytes(code)[-2:] == bytes(Op.REVERT + Op.STOP)
+            or bytes(code)[-2:] == bytes(Op.INVALID + Op.STOP)
+        ):
+            code = Bytecode(
+                bytes(code)[:-1],
+                popped_stack_items=code.popped_stack_items,
+                pushed_stack_items=code.pushed_stack_items,
+            )
+        else:
+            break
+
+    return Container.Code(code, max_stack_height=auto_max_height)
+
+
+def _inc_counter(d: dict, key: Any) -> None:
+    if key in d:
+        d[key] += 1
+    else:
+        d[key] = 1