Merge branch 'main' into kzgLibUpdate

Signed-off-by: Justin Florentine <[email protected]>

CHANGELOG.md

@@ -5,6 +5,7 @@
 ### Deprecations
 
 ### Additions and Improvements
+- TraceService: return results for transactions in block [#6086](https://github.com/hyperledger/besu/pull/6086)
 
 ### Bug Fixes
 
@@ -18,9 +19,15 @@
 
 ### Additions and Improvements
 - Ethereum Classic Spiral network upgrade [#6078](https://github.com/hyperledger/besu/pull/6078)
+- Add a method to read from a `Memory` instance without altering its inner state [#6073](https://github.com/hyperledger/besu/pull/6073)
 
 ### Bug fixes
 
+- Upgrade netty to address CVE-2023-44487, CVE-2023-34462 [#6100](https://github.com/hyperledger/besu/pull/6100)
+- Upgrade grpc to address CVE-2023-32731, CVE-2023-33953, CVE-2023-44487, CVE-2023-4785 [#6100](https://github.com/hyperledger/besu/pull/6100)
+
+---
+
 ### Download Links
 
 ## 23.10.1

besu/src/main/java/org/hyperledger/besu/services/TraceServiceImpl.java

@@ -26,13 +26,16 @@
 import org.hyperledger.besu.ethereum.chain.Blockchain;
 import org.hyperledger.besu.ethereum.core.Block;
 import org.hyperledger.besu.ethereum.core.BlockHeader;
+import org.hyperledger.besu.ethereum.core.Transaction;
 import org.hyperledger.besu.ethereum.mainnet.MainnetTransactionProcessor;
 import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule;
 import org.hyperledger.besu.ethereum.mainnet.ProtocolSpec;
 import org.hyperledger.besu.ethereum.processing.TransactionProcessingResult;
 import org.hyperledger.besu.ethereum.vm.CachingBlockHashLookup;
 import org.hyperledger.besu.evm.worldstate.WorldUpdater;
 import org.hyperledger.besu.plugin.Unstable;
+import org.hyperledger.besu.plugin.data.BlockTraceResult;
+import org.hyperledger.besu.plugin.data.TransactionTraceResult;
 import org.hyperledger.besu.plugin.services.TraceService;
 import org.hyperledger.besu.plugin.services.tracer.BlockAwareOperationTracer;
 
@@ -72,10 +75,9 @@ public TraceServiceImpl(
    * @param tracer an instance of OperationTracer
    */
   @Override
-  public void traceBlock(final long blockNumber, final BlockAwareOperationTracer tracer) {
-    checkArgument(tracer != null);
-    final Optional<Block> block = blockchainQueries.getBlockchain().getBlockByNumber(blockNumber);
-    block.ifPresent(value -> trace(value, tracer));
+  public BlockTraceResult traceBlock(
+      final long blockNumber, final BlockAwareOperationTracer tracer) {
+    return traceBlock(blockchainQueries.getBlockchain().getBlockByNumber(blockNumber), tracer);
   }
 
   /**
@@ -85,10 +87,41 @@ public void traceBlock(final long blockNumber, final BlockAwareOperationTracer t
    * @param tracer an instance of OperationTracer
    */
   @Override
-  public void traceBlock(final Hash hash, final BlockAwareOperationTracer tracer) {
+  public BlockTraceResult traceBlock(final Hash hash, final BlockAwareOperationTracer tracer) {
+    return traceBlock(blockchainQueries.getBlockchain().getBlockByHash(hash), tracer);
+  }
+
+  private BlockTraceResult traceBlock(
+      final Optional<Block> maybeBlock, final BlockAwareOperationTracer tracer) {
     checkArgument(tracer != null);
-    final Optional<Block> block = blockchainQueries.getBlockchain().getBlockByHash(hash);
-    block.ifPresent(value -> trace(value, tracer));
+    if (maybeBlock.isEmpty()) {
+      return BlockTraceResult.empty();
+    }
+
+    final Optional<List<TransactionProcessingResult>> results = trace(maybeBlock.get(), tracer);
+
+    if (results.isEmpty()) {
+      return BlockTraceResult.empty();
+    }
+
+    final BlockTraceResult.Builder builder = BlockTraceResult.builder();
+
+    final List<TransactionProcessingResult> transactionProcessingResults = results.get();
+    final List<Transaction> transactions = maybeBlock.get().getBody().getTransactions();
+    for (int i = 0; i < transactionProcessingResults.size(); i++) {
+      final TransactionProcessingResult transactionProcessingResult =
+          transactionProcessingResults.get(i);
+      final TransactionTraceResult transactionTraceResult =
+          transactionProcessingResult.isInvalid()
+              ? TransactionTraceResult.error(
+                  transactions.get(i).getHash(),
+                  transactionProcessingResult.getValidationResult().getErrorMessage())
+              : TransactionTraceResult.success(transactions.get(i).getHash());
+
+      builder.addTransactionTraceResult(transactionTraceResult);
+    }
+
+    return builder.build();
   }
 
   /**
@@ -136,15 +169,20 @@ public void trace(
         });
   }
 
-  private void trace(final Block block, final BlockAwareOperationTracer tracer) {
+  private Optional<List<TransactionProcessingResult>> trace(
+      final Block block, final BlockAwareOperationTracer tracer) {
     LOG.debug("Tracing block {}", block.toLogString());
     final Blockchain blockchain = blockchainQueries.getBlockchain();
-    Tracer.processTracing(
-        blockchainQueries,
-        block.getHash(),
-        traceableState ->
-            Optional.of(trace(blockchain, block, new ChainUpdater(traceableState), tracer)));
+
+    final Optional<List<TransactionProcessingResult>> results =
+        Tracer.processTracing(
+            blockchainQueries,
+            block.getHash(),
+            traceableState ->
+                Optional.of(trace(blockchain, block, new ChainUpdater(traceableState), tracer)));
     tracer.traceEndBlock(block.getHeader(), block.getBody());
+
+    return results;
   }
 
   private List<TransactionProcessingResult> trace(

besu/src/test/java/org/hyperledger/besu/services/TraceServiceImplTest.java

@@ -26,6 +26,8 @@
 import org.hyperledger.besu.ethereum.worldstate.WorldStateArchive;
 import org.hyperledger.besu.evm.log.Log;
 import org.hyperledger.besu.evm.worldstate.WorldView;
+import org.hyperledger.besu.plugin.data.BlockTraceResult;
+import org.hyperledger.besu.plugin.data.TransactionTraceResult;
 import org.hyperledger.besu.plugin.services.TraceService;
 import org.hyperledger.besu.plugin.services.tracer.BlockAwareOperationTracer;
 
@@ -115,7 +117,18 @@ void shouldReturnTheCorrectWorldViewForTxStartEnd() {
     final TxStartEndTracer txStartEndTracer = new TxStartEndTracer();
 
     // block contains 1 transaction
-    traceService.traceBlock(31, txStartEndTracer);
+    final BlockTraceResult blockTraceResult = traceService.traceBlock(31, txStartEndTracer);
+
+    assertThat(blockTraceResult).isNotNull();
+
+    final List<TransactionTraceResult> transactionTraceResults =
+        blockTraceResult.transactionTraceResults();
+    assertThat(transactionTraceResults.size()).isEqualTo(1);
+
+    assertThat(transactionTraceResults.get(0).getTxHash()).isNotNull();
+    assertThat(transactionTraceResults.get(0).getStatus())
+        .isEqualTo(TransactionTraceResult.Status.SUCCESS);
+    assertThat(transactionTraceResults.get(0).errorMessage()).isEmpty();
 
     assertThat(txStartEndTracer.txStartWorldView).isNotNull();
     assertThat(txStartEndTracer.txEndWorldView).isNotNull();

ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/MainnetTransactionProcessor.java

@@ -500,7 +500,8 @@ public TransactionProcessingResult processTransaction(
       LOG.error("Critical Exception Processing Transaction", re);
       return TransactionProcessingResult.invalid(
           ValidationResult.invalid(
-              TransactionInvalidReason.INTERNAL_ERROR, "Internal Error in Besu - " + re));
+              TransactionInvalidReason.INTERNAL_ERROR,
+              "Internal Error in Besu - " + re + "\n" + printableStackTraceFromThrowable(re)));
     }
   }
 
@@ -525,4 +526,14 @@ protected long refunded(
     final long refundAllowance = Math.min(maxRefundAllowance, gasRefund);
     return gasRemaining + refundAllowance;
   }
+
+  private String printableStackTraceFromThrowable(final RuntimeException re) {
+    final StringBuilder builder = new StringBuilder();
+
+    for (final StackTraceElement stackTraceElement : re.getStackTrace()) {
+      builder.append("\tat ").append(stackTraceElement.toString()).append("\n");
+    }
+
+    return builder.toString();
+  }
 }

ethereum/core/src/test/java/org/hyperledger/besu/ethereum/vm/MemoryTest.java

@@ -65,6 +65,22 @@ public void shouldSetMemoryWhenLengthGreaterThanSourceLength() {
     assertThat(memory.getWord(64)).isEqualTo(Bytes32.ZERO);
   }
 
+  @Test
+  public void shouldNotIncreaseActiveWordsIfGetBytesWithoutGrowth() {
+    final Bytes value = Bytes.concatenate(WORD1, WORD2);
+    memory.setBytes(0, value.size(), value);
+    final int initialActiveWords = memory.getActiveWords();
+
+    assertThat(memory.getBytesWithoutGrowth(64, Bytes32.SIZE)).isEqualTo((Bytes32.ZERO));
+    assertThat(memory.getActiveWords()).isEqualTo(initialActiveWords);
+
+    assertThat(memory.getBytes(32, Bytes32.SIZE)).isEqualTo((WORD2));
+    assertThat(memory.getActiveWords()).isEqualTo(initialActiveWords);
+
+    assertThat(memory.getBytes(64, Bytes32.SIZE)).isEqualTo((Bytes32.ZERO));
+    assertThat(memory.getActiveWords()).isEqualTo(initialActiveWords + 1);
+  }
+
   @Test
   public void shouldClearMemoryAfterSourceDataWhenLengthGreaterThanSourceLength() {
     memory.setWord(64, WORD3);

ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/ReferenceTestEnv.java

@@ -97,7 +97,7 @@ Withdrawal asWithdrawal() {
    */
   @JsonCreator
   public ReferenceTestEnv(
-      @JsonProperty("beaconRoot") final String beaconRoot,
+      @JsonProperty("parentBeaconBlockRoot") final String beaconRoot,
       @JsonProperty("blockHashes") final Map<String, String> blockHashes,
       @JsonProperty("ommers") final List<String> _ommers,
       @JsonProperty("previousHash") final String previousHash,

evm/src/main/java/org/hyperledger/besu/evm/frame/Memory.java

@@ -18,6 +18,7 @@
 
 import java.util.Arrays;
 
+import com.google.common.annotations.VisibleForTesting;
 import org.apache.tuweni.bytes.Bytes;
 import org.apache.tuweni.bytes.Bytes32;
 import org.apache.tuweni.bytes.MutableBytes;
@@ -179,7 +180,8 @@ int getActiveBytes() {
    *
    * @return The current number of active words stored in memory.
    */
-  int getActiveWords() {
+  @VisibleForTesting
+  public int getActiveWords() {
     return activeWords;
   }
 
@@ -202,11 +204,40 @@ public Bytes getBytes(final long location, final long numBytes) {
     }
 
     final int start = asByteIndex(location);
-
     ensureCapacityForBytes(start, length);
     return Bytes.wrap(Arrays.copyOfRange(memBytes, start, start + length));
   }
 
+  /**
+   * Returns a copy of bytes by peeking into memory without expanding the active words.
+   *
+   * @param location The location in memory to start with.
+   * @param numBytes The number of bytes to get.
+   * @return A fresh copy of the bytes from memory starting at {@code location} and extending {@code
+   *     numBytes}.
+   */
+  public Bytes getBytesWithoutGrowth(final long location, final long numBytes) {
+    // Note: if length == 0, we don't require any memory expansion, whatever location is. So
+    // we must call asByteIndex(location) after this check so as it doesn't throw if the location
+    // is too big but the length is 0 (which is somewhat nonsensical, but is exercise by some
+    // tests).
+    final int length = asByteLength(numBytes);
+    if (length == 0) {
+      return Bytes.EMPTY;
+    }
+
+    final int start = asByteIndex(location);
+
+    // Arrays.copyOfRange would throw if start > memBytes.length, so just return the expected
+    // number of zeros without expanding the memory.
+    // Otherwise, just follow the happy path.
+    if (start > memBytes.length) {
+      return Bytes.wrap(new byte[(int) numBytes]);
+    } else {
+      return Bytes.wrap(Arrays.copyOfRange(memBytes, start, start + length));
+    }
+  }
+
   /**
    * Returns a copy of bytes from memory.
    *

evm/src/main/java/org/hyperledger/besu/evm/frame/MessageFrame.java

@@ -654,6 +654,17 @@ public MutableBytes readMutableMemory(final long offset, final long length) {
     return readMutableMemory(offset, length, false);
   }
 
+  /**
+   * Read bytes in memory without expanding the word capacity.
+   *
+   * @param offset The offset in memory
+   * @param length The length of the bytes to read
+   * @return The bytes in the specified range
+   */
+  public Bytes shadowReadMemory(final long offset, final long length) {
+    return memory.getBytesWithoutGrowth(offset, length);
+  }
+
   /**
    * Read bytes in memory .
    *

evm/src/main/java/org/hyperledger/besu/evm/processor/AbstractMessageProcessor.java

@@ -184,8 +184,12 @@ private void codeExecute(final MessageFrame frame, final OperationTracer operati
    * @param operationTracer the operation tracer
    */
   public void process(final MessageFrame frame, final OperationTracer operationTracer) {
-    if (operationTracer != null && frame.getMessageStackSize() > 1) {
-      operationTracer.traceContextEnter(frame);
+    if (operationTracer != null) {
+      if (frame.getState() == MessageFrame.State.NOT_STARTED) {
+        operationTracer.traceContextEnter(frame);
+      } else {
+        operationTracer.traceContextReEnter(frame);
+      }
     }
 
     if (frame.getState() == MessageFrame.State.NOT_STARTED) {
@@ -213,13 +217,13 @@ public void process(final MessageFrame frame, final OperationTracer operationTra
     }
 
     if (frame.getState() == MessageFrame.State.COMPLETED_SUCCESS) {
-      if (operationTracer != null && frame.getMessageStackSize() > 1) {
+      if (operationTracer != null) {
         operationTracer.traceContextExit(frame);
       }
       completedSuccess(frame);
     }
     if (frame.getState() == MessageFrame.State.COMPLETED_FAILED) {
-      if (operationTracer != null && frame.getMessageStackSize() > 1) {
+      if (operationTracer != null) {
         operationTracer.traceContextExit(frame);
       }
       completedFailed(frame);

evm/src/main/java/org/hyperledger/besu/evm/tracing/OperationTracer.java

@@ -102,6 +102,13 @@ default void traceEndTransaction(
    */
   default void traceContextEnter(final MessageFrame frame) {}
 
+  /**
+   * Trace the re-entry in a context from a child context
+   *
+   * @param frame the frame
+   */
+  default void traceContextReEnter(final MessageFrame frame) {}
+
   /**
    * Trace the exiting of a context
    *