Skip to content

Commit

Permalink
Merge branch 'main' into zkbesu
Browse files Browse the repository at this point in the history
# Conflicts:
#	plugin-api/build.gradle
  • Loading branch information
fab-10 committed Jul 26, 2024
2 parents 26f89ac + 51bb6c7 commit f812936
Show file tree
Hide file tree
Showing 69 changed files with 1,277 additions and 268 deletions.
7 changes: 6 additions & 1 deletion CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
### Breaking Changes
- Remove deprecated sync modes (X_SNAP and X_CHECKPOINT). Use SNAP and CHECKPOINT instead [#7309](https://github.com/hyperledger/besu/pull/7309)
- Remove PKI-backed QBFT (deprecated in 24.5.1) Other forms of QBFT remain unchanged. [#7293](https://github.com/hyperledger/besu/pull/7293)
- Do not maintain connections to PoA bootnodes [#7358](https://github.com/hyperledger/besu/pull/7358). See [#7314](https://github.com/hyperledger/besu/pull/7314) for recommended alternative behaviour.

### Additions and Improvements
- `--Xsnapsync-bft-enabled` option enables experimental support for snap sync with IBFT/QBFT permissioned Bonsai-DB chains [#7140](https://github.com/hyperledger/besu/pull/7140)
Expand All @@ -23,12 +24,16 @@
- Added EIP-7702 [#7237](https://github.com/hyperledger/besu/pull/7237)
- Implement gnark-crypto for eip-196 [#7262](https://github.com/hyperledger/besu/pull/7262)
- Add trie log pruner metrics [#7352](https://github.com/hyperledger/besu/pull/7352)
- Force bonsai-limit-trie-logs-enabled=false when sync-mode=FULL instead of startup error [#7357](https://github.com/hyperledger/besu/pull/7357)
- `--Xbonsai-parallel-tx-processing-enabled` option enables executing transactions in parallel during block processing for Bonsai nodes

- Reduce default trie log pruning window size from 30,000 to 5,000 [#7365](https://github.com/hyperledger/besu/pull/7365)
- Add option `--poa-discovery-retry-bootnodes` for PoA networks to always use bootnodes during peer refresh, not just on first start [#7314](https://github.com/hyperledger/besu/pull/7314)

### Bug fixes
- Fix `eth_call` deserialization to correctly ignore unknown fields in the transaction object. [#7323](https://github.com/hyperledger/besu/pull/7323)
- Prevent Besu from starting up with sync-mode=FULL and bonsai-limit-trie-logs-enabled=true for private networks [#7357](https://github.com/hyperledger/besu/pull/7357)
- Add 30 second timeout to trie log pruner preload [#7365](https://github.com/hyperledger/besu/pull/7365)
- Avoid executing pruner preload during trie log subcommands [#7366](https://github.com/hyperledger/besu/pull/7366)

## 24.7.0

Expand Down
15 changes: 1 addition & 14 deletions besu/src/main/java/org/hyperledger/besu/RunnerBuilder.java
Original file line number Diff line number Diff line change
Expand Up @@ -821,20 +821,7 @@ public Runner build() {
LOG.debug("added ethash observer: {}", stratumServer.get());
}

final Stream<EnodeURL> maintainedPeers;
if (besuController.getGenesisConfigOptions().isPoa()) {
// In a permissioned chain Besu should maintain connections to both static nodes and
// bootnodes, which includes retries periodically
maintainedPeers =
sanitizePeers(
network,
Stream.concat(staticNodes.stream(), bootnodes.stream()).collect(Collectors.toList()));
LOG.debug("Added bootnodes to the maintained peer list");
} else {
// In a public chain only maintain connections to static nodes
maintainedPeers = sanitizePeers(network, staticNodes);
}
maintainedPeers
sanitizePeers(network, staticNodes)
.map(DefaultPeer::fromEnodeURL)
.forEach(peerNetwork::addMaintainedConnectionPeer);

Expand Down
36 changes: 34 additions & 2 deletions besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java
Original file line number Diff line number Diff line change
Expand Up @@ -141,6 +141,7 @@
import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueStorageProviderBuilder;
import org.hyperledger.besu.ethereum.transaction.TransactionSimulator;
import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration;
import org.hyperledger.besu.ethereum.worldstate.ImmutableDataStorageConfiguration;
import org.hyperledger.besu.evm.precompile.AbstractAltBnPrecompiledContract;
import org.hyperledger.besu.evm.precompile.BigIntegerModularExponentiationPrecompiledContract;
import org.hyperledger.besu.evm.precompile.KZGPointEvalPrecompiledContract;
Expand Down Expand Up @@ -1601,7 +1602,7 @@ private void validateTransactionPoolOptions() {
}

private void validateDataStorageOptions() {
dataStorageOptions.validate(commandLine, syncMode);
dataStorageOptions.validate(commandLine);
}

private void validateRequiredOptions() {
Expand Down Expand Up @@ -2261,10 +2262,41 @@ private MiningParameters getMiningParameters() {
return miningParameters;
}

private DataStorageConfiguration getDataStorageConfiguration() {
/**
* Get the data storage configuration
*
* @return the data storage configuration
*/
public DataStorageConfiguration getDataStorageConfiguration() {
if (dataStorageConfiguration == null) {
dataStorageConfiguration = dataStorageOptions.toDomainObject();
}

if (SyncMode.FULL.equals(getDefaultSyncModeIfNotSet())
&& DataStorageFormat.BONSAI.equals(dataStorageConfiguration.getDataStorageFormat())
&& dataStorageConfiguration.getBonsaiLimitTrieLogsEnabled()) {

if (CommandLineUtils.isOptionSet(
commandLine, DataStorageOptions.BONSAI_LIMIT_TRIE_LOGS_ENABLED)) {
throw new ParameterException(
commandLine,
String.format(
"Cannot enable %s with --sync-mode=%s and --data-storage-format=%s. You must set %s or use a different sync-mode",
DataStorageOptions.BONSAI_LIMIT_TRIE_LOGS_ENABLED,
SyncMode.FULL,
DataStorageFormat.BONSAI,
DataStorageOptions.BONSAI_LIMIT_TRIE_LOGS_ENABLED + "=false"));
}

dataStorageConfiguration =
ImmutableDataStorageConfiguration.copyOf(dataStorageConfiguration)
.withBonsaiLimitTrieLogsEnabled(false);
logger.warn(
"Forcing {}, since it cannot be enabled with --sync-mode={} and --data-storage-format={}.",
DataStorageOptions.BONSAI_LIMIT_TRIE_LOGS_ENABLED + "=false",
SyncMode.FULL,
DataStorageFormat.BONSAI);
}
return dataStorageConfiguration;
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,6 @@

import org.hyperledger.besu.cli.options.CLIOptions;
import org.hyperledger.besu.cli.util.CommandLineUtils;
import org.hyperledger.besu.ethereum.eth.sync.SyncMode;
import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration;
import org.hyperledger.besu.ethereum.worldstate.ImmutableDataStorageConfiguration;
import org.hyperledger.besu.plugin.services.storage.DataStorageFormat;
Expand Down Expand Up @@ -63,7 +62,8 @@ public class DataStorageOptions implements CLIOptions<DataStorageConfiguration>
arity = "1")
private Long bonsaiMaxLayersToLoad = DEFAULT_BONSAI_MAX_LAYERS_TO_LOAD;

private static final String BONSAI_LIMIT_TRIE_LOGS_ENABLED = "--bonsai-limit-trie-logs-enabled";
/** The bonsai limit trie logs enabled option name */
public static final String BONSAI_LIMIT_TRIE_LOGS_ENABLED = "--bonsai-limit-trie-logs-enabled";

/** The bonsai trie logs pruning window size. */
public static final String BONSAI_TRIE_LOG_PRUNING_WINDOW_SIZE =
Expand Down Expand Up @@ -147,20 +147,10 @@ public static DataStorageOptions create() {
* Validates the data storage options
*
* @param commandLine the full commandLine to check all the options specified by the user
* @param syncMode the sync mode
*/
public void validate(final CommandLine commandLine, final SyncMode syncMode) {
public void validate(final CommandLine commandLine) {
if (DataStorageFormat.BONSAI == dataStorageFormat) {
if (bonsaiLimitTrieLogsEnabled) {
if (SyncMode.FULL == syncMode) {
throw new CommandLine.ParameterException(
commandLine,
String.format(
"Cannot enable %s with sync-mode %s. You must set %s or use a different sync-mode",
BONSAI_LIMIT_TRIE_LOGS_ENABLED,
SyncMode.FULL,
BONSAI_LIMIT_TRIE_LOGS_ENABLED + "=false"));
}
if (bonsaiMaxLayersToLoad < MINIMUM_BONSAI_TRIE_LOG_RETENTION_LIMIT) {
throw new CommandLine.ParameterException(
commandLine,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
import org.hyperledger.besu.ethereum.trie.diffbased.bonsai.storage.BonsaiWorldStateKeyValueStorage;
import org.hyperledger.besu.ethereum.trie.diffbased.common.trielog.TrieLogPruner;
import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration;
import org.hyperledger.besu.ethereum.worldstate.ImmutableDataStorageConfiguration;
import org.hyperledger.besu.plugin.services.storage.DataStorageFormat;

import java.io.IOException;
Expand Down Expand Up @@ -82,7 +83,14 @@ public void run() {
}

private static BesuController createBesuController() {
return parentCommand.besuCommand.buildController();
final DataStorageConfiguration config = parentCommand.besuCommand.getDataStorageConfiguration();
// disable limit trie logs to avoid preloading during subcommand execution
return parentCommand
.besuCommand
.getControllerBuilder()
.dataStorageConfiguration(
ImmutableDataStorageConfiguration.copyOf(config).withBonsaiLimitTrieLogsEnabled(false))
.build();
}

@Command(
Expand Down
26 changes: 24 additions & 2 deletions besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,7 @@
import org.hyperledger.besu.metrics.StandardMetricCategory;
import org.hyperledger.besu.metrics.prometheus.MetricsConfiguration;
import org.hyperledger.besu.plugin.data.EnodeURL;
import org.hyperledger.besu.plugin.services.storage.DataStorageFormat;
import org.hyperledger.besu.util.number.Fraction;
import org.hyperledger.besu.util.number.Percentage;
import org.hyperledger.besu.util.number.PositiveNumber;
Expand Down Expand Up @@ -1303,13 +1304,34 @@ public void bonsaiLimitTrieLogsEnabledByDefault() {
}

@Test
public void parsesInvalidDefaultBonsaiLimitTrieLogsWhenFullSyncEnabled() {
public void bonsaiLimitTrieLogsDisabledWhenFullSyncEnabled() {
parseCommand("--sync-mode=FULL");

verify(mockControllerBuilder)
.dataStorageConfiguration(dataStorageConfigurationArgumentCaptor.capture());

final DataStorageConfiguration dataStorageConfiguration =
dataStorageConfigurationArgumentCaptor.getValue();
assertThat(dataStorageConfiguration.getDataStorageFormat()).isEqualTo(BONSAI);
assertThat(dataStorageConfiguration.getBonsaiLimitTrieLogsEnabled()).isFalse();
verify(mockLogger)
.warn(
"Forcing {}, since it cannot be enabled with --sync-mode={} and --data-storage-format={}.",
"--bonsai-limit-trie-logs-enabled=false",
SyncMode.FULL,
DataStorageFormat.BONSAI);
assertThat(commandErrorOutput.toString(UTF_8)).isEmpty();
}

@Test
public void parsesInvalidWhenFullSyncAndBonsaiLimitTrieLogsExplicitlyTrue() {
parseCommand("--sync-mode=FULL", "--bonsai-limit-trie-logs-enabled=true");

Mockito.verifyNoInteractions(mockRunnerBuilder);
assertThat(commandOutput.toString(UTF_8)).isEmpty();
assertThat(commandErrorOutput.toString(UTF_8))
.contains("Cannot enable --bonsai-limit-trie-logs-enabled with sync-mode FULL");
.contains(
"Cannot enable --bonsai-limit-trie-logs-enabled with --sync-mode=FULL and --data-storage-format=BONSAI. You must set --bonsai-limit-trie-logs-enabled=false or use a different sync-mode");
}

@Test
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -55,14 +55,6 @@ public void bonsaiTrieLogsEnabled_explicitlySetToFalse() {
"--bonsai-limit-trie-logs-enabled=false");
}

@Test
public void bonsaiTrieLogPruningWindowSizeShouldBePositive2() {
internalTestFailure(
"Cannot enable --bonsai-limit-trie-logs-enabled with sync-mode FULL. You must set --bonsai-limit-trie-logs-enabled=false or use a different sync-mode",
"--sync-mode",
"FULL");
}

@Test
public void bonsaiTrieLogPruningWindowSizeShouldBePositive() {
internalTestFailure(
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,74 @@
/*
* Copyright contributors to Hyperledger Besu.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
* specific language governing permissions and limitations under the License.
*
* SPDX-License-Identifier: Apache-2.0
*/
package org.hyperledger.besu.cli.subcommands.storage;

import static org.assertj.core.api.AssertionsForInterfaceTypes.assertThat;
import static org.mockito.Mockito.atLeastOnce;
import static org.mockito.Mockito.verify;

import org.hyperledger.besu.cli.CommandTestAbstract;
import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration;

import java.util.List;

import org.junit.jupiter.api.Test;

class TrieLogSubCommandTest extends CommandTestAbstract {

@Test
void limitTrieLogsDefaultDisabledForAllSubcommands() {
assertTrieLogSubcommand("prune");
assertTrieLogSubcommand("count");
assertTrieLogSubcommand("import");
assertTrieLogSubcommand("export");
}

@Test
void limitTrieLogsDisabledForAllSubcommands() {
assertTrieLogSubcommandWithExplicitLimitEnabled("prune");
assertTrieLogSubcommandWithExplicitLimitEnabled("count");
assertTrieLogSubcommandWithExplicitLimitEnabled("import");
assertTrieLogSubcommandWithExplicitLimitEnabled("export");
}

private void assertTrieLogSubcommand(final String trieLogSubcommand) {
parseCommand("storage", "trie-log", trieLogSubcommand);
assertConfigurationIsDisabledBySubcommand();
}

private void assertTrieLogSubcommandWithExplicitLimitEnabled(final String trieLogSubcommand) {
parseCommand("--bonsai-limit-trie-logs-enabled=true", "storage", "trie-log", trieLogSubcommand);
assertConfigurationIsDisabledBySubcommand();
}

private void assertConfigurationIsDisabledBySubcommand() {
verify(mockControllerBuilder, atLeastOnce())
.dataStorageConfiguration(dataStorageConfigurationArgumentCaptor.capture());
final List<DataStorageConfiguration> configs =
dataStorageConfigurationArgumentCaptor.getAllValues();
assertThat(configs.get(0).getBonsaiLimitTrieLogsEnabled()).isTrue();
assertThat(configs.get(1).getBonsaiLimitTrieLogsEnabled()).isFalse();
}

@Test
void limitTrieLogsDefaultEnabledForBesuMainCommand() {
parseCommand();
verify(mockControllerBuilder, atLeastOnce())
.dataStorageConfiguration(dataStorageConfigurationArgumentCaptor.capture());
final List<DataStorageConfiguration> configs =
dataStorageConfigurationArgumentCaptor.getAllValues();
assertThat(configs).allMatch(DataStorageConfiguration::getBonsaiLimitTrieLogsEnabled);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
package org.hyperledger.besu.ethereum.blockcreation.txselection;

import static org.hyperledger.besu.plugin.data.TransactionSelectionResult.BLOCK_SELECTION_TIMEOUT;
import static org.hyperledger.besu.plugin.data.TransactionSelectionResult.INVALID_TX_EVALUATION_TOO_LONG;
import static org.hyperledger.besu.plugin.data.TransactionSelectionResult.SELECTED;
import static org.hyperledger.besu.plugin.data.TransactionSelectionResult.TX_EVALUATION_TOO_LONG;

Expand Down Expand Up @@ -419,11 +420,14 @@ private TransactionSelectionResult handleTransactionNotSelected(

final var pendingTransaction = evaluationContext.getPendingTransaction();

// check if this tx took too much to evaluate, and in case remove it from the pool
// check if this tx took too much to evaluate, and in case it was invalid remove it from the
// pool, otherwise penalize it.
final TransactionSelectionResult actualResult =
isTimeout.get()
? transactionTookTooLong(evaluationContext)
? TX_EVALUATION_TOO_LONG
? transactionTookTooLong(evaluationContext, selectionResult)
? selectionResult.discard()
? INVALID_TX_EVALUATION_TOO_LONG
: TX_EVALUATION_TOO_LONG
: BLOCK_SELECTION_TIMEOUT
: selectionResult;

Expand All @@ -441,16 +445,21 @@ private TransactionSelectionResult handleTransactionNotSelected(
return actualResult;
}

private boolean transactionTookTooLong(final TransactionEvaluationContext evaluationContext) {
private boolean transactionTookTooLong(
final TransactionEvaluationContext evaluationContext,
final TransactionSelectionResult selectionResult) {
final var evaluationTimer = evaluationContext.getEvaluationTimer();
if (evaluationTimer.elapsed(TimeUnit.MILLISECONDS) > blockTxsSelectionMaxTime) {
LOG.atWarn()
.setMessage(
"Transaction {} is too late for inclusion, evaluated in {} that is over the max limit of {}ms"
+ ", removing it from the pool")
"Transaction {} is too late for inclusion, with result {}, evaluated in {} that is over the max limit of {}ms"
+ ", {}")
.addArgument(evaluationContext.getPendingTransaction()::getHash)
.addArgument(selectionResult)
.addArgument(evaluationTimer)
.addArgument(blockTxsSelectionMaxTime)
.addArgument(
selectionResult.discard() ? "removing it from the pool" : "penalizing it in the pool")
.log();
return true;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -110,7 +110,8 @@ private TransactionSelectionResult transactionSelectionResultForInvalidResult(
* @return True if the invalid reason is transient, false otherwise.
*/
private boolean isTransientValidationError(final TransactionInvalidReason invalidReason) {
return invalidReason.equals(TransactionInvalidReason.GAS_PRICE_BELOW_CURRENT_BASE_FEE)
return invalidReason.equals(TransactionInvalidReason.UPFRONT_COST_EXCEEDS_BALANCE)
|| invalidReason.equals(TransactionInvalidReason.GAS_PRICE_BELOW_CURRENT_BASE_FEE)
|| invalidReason.equals(TransactionInvalidReason.NONCE_TOO_HIGH);
}
}
Loading

0 comments on commit f812936

Please sign in to comment.